Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Refactor: knowledge compilation field name refactor (#20587)

Wang Qif307aba34fa6
diff --git a/conf/infinity_mapping.json b/conf/infinity_mapping.json
index 15ba437143..424a4ace26 100644
--- a/conf/infinity_mapping.json
+++ b/conf/infinity_mapping.json
@@ -26,7 +26,6 @@
"rank_int": {"type": "integer", "default": 0},
"rank_flt": {"type": "float", "default": 0},
"available_int": {"type": "integer", "default": 1, "index_type": {"type": "secondary", "cardinality": "low"}},
- "knowledge_graph_kwd": {"type": "varchar", "default": ""},
"entities_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},
"pagerank_fea": {"type": "integer", "default": 0},
"tag_feas": {"type": "varchar", "default": "", "analyzer": "rankfeatures"},
@@ -51,7 +50,6 @@
"source_chunk_ids": {"type": "json", "default": "[]", "index_type": {"type": "secondary", "cardinality": "low"}},
"source_doc_ids": {"type": "json", "default": "[]", "index_type": {"type": "secondary", "cardinality": "low"}},
"compilation_template_ids": {"type": "json", "default": "[]", "index_type": {"type": "secondary", "cardinality": "low"}},
- "compilation_template_kind_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},
"chunk_hash_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},
"input_hash_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},
"wiki_slug_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},
@@ -66,7 +64,6 @@
"topic_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},
"plan_group_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},
"generation_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},
- "page_type_kwd": {"type": "varchar", "default": "", "analyzer": "whitespace-#"},
"entity_names_kwd": {"type": "json", "default": "[]", "index_type": {"type": "secondary", "cardinality": "low"}},
"entity_names": {"type": "json", "default": "[]", "enabled": false},
"outlinks_kwd": {"type": "json", "default": "[]", "index_type": {"type": "secondary", "cardinality": "low"}},
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.go
index d987389572..eeff56e0b0 100644
--- a/internal/engine/elasticsearch/chunk.go
+++ b/internal/engine/elasticsearch/chunk.go
@@ -709,34 +709,34 @@ func conditionTermsList(value interface{}) ([]interface{}, bool) {
return out, true
}
-// mustNotExistsClauses renders the `must_not` exclusions a condition carries.
-// The only form RAGFlow produces is must_not={"exists": <field>} (e.g.
-// hybrid_search excluding compiled products); any other shape yields nothing.
+// mustNotExistsClauses renders field and existence exclusions.
func mustNotExistsClauses(value interface{}) []map[string]interface{} {
exclusions, ok := value.(map[string]interface{})
if !ok {
return nil
}
- clauses := make([]map[string]interface{}, 0, len(exclusions))
- for field, value := range exclusions {
- if field != "exists" {
- continue
- }
- clauses = append(clauses, map[string]interface{}{
- "exists": map[string]interface{}{"field": value},
- })
+ if field, ok := exclusions["exists"]; ok && len(exclusions) == 1 {
+ return []map[string]interface{}{{"exists": map[string]interface{}{"field": field}}}
}
- return clauses
+ if query := buildBoolQueryFromConditionRaw(exclusions, nil, false, false); query != nil {
+ return []map[string]interface{}{query}
+ }
+ return nil
}
// updateChunksByQuery handles multi-document update
func (e *Engine) updateChunksByQuery(ctx context.Context, indexName string, condition map[string]interface{}, newValue map[string]interface{}) error {
common.Debug("ElasticsearchConnection.updateChunksByQuery called", zap.String("indexName", indexName))
+ condition = types.CompilationFilter(condition)
// Build bool query from condition
var mustClauses []map[string]interface{}
var mustNotClauses []map[string]interface{}
for k, v := range condition {
+ if k == "and" || k == "or" {
+ mustClauses = append(mustClauses, buildBoolQueryFromConditionRaw(map[string]interface{}{k: v}, nil, false, false))
+ continue
+ }
if k == "exists" {
mustClauses = append(mustClauses, map[string]interface{}{
"exists": map[string]interface{}{"field": v},
@@ -1010,6 +1010,7 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfac
return 0, nil
}
+ condition = types.CompilationFilter(condition)
// Build bool query from condition
var mustClauses []map[string]interface{}
var filterClauses []map[string]interface{}
@@ -1045,6 +1046,10 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfac
// Add all other conditions as filters/must/must_not
for k, v := range condition {
+ if k == "and" || k == "or" {
+ filterClauses = append(filterClauses, buildBoolQueryFromConditionRaw(map[string]interface{}{k: v}, nil, false, false))
+ continue
+ }
if k == "id" || k == "kb_id" {
continue // Already handled above
}
@@ -2138,6 +2143,10 @@ func memoryMessageStatusBool(value interface{}) bool {
// Skill indexes use status, regular chunk indexes use kb_id, and memory
// message indexes use memory_id plus message-specific storage fields.
func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string, isSkillIndex, isMemoryIndex bool) map[string]interface{} {
+ return buildBoolQueryFromConditionRaw(types.CompilationFilter(filter), kbIDs, isSkillIndex, isMemoryIndex)
+}
+
+func buildBoolQueryFromConditionRaw(filter map[string]interface{}, kbIDs []string, isSkillIndex, isMemoryIndex bool) map[string]interface{} {
var mustClauses []interface{}
var mustNotClauses []interface{}
var filterClauses []interface{}
@@ -2168,6 +2177,20 @@ func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string,
}
for k, v := range filter {
+ if k == "and" || k == "or" {
+ var children []interface{}
+ for _, child := range types.FilterClauses(v) {
+ children = append(children, buildBoolQueryFromConditionRaw(child, nil, false, isMemoryIndex))
+ }
+ if len(children) > 0 {
+ clause := map[string]interface{}{"filter": children}
+ if k == "or" {
+ clause = map[string]interface{}{"should": children, "minimum_should_match": 1}
+ }
+ filterClauses = append(filterClauses, map[string]interface{}{"bool": clause})
+ }
+ continue
+ }
if isMemoryIndex {
k = mapMemoryMessageESField(k, false)
}
@@ -2215,13 +2238,15 @@ func buildBoolQueryFromCondition(filter map[string]interface{}, kbIDs []string,
continue
}
if k == "must_not" {
- if condition, ok := v.(map[string]interface{}); ok {
- if field, ok := condition["exists"].(string); ok && field != "" {
- mustNotClauses = append(mustNotClauses, map[string]interface{}{"exists": map[string]interface{}{"field": field}})
- }
+ for _, clause := range mustNotExistsClauses(v) {
+ mustNotClauses = append(mustNotClauses, clause)
}
continue
}
+ if k == "exists" {
+ filterClauses = append(filterClauses, map[string]interface{}{"exists": map[string]interface{}{"field": v}})
+ continue
+ }
if k == "knowledge_graph_kwd" {
var graphTypeClauses []interface{}
for _, field := range []string{"type_kwd", "knowledge_graph_kwd"} {
diff --git a/internal/engine/elasticsearch/chunk_helpers_test.go b/internal/engine/elasticsearch/chunk_helpers_test.go
index d28e630866..238c8f0aa2 100644
--- a/internal/engine/elasticsearch/chunk_helpers_test.go
+++ b/internal/engine/elasticsearch/chunk_helpers_test.go
@@ -195,8 +195,8 @@ func TestDeleteChunksPreservesStringSliceCondition(t *testing.T) {
}
engine := &Engine{client: client}
deleted, err := engine.DeleteChunks(ctx, map[string]interface{}{
- "kb_id": "kb-1",
- "compile_kwd": []string{"wiki_entity", "wiki_relation"},
+ "kb_id": "kb-1",
+ "source_doc_ids": []string{"doc-a", "doc-b"},
}, "ragflow_tenant", "kb-1")
if err != nil {
t.Fatalf("DeleteChunks: %v", err)
@@ -218,7 +218,7 @@ func TestDeleteChunksPreservesStringSliceCondition(t *testing.T) {
t.Fatalf("delete query must=%#v, want one terms clause", boolQuery["must"])
}
terms := must[0].(map[string]interface{})["terms"].(map[string]interface{})
- assertEqual(t, terms["compile_kwd"], []interface{}{"wiki_entity", "wiki_relation"})
+ assertEqual(t, terms["source_doc_ids"], []interface{}{"doc-a", "doc-b"})
}
// TestDeleteChunksIDStringSlice guards against the id filter being dropped when
diff --git a/internal/engine/elasticsearch/compilation_test.go b/internal/engine/elasticsearch/compilation_test.go
new file mode 100644
index 0000000000..e5132e81bb
--- /dev/null
+++ b/internal/engine/elasticsearch/compilation_test.go
@@ -0,0 +1,162 @@
+package elasticsearch
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "sync/atomic"
+ "testing"
+
+ "github.com/elastic/go-elasticsearch/v8"
+ "ragflow/internal/common"
+ "ragflow/internal/engine/types"
+)
+
+func TestCompilationSearchReadsBothGenerationsInOneRequest(t *testing.T) {
+ if err := common.InitLogger("info", common.FileOutput{}, "compilation_test"); err != nil {
+ t.Fatal(err)
+ }
+ rows := []map[string]interface{}{
+ {"id": "new", "kb_id": "kb1", "compile_kwd": "wiki", "type_kwd": "wiki_page"},
+ {"id": "old", "kb_id": "kb1", "compile_kwd": "wiki_page"},
+ {"id": "section", "kb_id": "kb1", "compile_kwd": "wiki", "type_kwd": "wiki_section"},
+ }
+ var requests atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests.Add(1)
+ var body map[string]interface{}
+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+ t.Error(err)
+ http.Error(w, "bad query", 400)
+ return
+ }
+ var hits []map[string]interface{}
+ for _, row := range rows {
+ if matchesCompilationQuery(body["query"].(map[string]interface{}), row) {
+ hits = append(hits, map[string]interface{}{"_id": row["id"], "_source": row})
+ }
+ }
+ w.Header().Set("X-Elastic-Product", "Elasticsearch")
+ w.Header().Set("Content-Type", "application/json")
+ if err := json.NewEncoder(w).Encode(map[string]interface{}{"hits": map[string]interface{}{"total": map[string]interface{}{"value": len(hits)}, "hits": hits}}); err != nil {
+ t.Error(err)
+ }
+ }))
+ defer server.Close()
+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ eng := &Engine{client: client}
+ result, err := eng.Search(t.Context(), &types.SearchRequest{IndexNames: []string{"ragflow_t1"}, KbIDs: []string{"kb1"}, Limit: 10, Filter: map[string]interface{}{"type_kwd": "wiki_page"}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if requests.Load() != 1 || result.Total != 2 || len(result.Chunks) != 2 {
+ t.Fatalf("requests=%d, result=%+v; want one request returning new and old pages", requests.Load(), result)
+ }
+}
+
+func TestCompilationQueryMatchesNewAndOldRows(t *testing.T) {
+ for _, test := range []struct {
+ name string
+ filter, row map[string]interface{}
+ want bool
+ }{
+ {"new page", map[string]interface{}{"compile_kwd": "wiki_page"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "wiki", "type_kwd": "wiki_page"}, true},
+ {"old page", map[string]interface{}{"compile_kwd": "wiki_page"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "wiki_page"}, true},
+ {"section excluded", map[string]interface{}{"compile_kwd": "wiki_page"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "wiki", "type_kwd": "wiki_section"}, false},
+ {"tenant dataset scope", map[string]interface{}{"compile_kwd": "wiki_page"}, map[string]interface{}{"kb_id": "kb2", "compile_kwd": "wiki", "type_kwd": "wiki_page"}, false},
+ {"old graph", map[string]interface{}{"compile_kwd": "graph"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "hypergraph", "compilation_template_kind_kwd": "knowledge_graph"}, true},
+ {"canonical row", map[string]interface{}{"compilation_template_kind_kwd": "timeline"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "graph"}, false},
+ {"old inferred timeline preserves graph", map[string]interface{}{"compile_kwd": "graph"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "timeline", "compilation_template_kind_kwd": "knowledge_graph"}, true},
+ {"old inferred timeline is not a timeline template", map[string]interface{}{"compile_kwd": "timeline"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "timeline", "compilation_template_kind_kwd": "knowledge_graph"}, false},
+ {"blank stamp", map[string]interface{}{"compile_kwd": "graph"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "graph", "compilation_template_kind_kwd": ""}, true},
+ {"navigation cleanup", map[string]interface{}{"compile_kwd": "dataset_nav"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "page_index", "type_kwd": "nav_cluster"}, true},
+ {"structure survives nav cleanup", map[string]interface{}{"compile_kwd": "dataset_nav"}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "page_index", "type_kwd": "entity"}, false},
+ {"wiki graph cleanup", map[string]interface{}{"compile_kwd": []string{"wiki_entity", "wiki_relation"}}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "wiki", "type_kwd": "wiki_relation"}, true},
+ {"page survives graph cleanup", map[string]interface{}{"compile_kwd": []string{"wiki_entity", "wiki_relation"}}, map[string]interface{}{"kb_id": "kb1", "compile_kwd": "wiki", "type_kwd": "wiki_page"}, false},
+ {"mind map type reads old spelling", map[string]interface{}{"entity_type_kwd": "mind_map"}, map[string]interface{}{"kb_id": "kb1", "entity_type_kwd": "mindmap"}, true},
+ {"mind map type reads new spelling", map[string]interface{}{"entity_type_kwd": "mindmap"}, map[string]interface{}{"kb_id": "kb1", "entity_type_kwd": "mind_map"}, true},
+ {"mind map type remains scoped", map[string]interface{}{"entity_type_kwd": "mind_map"}, map[string]interface{}{"kb_id": "kb2", "entity_type_kwd": "mindmap"}, false},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ query := buildBoolQueryFromCondition(test.filter, []string{"kb1"}, false, false)
+ if got := matchesCompilationQuery(query, test.row); got != test.want {
+ t.Fatalf("query %v matched = %v, want %v", query, got, test.want)
+ }
+ })
+ }
+}
+
+func matchesCompilationQuery(query, row map[string]interface{}) bool {
+ if clause, ok := query["bool"].(map[string]interface{}); ok {
+ for _, field := range []string{"must", "filter"} {
+ for _, child := range compilationQueryChildren(clause[field]) {
+ if !matchesCompilationQuery(child, row) {
+ return false
+ }
+ }
+ }
+ for _, child := range compilationQueryChildren(clause["must_not"]) {
+ if matchesCompilationQuery(child, row) {
+ return false
+ }
+ }
+ minimum := 0
+ switch value := clause["minimum_should_match"].(type) {
+ case int:
+ minimum = value
+ case float64:
+ minimum = int(value)
+ }
+ if minimum > 0 {
+ matched := 0
+ for _, child := range compilationQueryChildren(clause["should"]) {
+ if matchesCompilationQuery(child, row) {
+ matched++
+ }
+ }
+ if matched < minimum {
+ return false
+ }
+ }
+ return true
+ }
+ if clause, ok := query["exists"].(map[string]interface{}); ok {
+ return row[clause["field"].(string)] != nil
+ }
+ for _, operator := range []string{"term", "terms"} {
+ if clause, ok := query[operator].(map[string]interface{}); ok {
+ for field, expected := range clause {
+ if items, ok := conditionTermsList(expected); ok {
+ matched := false
+ for _, value := range items {
+ matched = matched || fmt.Sprint(row[field]) == fmt.Sprint(value)
+ }
+ if !matched {
+ return false
+ }
+ } else if fmt.Sprint(row[field]) != fmt.Sprint(expected) {
+ return false
+ }
+ }
+ return true
+ }
+ }
+ return false
+}
+
+func compilationQueryChildren(value interface{}) []map[string]interface{} {
+ if children, ok := value.([]map[string]interface{}); ok {
+ return children
+ }
+ var out []map[string]interface{}
+ if children, ok := value.([]interface{}); ok {
+ for _, child := range children {
+ out = append(out, child.(map[string]interface{}))
+ }
+ }
+ return out
+}
diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.go
index b3d5507360..83f9035d09 100644
--- a/internal/engine/infinity/chunk.go
+++ b/internal/engine/infinity/chunk.go
@@ -2536,6 +2536,13 @@ func buildGraphTypeFilterCondition(value interface{}) string {
func equivalentConditionToStr(condition map[string]interface{}, tableColumns map[string]struct {
Type string
Default interface{}
+}) string {
+ return equivalentConditionToStrRaw(types.CompilationFilter(condition), tableColumns)
+}
+
+func equivalentConditionToStrRaw(condition map[string]interface{}, tableColumns map[string]struct {
+ Type string
+ Default interface{}
}) string {
if len(condition) == 0 {
return ""
@@ -2544,6 +2551,22 @@ func equivalentConditionToStr(condition map[string]interface{}, tableColumns map
var cond []string
for k, v := range condition {
+ if k == "and" || k == "or" {
+ var children []string
+ for _, child := range types.FilterClauses(v) {
+ if expr := equivalentConditionToStrRaw(child, tableColumns); expr != "" {
+ children = append(children, "("+expr+")")
+ }
+ }
+ operator := " AND "
+ if k == "or" {
+ operator = " OR "
+ }
+ if len(children) > 0 {
+ cond = append(cond, joinBalanced(children, operator))
+ }
+ continue
+ }
if k == "_id" || utility.IsEmpty(v) {
continue
}
@@ -2551,11 +2574,8 @@ func equivalentConditionToStr(condition map[string]interface{}, tableColumns map
// Handle must_not specially
if k == "must_not" {
if m, ok := v.(map[string]interface{}); ok {
- for kk, vv := range m {
- if kk == "exists" {
- // For must_not exists, use !='' since we don't have table schema
- cond = append(cond, fmt.Sprintf("NOT (%v!='')", vv))
- }
+ if expr := equivalentConditionToStrRaw(m, tableColumns); expr != "" {
+ cond = append(cond, "NOT ("+expr+")")
}
}
continue
diff --git a/internal/engine/infinity/common.go b/internal/engine/infinity/common.go
index 8d648214d1..edeb53036e 100644
--- a/internal/engine/infinity/common.go
+++ b/internal/engine/infinity/common.go
@@ -22,6 +22,7 @@ import (
"encoding/json"
"fmt"
"ragflow/internal/common"
+ "ragflow/internal/engine/types"
"strings"
"unicode"
@@ -375,10 +376,33 @@ func existsCondition(field string, tableColumns map[string]struct {
func buildFilterFromCondition(condition map[string]interface{}, tableColumns map[string]struct {
Type string
Default interface{}
+}) string {
+ return buildFilterFromConditionRaw(types.CompilationFilter(condition), tableColumns)
+}
+
+func buildFilterFromConditionRaw(condition map[string]interface{}, tableColumns map[string]struct {
+ Type string
+ Default interface{}
}) string {
var conditions []string
for k, v := range condition {
+ if k == "and" || k == "or" {
+ var children []string
+ for _, child := range types.FilterClauses(v) {
+ if expr := buildFilterFromConditionRaw(child, tableColumns); expr != "" {
+ children = append(children, "("+expr+")")
+ }
+ }
+ operator := " AND "
+ if k == "or" {
+ operator = " OR "
+ }
+ if len(children) > 0 {
+ conditions = append(conditions, joinBalanced(children, operator))
+ }
+ continue
+ }
if v == nil {
continue
}
@@ -388,18 +412,21 @@ func buildFilterFromCondition(condition map[string]interface{}, tableColumns map
// Handle must_not conditions -> NOT (...)
if k == "must_not" {
- if mustNotMap, ok := v.(map[string]interface{}); ok {
- for kk, vv := range mustNotMap {
- if kk == "exists" {
- if existsField, ok := vv.(string); ok {
- conditions = append(conditions, fmt.Sprintf("NOT (%s)", existsCondition(existsField, tableColumns)))
- }
- }
+ if m, ok := v.(map[string]interface{}); ok {
+ if expr := buildFilterFromConditionRaw(m, tableColumns); expr != "" {
+ conditions = append(conditions, "NOT ("+expr+")")
}
}
continue
}
+ if k == "exists" {
+ if field, ok := v.(string); ok {
+ conditions = append(conditions, existsCondition(field, tableColumns))
+ }
+ continue
+ }
+
if k == "knowledge_graph_kwd" {
if graphTypeCondition := buildGraphTypeFilterCondition(v); graphTypeCondition != "" {
conditions = append(conditions, graphTypeCondition)
diff --git a/internal/engine/infinity/sql_test.go b/internal/engine/infinity/sql_test.go
index f66c51987a..495d62eaf5 100644
--- a/internal/engine/infinity/sql_test.go
+++ b/internal/engine/infinity/sql_test.go
@@ -716,6 +716,16 @@ func TestKeywordFilterConditionExactForWhitespaceValues(t *testing.T) {
// would write a doc's parent_kwd while the cluster's own update matched nothing.
func TestKeywordFilterRenderingBothPaths(t *testing.T) {
const name = "Imperial Memorial and Governance Advice 59cbfbef"
+ for _, alias := range []string{"mind_map", "mindmap"} {
+ condition := map[string]interface{}{"entity_type_kwd": alias}
+ for _, query := range []string{equivalentConditionToStr(condition, nil), buildFilterFromCondition(condition, nil)} {
+ for _, value := range []string{"mind_map", "mindmap"} {
+ if !strings.Contains(query, "filter_fulltext('entity_type_kwd', '"+value+"')") {
+ t.Errorf("entity type filter %q does not match %q", query, value)
+ }
+ }
+ }
+ }
wantParent := `(parent_kwd = '` + name + `')`
parentCond := map[string]interface{}{"parent_kwd": []string{name}}
diff --git a/internal/engine/types/compilation.go b/internal/engine/types/compilation.go
new file mode 100644
index 0000000000..20aa641ce5
--- /dev/null
+++ b/internal/engine/types/compilation.go
@@ -0,0 +1,237 @@
+package types
+
+import "strings"
+
+// CanonicalCompilationKind returns the persisted knowledge compilation kind.
+func CanonicalCompilationKind(kind string) string {
+ kind = strings.ToLower(strings.ReplaceAll(strings.TrimSpace(kind), "-", "_"))
+ switch kind {
+ case "knowledge_graph", "knowledgegraph", "structure", "hypergraph", "list", "set":
+ return "graph"
+ case "mindmap":
+ return "mind_map"
+ case "pageindex":
+ return "page_index"
+ case "wiki_page", "wiki_section", "wiki_entity", "wiki_relation", "wiki_contribution_state", "wiki_map_active", "wiki_map_extract":
+ return "wiki"
+ }
+ return kind
+}
+
+func compilationString(value any) string {
+ switch value := value.(type) {
+ case string:
+ return strings.TrimSpace(value)
+ case []string:
+ if len(value) > 0 {
+ return strings.TrimSpace(value[0])
+ }
+ case []any:
+ if len(value) > 0 {
+ return compilationString(value[0])
+ }
+ }
+ return ""
+}
+
+func isCompilationKind(kind string) bool {
+ switch kind {
+ case "graph", "mind_map", "page_index", "tree", "timeline", "wiki", "session_graph", "session_essence":
+ return true
+ }
+ return false
+}
+
+// CompilationKind reads the canonical field before the stored template kind.
+func CompilationKind(row map[string]any) string {
+ kind := compilationString(row["compile_kwd"])
+ template := compilationString(row["compilation_template_kind_kwd"])
+ if isCompilationKind(kind) && template == "" {
+ return kind
+ }
+ if template != "" {
+ return CanonicalCompilationKind(template)
+ }
+ return CanonicalCompilationKind(kind)
+}
+
+func CompilationRowType(row map[string]any) string {
+ kind := compilationString(row["compile_kwd"])
+ if kind == "wiki" {
+ return compilationString(row["type_kwd"])
+ }
+ if strings.HasPrefix(kind, "wiki_") {
+ return kind
+ }
+ return compilationString(row["type_kwd"])
+}
+
+func WikiPageCategory(row map[string]any) string {
+ if compilationString(row["compile_kwd"]) == "wiki" {
+ if category := compilationString(row["entity_type_kwd"]); category != "" {
+ return category
+ }
+ }
+ return compilationString(row["page_type_kwd"])
+}
+
+func IsNavigationRow(row map[string]any) bool {
+ role := compilationString(row["type_kwd"])
+ return role == "nav_doc" || role == "nav_cluster" || compilationString(row["compile_kwd"]) == "dataset_nav"
+}
+
+func compilationValues(value any) []string {
+ switch value := value.(type) {
+ case string:
+ return []string{value}
+ case []string:
+ return value
+ case []any:
+ var out []string
+ for _, v := range value {
+ if s, ok := v.(string); ok {
+ out = append(out, s)
+ }
+ }
+ return out
+ }
+ return nil
+}
+
+// CompilationFilter builds predicates for mixed stored generations in one query.
+// Each old-field branch is guarded so it cannot override a canonical value.
+func CompilationFilter(filter map[string]any) map[string]any {
+ out := make(map[string]any, len(filter))
+ var clauses []map[string]any
+ for field, value := range filter {
+ var alternatives []map[string]any
+ switch field {
+ case "compilation_template_kind_kwd":
+ for _, kind := range compilationValues(value) {
+ alternatives = append(alternatives, compilationKindFilter(kind))
+ }
+ if len(alternatives) == 0 {
+ out[field] = value
+ }
+ case "compile_kwd":
+ changed := false
+ for _, kind := range compilationValues(value) {
+ switch {
+ case kind == "dataset_nav":
+ changed = true
+ alternatives = append(alternatives, map[string]any{"type_kwd": []string{"nav_doc", "nav_cluster"}})
+ case strings.HasPrefix(kind, "wiki_") && CanonicalCompilationKind(kind) == "wiki":
+ changed = true
+ alternatives = append(alternatives, wikiRoleFilter(kind))
+ case isCompilationKind(CanonicalCompilationKind(kind)):
+ changed = true
+ alternatives = append(alternatives, compilationKindFilter(kind))
+ default:
+ alternatives = append(alternatives, map[string]any{field: kind})
+ }
+ }
+ if !changed {
+ alternatives = nil
+ out[field] = value
+ }
+ case "type_kwd":
+ changed := false
+ for _, role := range compilationValues(value) {
+ if strings.HasPrefix(role, "wiki_") && CanonicalCompilationKind(role) == "wiki" {
+ changed = true
+ alternatives = append(alternatives, wikiRoleFilter(role))
+ } else {
+ alternatives = append(alternatives, map[string]any{field: role})
+ }
+ }
+ if !changed {
+ alternatives = nil
+ out[field] = value
+ }
+ case "entity_type_kwd":
+ out[field] = value
+ for _, entityType := range compilationValues(value) {
+ if entityType == "mind_map" || entityType == "mindmap" {
+ entityTypes := append([]string(nil), compilationValues(value)...)
+ entityTypes = append(entityTypes, "mind_map", "mindmap")
+ out[field] = entityTypes
+ break
+ }
+ }
+ case "page_type_kwd":
+ alternatives = append(alternatives,
+ map[string]any{"compile_kwd": "wiki", "entity_type_kwd": value},
+ map[string]any{"page_type_kwd": value, "must_not": map[string]any{"compile_kwd": "wiki", "exists": "entity_type_kwd"}})
+ case "and", "or":
+ var children []map[string]any
+ for _, child := range FilterClauses(value) {
+ children = append(children, CompilationFilter(child))
+ }
+ out[field] = children
+ default:
+ out[field] = value
+ }
+ if len(alternatives) > 0 {
+ clauses = append(clauses, map[string]any{"or": alternatives})
+ }
+ }
+ if len(clauses) > 0 {
+ if len(out) > 0 {
+ clauses = append(clauses, out)
+ }
+ return map[string]any{"and": clauses}
+ }
+ return out
+}
+
+func wikiRoleFilter(role string) map[string]any {
+ return map[string]any{"or": []map[string]any{
+ {"compile_kwd": "wiki", "type_kwd": role},
+ {"compile_kwd": role},
+ }}
+}
+
+func compilationKindFilter(kind string) map[string]any {
+ canonical := CanonicalCompilationKind(kind)
+ aliases := []string{canonical}
+ switch canonical {
+ case "graph":
+ aliases = []string{"graph", "knowledge_graph", "knowledgegraph", "structure"}
+ case "mind_map":
+ aliases = []string{"mind_map", "mindmap"}
+ case "page_index":
+ aliases = []string{"page_index", "pageindex"}
+ }
+ oldKeywords := append([]string(nil), aliases...)
+ if canonical == "graph" {
+ oldKeywords = append(oldKeywords, "hypergraph", "list", "set")
+ }
+ if canonical == "wiki" {
+ oldKeywords = append(oldKeywords, "wiki_page", "wiki_section", "wiki_entity", "wiki_relation", "wiki_contribution_state", "wiki_map_active", "wiki_map_extract")
+ }
+ missingTemplateKind := []map[string]any{
+ {"must_not": map[string]any{"exists": "compilation_template_kind_kwd"}},
+ {"compilation_template_kind_kwd": []string{""}},
+ }
+ return map[string]any{"or": []map[string]any{
+ {"compile_kwd": canonical, "or": missingTemplateKind},
+ {"compilation_template_kind_kwd": aliases},
+ {"compile_kwd": oldKeywords, "or": missingTemplateKind},
+ }}
+}
+
+func FilterClauses(value any) []map[string]any {
+ switch value := value.(type) {
+ case []map[string]any:
+ return value
+ case []any:
+ var out []map[string]any
+ for _, child := range value {
+ if child, ok := child.(map[string]any); ok {
+ out = append(out, child)
+ }
+ }
+ return out
+ }
+ return nil
+}
diff --git a/internal/engine/types/compilation_test.go b/internal/engine/types/compilation_test.go
new file mode 100644
index 0000000000..694afa5b2a
--- /dev/null
+++ b/internal/engine/types/compilation_test.go
@@ -0,0 +1,97 @@
+package types
+
+import "testing"
+
+func TestCompilationKindPrefersCanonicalValue(t *testing.T) {
+ for _, test := range []struct {
+ row map[string]any
+ want string
+ }{
+ {map[string]any{"compile_kwd": "graph"}, "graph"},
+ {map[string]any{"compile_kwd": "timeline", "compilation_template_kind_kwd": "knowledge_graph"}, "graph"},
+ {map[string]any{"compile_kwd": "list", "compilation_template_kind_kwd": "page_index"}, "page_index"},
+ {map[string]any{"compile_kwd": "hypergraph"}, "graph"},
+ {map[string]any{"compile_kwd": []string{"mindmap"}}, "mind_map"},
+ {map[string]any{"compile_kwd": "wiki_section"}, "wiki"},
+ } {
+ if got := CompilationKind(test.row); got != test.want {
+ t.Errorf("CompilationKind(%v) = %q, want %q", test.row, got, test.want)
+ }
+ }
+}
+
+func TestCompilationFilterReadsMixedGenerations(t *testing.T) {
+ for _, test := range []struct {
+ name string
+ filter, row map[string]any
+ want bool
+ }{
+ {"new graph", map[string]any{"compile_kwd": "graph", "kb_id": "kb1"}, map[string]any{"compile_kwd": "graph", "kb_id": "kb1"}, true},
+ {"old graph", map[string]any{"compile_kwd": "graph"}, map[string]any{"compile_kwd": "hypergraph", "compilation_template_kind_kwd": "knowledge_graph"}, true},
+ {"old inferred graph", map[string]any{"compile_kwd": "graph"}, map[string]any{"compile_kwd": "hypergraph"}, true},
+ {"old page index inferred list", map[string]any{"compile_kwd": "page_index"}, map[string]any{"compile_kwd": "list", "compilation_template_kind_kwd": "page_index"}, true},
+ {"no category mixing", map[string]any{"compile_kwd": "graph"}, map[string]any{"compile_kwd": "list", "compilation_template_kind_kwd": "page_index"}, false},
+ {"canonical row has no old stamp", map[string]any{"compilation_template_kind_kwd": "timeline"}, map[string]any{"compile_kwd": "graph"}, false},
+ {"old stamp preserves original kind", map[string]any{"compile_kwd": "graph"}, map[string]any{"compile_kwd": "timeline", "compilation_template_kind_kwd": "knowledge_graph"}, true},
+ {"old subtype is not original kind", map[string]any{"compile_kwd": "timeline"}, map[string]any{"compile_kwd": "timeline", "compilation_template_kind_kwd": "knowledge_graph"}, false},
+ {"dataset stays scoped", map[string]any{"compile_kwd": "graph", "kb_id": "kb1"}, map[string]any{"compile_kwd": "graph", "kb_id": "kb2"}, false},
+ {"new wiki page", map[string]any{"compile_kwd": "wiki_page"}, map[string]any{"compile_kwd": "wiki", "type_kwd": "wiki_page"}, true},
+ {"old wiki page", map[string]any{"type_kwd": "wiki_page"}, map[string]any{"compile_kwd": "wiki_page"}, true},
+ {"section excluded", map[string]any{"compile_kwd": "wiki_page"}, map[string]any{"compile_kwd": "wiki", "type_kwd": "wiki_section"}, false},
+ {"state excluded", map[string]any{"compile_kwd": "wiki_page"}, map[string]any{"compile_kwd": "wiki", "type_kwd": "wiki_map_active"}, false},
+ {"new navigation", map[string]any{"compile_kwd": "dataset_nav"}, map[string]any{"compile_kwd": "page_index", "type_kwd": "nav_doc"}, true},
+ {"structure excluded", map[string]any{"compile_kwd": "dataset_nav"}, map[string]any{"compile_kwd": "page_index", "type_kwd": "entity"}, false},
+ {"new category", map[string]any{"page_type_kwd": "concept"}, map[string]any{"compile_kwd": "wiki", "entity_type_kwd": "concept"}, true},
+ {"old category", map[string]any{"page_type_kwd": "concept"}, map[string]any{"compile_kwd": "wiki_page", "page_type_kwd": "concept"}, true},
+ {"category wins", map[string]any{"page_type_kwd": "entity"}, map[string]any{"compile_kwd": "wiki", "entity_type_kwd": "concept", "page_type_kwd": "entity"}, false},
+ {"mind map type reads old value", map[string]any{"entity_type_kwd": "mind_map"}, map[string]any{"entity_type_kwd": "mindmap"}, true},
+ {"old mind map type reads new value", map[string]any{"entity_type_kwd": []string{"mindmap"}}, map[string]any{"entity_type_kwd": "mind_map"}, true},
+ {"mind map type excludes other types", map[string]any{"entity_type_kwd": []any{"mind_map"}}, map[string]any{"entity_type_kwd": "person"}, false},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ if got := matchesCompilationFilter(test.row, CompilationFilter(test.filter)); got != test.want {
+ t.Fatalf("matched = %v, want %v", got, test.want)
+ }
+ })
+ }
+}
+
+func matchesCompilationFilter(row, filter map[string]any) bool {
+ for field, value := range filter {
+ switch field {
+ case "and":
+ for _, child := range FilterClauses(value) {
+ if !matchesCompilationFilter(row, child) {
+ return false
+ }
+ }
+ case "or":
+ matched := false
+ for _, child := range FilterClauses(value) {
+ matched = matched || matchesCompilationFilter(row, child)
+ }
+ if !matched {
+ return false
+ }
+ case "must_not":
+ if matchesCompilationFilter(row, value.(map[string]any)) {
+ return false
+ }
+ case "exists":
+ if compilationString(row[value.(string)]) == "" {
+ return false
+ }
+ default:
+ matched := false
+ for _, actual := range compilationValues(row[field]) {
+ for _, expected := range compilationValues(value) {
+ matched = matched || actual == expected
+ }
+ }
+ if !matched {
+ return false
+ }
+ }
+ }
+ return true
+}
diff --git a/internal/ingestion/component/knowledge_compiler/common/types.go b/internal/ingestion/component/knowledge_compiler/common/types.go
index a7070a7471..34c2136bb6 100644
--- a/internal/ingestion/component/knowledge_compiler/common/types.go
+++ b/internal/ingestion/component/knowledge_compiler/common/types.go
@@ -127,10 +127,7 @@ type Product struct {
DocID string
TenantID string
Variant Variant
- // Kind is the original compilation_template.kind (e.g. "page_index"),
- // distinct from Variant (the collapsed Go strategy, e.g. "structure"). It
- // is stamped onto compilation_template_kind_kwd so the document-structure
- // endpoint can group rows by the true template kind.
+ // Kind identifies the producing template's knowledge compilation kind.
Kind string
// TemplateID is the compilation template that produced this row.
TemplateID string
diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.go
index 892a352488..49cbeae434 100644
--- a/internal/ingestion/component/knowledge_compiler/component.go
+++ b/internal/ingestion/component/knowledge_compiler/component.go
@@ -6,12 +6,14 @@ package knowledge_compiler
import (
"context"
+ "encoding/json"
"fmt"
"sort"
"strings"
"ragflow/internal/agent/runtime"
clog "ragflow/internal/common"
+ enginetypes "ragflow/internal/engine/types"
"ragflow/internal/ingestion/component/globals"
"ragflow/internal/ingestion/component/knowledge_compiler/common"
"ragflow/internal/ingestion/component/knowledge_compiler/mindmap"
@@ -367,9 +369,8 @@ func overlayTemplateConfig(param *common.Param, cfg map[string]any) {
}
}
-// kindOrVariant returns the original template kind when present (the true
-// compilation_template.kind, e.g. "page_index"), otherwise the collapsed Go
-// variant. It drives the compilation_template_kind_kwd stamp.
+// kindOrVariant returns the producing template kind, or the execution variant
+// when a product has no template kind.
func kindOrVariant(p common.Product) string {
if p.Kind != "" {
return p.Kind
@@ -377,20 +378,6 @@ func kindOrVariant(p common.Product) string {
return string(p.Variant)
}
-// variantCompileKWD maps each Go variant to the compile_kwd discriminator value
-// Python writes into ES (rag/advanced_rag/knowlege_compile). It is the primary
-// key that distinguishes compiled knowledge units from ordinary chunks and
-// routes retrieval-side filters. The wiki value MUST be "wiki_page" (Python's
-// canonical WIKI_PAGE_COMPILE_KWD in wiki.py:1661 / wiki_incremental.py:44 /
-// dataset_wiki_generator.py:108) so Go-produced wiki pages are visible to the
-// artifact API (dataset_artifact_service.go reads compile_kwd="wiki_page").
-var variantCompileKWD = map[common.Variant]string{
- common.VariantStructure: "structure",
- common.VariantWiki: "wiki_page",
- common.VariantTree: "tree",
- common.VariantMindmap: "mindmap",
-}
-
// productsToChunkDocs converts the internal compiled Product rows into
// schema.ChunkDoc values aligned to conf/infinity_mapping.json. Product.Meta is
// not persisted: only engine columns are written, because Infinity rejects an
@@ -434,29 +421,20 @@ func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {
return nil, err
}
}
- compileKWD := variantCompileKWD[p.Variant]
- // A variant may pin a finer-grained compile_kwd per row via Meta
- // (structure stamps the inferred compile kind — list/set/hypergraph —
- // mirroring Python's per-row autotype stamp).
- if v := metaString(p.Meta, "compile_kwd"); v != "" {
- compileKWD = v
- }
- // Wiki sub-parts: sections get their own compile_kwd so that a page
- // search on compile_kwd="wiki_page" returns pages only (page.go emits
- // both kind:"page" and kind:"section" rows under VariantWiki). This is
- // the schema-backed page/section discriminator: "wiki_page" == page,
- // "wiki_section" == a page sub-section.
- if p.Variant == common.VariantWiki && metaString(p.Meta, "kind") == "section" && compileKWD == "wiki_page" {
- compileKWD = "wiki_section"
- }
- if compileKWD == "" {
- compileKWD = string(p.Variant)
- }
+ compileKWD := enginetypes.CanonicalCompilationKind(kindOrVariant(p))
if err := doc.SetExtraValue("compile_kwd", compileKWD); err != nil {
return nil, err
}
- if err := doc.SetExtraValue("compilation_template_kind_kwd", kindOrVariant(p)); err != nil {
- return nil, err
+ if p.Variant == common.VariantStructure {
+ if inferred := metaString(p.Meta, "compile_kwd"); inferred != "" {
+ extra, err := json.Marshal(map[string]string{"compile_type": inferred})
+ if err != nil {
+ return nil, err
+ }
+ if err := doc.SetExtraValue("extra", string(extra)); err != nil {
+ return nil, err
+ }
+ }
}
// scope_kwd marks doc/dataset-level rows (B8/O1=B). A doc-level compiled
// product is always a per-document (scope="doc") input to the dataset-level
@@ -516,7 +494,12 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
return applyStructureGraphColumns(doc, p, kind)
case common.VariantWiki:
- // One artifact_page row per wiki page; section rows reuse the same
+ if kind != "" {
+ if err := doc.SetExtraValue("type_kwd", "wiki_"+kind); err != nil {
+ return err
+ }
+ }
+ // Page and section roles reuse the same
// page-level columns so retrieval-side filters work uniformly.
// Match the Python writer contract (api/db/db_models.py slug_kwd):
// slug_kwd stores the full "<page_type>/<slug>" form, so retrieval
@@ -543,7 +526,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
setTitleTokens(doc, v)
}
if v := metaString(p.Meta, "page_type"); v != "" {
- if err := doc.SetExtraValue("page_type_kwd", v); err != nil {
+ if err := doc.SetExtraValue("entity_type_kwd", v); err != nil {
return err
}
}
diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.go
index e26a3bcc1f..6a784fcc86 100644
--- a/internal/ingestion/component/knowledge_compiler/component_test.go
+++ b/internal/ingestion/component/knowledge_compiler/component_test.go
@@ -353,7 +353,7 @@ func TestKnowledgeCompiler_Wiki_EndToEnd(t *testing.T) {
// wiki produces a "page" chunk (kind carried by compile_kwd=wiki_page).
foundPage := false
for _, c := range chunks {
- if kind, _ := c["compile_kwd"].(string); kind == "wiki_page" {
+ if kind, _ := c["type_kwd"].(string); kind == "wiki_page" {
foundPage = true
}
}
@@ -431,7 +431,7 @@ func invokeWikiCompiler(t *testing.T, inputs map[string]any) []map[string]any {
func hasWikiPageChunk(chunks []map[string]any) bool {
for _, cm := range chunks {
- if kind, _ := cm["compile_kwd"].(string); kind == "wiki_page" {
+ if kind, _ := cm["type_kwd"].(string); kind == "wiki_page" {
return true
}
}
@@ -574,6 +574,9 @@ func TestKnowledgeCompiler_Mindmap_EndToEnd(t *testing.T) {
kind, _ := c["type_kwd"].(string)
if kind == "entity" {
entityCount++
+ if c["entity_type_kwd"] != "mind_map" {
+ t.Fatalf("mindmap entity_type_kwd = %v, want mind_map", c["entity_type_kwd"])
+ }
if _, ok := c["name_kwd"]; !ok {
t.Fatalf("mindmap entity chunk missing name_kwd: %+v", c)
}
@@ -629,7 +632,7 @@ func TestKnowledgeCompiler_EmitsChunks(t *testing.T) {
cm := r.(map[string]any)
// Structure rows carry the inferred compile kind (hypergraph here),
// mirroring Python's per-row autotype stamp.
- if ck, _ := cm["compile_kwd"].(string); ck == "hypergraph" {
+ if ck, _ := cm["compile_kwd"].(string); ck == "graph" {
compiled++
if cm["id"] == nil || cm["id"] == "" {
t.Fatal("compiled chunk missing id")
@@ -674,7 +677,7 @@ func TestKnowledgeCompiler_TemplateIDsAndProvenance(t *testing.T) {
}
for _, r := range out["chunks"].([]any) {
cm := r.(map[string]any)
- if ck, _ := cm["compile_kwd"].(string); ck != "hypergraph" {
+ if ck, _ := cm["compile_kwd"].(string); ck != "graph" {
continue
}
// Every compiled chunk must carry the resolved template id (one per
@@ -877,7 +880,7 @@ func TestKnowledgeCompiler_Wiki_UpdateMergesExistingPage(t *testing.T) {
if !ok {
continue
}
- if cm["compile_kwd"] == "wiki_page" && cm["slug_kwd"] == "entity/person/alpha" {
+ if cm["type_kwd"] == "wiki_page" && cm["slug_kwd"] == "entity/person/alpha" {
page = cm
break
}
@@ -1203,7 +1206,7 @@ func TestKnowledgeCompiler_GroupIDsResolvedToTemplateIDs(t *testing.T) {
cm := r.(map[string]any)
// No parser_config is supplied, so InferType returns "list" and the
// structure variant stamps compile_kwd="list" (not "structure").
- if cm["compile_kwd"] != "list" {
+ if cm["compile_kwd"] != "graph" {
continue
}
checked++
@@ -1444,7 +1447,7 @@ func TestKnowledgeCompiler_BuildInputsAcceptsMapSliceChunks(t *testing.T) {
// TestProductsToChunkDocs_PageVsSectionCompileKWD locks the page/section
// discriminator (compile_kwd) and the schema-column contract: the page body goes
// to md_with_weight and no Go-only field (kc_*, tenant_id) is emitted.
-func TestProductsToChunkDocs_PageVsSectionCompileKWD(t *testing.T) {
+func TestProductsToChunkDocs_WikiPageAndSectionRoles(t *testing.T) {
page := common.Product{
ID: "page-id", DocID: "d1", TenantID: "t1", Variant: common.VariantWiki,
Content: "# Alpha\n\nBody", ParentID: "",
@@ -1462,9 +1465,17 @@ func TestProductsToChunkDocs_PageVsSectionCompileKWD(t *testing.T) {
var pageKWD, sectionKWD string
var sectionParent, pageBody, sectionBody string
for _, d := range docs {
+ if compiled, _ := d.GetExtraString("compile_kwd"); compiled != "wiki" {
+ t.Fatalf("compile_kwd = %q, want wiki", compiled)
+ }
+ for _, field := range []string{"compilation_template_kind_kwd", "page_type_kwd"} {
+ if _, exists := d.Extra[field]; exists {
+ t.Fatalf("retired field %s was written", field)
+ }
+ }
// compile_kwd IS the page/section discriminator (wiki_page /
// wiki_section); there is no separate kind column.
- kind, _ := d.GetExtraString("compile_kwd")
+ kind, _ := d.GetExtraString("type_kwd")
if kind == "wiki_page" {
pageKWD = kind
pageBody, _ = d.GetExtraString("md_with_weight")
diff --git a/internal/ingestion/component/knowledge_compiler/golden_test.go b/internal/ingestion/component/knowledge_compiler/golden_test.go
index acbd2b621f..88d828fc9c 100644
--- a/internal/ingestion/component/knowledge_compiler/golden_test.go
+++ b/internal/ingestion/component/knowledge_compiler/golden_test.go
@@ -210,7 +210,7 @@ func TestGolden_Wiki_ProductCount(t *testing.T) {
foundPage := false
for _, p := range prods {
// compile_kwd=wiki_page is the page/section discriminator.
- if kwd, _ := p.GetExtraString("compile_kwd"); kwd == "wiki_page" {
+ if kwd, _ := p.GetExtraString("type_kwd"); kwd == "wiki_page" {
foundPage = true
}
}
diff --git a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go
index 2376bb2949..97d527342f 100644
--- a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go
+++ b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go
@@ -149,7 +149,7 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
//
// Mapping:
// - each node (including the root) → an entity product (kind="entity",
-// name = node id, type = "mindmap").
+// name = node id, type = "mind_map").
// - each parent→child edge → a relation product (kind="relation",
// from = parent id, to = child id, type = "related", matching Python's
// mindmap structure-graph projection).
@@ -165,12 +165,12 @@ func treeToProducts(tenantID, docID string, root *utility.Node, fallbackSourceCh
seen := map[string]bool{}
// Entity: root node.
rootSourceChunkIDs := nodeSourceChunkIDs(root, fallbackSourceChunkIDs)
- rootPayload := map[string]any{"name": root.ID, "type": "mindmap"}
+ rootPayload := map[string]any{"name": root.ID, "type": "mind_map"}
rootMeta := map[string]any{
"kind": "entity",
"name": root.ID,
- "entity_type": "mindmap",
- "compile_kwd": "mindmap",
+ "entity_type": "mind_map",
+ "compile_kwd": "mind_map",
}
if len(rootSourceChunkIDs) > 0 {
rootPayload["source_chunk_ids"] = rootSourceChunkIDs
@@ -203,12 +203,12 @@ func treeToProducts(tenantID, docID string, root *utility.Node, fallbackSourceCh
if !seen[child.ID] {
seen[child.ID] = true
childSourceChunkIDs := nodeSourceChunkIDs(child, fallbackSourceChunkIDs)
- childPayload := map[string]any{"name": child.ID, "type": "mindmap"}
+ childPayload := map[string]any{"name": child.ID, "type": "mind_map"}
childMeta := map[string]any{
"kind": "entity",
"name": child.ID,
- "entity_type": "mindmap",
- "compile_kwd": "mindmap",
+ "entity_type": "mind_map",
+ "compile_kwd": "mind_map",
}
if len(childSourceChunkIDs) > 0 {
childPayload["source_chunk_ids"] = childSourceChunkIDs
@@ -242,7 +242,7 @@ func treeToProducts(tenantID, docID string, root *utility.Node, fallbackSourceCh
"from": p.parent,
"to": child.ID,
"relation_type": "related",
- "compile_kwd": "mindmap",
+ "compile_kwd": "mind_map",
},
})
}
diff --git a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.go b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.go
index 7afc3944b5..99f3e95ce3 100644
--- a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.go
+++ b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.go
@@ -62,6 +62,9 @@ func TestTreeToProducts_ParentLinks(t *testing.T) {
entCount, relCount := 0, 0
fromTo := map[string]bool{}
for _, p := range products {
+ if p.Meta["compile_kwd"] != "mind_map" {
+ t.Errorf("product %s compile_kwd = %v, want mind_map", p.ID, p.Meta["compile_kwd"])
+ }
kind, _ := p.Meta["kind"].(string)
switch kind {
case "entity":
@@ -70,14 +73,11 @@ func TestTreeToProducts_ParentLinks(t *testing.T) {
if err := json.Unmarshal([]byte(p.Content), &payload); err != nil {
t.Fatalf("entity %q content is not JSON: %v", p.Meta["name"], err)
}
- if payload["name"] != p.Meta["name"] || payload["type"] != "mindmap" {
- t.Errorf("entity payload = %v, want name=%v and type=mindmap", payload, p.Meta["name"])
- }
- if p.Meta["entity_type"] != "mindmap" {
- t.Errorf("entity %v type = %v, want mindmap", p.Meta["name"], p.Meta["entity_type"])
+ if payload["name"] != p.Meta["name"] || payload["type"] != "mind_map" {
+ t.Errorf("entity payload = %v, want name=%v and type=mind_map", payload, p.Meta["name"])
}
- if p.Meta["compile_kwd"] != "mindmap" {
- t.Errorf("entity %v compile_kwd = %v, want mindmap", p.Meta["name"], p.Meta["compile_kwd"])
+ if p.Meta["entity_type"] != "mind_map" {
+ t.Errorf("entity %v type = %v, want mind_map", p.Meta["name"], p.Meta["entity_type"])
}
case "relation":
relCount++
diff --git a/internal/ingestion/component/knowledge_compiler/storage_test.go b/internal/ingestion/component/knowledge_compiler/storage_test.go
new file mode 100644
index 0000000000..1cb27a3fd3
--- /dev/null
+++ b/internal/ingestion/component/knowledge_compiler/storage_test.go
@@ -0,0 +1,54 @@
+package knowledge_compiler
+
+import (
+ "encoding/json"
+ "testing"
+
+ "ragflow/internal/ingestion/component/knowledge_compiler/common"
+)
+
+func TestProductsToChunkDocsCanonicalKinds(t *testing.T) {
+ for _, test := range []struct {
+ kind string
+ variant common.Variant
+ want string
+ }{
+ {"knowledge_graph", common.VariantStructure, "graph"},
+ {"graph", common.VariantStructure, "graph"},
+ {"mind_map", common.VariantMindmap, "mind_map"},
+ {"mindmap", common.VariantMindmap, "mind_map"},
+ {"page_index", common.VariantStructure, "page_index"},
+ {"tree", common.VariantTree, "tree"},
+ {"timeline", common.VariantStructure, "timeline"},
+ {"wiki", common.VariantWiki, "wiki"},
+ } {
+ t.Run(test.kind, func(t *testing.T) {
+ docs, err := productsToChunkDocs([]common.Product{{ID: "row", Kind: test.kind, Variant: test.variant, Content: "summary",
+ Meta: map[string]any{"compile_kwd": "list", "kind": "entity", "page_type": "concept"}}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ row := docs[0].ToMap()
+ if row["compile_kwd"] != test.want {
+ t.Fatalf("compile_kwd = %v, want %s", row["compile_kwd"], test.want)
+ }
+ for _, field := range []string{"compilation_template_kind_kwd", "page_type_kwd"} {
+ if _, exists := row[field]; exists {
+ t.Fatalf("retired field %s was written", field)
+ }
+ }
+ if test.variant == common.VariantStructure {
+ var extra map[string]string
+ if err := json.Unmarshal([]byte(row["extra"].(string)), &extra); err != nil {
+ t.Fatal(err)
+ }
+ if extra["compile_type"] != "list" {
+ t.Fatalf("compile type = %v", extra)
+ }
+ }
+ if test.variant == common.VariantWiki && (row["type_kwd"] != "wiki_entity" || row["entity_type_kwd"] != "concept") {
+ t.Fatalf("wiki role/category = %v/%v", row["type_kwd"], row["entity_type_kwd"])
+ }
+ })
+ }
+}
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go
index 08810af73d..279c65371d 100644
--- a/internal/ingestion/knowledge_compile/consumer.go
+++ b/internal/ingestion/knowledge_compile/consumer.go
@@ -28,6 +28,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/engine"
+ enginetypes "ragflow/internal/engine/types"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
"ragflow/internal/service/file"
"ragflow/internal/service/nav"
@@ -377,8 +378,8 @@ func (c *Consumer) RebuildDataset(ctx context.Context, tenant, kb, mode string)
// carry the doc's compile variants so the consumer can route to the correct
// dataset-level path (a nil/empty Variants would fall back to the legacy
// unified path and tree nav could not be rebuilt). The variants are recovered
- // from the doc-level compiled products' authoritative `compilation_template_kind_kwd`
- // (via KindToVariant, O2a whitelist hard-fail), not re-derived from `compile_kwd`.
+ // from the doc-level compiled products' canonical compilation kind
+ // via the shared variant mapping.
docs, err := c.docLister(ctx, tenant, kb)
if err != nil {
return fmt.Errorf("knowledge_compile: rebuild list docs: %w", err)
@@ -421,7 +422,7 @@ func defaultDocLister(ctx context.Context, tenant, kb string) ([]string, error)
// recoverDocTypes reconstructs the compile-type and frontend task-type sets a
// document produced by reading its persisted doc-level products and mapping
-// each product's authoritative `compilation_template_kind_kwd` through the
+// each product's canonical compilation kind through the
// shared kind mappings.
// It is used by RebuildDataset (B1) so the republished completed events carry
// the variants needed to route the dataset-level re-compile (tree nav, structure
@@ -444,7 +445,7 @@ func (c *Consumer) recoverDocTypes(ctx context.Context, tenant, kb, docID string
seenTaskTypes := map[string]struct{}{}
var variants []string
for _, p := range products {
- // Authoritative: product.Kind (compilation_template_kind_kwd) when present.
+ // Use the compilation kind restored by the reader.
if p.Kind != "" {
v, err := kccommon.KindToVariant(p.Kind)
if err != nil {
@@ -884,7 +885,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb, token string, e
for _, cand := range candidates {
v := string(cand.Variant)
byVariant[v]++
- if v == "wiki_page" {
+ if cand.Variant == kccommon.VariantWiki {
wikiPageSlugs = append(wikiPageSlugs, candidateIdentity(cand))
}
}
@@ -1523,9 +1524,11 @@ func navInputFromProducts(kb string, products []kccommon.Product) []nav.UpsertDo
// otherwise use the folded structure/page-index summary. This makes the
// result independent of product iteration order.
if strings.TrimSpace(a.treeSummary) != "" {
+ a.in.CompileKind = "tree"
a.in.Summary = a.treeSummary
a.in.Embedd = a.treeEmbedd
} else {
+ a.in.CompileKind = "page_index"
a.in.Summary = strings.Join(a.structureLines, "\n")
}
if strings.TrimSpace(a.in.Summary) == "" {
@@ -1585,16 +1588,9 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,
if p.Variant != kccommon.VariantStructure && p.Variant != kccommon.VariantMindmap {
continue
}
- // Dataset rows carry the SAME compile_kwd as the doc rows (the inferred
- // compile type / autotype: "hypergraph"/"timeline"/"mindmap"/"list"/…),
- // mirroring Python dataset_structure_merger._do_build which passes the doc
- // row's compile_kwd through verbatim. The template kind (p.Kind, restored
- // from compilation_template_kind_kwd) is stored on the SEPARATE
- // compilation_template_kind_kwd field and is what read/delete paths match
- // on — NOT compile_kwd. Do not rewrite compile_kwd to the template kind:
- // that would diverge from Python (which never does) and split doc vs
- // dataset rows.
- ckwd := metaString(p.Meta, "compile_kwd")
+ // Dataset rows retain the producing compilation kind. Extraction shape
+ // is separate metadata and does not determine the stored kind.
+ ckwd := enginetypes.CanonicalCompilationKind(kccommon.FirstNonEmpty(p.Kind, metaString(p.Meta, "compile_kwd")))
if ckwd == "" {
ckwd = compileKwdForVariant(p.Variant)
}
@@ -1619,7 +1615,7 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,
k := rkey{template: template, from: normalizedStructureEntityName(from), typ: structureRelationType(relType), to: normalizedStructureEntityName(to), ckwd: ckwd}
b := relByKey[k]
if b == nil {
- b = &StructureBucket{Name: from + " -> " + to, Type: "relation", FromEntity: from, ToEntity: to, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind, RelationType: relType}
+ b = &StructureBucket{Name: from + " -> " + to, Type: "relation", FromEntity: from, ToEntity: to, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind, CompileType: metaString(p.Meta, "compile_type"), RelationType: relType}
relByKey[k] = b
}
appendBucket(b, p)
@@ -1637,7 +1633,7 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,
k := ekey{template: template, name: normalizedStructureEntityName(name), ckwd: ckwd}
b := entByKey[k]
if b == nil {
- b = &StructureBucket{Name: name, Type: typ, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind}
+ b = &StructureBucket{Name: name, Type: typ, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind, CompileType: metaString(p.Meta, "compile_type")}
entByKey[k] = b
} else {
b.Type = preferredStructureEntityType(b.Type, typ)
@@ -1675,6 +1671,9 @@ func normalizedStructureEntityName(name string) string {
func preferredStructureEntityType(existing, incoming string) string {
existing = strings.TrimSpace(existing)
incoming = strings.TrimSpace(incoming)
+ if existing == "mindmap" && incoming == "mind_map" {
+ return incoming
+ }
if existing == "" || strings.EqualFold(existing, "other") {
if incoming != "" && !strings.EqualFold(incoming, "other") {
return incoming
@@ -1690,7 +1689,7 @@ func structureTemplateIdentity(templateID, templateKind string) string {
if templateID = strings.TrimSpace(templateID); templateID != "" {
return templateID
}
- return strings.TrimSpace(templateKind)
+ return enginetypes.CanonicalCompilationKind(templateKind)
}
// appendBucket folds a structure product into a bucket: concatenates its
diff --git a/internal/ingestion/knowledge_compile/nav_dispatch_test.go b/internal/ingestion/knowledge_compile/nav_dispatch_test.go
index 36e146ba98..9763019af5 100644
--- a/internal/ingestion/knowledge_compile/nav_dispatch_test.go
+++ b/internal/ingestion/knowledge_compile/nav_dispatch_test.go
@@ -60,6 +60,9 @@ func TestNavInputFromProducts_TreeAndPageIndex(t *testing.T) {
if !ok {
t.Fatal("missing PageIndex nav input for d2")
}
+ if treeIn.CompileKind != "tree" || pageIndexIn.CompileKind != "page_index" {
+ t.Fatalf("navigation kinds = %q/%q", treeIn.CompileKind, pageIndexIn.CompileKind)
+ }
if _, found := byDoc["d3"]; found {
t.Fatal("Graph structure product must not produce navigation input")
}
diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.go
index 374dff8155..fcd401fce6 100644
--- a/internal/ingestion/knowledge_compile/reader.go
+++ b/internal/ingestion/knowledge_compile/reader.go
@@ -102,7 +102,7 @@ type engineReader struct {
// unknown column); payload = content_with_weight, kind = compile_kwd.
var compiledSelectFields = []string{
"id", "doc_id", "compile_kwd",
- "available_int",
+ "available_int", "extra",
"content_with_weight",
"source_chunk_ids", "source_doc_ids",
"name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd",
@@ -133,7 +133,7 @@ var compiledSelectFields = []string{
// in _source; the Infinity engine expands it against the table's real columns
// because Infinity's SQL binder rejects a partial wildcard (3013).
var wikiSelectFields = []string{
- "page_type_kwd", "topic_kwd", "plan_group_kwd", "generation_kwd", "title_kwd",
+ "page_type_kwd", "entity_type_kwd", "type_kwd", "topic_kwd", "plan_group_kwd", "generation_kwd", "title_kwd",
"entity_names_kwd", "summary_with_weight",
"related_kb_pages_kwd", "outlinks_kwd", "depth_int",
"q_*_vec",
@@ -172,7 +172,7 @@ func (r engineReader) LoadDocProducts(ctx context.Context, tenant, kb, docID str
}
for _, c := range res.Chunks {
// Only compiled products carry compile_kwd; skip ordinary source chunks.
- if _, ok := c["compile_kwd"]; !ok {
+ if _, ok := c["compile_kwd"]; !ok || types.IsNavigationRow(c) {
continue
}
// Reverse-map the row's compile_kwd; reject dirty/unknown kinds
@@ -246,7 +246,7 @@ func (r engineReader) LoadMergedWikiPages(ctx context.Context, tenant, kb string
if eng == nil {
return nil, nil
}
- filter := map[string]interface{}{"available_int": 1, "scope_kwd": "dataset", "compile_kwd": compileKwdWikiPage}
+ filter := map[string]interface{}{"available_int": 1, "scope_kwd": "dataset", "type_kwd": compileKwdWikiPage}
var pages []kccommon.Product
for offset := 0; ; offset += loadDocProductsLimit {
res, err := eng.Search(ctx, &types.SearchRequest{
@@ -291,7 +291,7 @@ func (r engineReader) LoadDocumentWikiPagesBySlugs(ctx context.Context, tenant,
}
filter := map[string]interface{}{
"available_int": 0,
- "compile_kwd": compileKwdWikiPage,
+ "type_kwd": compileKwdWikiPage,
"slug_kwd": slugs,
}
if statusAvailable {
@@ -352,25 +352,32 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
if err != nil || mapped != expect {
return kccommon.Product{}, false
}
- // Round-trip the raw compile_kwd (the inferred compile type / autotype:
- // list/set/hypergraph/timeline/mindmap/…) so the dataset-level merge can
- // stamp the SAME value on the dataset row as the doc row (Python _do_build
- // carries the doc row's compile_kwd through verbatim). Without this, the
- // merge falls back to compileKwdForVariant ("structure"/"mindmap"), which
- // diverges from the doc row's autotype ("hypergraph"/"timeline").
+ // Preserve compilation metadata for the dataset merge.
merged := isAvailable(c["available_int"])
meta := map[string]any{}
- // Preserve the raw compile_kwd (autotype) for the dataset merge (see the
- // round-trip note above). A non-string scalar from the engine is normalized
- // via asString, matching the variant reverse-map at the top of this func.
+ if raw := asString(c["extra"]); raw != "" {
+ var extra map[string]any
+ if err := json.Unmarshal([]byte(raw), &extra); err != nil {
+ return kccommon.Product{}, false
+ }
+ if inferred, ok := extra["compile_type"].(string); ok {
+ meta["compile_type"] = inferred
+ }
+ }
if v := asString(c["compile_kwd"]); v != "" {
meta["compile_kwd"] = v
+ if _, ok := meta["compile_type"]; !ok && types.CanonicalCompilationKind(v) != v {
+ meta["compile_type"] = v
+ }
}
if v, ok := c["name_kwd"].(string); ok && v != "" {
meta["name"] = v
}
if v, ok := c["entity_type_kwd"].(string); ok && v != "" {
+ if expect == kccommon.VariantMindmap && v == "mindmap" {
+ v = "mind_map"
+ }
meta["entity_type"] = v
}
if v, ok := c["from_entity_kwd"].(string); ok && v != "" {
@@ -391,7 +398,7 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
}
// Restore wiki page fields so the merged product (and hence the dataset-level
// merged row) retains the metadata the artifact API and page renderers read.
- if v, ok := c["page_type_kwd"].(string); ok && v != "" {
+ if v := types.WikiPageCategory(c); v != "" {
meta["page_type"] = v
}
if v, ok := c["topic_kwd"].(string); ok && v != "" {
@@ -437,12 +444,10 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
meta["kind"] = "entity"
}
}
- // compile_kwd IS the page/section discriminator (there is no kind column):
- // wiki_page -> "page", wiki_section -> "section". Without it the
- // processBatch "Meta.kind==page" filter would drop every wiki page.
- if variant == compileKwdWikiPage {
+ // Page and section roles share the wiki compilation kind.
+ if types.CompilationRowType(c) == compileKwdWikiPage {
meta["kind"] = "page"
- } else if variant == compileKwdWikiSection {
+ } else if types.CompilationRowType(c) == compileKwdWikiSection {
meta["kind"] = "section"
}
if v := asString(c["plan_group_kwd"]); v != "" {
@@ -469,10 +474,8 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo
}
vec, _ := kccommon.VectorFromChunkMap(c, 0)
- // Restore the authoritative template kind (compilation_template_kind_kwd)
- // so callers (e.g. RebuildDataset's variant recovery, B1a) can map it back
- // via KindToVariant instead of re-deriving from the ambiguous compile_kwd.
- kind := asString(c["compilation_template_kind_kwd"])
+ // Restore the producing compilation kind for routing and merging.
+ kind := types.CompilationKind(c)
// Restore the compilation template id (from compilation_template_ids) so the
// dataset merge can bucket structure rows per template and read/delete paths
// can filter by it. A row should carry exactly one template id; if it carries
@@ -516,7 +519,7 @@ func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, vari
Limit: topN,
SelectFields: append([]string{"id", "doc_id", "kb_id", "content_with_weight",
"name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd", "slug_kwd",
- "source_chunk_ids", "source_doc_ids", "available_int", "compile_kwd",
+ "source_chunk_ids", "source_doc_ids", "available_int", "compile_kwd", "type_kwd",
"create_timestamp_flt", "create_time"},
wikiSelectFields...),
Filter: map[string]interface{}{
@@ -533,24 +536,17 @@ func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, vari
},
},
}
+ if variant == kccommon.VariantWiki {
+ req.Filter["type_kwd"] = compileKwdWikiPage
+ }
res, err := eng.Search(ctx, req)
if err != nil {
return kccommon.Product{}, 0, err
}
- // The KNN Filter scopes compile_kwd, but a foreign/legacy row that slipped
- // past it must not poison the merge candidate. Reject by the raw compile_kwd
- // (wiki_page vs wiki_section are distinct keywords even though both map to
- // VariantWiki); the page/section distinction is resolved downstream by
- // Meta.kind (see productFromChunkMap).
+ // Verify the row family and role even when a backend returns foreign rows.
expectKwd := compileKwdForVariant(variant)
for _, c := range res.Chunks {
- // The KNN Filter already scopes compile_kwd, but a foreign/legacy row that
- // slipped past it must not poison the merge candidate. Reject by the
- // reverse-mapped variant (the dirty-row contract): productFromChunkMap
- // validates KwdToVariant(c) == variant and drops dirty/unknown kinds. The
- // raw-keyword check below is a fast pre-filter before the full product
- // reconstruction.
- if asString(c["compile_kwd"]) != expectKwd {
+ if types.CompilationKind(c) != expectKwd || (variant == kccommon.VariantWiki && types.CompilationRowType(c) != compileKwdWikiPage) {
continue
}
p, ok := productFromChunkMap(c, tenant, variant)
diff --git a/internal/ingestion/knowledge_compile/reader_test.go b/internal/ingestion/knowledge_compile/reader_test.go
index ae62dffa0a..42a21289aa 100644
--- a/internal/ingestion/knowledge_compile/reader_test.go
+++ b/internal/ingestion/knowledge_compile/reader_test.go
@@ -131,7 +131,7 @@ func TestSearchSimilarFiltersByVariant(t *testing.T) {
if eng.lastSearchReq.Filter["available_int"] != 1 {
t.Fatalf("expected available_int=1 filter, got %v", eng.lastSearchReq.Filter["available_int"])
}
- if eng.lastSearchReq.Filter["compile_kwd"] != string(compileKwdWikiPage) {
+ if eng.lastSearchReq.Filter["compile_kwd"] != "wiki" || eng.lastSearchReq.Filter["type_kwd"] != compileKwdWikiPage {
t.Fatalf("expected compile_kwd=%q filter, got %v", compileKwdWikiPage, eng.lastSearchReq.Filter["compile_kwd"])
}
// SelectFields must include the round-trip columns added in the 8th review.
diff --git a/internal/ingestion/knowledge_compile/storage.md b/internal/ingestion/knowledge_compile/storage.md
new file mode 100644
index 0000000000..8d8f82c1af
--- /dev/null
+++ b/internal/ingestion/knowledge_compile/storage.md
@@ -0,0 +1,42 @@
+# Knowledge compilation storage
+
+`compile_kwd` identifies the producing knowledge compilation kind. New rows use
+`graph`, `mind_map`, `page_index`, `tree`, `timeline`, or `wiki`. The existing
+session kinds remain `session_graph` and `session_essence`.
+
+`type_kwd` identifies the row's role:
+
+| Compilation | Roles |
+| --- | --- |
+| Graph, mind map, page index | `entity`, `relation`, `kg_build_meta` |
+| Tree | Existing entity/relation roles; root/summary metadata remains in `raptor_kwd` |
+| Navigation from tree/page index | `nav_doc`, `nav_cluster` |
+| Wiki | `wiki_page`, `wiki_section`, `wiki_entity`, `wiki_relation`, `wiki_contribution_state`, `wiki_map_active`, `wiki_map_extract` |
+
+Wiki page categories such as `concept`, `entity`, and `topic` are stored in
+`entity_type_kwd`. Inferred structure extraction types such as `list`, `set`,
+and `hypergraph` are retained as `compile_type` inside the JSON string in `extra`.
+Neither changes the compilation kind.
+
+New writes omit `compilation_template_kind_kwd` and `page_type_kwd`.
+Their engine columns remain available for reading existing data.
+
+Navigation document leaves retain the source compilation kind. Topic clusters
+retain the kind of the compilation that created them; clusters can contain
+documents compiled by both tree and page index. Navigation selection uses the
+row role, so these clusters remain accessible across both kinds. Navigation
+rows remain hidden from ordinary retrieval with `available_int=0`.
+
+Elasticsearch and Infinity translate compilation predicates into one query
+covering both stored generations. For example, selecting the `wiki_page` role
+matches either `(compile_kwd=wiki AND type_kwd=wiki_page)` or the old
+`compile_kwd=wiki_page`. Compilation-kind reads prefer canonical `compile_kwd`
+values on rows without the retired template-kind stamp. Stamped rows retain
+their original template kind, including when their inferred type happens to
+equal a canonical kind. Wiki category reads prefer `entity_type_kwd` and
+otherwise read `page_type_kwd`. Tenant/dataset and document constraints remain
+outside the alternative branches.
+
+Rebuild cleanup selects navigation by role and merged structure rows by dataset
+scope. It must not delete all rows sharing a compilation kind, since document
+inputs and dataset outputs now use the same canonical value.
diff --git a/internal/ingestion/knowledge_compile/structure_merge_test.go b/internal/ingestion/knowledge_compile/structure_merge_test.go
index eb30b2d8e2..33e7c839e0 100644
--- a/internal/ingestion/knowledge_compile/structure_merge_test.go
+++ b/internal/ingestion/knowledge_compile/structure_merge_test.go
@@ -252,13 +252,8 @@ func TestMergeStructureDataset_DescriptionFallbacks(t *testing.T) {
}
// TestMergeStructureDataset_CompileKwdIsAutotypeNotTemplateKind covers the
-// option-A alignment: the dataset row's compile_kwd is the doc row's inferred
-// compile type (autotype, e.g. "hypergraph"/"mindmap"), NOT the template kind.
-// The template kind travels on the separate TemplateKind field (stamped to
-// compilation_template_kind_kwd by WriteMergedStructure), which is what read/
-// delete paths match on. This mirrors Python _do_build (compile_kwd passes
-// through verbatim) + get_dataset_structure (matches template kind).
-func TestMergeStructureDataset_CompileKwdIsAutotypeNotTemplateKind(t *testing.T) {
+// Dataset rows retain the producing compilation kind, independent of extraction shape.
+func TestMergeStructureDataset_CompileKwdIsCanonicalKind(t *testing.T) {
c := &Consumer{writer: &fakeWriter{}}
products := []kccommon.Product{
{Variant: kccommon.VariantStructure, DocID: "d1", Kind: "knowledge_graph", TemplateID: "tpl-graph",
@@ -279,8 +274,8 @@ func TestMergeStructureDataset_CompileKwdIsAutotypeNotTemplateKind(t *testing.T)
b := fw.buckets[i]
switch b.Name {
case "Engine":
- if b.CompileKwd != "hypergraph" {
- t.Errorf("Engine compile_kwd = %q, want autotype \"hypergraph\" (not template kind)", b.CompileKwd)
+ if b.CompileKwd != "graph" {
+ t.Errorf("Engine compile_kwd = %q, want \"graph\"", b.CompileKwd)
}
if b.TemplateKind != "knowledge_graph" {
t.Errorf("Engine TemplateKind = %q, want \"knowledge_graph\"", b.TemplateKind)
@@ -289,8 +284,8 @@ func TestMergeStructureDataset_CompileKwdIsAutotypeNotTemplateKind(t *testing.T)
t.Errorf("Engine TemplateID = %q, want \"tpl-graph\"", b.TemplateID)
}
case "Fuel":
- if b.CompileKwd != "mindmap" {
- t.Errorf("Fuel compile_kwd = %q, want autotype \"mindmap\" (not template kind)", b.CompileKwd)
+ if b.CompileKwd != "mind_map" {
+ t.Errorf("Fuel compile_kwd = %q, want \"mind_map\"", b.CompileKwd)
}
if b.TemplateKind != "mind_map" {
t.Errorf("Fuel TemplateKind = %q, want \"mind_map\"", b.TemplateKind)
diff --git a/internal/ingestion/knowledge_compile/wiki_contribution.go b/internal/ingestion/knowledge_compile/wiki_contribution.go
index ff3ba9fa8e..ad872543e1 100644
--- a/internal/ingestion/knowledge_compile/wiki_contribution.go
+++ b/internal/ingestion/knowledge_compile/wiki_contribution.go
@@ -68,10 +68,10 @@ func (s *engineWikiContributionStore) Get(ctx context.Context, tenantID, dataset
IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},
KbIDs: []string{datasetID},
Limit: 1,
- SelectFields: []string{"id", "compile_kwd", "content_with_weight"},
+ SelectFields: []string{"id", "compile_kwd", "type_kwd", "content_with_weight"},
Filter: map[string]any{
"id": []string{key},
- "compile_kwd": wikiContributionCompileKWD,
+ "type_kwd": wikiContributionCompileKWD,
"available_int": 0,
},
})
@@ -100,7 +100,8 @@ func (s *engineWikiContributionStore) Put(ctx context.Context, tenantID, dataset
"id": wikiContributionStateID(datasetID, contribution.DocumentID),
"doc_id": "wiki_contribution:" + contribution.DocumentID,
"kb_id": datasetID,
- "compile_kwd": wikiContributionCompileKWD,
+ "compile_kwd": "wiki",
+ "type_kwd": wikiContributionCompileKWD,
"scope_kwd": "doc",
"source_doc_ids": []string{contribution.DocumentID},
"content_with_weight": string(payload),
@@ -121,7 +122,7 @@ func (s *engineWikiContributionStore) Delete(ctx context.Context, tenantID, data
func (s *engineWikiContributionStore) ClearDataset(ctx context.Context, tenantID, datasetID string) error {
_, err := s.engine.DeleteChunks(ctx, map[string]any{
"kb_id": datasetID,
- "compile_kwd": wikiContributionCompileKWD,
+ "type_kwd": wikiContributionCompileKWD,
"available_int": 0,
}, fmt.Sprintf("ragflow_%s", tenantID), datasetID)
return err
diff --git a/internal/ingestion/knowledge_compile/wiki_graph_test.go b/internal/ingestion/knowledge_compile/wiki_graph_test.go
index fe0d516a88..e79e1f4566 100644
--- a/internal/ingestion/knowledge_compile/wiki_graph_test.go
+++ b/internal/ingestion/knowledge_compile/wiki_graph_test.go
@@ -34,7 +34,7 @@ func page(pageType, slug string, outlinks ...string) wikiPageProjection {
func findEntity(rows []map[string]interface{}, slug string) map[string]interface{} {
for _, r := range rows {
- if r["compile_kwd"] == compileKwdWikiEntity && r["slug_kwd"] == slug {
+ if r["compile_kwd"] == "wiki" && r["type_kwd"] == compileKwdWikiEntity && r["slug_kwd"] == slug {
return r
}
}
@@ -43,7 +43,7 @@ func findEntity(rows []map[string]interface{}, slug string) map[string]interface
func findRelation(rows []map[string]interface{}, from, to string) map[string]interface{} {
for _, r := range rows {
- if r["compile_kwd"] == compileKwdWikiRelation && r["from_kwd"] == from && r["to_kwd"] == to {
+ if r["compile_kwd"] == "wiki" && r["type_kwd"] == compileKwdWikiRelation && r["from_kwd"] == from && r["to_kwd"] == to {
return r
}
}
@@ -252,8 +252,8 @@ func TestProjectWikiGraphRowsFieldMapping(t *testing.T) {
if e == nil {
t.Fatalf("missing entity")
}
- if e["entity_type_kwd"] != "wiki_concept" {
- t.Fatalf("expected entity_type_kwd=wiki_concept, got %v", e["entity_type_kwd"])
+ if e["entity_type_kwd"] != "concept" {
+ t.Fatalf("expected entity_type_kwd=concept, got %v", e["entity_type_kwd"])
}
// id format: wiki_entity:{kb}:{slug} xxhashed to 16 hex chars.
id, _ := e["id"].(string)
diff --git a/internal/ingestion/knowledge_compile/wiki_map_store.go b/internal/ingestion/knowledge_compile/wiki_map_store.go
index 6d72e3e843..6c6921008e 100644
--- a/internal/ingestion/knowledge_compile/wiki_map_store.go
+++ b/internal/ingestion/knowledge_compile/wiki_map_store.go
@@ -46,10 +46,10 @@ func (s *wikiMapVersionStore) GetWikiMapActiveState(ctx context.Context, tenantI
IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},
KbIDs: []string{datasetID},
Limit: 1,
- SelectFields: []string{"id", "compile_kwd", "content_with_weight"},
+ SelectFields: []string{"id", "compile_kwd", "type_kwd", "content_with_weight"},
Filter: map[string]interface{}{
"id": []string{key},
- "compile_kwd": wikiMapActiveCompileKWD,
+ "type_kwd": wikiMapActiveCompileKWD,
"available_int": 0,
},
})
@@ -77,7 +77,8 @@ func (s *wikiMapVersionStore) PutWikiMapActiveState(ctx context.Context, state k
"id": state.Key,
"doc_id": "wiki_map_active:" + state.DocumentID,
"kb_id": state.DatasetID,
- "compile_kwd": wikiMapActiveCompileKWD,
+ "compile_kwd": "wiki",
+ "type_kwd": wikiMapActiveCompileKWD,
"scope_kwd": "doc",
"source_doc_ids": []string{state.DocumentID},
"content_with_weight": string(state.Payload),
@@ -147,11 +148,11 @@ func (s *wikiMapVersionStore) GetWikiMapVersions(ctx context.Context, tenantID,
KbIDs: []string{datasetID},
Limit: end - start,
SelectFields: []string{
- "id", "compile_kwd", "content_with_weight",
+ "id", "compile_kwd", "type_kwd", "content_with_weight",
},
Filter: map[string]interface{}{
"id": keys[start:end],
- "compile_kwd": wikiMapExtractCompileKWD,
+ "type_kwd": wikiMapExtractCompileKWD,
"available_int": 0,
},
})
@@ -162,7 +163,7 @@ func (s *wikiMapVersionStore) GetWikiMapVersions(ctx context.Context, tenantID,
continue
}
for _, row := range result.Chunks {
- if mapStoreString(row["compile_kwd"]) != wikiMapExtractCompileKWD {
+ if types.CompilationRowType(row) != wikiMapExtractCompileKWD {
continue
}
id := mapStoreString(row["id"])
@@ -229,7 +230,8 @@ func wikiMapVersionRow(version kccommon.WikiMapVersion) map[string]interface{} {
// document deletion cannot remove a reusable chunk/hash version.
"doc_id": wikiMapCacheDocID(version.DocumentID),
"kb_id": version.DatasetID,
- "compile_kwd": wikiMapExtractCompileKWD,
+ "compile_kwd": "wiki",
+ "type_kwd": wikiMapExtractCompileKWD,
"scope_kwd": "doc",
"source_chunk_ids": []string{version.ChunkID},
"source_doc_ids": []string{version.DocumentID},
diff --git a/internal/ingestion/knowledge_compile/wiki_map_store_test.go b/internal/ingestion/knowledge_compile/wiki_map_store_test.go
index 1d777c231f..5165608a41 100644
--- a/internal/ingestion/knowledge_compile/wiki_map_store_test.go
+++ b/internal/ingestion/knowledge_compile/wiki_map_store_test.go
@@ -81,21 +81,46 @@ func (e *wikiMapStoreEngine) Search(_ context.Context, req *types.SearchRequest)
}
func rowMatchesFilter(row map[string]interface{}, filter map[string]interface{}) bool {
- if expected, ok := filter["available_int"].(int); ok {
- if value, ok := row["available_int"].(int); !ok || value != expected {
- return false
- }
- }
- if mustNot, ok := filter["must_not"].(map[string]interface{}); ok {
- if exists, ok := mustNot["exists"].(string); ok {
- if _, present := row[exists]; present {
+ return matchesStoredCompilationRow(row, types.CompilationFilter(filter))
+}
+
+func matchesStoredCompilationRow(row, filter map[string]interface{}) bool {
+ for field, value := range filter {
+ switch field {
+ case "and":
+ for _, child := range types.FilterClauses(value) {
+ if !matchesStoredCompilationRow(row, child) {
+ return false
+ }
+ }
+ case "or":
+ matched := false
+ for _, child := range types.FilterClauses(value) {
+ matched = matched || matchesStoredCompilationRow(row, child)
+ }
+ if !matched {
+ return false
+ }
+ case "must_not":
+ if matchesStoredCompilationRow(row, value.(map[string]interface{})) {
+ return false
+ }
+ case "exists":
+ if mapStoreString(row[value.(string)]) == "" {
+ return false
+ }
+ default:
+ if expected, ok := value.([]string); ok {
+ matched := false
+ for _, v := range expected {
+ matched = matched || mapStoreString(row[field]) == v
+ }
+ if !matched {
+ return false
+ }
+ } else if !reflect.DeepEqual(row[field], value) {
return false
}
- }
- }
- for _, field := range []string{"compile_kwd", "type_kwd"} {
- if expected, ok := filter[field].(string); ok && mapStoreString(row[field]) != expected {
- return false
}
}
return true
@@ -143,7 +168,8 @@ func TestWikiMapVersionStoreUsesNonSearchableDocStoreRows(t *testing.T) {
"id": "version-a",
"doc_id": "wiki_map_cache:doc-1",
"kb_id": "kb-1",
- "compile_kwd": wikiMapExtractCompileKWD,
+ "compile_kwd": "wiki",
+ "type_kwd": wikiMapExtractCompileKWD,
"available_int": 0,
"chunk_hash_kwd": "hash-a",
} {
diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go
index 85ec2c764d..24bfd3079e 100644
--- a/internal/ingestion/knowledge_compile/writer.go
+++ b/internal/ingestion/knowledge_compile/writer.go
@@ -95,6 +95,7 @@ type Writer interface {
// folded descriptions and the union of source docs/chunks. It is written as a
// scope_kwd="dataset" row. Relations carry FromEntity/ToEntity instead of Name.
type StructureBucket struct {
+ CompileType string
Name string
Type string
Description string // folded entity descriptions
@@ -104,22 +105,13 @@ type StructureBucket struct {
VecCount int // number of vectors folded into Vector (for true mean)
FromEntity string // relation only
ToEntity string // relation only
- // CompileKwd is the raw compile keyword (the inferred compile type / autotype,
- // e.g. "hypergraph", "timeline", "mindmap", "list") that produced this bucket.
- // It is stamped on the stored row's compile_kwd so distinct structure kinds
- // never collide in the same dataset namespace (plan §1.1). It matches the doc
- // row's own compile_kwd — Python _do_build carries the doc row's compile_kwd
- // verbatim onto the dataset row, it does NOT rewrite it to the template kind.
+ // CompileKwd is the canonical knowledge compilation kind for this bucket.
CompileKwd string
// TemplateID is the compilation template id (compilation_template_ids[0]) that
// produced this bucket. Stamped on the dataset row so read/delete paths can
// filter per template (mirror Python get_dataset_structure).
TemplateID string
- // TemplateKind is the authoritative template kind (compilation_template_kind_kwd,
- // e.g. "knowledge_graph", "mind_map", "timeline"). This — NOT compile_kwd — is
- // the field read/delete paths match on for the dataset-structure kind
- // (mirror Python get_dataset_structure._discover_scope_templates, which
- // matches _resolve_dataset_structure_kind against compilation_template_kind_kwd).
+ // TemplateKind identifies the producing template and determines CompileKwd.
TemplateKind string
// RelationType is the relation type (Python default "related"), persisted as
// relation_type_kwd and part of the relation bucket/ID identity.
@@ -154,7 +146,7 @@ func (w engineWriter) DeleteMergedWikiPages(ctx context.Context, tenant, kb stri
}
baseName := fmt.Sprintf("ragflow_%s", tenant)
_, err := eng.DeleteChunks(ctx, map[string]interface{}{
- "id": ids, "kb_id": kb, "available_int": 1, "scope_kwd": "dataset", "compile_kwd": compileKwdWikiPage,
+ "id": ids, "kb_id": kb, "available_int": 1, "scope_kwd": "dataset", "type_kwd": compileKwdWikiPage,
}, baseName, kb)
return err
}
@@ -341,10 +333,10 @@ func (w engineWriter) loadWikiPageContent(ctx context.Context, tenant, kb string
Offset: offset,
Limit: pageBatchSize,
SelectFields: []string{
- "slug_kwd", "page_type_kwd", "md_with_weight", "content_with_weight",
+ "slug_kwd", "page_type_kwd", "compile_kwd", "entity_type_kwd", "type_kwd", "md_with_weight", "content_with_weight",
},
Filter: map[string]interface{}{
- "compile_kwd": compileKwdWikiPage,
+ "type_kwd": compileKwdWikiPage,
"available_int": 1,
"kb_id": kb,
"slug_kwd": slugs,
@@ -359,7 +351,7 @@ func (w engineWriter) loadWikiPageContent(ctx context.Context, tenant, kb string
}
for _, row := range res.Chunks {
target, ok := wikiPageCommitTargetForRow(
- pageEngineString(row["page_type_kwd"]),
+ types.WikiPageCategory(row),
pageEngineString(row["slug_kwd"]),
)
if !ok {
@@ -540,7 +532,7 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
"doc_id": kb,
"kb_id": kb,
"available_int": 1,
- "compile_kwd": ckwd,
+ "compile_kwd": types.CanonicalCompilationKind(kccommon.FirstNonEmpty(b.TemplateKind, ckwd)),
"scope_kwd": "dataset",
"content_with_weight": payload,
"source_doc_ids": b.SourceDocIDs,
@@ -548,12 +540,12 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
"create_time": now.Format("2006-01-02 15:04:05"),
"create_timestamp_flt": float64(now.Unix()),
}
- // Stamp the authoritative template identity so read/delete paths can match
- // the dataset-structure kind by compilation_template_kind_kwd (mirror Python
- // get_dataset_structure._discover_scope_templates), independent of the
- // autotype-valued compile_kwd above.
- if b.TemplateKind != "" {
- row["compilation_template_kind_kwd"] = b.TemplateKind
+ if b.CompileType != "" {
+ extra, err := json.Marshal(map[string]string{"compile_type": b.CompileType})
+ if err != nil {
+ return err
+ }
+ row["extra"] = string(extra)
}
if b.TemplateID != "" {
row["compilation_template_ids"] = []string{b.TemplateID}
@@ -599,7 +591,7 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strin
"doc_id": kb,
"kb_id": kb,
"available_int": 0,
- "compile_kwd": ckwd,
+ "compile_kwd": types.CanonicalCompilationKind(kccommon.FirstNonEmpty(b.TemplateKind, ckwd)),
"scope_kwd": "dataset",
"type_kwd": "kg_build_meta",
"create_time": now.Format("2006-01-02 15:04:05"),
@@ -707,9 +699,9 @@ func mergeExistingStructureBuckets(ctx context.Context, eng engine.DocEngine, ba
func structureCompileKind(compileKwd string) string {
if compileKwd = strings.TrimSpace(compileKwd); compileKwd != "" {
- return compileKwd
+ return types.CanonicalCompilationKind(compileKwd)
}
- return compileKwdStructure
+ return "graph"
}
func structureRelationType(value string) string {
@@ -738,8 +730,8 @@ func structureRowIdentity(row map[string]interface{}) string {
if ids := firstStringSlice(row["compilation_template_ids"]); len(ids) > 0 {
templateID = ids[0]
}
- template := structureTemplateIdentity(templateID, structureString(row["compilation_template_kind_kwd"]))
- compileKwd := structureCompileKind(structureString(row["compile_kwd"]))
+ template := structureTemplateIdentity(templateID, types.CompilationKind(row))
+ compileKwd := structureCompileKind(types.CompilationKind(row))
payload := structureRowPayload(row)
if strings.EqualFold(structureRowType(row), "relation") {
from := structureString(row["from_entity_kwd"])
@@ -966,7 +958,7 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm
// carry scope_kwd="dataset" alongside available_int=1 so the consumer and
// clean paths can filter by scope instead of (only) available_int.
"scope_kwd": "dataset",
- "compile_kwd": compileKwdForVariant(p.Variant),
+ "compile_kwd": types.CanonicalCompilationKind(kccommon.FirstNonEmpty(p.Kind, string(p.Variant))),
"content_with_weight": p.Content,
"source_doc_ids": srcDocIDs,
"source_chunk_ids": srcChunkIDs,
@@ -975,9 +967,9 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm
"create_time": now.Format("2006-01-02 15:04:05"),
"create_timestamp_flt": float64(now.Unix()),
}
- // The product kind (page vs section) is carried by compile_kwd
- // (wiki_page / wiki_section), which is what Python stores and what
- // productFromChunkMap derives Meta["kind"] from — there is no kind column.
+ if p.Variant == kccommon.VariantWiki {
+ m["type_kwd"] = "wiki_" + kccommon.FirstNonEmpty(metaString(p.Meta, "kind"), "page")
+ }
// wiki_incremental port: preserve the original creation timestamp across a
// page merge. If the incoming merged product already carries
// created_at_unix (restored by the Reader from create_timestamp_flt), reuse
@@ -988,11 +980,8 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm
// Rebuild the human-readable form from the preserved unix time.
m["create_time"] = time.Unix(int64(v), 0).Format("2006-01-02 15:04:05")
}
- // Carry the wiki page metadata onto the merged row so the dataset-level
- // products keep the fields the artifact API (ListArtifacts/ListWikiTopics)
- // and page renderers read. Without this the merged rows lose page_type_kwd /
- // topic_kwd / title_kwd and the compilation page would show no wiki pages
- // even though per-document products carry them.
+ // Carry wiki metadata onto the merged row so the dataset-level
+ // products retain their category, topic, title, and rendered page body.
//
// slug_kwd follows the Python writer contract (api/db/db_models.py): it is
// stored as the full "<page_type>/<slug>" form so GetWikiPage's filter
@@ -1011,7 +1000,7 @@ func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccomm
m["title_kwd"] = v
}
if pageType != "" {
- m["page_type_kwd"] = pageType
+ m["entity_type_kwd"] = pageType
}
if v := kccommon.NormalizeWikiTopicPath(metaString(p.Meta, "topic")); v != "" {
m["topic_kwd"] = v
@@ -1291,14 +1280,7 @@ func metaFloat(m map[string]any, key string) (float64, bool) {
return 0, false
}
-// KwdToVariant is the inverse of compileKwdForVariant: it maps a stored
-// compile_kwd back to its compiler Variant. Both wiki_page and wiki_section
-// map to VariantWiki (same product family); the page/section distinction is
-// carried by compile_kwd itself, not the variant. Returns an error for an
-// unknown kwd so callers can reject dirty/foreign rows. Structure products
-// stamp the inferred compile kind verbatim (list/set/hypergraph), which are NOT
-// in the KindToVariant whitelist, so they are mapped to VariantStructure
-// explicitly before the whitelist lookup; unknown kinds hard-fail (O2a).
+// KwdToVariant maps a stored compilation kind to its execution variant.
func KwdToVariant(kwd string) (kccommon.Variant, error) {
switch kwd {
case compileKwdWikiPage, compileKwdWikiSection, compileKwdWikiEntity, compileKwdWikiRelation:
@@ -1321,22 +1303,15 @@ func KwdToVariant(kwd string) (kccommon.Variant, error) {
// --- Wiki page graph materialization (wiki_entity / wiki_relation) ---
-// compile_kwd values for the dataset-level products this package writes. The
-// wiki variant compiles into "wiki_page" rows (per the Python writer contract
-// that GetWikiAlteration / ListArtifacts / GetWikiGraph all filter on
-// compile_kwd = "wiki_page"), while the page graph is materialized as the
-// dedicated wiki_entity / wiki_relation buckets.
+// Wiki row-role values used by readers and graph projection.
const (
compileKwdWikiPage = "wiki_page"
compileKwdWikiSection = "wiki_section"
compileKwdWikiEntity = "wiki_entity"
compileKwdWikiRelation = "wiki_relation"
compileKwdWikiPageGraph = "wiki_page_graph" // legacy Python blob, swept on drop
- // compileKwdStructure tags structure dataset-level merged rows
- // (scope_kwd="dataset"); compileKwdNav tags the dataset-navigation rows
- // written by NavService. Both are targets of per-variant clean (B4).
+ // Structure is the default execution family when no template kind is supplied.
compileKwdStructure = "structure"
- compileKwdNav = "dataset_nav"
)
// wikiGraphBatchSize bounds how many graph rows each parallel InsertChunks call
@@ -1347,14 +1322,12 @@ const wikiGraphBatchSize = 200
// a heavily-shared page does not accumulate an unbounded id list.
const wikiGraphSourceDocCap = 64
-// compileKwdForVariant maps a compiler Variant to the compile_kwd stamped on the
-// merged chunk document. The wiki variant is special-cased because all read
-// paths filter on compile_kwd = "wiki_page", not the raw variant string "wiki".
+// compileKwdForVariant returns the canonical storage kind of an execution variant.
func compileKwdForVariant(v kccommon.Variant) string {
if v == kccommon.VariantWiki {
- return compileKwdWikiPage
+ return "wiki"
}
- return string(v)
+ return types.CanonicalCompilationKind(string(v))
}
// wikiGraphXXHash derives a stable 16-char hex id (matches Python
@@ -1475,7 +1448,7 @@ func (w engineWriter) loadActiveDocumentWikiPages(ctx context.Context, tenant, k
bySlug := make(map[string]wikiPageProjection)
for offset := 0; ; offset += batchSize {
filter := map[string]interface{}{
- "compile_kwd": compileKwdWikiPage, "available_int": 0,
+ "type_kwd": compileKwdWikiPage, "available_int": 0,
"scope_kwd": "doc", "kb_id": kb,
}
if statusAvailable {
@@ -1486,7 +1459,7 @@ func (w engineWriter) loadActiveDocumentWikiPages(ctx context.Context, tenant, k
KbIDs: []string{kb},
Filter: filter,
SelectFields: []string{
- "slug_kwd", "page_type_kwd", "title_kwd", "entity_names_kwd",
+ "slug_kwd", "page_type_kwd", "compile_kwd", "entity_type_kwd", "type_kwd", "title_kwd", "entity_names_kwd",
"summary_with_weight", "outlinks_kwd", "source_doc_ids", "source_chunk_ids",
},
Limit: batchSize, Offset: offset,
@@ -1506,7 +1479,7 @@ func (w engineWriter) loadActiveDocumentWikiPages(ctx context.Context, tenant, k
page := bySlug[slug]
if page.Slug == "" {
page.Slug = slug
- page.PageType = metaString(row, "page_type_kwd")
+ page.PageType = types.WikiPageCategory(row)
page.Title = metaString(row, "title_kwd")
page.Summary = metaString(row, "summary_with_weight")
}
@@ -1546,13 +1519,8 @@ func (w engineWriter) DeleteMerged(ctx context.Context, tenant, kb string) error
return w.DeleteMergedForVariant(ctx, tenant, kb, []kccommon.Variant{kccommon.VariantWiki})
}
-// DeleteMergedForVariant deletes the dataset-level merged rows for a KB across
-// the given compile variants (B4). When variants is empty it clears the full set
-// the consumer manages (B1b: a full rebuild always clears everything, so an
-// empty set still means "clear all"). Row scope: structure dataset rows are
-// tagged scope_kwd="dataset"; wiki merged rows carry available_int=1; nav rows
-// carry compile_kwd="dataset_nav". The filter union is OR-ed across variants so
-// one call clears every relevant row in a single engine delete.
+// DeleteMergedForVariant clears the selected dataset outputs, preserving
+// document-level compilation inputs. An empty variant set clears all outputs.
func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb string, variants []kccommon.Variant) error {
eng := w.eng
if eng == nil {
@@ -1563,30 +1531,22 @@ func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb str
}
baseName := fmt.Sprintf("ragflow_%s", tenant)
if len(variants) == 0 {
- // Full-set clean (B1b): everything the consumer manages. Structure
- // dataset rows are tagged scope_kwd="dataset" (not a fixed compile_kwd),
- // so the full clean deletes by kb_id + compile_kwd IN (the fixed-kwd
- // buckets) OR scope_kwd="dataset". The scope_kwd="dataset" sweep requires
- // the dataset-structure filter keys, so it is guarded like the per-variant
- // structure branch (review fix: this is the highest-impact destructive
- // path and must fail loudly on unsupported engines).
+ // Navigation roles and searchable wiki outputs are separate from the
+ // dataset-scoped structure sweep.
if !datasetStructureSupported() {
return errDatasetStructureUnsupported()
}
_, err := eng.DeleteChunks(ctx, map[string]interface{}{
"kb_id": kb,
- "compile_kwd": []string{
- compileKwdNav,
- compileKwdWikiPage,
- compileKwdWikiSection,
- compileKwdStructure,
+ "or": []map[string]interface{}{
+ {"type_kwd": []string{"nav_doc", "nav_cluster"}},
+ {"type_kwd": []string{compileKwdWikiPage, compileKwdWikiSection}, "available_int": 1},
},
}, baseName, kb)
if err != nil {
return fmt.Errorf("delete merged (all variants): %w", err)
}
- // structure dataset rows carry scope_kwd="dataset" + compile_kwd="structure";
- // sweep them too (idempotent with the compile_kwd filter above).
+ // Clear dataset-scoped outputs after preserving document inputs.
_, err = eng.DeleteChunks(ctx, map[string]interface{}{
"kb_id": kb,
"scope_kwd": "dataset",
@@ -1596,12 +1556,7 @@ func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb str
}
return nil
}
- // Issue ONE delete per distinct variant bucket rather than one AND-ed filter:
- // different variants target different columns (wiki: available_int=1 +
- // compile_kwd; structure: scope_kwd="dataset"; nav: compile_kwd="dataset_nav"
- // with available_int=0), and AND-ing them would exclude the others' rows.
- // RebuildDataset (B1b) passes the full managed set, so this per-bucket sweep
- // is correct for both full and per-variant rebuilds.
+ // Each output family has its own role, scope, and visibility constraints.
for _, v := range variants {
switch v {
case kccommon.VariantWiki:
@@ -1610,7 +1565,7 @@ func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb str
if _, err := eng.DeleteChunks(ctx, map[string]interface{}{
"kb_id": kb,
"available_int": 1,
- "compile_kwd": []string{compileKwdWikiPage, compileKwdWikiSection},
+ "type_kwd": []string{compileKwdWikiPage, compileKwdWikiSection},
}, baseName, kb); err != nil {
return fmt.Errorf("delete merged (wiki): %w", err)
}
@@ -1633,11 +1588,11 @@ func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb str
return fmt.Errorf("delete merged (structure/mindmap): %w", err)
}
case kccommon.VariantTree:
- // tree/nav rows are compile_kwd="dataset_nav" with available_int=0;
+ // Navigation rows have available_int=0;
// do NOT add available_int=1 (that would exclude them).
if _, err := eng.DeleteChunks(ctx, map[string]interface{}{
- "kb_id": kb,
- "compile_kwd": []string{compileKwdNav},
+ "kb_id": kb,
+ "type_kwd": []string{"nav_doc", "nav_cluster"},
}, baseName, kb); err != nil {
return fmt.Errorf("delete merged (nav): %w", err)
}
@@ -1694,12 +1649,12 @@ func (w engineWriter) loadMergedWikiPages(ctx context.Context, tenant, kb string
IndexNames: []string{baseName},
KbIDs: []string{kb},
Filter: map[string]interface{}{
- "compile_kwd": compileKwdWikiPage,
+ "type_kwd": compileKwdWikiPage,
"available_int": 1,
"kb_id": kb,
},
SelectFields: []string{
- "slug_kwd", "page_type_kwd", "title_kwd",
+ "slug_kwd", "page_type_kwd", "entity_type_kwd", "type_kwd", "title_kwd",
"entity_names_kwd", "summary_with_weight", "outlinks_kwd",
"source_doc_ids", "source_chunk_ids",
// compile_kwd is selected purely so the query-result telemetry
@@ -1730,7 +1685,7 @@ func (w engineWriter) loadMergedWikiPages(ctx context.Context, tenant, kb string
for _, c := range res.Chunks {
p := wikiPageProjection{
Slug: metaString(c, "slug_kwd"),
- PageType: metaString(c, "page_type_kwd"),
+ PageType: types.WikiPageCategory(c),
Title: metaString(c, "title_kwd"),
Aliases: metaStringSlice(c, "entity_names_kwd"),
Summary: metaString(c, "summary_with_weight"),
@@ -1835,9 +1790,9 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string,
"doc_id": kb,
"kb_id": kb,
"available_int": 1,
- "compile_kwd": compileKwdWikiEntity,
- "type_kwd": "wiki_" + p.PageType,
- "entity_type_kwd": "wiki_" + p.PageType,
+ "compile_kwd": "wiki",
+ "type_kwd": compileKwdWikiEntity,
+ "entity_type_kwd": p.PageType,
"slug_kwd": p.Slug,
"title_kwd": p.Title,
"weight_int": weight,
@@ -1891,7 +1846,7 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string,
"doc_id": kb,
"kb_id": kb,
"available_int": 1,
- "compile_kwd": compileKwdWikiRelation,
+ "compile_kwd": "wiki",
"type_kwd": compileKwdWikiRelation,
"from_kwd": p.Slug,
"to_kwd": tgt,
@@ -1902,7 +1857,7 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string,
}
var relCount, entCount int
for _, r := range rows {
- switch r["compile_kwd"] {
+ switch r["type_kwd"] {
case compileKwdWikiRelation:
relCount++
case compileKwdWikiEntity:
diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.go
index caab047ff4..cf85feac25 100644
--- a/internal/ingestion/knowledge_compile/writer_test.go
+++ b/internal/ingestion/knowledge_compile/writer_test.go
@@ -88,7 +88,7 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {
cases := map[string]string{
"slug_kwd": "entity/alpha",
"title_kwd": "Alpha",
- "page_type_kwd": "entity",
+ "entity_type_kwd": "entity",
"topic_kwd": "Knowledge/Core/Alpha",
"summary_with_weight": "A page about Alpha",
// The wiki page body goes to md_with_weight, the column Python writes
@@ -284,9 +284,9 @@ func TestDeleteMergedScopesToMergedWikiRows(t *testing.T) {
if cond["available_int"] != 1 {
t.Errorf("DeleteMerged must scope to available_int=1, got %v", cond["available_int"])
}
- variants, ok := cond["compile_kwd"].([]string)
+ variants, ok := cond["type_kwd"].([]string)
if !ok {
- t.Fatalf("DeleteMerged must pass a compile_kwd string slice, got %T", cond["compile_kwd"])
+ t.Fatalf("DeleteMerged must pass a compile_kwd string slice, got %T", cond["type_kwd"])
}
if len(variants) != 2 || variants[0] != compileKwdWikiPage || variants[1] != compileKwdWikiSection {
t.Errorf("DeleteMerged compile_kwd = %v, want [%q %q]", variants, compileKwdWikiPage, compileKwdWikiSection)
@@ -302,26 +302,22 @@ func TestDeleteMergedForVariant_EmptySetClearsAll(t *testing.T) {
if err := w.DeleteMergedForVariant(context.Background(), "t1", "kb1", nil); err != nil {
t.Fatalf("DeleteMergedForVariant: %v", err)
}
- // The full clean issues two deletes: the compile_kwd bucket first, then the
- // scope_kwd="dataset" sweep. Assert on the first (compile_kwd) delete.
- if len(eng.deleteConds) < 1 {
- t.Fatal("engine.DeleteChunks was not called")
- }
- kwds, ok := eng.deleteConds[0]["compile_kwd"].([]string)
- if !ok {
- t.Fatalf("compile_kwd filter not a string slice, got %T", eng.deleteConds[0]["compile_kwd"])
- }
- want := []string{compileKwdNav, compileKwdWikiPage, compileKwdWikiSection, compileKwdStructure}
- if len(kwds) != len(want) {
- t.Fatalf("full clean compile_kwd = %v, want %v", kwds, want)
- }
- got := map[string]bool{}
- for _, k := range kwds {
- got[k] = true
- }
- for _, k := range want {
- if !got[k] {
- t.Errorf("full clean missing compile_kwd %q", k)
+ if len(eng.deleteConds) != 2 {
+ t.Fatalf("full clean issued %d deletes, want 2", len(eng.deleteConds))
+ }
+ for _, test := range []struct {
+ row map[string]interface{}
+ want bool
+ }{
+ {map[string]interface{}{"kb_id": "kb1", "compile_kwd": "wiki", "type_kwd": "wiki_page", "available_int": 1}, true},
+ {map[string]interface{}{"kb_id": "kb1", "compile_kwd": "wiki_page", "available_int": 1}, true},
+ {map[string]interface{}{"kb_id": "kb1", "compile_kwd": "page_index", "type_kwd": "nav_doc", "available_int": 0}, true},
+ {map[string]interface{}{"kb_id": "kb1", "compile_kwd": "graph", "type_kwd": "entity", "scope_kwd": "doc"}, false},
+ {map[string]interface{}{"kb_id": "kb1", "compile_kwd": "wiki", "type_kwd": "wiki_page", "available_int": 0}, false},
+ {map[string]interface{}{"kb_id": "kb2", "compile_kwd": "tree", "type_kwd": "nav_doc"}, false},
+ } {
+ if got := rowMatchesFilter(test.row, eng.deleteConds[0]); got != test.want {
+ t.Errorf("full clean matched %v = %v, want %v", test.row, got, test.want)
}
}
// The second delete sweeps structure scope_kwd="dataset".
@@ -352,9 +348,9 @@ func TestDeleteMergedForVariant_WikiScopesToAvailableOne(t *testing.T) {
if cond["available_int"] != 1 {
t.Errorf("wiki clean must scope to available_int=1, got %v", cond["available_int"])
}
- kwds, ok := cond["compile_kwd"].([]string)
+ kwds, ok := cond["type_kwd"].([]string)
if !ok || len(kwds) != 2 {
- t.Fatalf("wiki clean compile_kwd = %v, want [wiki_page wiki_section]", cond["compile_kwd"])
+ t.Fatalf("wiki clean compile_kwd = %v, want [wiki_page wiki_section]", cond["type_kwd"])
}
}
@@ -392,11 +388,8 @@ func TestDeleteMergedForVariant_WikiPlusNavCoversNavRows(t *testing.T) {
}
var wikiCond, navCond map[string]interface{}
for _, cond := range eng.deleteConds {
- kwds, _ := cond["compile_kwd"].([]string)
- for _, k := range kwds {
- if k == compileKwdNav {
- navCond = cond
- }
+ if roles, ok := cond["type_kwd"].([]string); ok && len(roles) == 2 && roles[0] == "nav_doc" && roles[1] == "nav_cluster" {
+ navCond = cond
}
if cond["available_int"] == 1 {
wikiCond = cond
@@ -406,7 +399,7 @@ func TestDeleteMergedForVariant_WikiPlusNavCoversNavRows(t *testing.T) {
t.Error("expected a wiki delete scoped to available_int=1")
}
if navCond == nil {
- t.Error("expected a nav delete (compile_kwd=dataset_nav)")
+ t.Error("expected a nav delete scoped to nav_doc/nav_cluster")
} else if _, hasAvail := navCond["available_int"]; hasAvail {
t.Errorf("nav delete must not carry available_int (nav rows are 0), got %v", navCond["available_int"])
}
diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.go
index ddc72e59b2..aab3a92ba4 100644
--- a/internal/ingestion/task/knowledge_compiler_wiring.go
+++ b/internal/ingestion/task/knowledge_compiler_wiring.go
@@ -258,7 +258,7 @@ func wikiCompiledRows(output map[string]any) []map[string]any {
if !ok {
continue
}
- compileKWD := strings.TrimSpace(anyString(row["compile_kwd"]))
+ compileKWD := enginetypes.CompilationRowType(row)
if compileKWD != "wiki_page" && compileKWD != "wiki_section" {
continue
}
@@ -291,7 +291,7 @@ func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.Wik
SelectFields: []string{"id", "compile_kwd", "slug_kwd", "parent_id"},
Filter: map[string]any{
"doc_id": []string{request.DocumentID},
- "compile_kwd": []string{"wiki_page", "wiki_section"},
+ "type_kwd": []string{"wiki_page", "wiki_section"},
"available_int": 0,
},
})
@@ -316,7 +316,7 @@ func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.Wik
affectedPageIDs := make(map[string]struct{})
if existing != nil && !fullReplace {
for _, row := range existing.Chunks {
- if anyString(row["compile_kwd"]) != "wiki_page" {
+ if enginetypes.CompilationRowType(row) != "wiki_page" {
continue
}
if _, ok := affected[anyString(row["slug_kwd"])]; ok {
@@ -326,7 +326,7 @@ func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.Wik
}
if existing != nil && fullReplace {
for _, row := range existing.Chunks {
- if anyString(row["compile_kwd"]) == "wiki_page" {
+ if enginetypes.CompilationRowType(row) == "wiki_page" {
removedSlugs = append(removedSlugs, anyString(row["slug_kwd"]))
}
}
@@ -369,7 +369,7 @@ func replaceDirtyWikiProducts(ctx context.Context, request knowledge_compile.Wik
func clearWikiActiveStates(ctx context.Context, docEngine engine.DocEngine, request knowledge_compile.WikiDirtyRequest) error {
_, err := docEngine.DeleteChunks(ctx, map[string]any{
"kb_id": request.DatasetID,
- "compile_kwd": "wiki_map_active",
+ "type_kwd": "wiki_map_active",
"available_int": 0,
"source_doc_ids": []string{request.DocumentID},
}, fmt.Sprintf("ragflow_%s", request.TenantID), request.DatasetID)
@@ -845,13 +845,13 @@ func (s *kcWikiPageStore) FindSimilarPages(ctx context.Context, tenantID, datase
IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},
KbIDs: []string{datasetID},
Limit: k,
- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},
+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "type_kwd", "entity_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},
// compile_kwd="wiki_page" is the schema-backed discriminator for wiki
// pages (sections carry compile_kwd="wiki_section"); there is no
// "kc_kind" column in the chunk schema, so filtering on it would return
// empty on Infinity.
Filter: map[string]interface{}{
- "compile_kwd": "wiki_page",
+ "type_kwd": "wiki_page",
},
MatchExprs: []interface{}{&enginetypes.MatchDenseExpr{
VectorColumnName: fmt.Sprintf("q_%d_vec", len(vec)),
@@ -881,10 +881,10 @@ func (s *kcWikiPageStore) GetPageBySlug(ctx context.Context, tenantID, datasetID
IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},
KbIDs: []string{datasetID},
Limit: 1,
- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},
+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "type_kwd", "entity_type_kwd", "topic_kwd", "plan_group_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},
Filter: map[string]interface{}{
- "compile_kwd": "wiki_page",
- "slug_kwd": slug,
+ "type_kwd": "wiki_page",
+ "slug_kwd": slug,
},
}
res, err := s.docEngine.Search(ctx, req)
@@ -903,9 +903,9 @@ func (s *kcWikiPageStore) FindPagesBySourceChunks(ctx context.Context, tenantID,
IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},
KbIDs: []string{datasetID},
Limit: k,
- SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},
+ SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "type_kwd", "entity_type_kwd", "topic_kwd", "summary_with_weight", "content_with_weight", "md_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "source_chunk_ids", "_score"},
Filter: map[string]interface{}{
- "compile_kwd": "wiki_page",
+ "type_kwd": "wiki_page",
"source_chunk_ids": chunkIDs,
},
}
@@ -929,7 +929,7 @@ func wikiPageCandidateFromRow(row map[string]interface{}) kc.WikiPageCandidate {
ID: strings.TrimSpace(anyString(row["id"])),
Slug: strings.TrimSpace(anyString(row["slug_kwd"])),
Title: strings.TrimSpace(anyString(row["title_kwd"])),
- PageType: strings.TrimSpace(anyString(row["page_type_kwd"])),
+ PageType: enginetypes.WikiPageCategory(row),
Topic: strings.TrimSpace(anyString(row["topic_kwd"])),
PlanGroup: strings.TrimSpace(anyString(row["plan_group_kwd"])),
Summary: strings.TrimSpace(anyString(row["summary_with_weight"])),
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index a32797dd0a..236f05fdb1 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -528,7 +528,7 @@ func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]str
KbIDs: []string{s.taskCtx.Doc.KbID},
Offset: offset,
Limit: pageSize,
- SelectFields: []string{"id", "compile_kwd", "compilation_template_kind_kwd"},
+ SelectFields: []string{"id", "compile_kwd", "compilation_template_kind_kwd", "type_kwd"},
Filter: map[string]any{"doc_id": []string{s.taskCtx.Doc.ID}},
})
if err != nil {
@@ -538,7 +538,7 @@ func (s *PipelineExecutor) loadDocumentCompiledState(ctx context.Context) ([]str
break
}
for _, row := range result.Chunks {
- if strings.TrimSpace(asCompiledKwd(row)) == "" {
+ if strings.TrimSpace(asCompiledKwd(row)) == "" || enginetypes.IsNavigationRow(row) {
continue
}
oldProducts = append(oldProducts, row)
@@ -609,22 +609,16 @@ func applyDocumentAvailability(chunks []map[string]any, status *string) {
}
}
-// compiledVariants returns the sorted, de-duplicated set of compile types a
-// document's compiled products carry. It reads the authoritative
-// `compilation_template_kind_kwd` the KnowledgeCompiler component stamps on each
-// compiled product (the resolved template's kind) and maps it through
-// common.KindToVariant (O2a whitelist: unknown kinds are skipped). Products
-// without an authoritative kind fall back to their `compile_kwd`-derived variant.
-// This surfaces the compiler's runtime variant inference to PublishCompleted so
-// the consumer can route the dataset-level re-compile per compile type.
+// compiledVariants returns the sorted, de-duplicated execution variants needed
+// to route document completion to the dataset consumer.
func compiledVariants(chunks []map[string]any) []string {
seen := map[string]struct{}{}
for _, ck := range chunks {
- if _, ok := ck["compile_kwd"]; !ok {
+ if _, ok := ck["compile_kwd"]; !ok || enginetypes.IsNavigationRow(ck) {
continue
}
var v kccommon.Variant
- if kind, ok := ck["compilation_template_kind_kwd"].(string); ok && kind != "" {
+ if kind := enginetypes.CompilationKind(ck); kind != "" {
mapped, err := kccommon.KindToVariant(kind)
if err != nil {
continue // unknown template kind (O2a): skip, do not mis-route
@@ -662,11 +656,11 @@ func compiledVariants(chunks []map[string]any) []string {
func compiledTaskTypes(chunks []map[string]any) []string {
seen := map[string]struct{}{}
for _, ck := range chunks {
- if _, ok := ck["compile_kwd"]; !ok {
+ if _, ok := ck["compile_kwd"]; !ok || enginetypes.IsNavigationRow(ck) {
continue
}
taskType := ""
- if kind, ok := ck["compilation_template_kind_kwd"].(string); ok && strings.TrimSpace(kind) != "" {
+ if kind := enginetypes.CompilationKind(ck); kind != "" {
if mapped, err := kccommon.KindToTaskType(kind); err == nil {
taskType = mapped
}
diff --git a/internal/rag/agentic-rag/runtime/tool_compiled_expansion.go b/internal/rag/agentic-rag/runtime/tool_compiled_expansion.go
index a38a5f12c5..e81a47d488 100644
--- a/internal/rag/agentic-rag/runtime/tool_compiled_expansion.go
+++ b/internal/rag/agentic-rag/runtime/tool_compiled_expansion.go
@@ -22,6 +22,7 @@ import (
"encoding/json"
"fmt"
"math"
+ enginetypes "ragflow/internal/engine/types"
"sort"
"strings"
@@ -597,14 +598,14 @@ func asString(v any) string {
}
// seedFilters builds the entity/relation condition: {"knowledge_graph_kwd": [kind]} plus an
-// optional compile_kwd ("tree") or compilation_template_kind_kwd selector.
+// optional canonical compilation-kind selector.
func seedFilters(kind, compileKwd, templateKwd string) map[string][]string {
f := map[string][]string{"knowledge_graph_kwd": {kind}}
if compileKwd != "" {
f["compile_kwd"] = []string{compileKwd}
}
if templateKwd != "" {
- f["compilation_template_kind_kwd"] = []string{templateKwd}
+ f["compile_kwd"] = []string{enginetypes.CanonicalCompilationKind(templateKwd)}
}
return f
}
diff --git a/internal/rag/agentic-rag/runtime/tool_compiled_expansion_test.go b/internal/rag/agentic-rag/runtime/tool_compiled_expansion_test.go
index d81044f0bc..691654b663 100644
--- a/internal/rag/agentic-rag/runtime/tool_compiled_expansion_test.go
+++ b/internal/rag/agentic-rag/runtime/tool_compiled_expansion_test.go
@@ -21,6 +21,7 @@ import (
"context"
"errors"
"log"
+ enginetypes "ragflow/internal/engine/types"
"strings"
"testing"
@@ -248,6 +249,12 @@ func TestCompiledExpanderGroupsDocScopeByRealOwner(t *testing.T) {
func rowMatchesFilters(r map[string]any, filters map[string][]string) bool {
for k, vals := range filters {
rv := asString(r[k])
+ if k == "compile_kwd" {
+ rv = enginetypes.CompilationKind(r)
+ if len(vals) > 0 && strings.HasPrefix(vals[0], "wiki_") {
+ rv = enginetypes.CompilationRowType(r)
+ }
+ }
matched := false
for _, v := range vals {
if rv == v {
diff --git a/internal/rag/agentic-rag/runtime/tool_navigation.go b/internal/rag/agentic-rag/runtime/tool_navigation.go
index 72dfc98424..3f1f933111 100644
--- a/internal/rag/agentic-rag/runtime/tool_navigation.go
+++ b/internal/rag/agentic-rag/runtime/tool_navigation.go
@@ -117,7 +117,7 @@ func loadStructureGraph(ctx context.Context, indexName, docID string, kinds map[
}
sr := StructureRow{
CompileKwd: fmt.Sprint(row["compile_kwd"]),
- TemplateKind: fmt.Sprint(row["compilation_template_kind_kwd"]),
+ TemplateKind: types.CompilationKind(row),
KnowledgeGraphKwd: kg,
Content: fmt.Sprint(row["content_with_weight"]),
}
@@ -213,14 +213,13 @@ func parseFloats(v any) ([]float64, bool) {
}
func normalizeKind(row map[string]interface{}) string {
- kind, _ := row["compilation_template_kind_kwd"].(string)
- if kind == "" {
- kind, _ = row["compile_kwd"].(string)
- }
- kind = strings.ToLower(strings.TrimSpace(strings.ReplaceAll(kind, "-", "_")))
- if kind == "pageindex" || kind == "page_index" || kind == "knowledge_graph" {
+ kind := types.CompilationKind(row)
+ if kind == "page_index" || kind == "graph" {
return "timeline"
}
+ if kind == "mind_map" {
+ return "mindmap"
+ }
return kind
}
diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.go
index dbc62a0e46..b013c60ce4 100644
--- a/internal/service/dataset_artifact_service.go
+++ b/internal/service/dataset_artifact_service.go
@@ -139,7 +139,7 @@ func intValue(v interface{}) int {
// HasWiki reports whether the dataset has any compiled wiki artifact.
func (s *DatasetArtifactService) HasWiki(ctx context.Context, tenantID, datasetID string) (bool, error) {
_, total, err := s.searchCompiled(ctx, tenantID, datasetID,
- map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage}}, nil, 0, 1, nil)
+ map[string]interface{}{"type_kwd": []string{CompileKwdWikiPage}}, nil, 0, 1, nil)
if err != nil {
return false, err
}
@@ -166,7 +166,7 @@ type WikiPageItem struct {
}
// wikiPageSelectFields are the engine fields ListWikiPages reads.
-var wikiPageSelectFields = []string{"slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "outlinks_int", "summary_with_weight"}
+var wikiPageSelectFields = []string{"slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "entity_type_kwd", "type_kwd", "topic_kwd", "outlinks_int", "summary_with_weight"}
// wikiPageOrderBy ranks wiki pages by connectivity then title.
func wikiPageOrderBy() *types.OrderByExpr {
@@ -183,7 +183,7 @@ func (s *DatasetArtifactService) ListWikiPages(ctx context.Context, tenantID, da
// list_wiki_pages has no such duplication because its writer emits one row
// per page, so mirror that by selecting the merged rows (available_int=1).
filter := map[string]interface{}{
- "compile_kwd": []string{CompileKwdWikiPage},
+ "type_kwd": []string{CompileKwdWikiPage},
"available_int": 1, // merged dataset-level rows only (see engine available_int handling)
}
if pageType != "" {
@@ -249,7 +249,7 @@ func wikiPageItemsFromChunks(chunks []map[string]interface{}) []WikiPageItem {
}
func wikiPageItemFromChunk(c map[string]interface{}) WikiPageItem {
- pageType := firstStringValue(c["page_type_kwd"])
+ pageType := types.WikiPageCategory(c)
// slug_kwd is stored as the full "<page_type>/<slug>" form (Python
// contract); expose the bare slug to the frontend so it can be placed in
// a single URL path segment (gin :slug does not match '/').
@@ -305,13 +305,13 @@ func (s *DatasetArtifactService) GetWikiPage(ctx context.Context, tenantID, data
// filter matches the stored value exactly.
slugKwd := pageType + "/" + slug
filter := map[string]interface{}{
- "compile_kwd": []string{CompileKwdWikiPage},
+ "type_kwd": []string{CompileKwdWikiPage},
"page_type_kwd": []string{pageType},
"slug_kwd": []string{slugKwd},
"available_int": 1, // merged dataset-level page, not the per-doc source row
}
chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
- []string{"slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "md_with_weight",
+ []string{"slug_kwd", "title_kwd", "page_type_kwd", "compile_kwd", "entity_type_kwd", "type_kwd", "topic_kwd", "md_with_weight",
"content_with_weight", "summary_with_weight", "entity_names_kwd", "outlinks_kwd",
"related_kb_pages_kwd", "source_chunk_ids", "source_doc_ids"},
0, 1, nil)
@@ -331,7 +331,7 @@ func (s *DatasetArtifactService) GetWikiPage(ctx context.Context, tenantID, data
// slug_kwd is the full "<page_type>/<slug>" form; expose the bare slug so a
// client can pass it straight back to GetWikiPage/UpdateWikiPage without the
// "<page_type>/" prefix being doubled (matches ListWikiPages).
- detailPageType := firstStringValue(c["page_type_kwd"])
+ detailPageType := types.WikiPageCategory(c)
detailSlug := firstStringValue(c["slug_kwd"])
if detailPageType != "" {
detailSlug = strings.TrimPrefix(detailSlug, detailPageType+"/")
@@ -364,7 +364,7 @@ func (s *DatasetArtifactService) UpdateWikiPage(ctx context.Context, tenantID, d
// deterministically from the bare slug (see GetWikiPage).
slugKwd := pageType + "/" + slug
filter := map[string]interface{}{
- "compile_kwd": []string{CompileKwdWikiPage},
+ "type_kwd": []string{CompileKwdWikiPage},
"page_type_kwd": []string{pageType},
"slug_kwd": []string{slugKwd},
"available_int": 1, // merged dataset-level page only
@@ -417,7 +417,7 @@ type WikiTopicItem struct {
// title/slug/summary, mirroring Python list_wiki_topics.
func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, datasetID, keywords string) ([]WikiTopicItem, int64, error) {
filter := map[string]interface{}{
- "compile_kwd": []string{CompileKwdWikiPage},
+ "type_kwd": []string{CompileKwdWikiPage},
"page_type_kwd": []string{"concept", "entity", "topic"},
"available_int": 1, // count only merged pages, not per-doc source rows
}
@@ -461,7 +461,7 @@ func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, d
// contains the keyword.
func (s *DatasetArtifactService) wikiTopicsWithKeywordPages(ctx context.Context, tenantID, datasetID, lowerKeyword string) (map[string]bool, error) {
filter := map[string]interface{}{
- "compile_kwd": []string{CompileKwdWikiPage},
+ "type_kwd": []string{CompileKwdWikiPage},
"page_type_kwd": []string{"concept", "entity"},
}
fields := []string{"topic_kwd", "title_kwd", "slug_kwd", "summary_with_weight"}
@@ -579,8 +579,8 @@ type WikiGraphRelation struct {
// GetWikiGraph returns an incremental wiki graph slice for a dataset. keywords
// seeds the overview with BM25 entity matches; topN overrides the entity cap.
func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, datasetID, keywords string, topN *int) (*WikiGraph, error) {
- entityFilter := map[string]interface{}{"compile_kwd": []string{CompileKwdWikiEntity}, "available_int": 1}
- relationFilter := map[string]interface{}{"compile_kwd": []string{CompileKwdWikiRelation}, "available_int": 1}
+ entityFilter := map[string]interface{}{"type_kwd": []string{CompileKwdWikiEntity}, "available_int": 1}
+ relationFilter := map[string]interface{}{"type_kwd": []string{CompileKwdWikiRelation}, "available_int": 1}
_, totalEntities, err := s.searchCompiled(ctx, tenantID, datasetID, entityFilter, []string{"id"}, 0, 1, nil)
if err != nil {
return nil, err
@@ -716,7 +716,7 @@ func (s *DatasetArtifactService) GetWikiAlteration(ctx context.Context, tenantID
involved := map[string]struct{}{}
for offset := 0; ; offset += 1000 {
chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID,
- map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage}},
+ map[string]interface{}{"type_kwd": []string{CompileKwdWikiPage}},
[]string{"source_doc_ids"}, offset, 1000, nil)
if err != nil {
return nil, err
@@ -990,7 +990,7 @@ func (s *DatasetArtifactService) GetSkillTree(ctx context.Context, tenantID, dat
filter["kwd"] = []string{kwd}
}
chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
- []string{"kwd", "title_kwd", "page_type_kwd", "outlinks_int", "inlinks_int"}, 0, 10000, nil)
+ []string{"kwd", "title_kwd", "page_type_kwd", "compile_kwd", "entity_type_kwd", "type_kwd", "outlinks_int", "inlinks_int"}, 0, 10000, nil)
if err != nil {
return nil, 0, err
}
@@ -1000,7 +1000,7 @@ func (s *DatasetArtifactService) GetSkillTree(ctx context.Context, tenantID, dat
items = append(items, SkillTreeItem{
Kwd: firstStringValue(c["kwd"]),
Title: firstStringValue(c["title_kwd"]),
- PageType: firstStringValue(c["page_type_kwd"]),
+ PageType: types.WikiPageCategory(c),
Outlinks: outlinks,
Inlinks: inlinks,
})
diff --git a/internal/service/dataset_structure_graph.go b/internal/service/dataset_structure_graph.go
index 8c0a657590..ffdf08689b 100644
--- a/internal/service/dataset_structure_graph.go
+++ b/internal/service/dataset_structure_graph.go
@@ -664,7 +664,7 @@ func resolveDatasetStructureKind(kind string) string {
return "knowledge_graph"
case "mindmap", "mind_map":
// Align with Python _DATASET_STRUCTURE_KIND_ALIASES: resolve the
- // user-facing "mindmap" to the stored compilation_template_kind_kwd value
+ // user-facing "mindmap" to the canonical compilation kind
// "mind_map" (the template kind, which is what read/delete paths match on).
return "mind_map"
case "timeline":
@@ -880,7 +880,7 @@ func (s *DatasetArtifactService) discoverDocumentGraphTemplateMeta(ctx context.C
}
for _, row := range rows {
templateID := rowTemplateID(row)
- kind := firstStringValue(row["compilation_template_kind_kwd"])
+ kind := types.CompilationKind(row)
if kind == "" {
kind = firstStringValue(row["compile_kwd"])
}
@@ -947,17 +947,8 @@ type DatasetStructureGraphResponse struct {
Templates []DocumentStructureGraphTemplate `json:"templates"`
}
-// GetDatasetStructure returns the dataset-scope structure graph for a resolved
-// kind, mirroring Python get_dataset_structure (dataset_api_service.py). Discovery
-// scans knowledge_graph_kwd=["entity"] dataset rows (scope_kwd="dataset") and
-// matches the resolved kind against the stamped compilation_template_kind_kwd —
-// NOT compile_kwd (compile_kwd holds the autotype "hypergraph"/"list"/"mindmap",
-// which is never the kind discriminator; Python _discover_scope_templates matches
-// _resolve_dataset_structure_kind against compilation_template_kind_kwd the same
-// way). It collects distinct template ids, then reads each template's dataset
-// entity/relation rows via buildBucket. It does NOT read kg_build_meta (write/
-// delete-side only). When Keywords is set, it returns the keyword-matched
-// entity subgraph using the name/BM25 and relation expansion path.
+// GetDatasetStructure returns dataset-scope entity/relation graphs for the
+// requested compilation kind, optionally narrowed by keyword matching.
func (s *DatasetArtifactService) GetDatasetStructure(ctx context.Context, in DatasetStructureGraphInput) (*DatasetStructureGraphResponse, error) {
resolved := resolveDatasetStructureKind(in.Kind)
if resolved == "" {
@@ -968,31 +959,27 @@ func (s *DatasetArtifactService) GetDatasetStructure(ctx context.Context, in Dat
}
resp := &DatasetStructureGraphResponse{Kind: resolved, Templates: []DocumentStructureGraphTemplate{}}
- // Discover distinct template ids whose stamped template kind resolves to the
- // requested kind, scanning dataset-scope entity rows only. scope_kwd="dataset"
- // is required here (unlike the legacy doc_graph fallback) because dataset rows
- // are the only ones carrying compilation_template_kind_kwd we can trust for the
- // dataset-scope kind match.
+ // Discover templates from dataset entity rows of the requested kind.
templateIDs := map[string]struct{}{}
// "id" must be projected: graphRowSearch keys its result map by the row id,
// and Infinity only returns fields listed in SelectFields (it does not
// synthesize id), so omitting it silently drops every row (review Major).
// The resolved kind is pushed into the filter so the engine applies the
// predicate instead of scanning all entity rows and discarding them in Go.
- metaFields := []string{"id", "compilation_template_kind_kwd", "compilation_template_ids"}
+ metaFields := []string{"id", "compile_kwd", "compilation_template_kind_kwd", "compilation_template_ids"}
entityCountFilter := map[string]interface{}{
- "knowledge_graph_kwd": []string{"entity"},
- "scope_kwd": []string{"dataset"},
- "compilation_template_kind_kwd": []string{resolved},
+ "knowledge_graph_kwd": []string{"entity"},
+ "scope_kwd": []string{"dataset"},
+ "compile_kwd": []string{types.CanonicalCompilationKind(resolved)},
}
_, entityTotal, err := graphRowSearch(ctx, in.TenantID, in.DatasetID, []string{"id"}, entityCountFilter, nil, 0, 1, nil)
if err != nil {
return nil, err
}
relationCountFilter := map[string]interface{}{
- "knowledge_graph_kwd": []string{"relation"},
- "scope_kwd": []string{"dataset"},
- "compilation_template_kind_kwd": []string{resolved},
+ "knowledge_graph_kwd": []string{"relation"},
+ "scope_kwd": []string{"dataset"},
+ "compile_kwd": []string{types.CanonicalCompilationKind(resolved)},
}
_, relationTotal, err := graphRowSearch(ctx, in.TenantID, in.DatasetID, []string{"id"}, relationCountFilter, nil, 0, 1, nil)
if err != nil {
@@ -1006,7 +993,7 @@ func (s *DatasetArtifactService) GetDatasetStructure(ctx context.Context, in Dat
return nil, err
}
for _, row := range rows {
- tkind := firstStringValue(row["compilation_template_kind_kwd"])
+ tkind := types.CompilationKind(row)
if tkind == "" || resolveDatasetStructureKind(tkind) != resolved {
continue
}
@@ -1041,10 +1028,10 @@ func (s *DatasetArtifactService) GetDatasetStructure(ctx context.Context, in Dat
}
}
bucketMeta, entities, relations, err := s.keywordSubgraph(ctx, in.TenantID, in.DatasetID, "", keywords, templateMeta, map[string]interface{}{
- "scope_kwd": []string{"dataset"},
- "compilation_template_ids": []string{tid},
- "compilation_template_kind_kwd": []string{resolved},
- "knowledge_graph_kwd": []string{"entity"},
+ "scope_kwd": []string{"dataset"},
+ "compilation_template_ids": []string{tid},
+ "compile_kwd": []string{types.CanonicalCompilationKind(resolved)},
+ "knowledge_graph_kwd": []string{"entity"},
})
if err != nil {
return nil, err
@@ -1068,9 +1055,9 @@ func (s *DatasetArtifactService) GetDatasetStructure(ctx context.Context, in Dat
// Read each template's dataset entity/relation rows.
for tid := range templateIDs {
scope := map[string]interface{}{
- "scope_kwd": []string{"dataset"},
- "compilation_template_ids": []string{tid},
- "compilation_template_kind_kwd": []string{resolved},
+ "scope_kwd": []string{"dataset"},
+ "compilation_template_ids": []string{tid},
+ "compile_kwd": []string{types.CanonicalCompilationKind(resolved)},
}
entities, relations, err := s.buildBucket(ctx, in.TenantID, in.DatasetID, scope, nil)
if err != nil {
@@ -1096,7 +1083,7 @@ func (s *DatasetArtifactService) GetDatasetStructure(ctx context.Context, in Dat
// resolveGraphBucket mirrors Python _resolve_bucket.
func resolveGraphBucket(row map[string]interface{}, templateMeta map[string]map[string]interface{}, documentID string) (map[string]interface{}, map[string]interface{}) {
compileKwd := firstStringValue(row["compile_kwd"])
- kindVal := firstStringValue(row["compilation_template_kind_kwd"])
+ kindVal := types.CompilationKind(row)
if kindVal == "" {
kindVal = compileKwd
}
@@ -1292,7 +1279,7 @@ func (s *DatasetArtifactService) keywordSubgraph(ctx context.Context, tenantID,
name := strings.ToLower(strings.Join(strings.Fields(graphStr(candidate.node["name"])), " "))
template := rowTemplateID(candidate.row)
if template == "" {
- template = firstStringValue(candidate.row["compilation_template_kind_kwd"])
+ template = types.CompilationKind(candidate.row)
}
if template == "" {
template = firstStringValue(candidate.row["compile_kwd"])
diff --git a/internal/service/document/knowledge_compile_status.go b/internal/service/document/knowledge_compile_status.go
index 0683d78484..ad04de0df8 100644
--- a/internal/service/document/knowledge_compile_status.go
+++ b/internal/service/document/knowledge_compile_status.go
@@ -69,7 +69,7 @@ func (s *DocumentService) documentKnowledgeCompileTypes(ctx context.Context, ten
KbIDs: []string{datasetID},
Offset: offset,
Limit: 1000,
- SelectFields: []string{"compile_kwd", "compilation_template_kind_kwd"},
+ SelectFields: []string{"compile_kwd", "compilation_template_kind_kwd", "type_kwd"},
Filter: map[string]any{"doc_id": []string{documentID}},
IncludeUnavailable: true,
})
@@ -81,7 +81,10 @@ func (s *DocumentService) documentKnowledgeCompileTypes(ctx context.Context, ten
break
}
for _, row := range result.Chunks {
- kind := strings.TrimSpace(documentStoreString(row["compilation_template_kind_kwd"]))
+ if types.IsNavigationRow(row) {
+ continue
+ }
+ kind := types.CompilationKind(row)
variant, variantErr := kccommon.KindToVariant(kind)
if variantErr != nil {
variant, variantErr = knowledge_compile.KwdToVariant(strings.TrimSpace(documentStoreString(row["compile_kwd"])))
diff --git a/internal/service/document/wiki_refresh.go b/internal/service/document/wiki_refresh.go
index a1fa55f6c9..91ec114c8a 100644
--- a/internal/service/document/wiki_refresh.go
+++ b/internal/service/document/wiki_refresh.go
@@ -74,7 +74,7 @@ func (s *DocumentService) loadAvailabilityToggleChunkIDs(ctx context.Context, in
KbIDs: []string{datasetID},
Offset: offset,
Limit: sourceChunkAvailabilityBatchSize,
- SelectFields: []string{"id", "compile_kwd"},
+ SelectFields: []string{"id", "compile_kwd", "type_kwd"},
Filter: map[string]any{"doc_id": []string{documentID}},
})
cancel()
@@ -85,6 +85,9 @@ func (s *DocumentService) loadAvailabilityToggleChunkIDs(ctx context.Context, in
break
}
for _, row := range result.Chunks {
+ if enginetypes.IsNavigationRow(row) {
+ continue
+ }
if kwd := strings.TrimSpace(documentStoreString(row["compile_kwd"])); kwd != "" {
variant, variantErr := knowledge_compile.KwdToVariant(kwd)
if variantErr != nil || variant == kccommon.VariantWiki {
@@ -111,7 +114,7 @@ func (s *DocumentService) loadSourceChunkIDs(ctx context.Context, indexName, dat
KbIDs: []string{datasetID},
Offset: offset,
Limit: sourceChunkAvailabilityBatchSize,
- SelectFields: []string{"id", "compile_kwd"},
+ SelectFields: []string{"id", "compile_kwd", "type_kwd"},
Filter: map[string]any{"doc_id": []string{documentID}},
IncludeUnavailable: includeUnavailable,
})
@@ -123,6 +126,9 @@ func (s *DocumentService) loadSourceChunkIDs(ctx context.Context, indexName, dat
break
}
for _, row := range result.Chunks {
+ if enginetypes.IsNavigationRow(row) {
+ continue
+ }
if strings.TrimSpace(documentStoreString(row["compile_kwd"])) != "" {
continue
}
diff --git a/internal/service/file/file_commit.go b/internal/service/file/file_commit.go
index 2c266575ec..81c805390a 100644
--- a/internal/service/file/file_commit.go
+++ b/internal/service/file/file_commit.go
@@ -709,7 +709,7 @@ func (s *FileCommitService) readCurrentPageContent(ctx context.Context, tenantID
Limit: 1,
SelectFields: []string{"md_with_weight", "content_with_weight"},
Filter: map[string]interface{}{
- "compile_kwd": []string{"wiki_page"},
+ "type_kwd": []string{"wiki_page"},
"page_type_kwd": []string{pageType},
"slug_kwd": []string{slugKwd},
"available_int": 1,
diff --git a/internal/service/nav/nav.go b/internal/service/nav/nav.go
index c1d77d658f..ad411533bf 100644
--- a/internal/service/nav/nav.go
+++ b/internal/service/nav/nav.go
@@ -64,11 +64,12 @@ type NavHit struct {
// UpsertDocInput carries the per-document summary to place into the nav tree.
type UpsertDocInput struct {
- TenantID string
- KbID string
- DocID string
- Summary string // document summary text (tree product or page_index summary)
- Embedd []float32 // optional precomputed embedding
+ CompileKind string // tree or page_index
+ TenantID string
+ KbID string
+ DocID string
+ Summary string // document summary text (tree product or page_index summary)
+ Embedd []float32 // optional precomputed embedding
}
// NavMergeLLM summarizes or merges nav text via an LLM. It mirrors the Python
diff --git a/internal/service/nlp/datasetnav.go b/internal/service/nlp/datasetnav.go
index 2cad92b644..06a5fb4e74 100644
--- a/internal/service/nlp/datasetnav.go
+++ b/internal/service/nlp/datasetnav.go
@@ -33,11 +33,10 @@ import (
)
// Dataset-nav constants. These mirror Python's dataset_nav.py: nav rows live in
-// the document index as compile_kwd="dataset_nav" rows with available_int=0
+// the document index with type_kwd nav_doc/nav_cluster and available_int=0
// (invisible to the default retriever, which filters available_int=1), and the
// tree is threaded through parent_kwd ("root" for depth-0 clusters).
const (
- navCompileKwd = "dataset_nav"
navRootParent = "root"
navMergeThreshold = 0.80 // sim >= this -> merge doc into cluster
@@ -128,9 +127,9 @@ func (s *NavService) navIndexName(tenantID string) string {
return fmt.Sprintf("ragflow_%s", tenantID)
}
-// navFilter builds the common nav filter that pins compile_kwd.
+// navFilter scopes navigation reads and writes to navigation row roles.
func navFilter(extra map[string]interface{}) map[string]interface{} {
- f := map[string]interface{}{"compile_kwd": []string{navCompileKwd}}
+ f := map[string]interface{}{"type_kwd": []string{nav.TypeNavDoc, nav.TypeNavCluster}}
for k, v := range extra {
f[k] = v
}
@@ -648,7 +647,7 @@ func navIntersectScope(docIDs []string, allowed map[string]struct{}) []string {
// SummariesByDocIDs returns the nav_doc description — the document's overall
// summary — keyed by doc_id for the given documents. It mirrors Python
// dataset_api_service._nav_doc_summaries: it reads the nav_doc rows
-// (compile_kwd=dataset_nav, type_kwd=nav_doc) for the doc_ids and yields, per
+// (type_kwd=nav_doc) for the doc_ids and yields, per
// doc, the payload description. The description wins over the row's label on
// purpose: callers render this value to the model as the routed document's
// <summary>, and title_kwd holds only the summary's first line (length-capped).
@@ -706,6 +705,13 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
if strings.TrimSpace(in.Summary) == "" {
return nil
}
+ in.CompileKind = types.CanonicalCompilationKind(in.CompileKind)
+ if in.CompileKind == "" {
+ in.CompileKind = "tree"
+ }
+ if in.CompileKind != "tree" && in.CompileKind != "page_index" {
+ return fmt.Errorf("datasetnav: invalid compilation kind %q", in.CompileKind)
+ }
de, err := s.docEngine()
if err != nil {
return err
@@ -739,7 +745,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
"type_kwd": []string{nav.TypeNavDoc},
"doc_id": []string{in.DocID},
}),
- []string{"content_with_weight", "content_ltks", "content_sm_ltks", "title_kwd", "docnm_kwd"}, 0, 1, nil)
+ []string{"content_with_weight", "content_ltks", "content_sm_ltks", "title_kwd", "docnm_kwd", "compile_kwd"}, 0, 1, nil)
if err != nil {
return err
}
@@ -751,16 +757,20 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
// file name. The tokenized text is (re)built when it is missing.
title := navDocTitle(in.Summary)
titleStale := navRowLabel(existing[0]) != title
+ kindStale := firstStringValue(existing[0]["compile_kwd"]) != in.CompileKind
ltksStale := firstStringValue(existing[0]["content_ltks"]) == "" ||
firstStringValue(existing[0]["content_sm_ltks"]) == ""
if payload, ok := existing[0]["content_with_weight"].(string); ok {
var m map[string]interface{}
if err := json.Unmarshal([]byte(payload), &m); err == nil {
if d, _ := m["description"].(string); d == in.Summary {
- if !titleStale && !ltksStale {
+ if !titleStale && !ltksStale && !kindStale {
return nil // unchanged
}
updates := map[string]interface{}{}
+ if kindStale {
+ updates["compile_kwd"] = in.CompileKind
+ }
if titleStale {
updates["title_kwd"] = title
}
@@ -770,10 +780,9 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
}
return de.UpdateChunks(ctx,
map[string]interface{}{
- "compile_kwd": []string{navCompileKwd},
- "type_kwd": []string{nav.TypeNavDoc},
- "doc_id": []string{in.DocID},
- "kb_id": in.KbID,
+ "type_kwd": []string{nav.TypeNavDoc},
+ "doc_id": []string{in.DocID},
+ "kb_id": in.KbID,
},
updates, s.navIndexName(in.TenantID), in.KbID)
}
@@ -813,7 +822,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
return err
}
if err := s.appendDocToCluster(ctx, de, in.TenantID, in.KbID, bestName, in.DocID,
- fmt.Sprintf("q_%d_vec", len(vec))); err != nil {
+ fmt.Sprintf("q_%d_vec", len(vec)), in.CompileKind); err != nil {
return err
}
// Explicit stable id (A5): the nav_doc row is addressable by a
@@ -822,7 +831,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
// instead of relying on a doc_id + type filter alone.
row := map[string]interface{}{
"id": navDocID(in.TenantID, in.KbID, in.DocID),
- "compile_kwd": navCompileKwd,
+ "compile_kwd": in.CompileKind,
"available_int": 0,
"type_kwd": "nav_doc",
// The nav_doc's label is a readable title derived from the summary
@@ -882,7 +891,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
clusterRow := map[string]interface{}{
"id": navClusterID(in.TenantID, in.KbID, name),
"doc_id": in.KbID, // cluster rows carry the kb as doc_id (Python _build_nav_cluster_row), so ES InsertChunks does not skip them
- "compile_kwd": navCompileKwd,
+ "compile_kwd": in.CompileKind,
"available_int": 0,
"type_kwd": "nav_cluster",
"title_kwd": name,
@@ -906,7 +915,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
// empty. The nav_doc carries a readable title + parent_kwd = the cluster name.
docRow := map[string]interface{}{
"id": navDocID(in.TenantID, in.KbID, in.DocID),
- "compile_kwd": navCompileKwd,
+ "compile_kwd": in.CompileKind,
"available_int": 0,
"type_kwd": "nav_doc",
"title_kwd": navDocTitle(in.Summary),
@@ -964,8 +973,8 @@ func navChildRehomeCondition(child map[string]interface{}, kbID string) map[stri
return nil
}
cond := map[string]interface{}{
- "compile_kwd": []string{navCompileKwd},
- "kb_id": kbID,
+ "type_kwd": []string{nav.TypeNavDoc, nav.TypeNavCluster},
+ "kb_id": kbID,
}
if rowID := firstStringValue(child["id"]); rowID != "" {
cond["id"] = []string{rowID}
@@ -1148,7 +1157,7 @@ func (s *NavService) maybeSplitCluster(ctx context.Context, tenantID, kbID, clus
// engine-assigned row id and inherit the cluster's directly-held documents.
clusterChunks, _, err := s.navSearch(ctx, tenantID, kbID,
navClusterCondition(tenantID, kbID, clusterName),
- []string{"id", "doc_count_int", "doc_ids_kwd", "parent_kwd", "depth_int"}, 0, 1, nil)
+ []string{"id", "doc_count_int", "doc_ids_kwd", "parent_kwd", "depth_int", "compile_kwd"}, 0, 1, nil)
if err != nil {
return err
}
@@ -1242,6 +1251,10 @@ func (s *NavService) maybeSplitCluster(ctx context.Context, tenantID, kbID, clus
if err != nil {
return err
}
+ splitCompileKind := "tree"
+ if len(clusterChunks) > 0 && types.CompilationKind(clusterChunks[0]) == "page_index" {
+ splitCompileKind = "page_index"
+ }
for _, spl := range []struct {
name string
count int
@@ -1269,7 +1282,7 @@ func (s *NavService) maybeSplitCluster(ctx context.Context, tenantID, kbID, clus
}
row := map[string]interface{}{
"id": navClusterID(tenantID, kbID, spl.name),
- "compile_kwd": navCompileKwd,
+ "compile_kwd": splitCompileKind,
"available_int": 0,
"type_kwd": "nav_cluster",
"title_kwd": spl.name,
@@ -1507,10 +1520,10 @@ func (s *NavService) llmCreateSummary(ctx context.Context, tenantID, text string
// doc_count_int. Implemented as a read-modify-write. vecCol names the engine
// vector column (q_<dim>_vec) so an overfull-cluster split can inherit a
// representative vector; pass "" when the caller has no known dimension.
-func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine, tenantID, kbID, clusterName, docID, vecCol string) error {
+func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine, tenantID, kbID, clusterName, docID, vecCol, compileKind string) error {
chunks, _, err := s.navSearch(ctx, tenantID, kbID,
navClusterCondition(tenantID, kbID, clusterName),
- []string{"doc_ids_kwd", "doc_count_int"}, 0, 1, nil)
+ []string{"doc_ids_kwd", "doc_count_int", "compile_kwd"}, 0, 1, nil)
if err != nil {
return err
}
@@ -1546,9 +1559,13 @@ func (s *NavService) appendDocToCluster(ctx context.Context, de engine.DocEngine
// this one. The read-modify-write here is expected to run under a per-dataset
// lock held by the UpsertDoc caller; without it, concurrent appends to the
// same cluster can lose updates.
+ updates := map[string]interface{}{"doc_ids_kwd": ids, "doc_count_int": count}
+ if kind := types.CompilationKind(chunks[0]); kind != "tree" && kind != "page_index" {
+ updates["compile_kwd"] = compileKind
+ }
if err := de.UpdateChunks(ctx,
navClusterCondition(tenantID, kbID, clusterName),
- map[string]interface{}{"doc_ids_kwd": ids, "doc_count_int": count},
+ updates,
s.navIndexName(tenantID), kbID); err != nil {
return err
}
@@ -1649,7 +1666,7 @@ func (s *NavService) findClusterContainingDoc(ctx context.Context, tenantID, kbI
IndexNames: []string{s.navIndexName(tenantID)},
KbIDs: []string{kbID},
SelectFields: []string{"title_kwd", "docnm_kwd", "doc_ids_kwd"},
- Filter: map[string]interface{}{"type_kwd": "nav_cluster", "compile_kwd": []string{navCompileKwd}},
+ Filter: map[string]interface{}{"type_kwd": "nav_cluster"},
Offset: offset,
Limit: pageSize,
})
diff --git a/internal/service/nlp/datasetnav_test.go b/internal/service/nlp/datasetnav_test.go
index 56e03bef5c..48a024b1eb 100644
--- a/internal/service/nlp/datasetnav_test.go
+++ b/internal/service/nlp/datasetnav_test.go
@@ -188,6 +188,8 @@ func (m *memNavEngine) FilterDocIdsByMetaPushdown(context.Context, *gorm.DB, []s
return nil
}
+const navCompileKwd = "dataset_nav"
+
// matchNavRow mirrors dataset_nav._matches_condition (dataset_nav.py:598-608):
// every field must have one of the row's values equal to one of the condition's
// values. A list condition therefore means intersection when the row's field is
@@ -307,6 +309,49 @@ func newTestNav(eng *memNavEngine) *NavService {
return ns
}
+func TestNavServicePageIndexRowsAndCleanup(t *testing.T) {
+ eng := &memNavEngine{}
+ ns := newTestNav(eng)
+ if err := ns.UpsertDoc(t.Context(), nav.UpsertDocInput{TenantID: "t1", KbID: "kb1", DocID: "d1", CompileKind: "page_index", Summary: "Page index summary"}); err != nil {
+ t.Fatal(err)
+ }
+ if len(eng.rows) != 2 {
+ t.Fatalf("rows = %d, want document and cluster", len(eng.rows))
+ }
+ for _, row := range eng.rows {
+ if row["compile_kwd"] != "page_index" {
+ t.Fatalf("compile_kwd = %v, want page_index", row["compile_kwd"])
+ }
+ if row["type_kwd"] == "nav_cluster" {
+ row["compile_kwd"] = "dataset_nav"
+ }
+ }
+ if err := ns.UpsertDoc(t.Context(), nav.UpsertDocInput{TenantID: "t1", KbID: "kb1", DocID: "d2", CompileKind: "page_index", Summary: "Page index summary"}); err != nil {
+ t.Fatal(err)
+ }
+ if len(eng.rows) != 3 {
+ t.Fatalf("rows = %d, want two documents in one cluster", len(eng.rows))
+ }
+ for _, row := range eng.rows {
+ if row["compile_kwd"] != "page_index" {
+ t.Fatalf("updated compile_kwd = %v, want page_index", row["compile_kwd"])
+ }
+ }
+ hits, err := ns.Search(t.Context(), "t1", "kb1", "Page index", nil, nil, 5)
+ if err != nil || len(hits) == 0 {
+ t.Fatalf("Search = %v, %v", hits, err)
+ }
+ if err := ns.RemoveDoc(t.Context(), "t1", "kb1", "d1"); err != nil {
+ t.Fatal(err)
+ }
+ if err := ns.RemoveDoc(t.Context(), "t1", "kb1", "d2"); err != nil {
+ t.Fatal(err)
+ }
+ if len(eng.rows) != 0 {
+ t.Fatalf("rows after RemoveDoc = %d, want 0", len(eng.rows))
+ }
+}
+
// TestNavService_UpsertDoc_WritesNavRow asserts acceptance #1: after UpsertDoc
// the row carries both compile_kwd=dataset_nav and available_int=0.
func TestNavService_UpsertDoc_WritesNavRow(t *testing.T) {
@@ -319,8 +364,8 @@ func TestNavService_UpsertDoc_WritesNavRow(t *testing.T) {
t.Fatal("expected at least one nav row")
}
row := eng.rows[0]
- if row["compile_kwd"] != "dataset_nav" {
- t.Errorf("compile_kwd = %v, want dataset_nav", row["compile_kwd"])
+ if row["compile_kwd"] != "tree" {
+ t.Errorf("compile_kwd = %v, want tree", row["compile_kwd"])
}
if row["available_int"] != 0 {
t.Errorf("available_int = %v, want 0", row["available_int"])
diff --git a/internal/service/wikisearch/engine_service.go b/internal/service/wikisearch/engine_service.go
index 9eac92b799..a10e24a1a5 100644
--- a/internal/service/wikisearch/engine_service.go
+++ b/internal/service/wikisearch/engine_service.go
@@ -9,10 +9,7 @@ import (
"ragflow/internal/engine/types"
)
-// compileKWDWikiPage is the canonical compile_kwd for compiled wiki pages. It
-// must match Python's WIKI_PAGE_COMPILE_KWD ("wiki_page", wiki.py:1661) AND the
-// Go compiler's variantCompileKWD[VariantWiki] (component.go), so both Python-
-// and Go-produced wiki pages are surfaced by this service.
+// compileKWDWikiPage identifies the wiki page role.
const compileKWDWikiPage = "wiki_page"
// tenantIndexName returns the tenant-scoped chunk index name
@@ -59,7 +56,7 @@ func (s *engineWikiService) AvailableFor(ctx context.Context, tenantID string, d
KbIDs: datasetIDs,
Limit: 1,
Filter: map[string]interface{}{
- "compile_kwd": compileKWDWikiPage,
+ "type_kwd": compileKWDWikiPage,
"available_int": 1,
},
})
@@ -91,12 +88,9 @@ func (s *engineWikiService) QueryPages(ctx context.Context, tenantID string, dat
"id", "kb_id", "doc_id", "docnm_kwd", "content_with_weight",
"slug_kwd", "source_chunk_ids",
},
- // Discriminate wiki pages by compile_kwd="wiki_page". There is NO
- // "kc_kind" column in the chunk schema (infinity_mapping.json:47-56), so
- // it must not be used as a filter. Sections (page.go kind:"section") are
- // stamped compile_kwd="wiki_section", so this filter returns pages only.
+ // Page roles exclude sections and internal wiki state.
Filter: map[string]interface{}{
- "compile_kwd": compileKWDWikiPage,
+ "type_kwd": compileKWDWikiPage,
"available_int": 1,
},
MatchExprs: []interface{}{
diff --git a/internal/service/wikisearch/engine_service_test.go b/internal/service/wikisearch/engine_service_test.go
index fb71c681cc..700d4d5180 100644
--- a/internal/service/wikisearch/engine_service_test.go
+++ b/internal/service/wikisearch/engine_service_test.go
@@ -44,10 +44,10 @@ func (f *fakeDocEngine) Search(_ context.Context, req *types.SearchRequest) (*ty
rows := f.searchRows
// Honor the compile_kwd filter so availability semantics are exercised
// faithfully (only rows carrying the requested compile_kwd match).
- if kwd, ok := req.Filter["compile_kwd"].(string); ok {
+ if kwd, ok := req.Filter["type_kwd"].(string); ok {
filtered := make([]map[string]interface{}, 0, len(rows))
for _, r := range rows {
- if r["compile_kwd"] == kwd {
+ if types.CompilationRowType(r) == kwd {
filtered = append(filtered, r)
}
}
@@ -109,8 +109,8 @@ func TestEngineService_QueryPages_NormalizesEngineRowShape(t *testing.T) {
if !reflect.DeepEqual(eng.searchReq.KbIDs, []string{"kb1"}) {
t.Errorf("KbIDs = %v, want [kb1]", eng.searchReq.KbIDs)
}
- if f, ok := eng.searchReq.Filter["compile_kwd"].(string); !ok || f != "wiki_page" {
- t.Errorf("compile_kwd filter = %v, want wiki_page", eng.searchReq.Filter["compile_kwd"])
+ if f, ok := eng.searchReq.Filter["type_kwd"].(string); !ok || f != "wiki_page" {
+ t.Errorf("compile_kwd filter = %v, want wiki_page", eng.searchReq.Filter["type_kwd"])
}
// SelectFields must project slug_kwd + source_chunk_ids (Infinity's default
// projection omits them), otherwise the page slug and P7 provenance are lost.
@@ -238,8 +238,8 @@ func TestEngineService_AvailableFor_BoundedExistenceSearch(t *testing.T) {
if eng.searchReq.Limit != 1 {
t.Errorf("existence Limit = %d, want 1 (bounded)", eng.searchReq.Limit)
}
- if f, ok := eng.searchReq.Filter["compile_kwd"].(string); !ok || f != "wiki_page" {
- t.Errorf("existence compile_kwd filter = %v, want wiki_page", eng.searchReq.Filter["compile_kwd"])
+ if f, ok := eng.searchReq.Filter["type_kwd"].(string); !ok || f != "wiki_page" {
+ t.Errorf("existence compile_kwd filter = %v, want wiki_page", eng.searchReq.Filter["type_kwd"])
}
if !reflect.DeepEqual(eng.searchReq.KbIDs, []string{"kb1"}) {
t.Errorf("existence KbIDs = %v, want [kb1]", eng.searchReq.KbIDs)