Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Port dataset nav and structure graph fixes to Go, merge agents list (#18183)

Zhichang Yu6677f14bdff0
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go
index 75898b31f9..e7c6fab1db 100644
--- a/cmd/ragflow_server.go
+++ b/cmd/ragflow_server.go
@@ -569,6 +569,14 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArg
globalConfig.GetDefaultChatModel().Name,
globalConfig.GetDefaultEmbeddingModel().Name,
)
+ // The dataset-level knowledge-compile consumer (tree/structure products) upserts
+ // into the dataset-nav tree, so the Ingestor must install the same ES-backed
+ // NavService the API server installs. Without this, nav.GetNavService() returns
+ // nil and tree/structure products are dropped (the consumer logs "nav service
+ // unavailable, skipping dataset-nav upsert"), leaving the dataset tree empty.
+ // The embedder resolves the tenant's embedding model on demand, so both
+ // Search and UpsertDoc can embed queries/summaries automatically.
+ nav.SetNavService(nlp.NewNavService(service.NewNavEmbedder(service.NewModelProviderService(), "")))
// Memory extraction runs on the Ingestor's shared NATS consumer + worker
// pool (task_type="memory" dispatched by processMessage -> executeMemoryTask),
// so there is no longer a dedicated Redis memory consumer to start.
diff --git a/internal/dao/compilation_template_group.go b/internal/dao/compilation_template_group.go
index 76826d6b9d..2075a292b6 100644
--- a/internal/dao/compilation_template_group.go
+++ b/internal/dao/compilation_template_group.go
@@ -59,6 +59,33 @@ func (dao *CompilationTemplateGroupDAO) ListSaved(ctx context.Context, db *gorm.
return groups, nil
}
+// ListOwnedSaved returns only the tenant's own valid groups (built-in groups
+// with empty tenant_id are excluded), mirroring the Python list_saved() query
+// (cls.model.tenant_id == tenant_id). The merged /agents list uses this so
+// built-in catalogue groups do not leak into a tenant's canvas list.
+func (dao *CompilationTemplateGroupDAO) ListOwnedSaved(ctx context.Context, db *gorm.DB, tenantID, keywords, scope, orderby string, desc bool) ([]*entity.CompilationTemplateGroup, error) {
+ q := db.WithContext(ctx).
+ Where("tenant_id = ? AND status = ?", tenantID, string(entity.StatusValid))
+ if keywords != "" {
+ q = q.Where("name LIKE ?", "%"+keywords+"%")
+ }
+ if scope != "" {
+ q = q.Where("scope = ?", scope)
+ }
+ if orderby != "name" && orderby != "scope" && orderby != "create_time" && orderby != "update_time" {
+ orderby = "create_time"
+ }
+ dir := "asc"
+ if desc {
+ dir = "desc"
+ }
+ var groups []*entity.CompilationTemplateGroup
+ if err := q.Order(orderby + " " + dir).Find(&groups).Error; err != nil {
+ return nil, err
+ }
+ return groups, nil
+}
+
// CountSavedByTenant counts the tenant's own valid groups; built-in groups
// (empty tenant_id) are excluded, mirroring the group_count query in Python
// get_owner_filter / get_category_filter.
diff --git a/internal/handler/agent_test.go b/internal/handler/agent_test.go
index da88e819c4..1873868473 100644
--- a/internal/handler/agent_test.go
+++ b/internal/handler/agent_test.go
@@ -393,9 +393,12 @@ func setupAgentRouter(svc agentServiceIface) *gin.Engine {
func TestListAgents_Success(t *testing.T) {
title := "My Agent"
+ agentJSON, _ := json.Marshal(service.AgentItem{
+ ID: "canvas-1", Title: &title, Permission: "me", CanvasCategory: "agent_canvas",
+ })
svc := &fakeAgentService{
result: &service.ListAgentsResponse{
- Canvas: []*service.AgentItem{{ID: "canvas-1", Title: &title, Permission: "me", CanvasCategory: "agent_canvas"}},
+ Canvas: []json.RawMessage{agentJSON},
Total: 1,
},
code: common.CodeSuccess,
diff --git a/internal/handler/dataset_artifact.go b/internal/handler/dataset_artifact.go
index dcddfddda7..3ac47a86f2 100644
--- a/internal/handler/dataset_artifact.go
+++ b/internal/handler/dataset_artifact.go
@@ -339,7 +339,9 @@ func (h *DatasetArtifactHandler) ListNavigation(c *gin.Context) {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
- common.SuccessWithData(c, gin.H{"total": total, "nav": items}, "success")
+ // Response key is "items" (not "nav") — the frontend DatasetNavList reads
+ // data.items and Python list_nav_clusters/_nav_search return {"total","items"}.
+ common.SuccessWithData(c, gin.H{"total": total, "items": items}, "success")
}
// DeleteNavigation handles DELETE /navigation — delete all navigation clusters.
@@ -381,7 +383,9 @@ func (h *DatasetArtifactHandler) ListNavigationChildren(c *gin.Context) {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
- common.SuccessWithData(c, gin.H{"total": total, "children": items}, "success")
+ // Same contract as the top-level nav list: Python list_nav_children returns
+ // {"total","items"} and the frontend reads data.items for child expansion.
+ common.SuccessWithData(c, gin.H{"total": total, "items": items}, "success")
}
// GetSkillTree handles GET /skills — skill tree.
@@ -453,13 +457,20 @@ func (h *DatasetArtifactHandler) GetDocumentGraph(c *gin.Context) {
}
datasetID := c.Param("dataset_id")
documentID := c.Param("document_id")
- graphType := c.Query("graph_type")
- items, total, err := h.svc.GetDocumentGraph(c.Request.Context(), tenantID, datasetID, documentID, graphType)
+ resp, err := h.svc.GetDocumentGraph(c.Request.Context(), service.DocumentStructureGraphInput{
+ TenantID: tenantID,
+ DatasetID: datasetID,
+ DocumentID: documentID,
+ Keywords: c.Query("keywords"),
+ })
if err != nil {
common.ErrorWithCode(c, common.CodeDataError, err.Error())
return
}
- common.SuccessWithData(c, gin.H{"total": total, "graph": items}, "success")
+ if resp == nil {
+ resp = &service.DocumentStructureGraphResponse{Templates: []service.DocumentStructureGraphTemplate{}}
+ }
+ common.SuccessWithData(c, resp, "success")
}
// DeleteDocumentGraph handles DELETE /documents/<document_id>/structure/graph — delete document structure graph.
diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.go
index aa0b23357b..8b29cbebb1 100644
--- a/internal/ingestion/component/knowledge_compiler/component.go
+++ b/internal/ingestion/component/knowledge_compiler/component.go
@@ -449,44 +449,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
switch p.Variant {
case common.VariantStructure:
// knowledge_graph_kwd: "entity" | "relation" | "graph".
- if kind != "" {
- if err := doc.SetExtraValue("knowledge_graph_kwd", kind); err != nil {
- return err
- }
- }
- // Relations carry from/to entity endpoints (from_entity_kwd / to_entity_kwd).
- if kind == "relation" {
- if v := metaString(p.Meta, "from"); v != "" {
- if err := doc.SetExtraValue("from_entity_kwd", v); err != nil {
- return err
- }
- }
- if v := metaString(p.Meta, "to"); v != "" {
- if err := doc.SetExtraValue("to_entity_kwd", v); err != nil {
- return err
- }
- }
- }
- // Entities carry their canonical name on name_kwd (lowercased, mirroring
- // Python's _struct_to_doc_storage_doc; the structure-graph endpoints
- // filter/sort on it) plus entity_type_kwd and mention_count_int.
- if kind == "entity" {
- if v := metaString(p.Meta, "name"); v != "" {
- if err := doc.SetExtraValue("name_kwd", strings.ToLower(v)); err != nil {
- return err
- }
- }
- if v := metaString(p.Meta, "entity_type"); v != "" {
- if err := doc.SetExtraValue("entity_type_kwd", v); err != nil {
- return err
- }
- }
- }
- if v, ok := metaInt(p.Meta, "mention_count"); ok {
- if err := doc.SetExtraValue("mention_count_int", v); err != nil {
- return err
- }
- }
+ return applyStructureGraphColumns(doc, p, kind)
case common.VariantWiki:
// One artifact_page row per wiki page; section rows reuse the same
@@ -568,23 +531,35 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
}
case common.VariantTree:
- // raptor_kwd tags summary/root nodes; raptor_layer_int records tree depth.
- if kind != "" {
- if err := doc.SetExtraValue("raptor_kwd", kind); err != nil {
- return err
- }
- }
- if v, ok := metaInt(p.Meta, "level"); ok {
- if err := doc.SetExtraValue("raptor_layer_int", v); err != nil {
- return err
+ switch kind {
+ case "entity", "relation", "graph":
+ // The tree is also projected onto the structure-graph shape (Python
+ // raptor_tree_to_graph + _struct_upsert_tree_graph_rows): entity /
+ // relation rows carry knowledge_graph_kwd and the compact graph blob
+ // (kind "graph") is the /structure/graph discovery row. This is the
+ // same storage contract as the structure variant, so both share
+ // applyStructureGraphColumns.
+ return applyStructureGraphColumns(doc, p, kind)
+ default:
+ // RAPTOR summary/root rows: raptor_kwd tags the node kind;
+ // raptor_layer_int records tree depth.
+ if kind != "" {
+ if err := doc.SetExtraValue("raptor_kwd", kind); err != nil {
+ return err
+ }
}
- if err := doc.SetExtraValue("depth_int", v); err != nil {
- return err
+ if v, ok := metaInt(p.Meta, "level"); ok {
+ if err := doc.SetExtraValue("raptor_layer_int", v); err != nil {
+ return err
+ }
+ if err := doc.SetExtraValue("depth_int", v); err != nil {
+ return err
+ }
}
- }
- if v := metaStringSlice(p.Meta, "children"); len(v) > 0 {
- if err := doc.SetExtraValue("children_kwd", v); err != nil {
- return err
+ if v := metaStringSlice(p.Meta, "children"); len(v) > 0 {
+ if err := doc.SetExtraValue("children_kwd", v); err != nil {
+ return err
+ }
}
}
@@ -611,6 +586,53 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
return nil
}
+// applyStructureGraphColumns emits the structure-graph row columns shared by the
+// structure and tree variants (Python _struct_to_doc_storage_doc contract):
+// - knowledge_graph_kwd: "entity" | "relation" | "graph"
+// - relations: from_entity_kwd / to_entity_kwd
+// - entities: name_kwd (lowercased) / entity_type_kwd
+// - mention_count_int
+//
+// Keeping this in one helper prevents the two variants' storage contracts from
+// diverging (review Major).
+func applyStructureGraphColumns(doc *schema.ChunkDoc, p common.Product, kind string) error {
+ if kind != "" {
+ if err := doc.SetExtraValue("knowledge_graph_kwd", kind); err != nil {
+ return err
+ }
+ }
+ if kind == "relation" {
+ if v := metaString(p.Meta, "from"); v != "" {
+ if err := doc.SetExtraValue("from_entity_kwd", v); err != nil {
+ return err
+ }
+ }
+ if v := metaString(p.Meta, "to"); v != "" {
+ if err := doc.SetExtraValue("to_entity_kwd", v); err != nil {
+ return err
+ }
+ }
+ }
+ if kind == "entity" {
+ if v := metaString(p.Meta, "name"); v != "" {
+ if err := doc.SetExtraValue("name_kwd", strings.ToLower(v)); err != nil {
+ return err
+ }
+ }
+ if v := metaString(p.Meta, "entity_type"); v != "" {
+ if err := doc.SetExtraValue("entity_type_kwd", v); err != nil {
+ return err
+ }
+ }
+ }
+ if v, ok := metaInt(p.Meta, "mention_count"); ok {
+ if err := doc.SetExtraValue("mention_count_int", v); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
// metaString reads a string-valued Product.Meta key.
func metaString(m map[string]any, key string) string {
v, _ := m[key].(string)
diff --git a/internal/ingestion/component/knowledge_compiler/golden/metrics.go b/internal/ingestion/component/knowledge_compiler/golden/metrics.go
index 753cfa8ff2..27224bb628 100644
--- a/internal/ingestion/component/knowledge_compiler/golden/metrics.go
+++ b/internal/ingestion/component/knowledge_compiler/golden/metrics.go
@@ -47,10 +47,18 @@ func AnalyzeTreeProducts(chunks []schema.ChunkDoc, validSourceIDs ...string) Tre
validSet[id] = true
}
checkValid := len(validSet) > 0
- m := TreeMetrics{ProductCount: len(chunks), AllParented: true, VectorOK: true, SchemaOK: true, covered: make(map[string]bool)}
+ m := TreeMetrics{AllParented: true, VectorOK: true, SchemaOK: true, covered: make(map[string]bool)}
maxLevel := -1
for _, c := range chunks {
kind, _ := c.GetExtraString("kc_kind")
+ // Auxiliary tree-graph rows (kind entity/relation/graph, from the
+ // structure-graph projection) are not part of the RAPTOR tree structure,
+ // so they are excluded from the structural metrics (ProductCount,
+ // AllParented, SchemaOK, ...). The tree itself is the root/summary set.
+ if kind != "root" && kind != "summary" {
+ continue
+ }
+ m.ProductCount++
level := 0
if lf, ok := extraFloat(c, "kc_level"); ok {
level = int(lf)
diff --git a/internal/ingestion/component/knowledge_compiler/tree/graph.go b/internal/ingestion/component/knowledge_compiler/tree/graph.go
new file mode 100644
index 0000000000..a6b3a678ac
--- /dev/null
+++ b/internal/ingestion/component/knowledge_compiler/tree/graph.go
@@ -0,0 +1,356 @@
+package tree
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "sort"
+ "strings"
+
+ "ragflow/internal/ingestion/component/knowledge_compiler/common"
+)
+
+// stringsContains reports whether s contains substring sub.
+func stringsContains(s, sub string) bool {
+ return strings.Contains(s, sub)
+}
+
+// strSliceContains reports whether s contains v.
+func strSliceContains(s []string, v string) bool {
+ for _, e := range s {
+ if e == v {
+ return true
+ }
+ }
+ return false
+}
+
+// stringsJoinNonEmpty joins the non-empty parts with sep.
+func stringsJoinNonEmpty(parts []string, sep string) string {
+ var kept []string
+ for _, p := range parts {
+ if p != "" {
+ kept = append(kept, p)
+ }
+ }
+ return strings.Join(kept, sep)
+}
+
+// stringMetaSlice coerces a Product.Meta value ([]string or []any of strings)
+// into a []string.
+func stringMetaSlice(v any) []string {
+ switch x := v.(type) {
+ case []string:
+ return x
+ case []any:
+ out := make([]string, 0, len(x))
+ for _, e := range x {
+ if s, ok := e.(string); ok && s != "" {
+ out = append(out, s)
+ }
+ }
+ return out
+ }
+ return nil
+}
+
+// payloadChunkIDs extracts the source_chunk_ids from a tree-graph payload.
+func payloadChunkIDs(payload map[string]any) []string {
+ switch v := payload["source_chunk_ids"].(type) {
+ case []string:
+ return v
+ case []any:
+ var out []string
+ for _, e := range v {
+ if s, ok := e.(string); ok && s != "" {
+ out = append(out, s)
+ }
+ }
+ return out
+ }
+ return nil
+}
+
+// payloadDescription is the embedding input for a tree-graph entity/relation:
+// the concatenated string values of every field except description (lists
+// flattened), matching Python _struct_payload_description.
+func payloadDescription(payload map[string]any) string {
+ keys := make([]string, 0, len(payload))
+ for k := range payload {
+ if k != "description" {
+ keys = append(keys, k)
+ }
+ }
+ sort.Strings(keys)
+ var parts []string
+ for _, k := range keys {
+ switch v := payload[k].(type) {
+ case string:
+ if v != "" {
+ parts = append(parts, v)
+ }
+ case []string:
+ for _, e := range v {
+ if e != "" {
+ parts = append(parts, e)
+ }
+ }
+ case []any:
+ for _, e := range v {
+ if s, ok := e.(string); ok && s != "" {
+ parts = append(parts, s)
+ }
+ }
+ }
+ }
+ return strings.Join(parts, " ")
+}
+
+// payloadJSON serialises a payload the way Python's json.dumps(ensure_ascii=
+// False) does (no HTML escaping), with alphabetically sorted keys for a
+// canonical, hash-stable form.
+func payloadJSON(payload map[string]any) string {
+ var b bytes.Buffer
+ enc := json.NewEncoder(&b)
+ enc.SetEscapeHTML(false)
+ if err := enc.Encode(payload); err != nil {
+ return "{}"
+ }
+ return strings.TrimSpace(b.String())
+}
+
+// graphNode mirrors Python's RAPTOR tree node dict (title/description/children/
+// source_chunk_ids) reconstructed from the flat products emitted by buildTree,
+// so the tree can be projected to a {entities, relations} graph exactly like
+// Python's raptor_tree_to_graph (chunk_post_processor.py:470).
+type graphNode struct {
+ title string
+ description string
+ sourceChunkIDs []string
+ children []*graphNode
+}
+
+// collapseUnary merges a node that wraps exactly one child into that child,
+// mirroring Python raptor_tree_to_graph._collapse_unary: the parent's and the
+// child's descriptions/source-chunk-ids are concatenated (dedup'd), then the
+// collapsed node adopts the child's children.
+func collapseUnary(node *graphNode) *graphNode {
+ collapsed := &graphNode{
+ title: node.title,
+ description: node.description,
+ sourceChunkIDs: node.sourceChunkIDs,
+ }
+ for _, c := range node.children {
+ collapsed.children = append(collapsed.children, collapseUnary(c))
+ }
+ for len(collapsed.children) == 1 {
+ child := collapsed.children[0]
+ parentTitle := collapsed.title
+ childTitle := child.title
+ parentDesc := collapsed.description
+ if parentDesc == "" {
+ parentDesc = parentTitle
+ }
+ childDesc := child.description
+ if childDesc == "" {
+ childDesc = childTitle
+ }
+
+ var descriptions []string
+ descriptions = append(descriptions, parentDesc)
+ if childTitle != "" && childTitle != parentTitle &&
+ !stringsContains(childDesc, childTitle) {
+ descriptions = append(descriptions, childTitle)
+ }
+ if childDesc != "" && !strSliceContains(descriptions, childDesc) {
+ descriptions = append(descriptions, childDesc)
+ }
+
+ sourceChunkIDs := append([]string{}, collapsed.sourceChunkIDs...)
+ for _, id := range child.sourceChunkIDs {
+ if id != "" && !strSliceContains(sourceChunkIDs, id) {
+ sourceChunkIDs = append(sourceChunkIDs, id)
+ }
+ }
+
+ collapsed.description = stringsJoinNonEmpty(descriptions, "\n\n")
+ collapsed.sourceChunkIDs = sourceChunkIDs
+ collapsed.children = child.children
+ }
+ return collapsed
+}
+
+// raptorTreeToGraph projects a RAPTOR tree onto {entities, relations}, matching
+// Python raptor_tree_to_graph: every node becomes an entity of type "tree_node";
+// every parent→child edge (that is not a self-loop) becomes a "child" relation.
+func raptorTreeToGraph(root *graphNode) ([]map[string]any, []map[string]any) {
+ var entities []map[string]any
+ var relations []map[string]any
+ var walk func(node *graphNode, parentTitle string)
+ walk = func(node *graphNode, parentTitle string) {
+ if node == nil {
+ return
+ }
+ title := node.title
+ ent := map[string]any{
+ "name": title,
+ "type": "tree_node",
+ "description": firstNonEmpty(node.description, title),
+ "mention_count": 1,
+ }
+ if len(node.sourceChunkIDs) > 0 {
+ ent["source_chunk_ids"] = node.sourceChunkIDs
+ }
+ entities = append(entities, ent)
+ if parentTitle != "" && parentTitle != title {
+ relations = append(relations, map[string]any{
+ "from": parentTitle,
+ "to": title,
+ "type": "child",
+ })
+ }
+ for _, child := range node.children {
+ walk(child, title)
+ }
+ }
+ walk(root, "")
+ return entities, relations
+}
+
+// buildTreeGraph reconstructs the tree from the flat summary products and
+// produces the entity/relation/graph products Python writes for a tree variant
+// (_struct_upsert_tree_graph_rows + _struct_upsert_graph_json):
+// - one entity product per tree node (kind "entity", knowledge_graph_kwd via
+// the writer), type "tree_node";
+// - one relation product per parent→child edge (kind "relation");
+// - one compact graph blob product (kind "graph") carrying the whole
+// {entities, relations} projection, which is the /structure/graph discovery
+// row (Python scans knowledge_graph_kwd="graph").
+//
+// templateID is stamped into each row so the document-structure endpoint can
+// group by template id; compileKWD is "tree".
+func buildTreeGraph(ctx context.Context, deps common.Deps, docID string, products []common.Product) ([]common.Product, error) {
+ if deps.Embed == nil {
+ return nil, fmt.Errorf("tree: embedder required to build the tree graph")
+ }
+ root := reconstructTree(products)
+ if root == nil {
+ // No root summary survived; there is no tree to project.
+ return nil, nil
+ }
+ root = collapseUnary(root)
+ entities, relations := raptorTreeToGraph(root)
+
+ var out []common.Product
+ var descs []string
+ var payloads []map[string]any
+ var kinds []string
+ for _, ent := range entities {
+ descs = append(descs, payloadDescription(ent))
+ payloads = append(payloads, ent)
+ kinds = append(kinds, "entity")
+ }
+ for _, rel := range relations {
+ descs = append(descs, payloadDescription(rel))
+ payloads = append(payloads, rel)
+ kinds = append(kinds, "relation")
+ }
+ vecs, err := deps.Embed.Encode(ctx, descs)
+ if err != nil {
+ return nil, err
+ }
+ for i, payload := range payloads {
+ kind := kinds[i]
+ var vec []float32
+ if i < len(vecs) {
+ vec = vecs[i]
+ }
+ meta := map[string]any{
+ "kind": kind,
+ "compile_kwd": "tree",
+ "source_chunk_ids": payloadChunkIDs(payload),
+ "mention_count": 1,
+ }
+ if kind == "entity" {
+ if name, ok := payload["name"].(string); ok && name != "" {
+ meta["name"] = name
+ }
+ if typ, ok := payload["type"].(string); ok && typ != "" {
+ meta["entity_type"] = typ
+ } else {
+ meta["entity_type"] = "other"
+ }
+ } else {
+ if from, ok := payload["from"].(string); ok {
+ meta["from"] = from
+ }
+ if to, ok := payload["to"].(string); ok {
+ meta["to"] = to
+ }
+ }
+ out = append(out, common.Product{
+ ID: common.StableRowID(payloadJSON(payload), docID),
+ DocID: docID,
+ TenantID: deps.TenantID,
+ Variant: common.VariantTree,
+ Content: payloadJSON(payload),
+ Vector: vec,
+ Meta: meta,
+ })
+ }
+
+ // Compact graph blob discovery row (knowledge_graph_kwd="graph").
+ graph := map[string]any{"entities": entities, "relations": relations}
+ graphContent := payloadJSON(graph)
+ graphVecs, err := deps.Embed.Encode(ctx, []string{graphContent})
+ if err != nil {
+ return nil, err
+ }
+ var gv []float32
+ if len(graphVecs) > 0 {
+ gv = graphVecs[0]
+ }
+ out = append(out, common.Product{
+ ID: common.StableRowID(docID, "tree", "structure_graph"),
+ DocID: docID,
+ TenantID: deps.TenantID,
+ Variant: common.VariantTree,
+ Content: graphContent,
+ Vector: gv,
+ Meta: map[string]any{
+ "kind": "graph",
+ "compile_kwd": "tree",
+ },
+ })
+ return out, nil
+}
+
+// reconstructTree assembles a graphNode tree from the flat summary products:
+// the root has Meta.kind=="root"; every other node's parent is the product with
+// ID == node.ParentID. Node title comes from Meta.title, description from
+// Content, source chunk ids from Meta.source_chunk_ids.
+func reconstructTree(products []common.Product) *graphNode {
+ byID := make(map[string]*graphNode, len(products))
+ for _, p := range products {
+ title, _ := p.Meta["title"].(string)
+ byID[p.ID] = &graphNode{
+ title: title,
+ description: p.Content,
+ sourceChunkIDs: stringMetaSlice(p.Meta["source_chunk_ids"]),
+ }
+ }
+ var root *graphNode
+ for _, p := range products {
+ kind, _ := p.Meta["kind"].(string)
+ node := byID[p.ID]
+ if kind == "root" {
+ root = node
+ continue
+ }
+ if parent := byID[p.ParentID]; parent != nil {
+ parent.children = append(parent.children, node)
+ }
+ }
+ return root
+}
diff --git a/internal/ingestion/component/knowledge_compiler/tree/graph_test.go b/internal/ingestion/component/knowledge_compiler/tree/graph_test.go
new file mode 100644
index 0000000000..5aa56797ac
--- /dev/null
+++ b/internal/ingestion/component/knowledge_compiler/tree/graph_test.go
@@ -0,0 +1,107 @@
+package tree
+
+import (
+ "reflect"
+ "testing"
+
+ "ragflow/internal/ingestion/component/knowledge_compiler/common"
+)
+
+// TestRaptorTreeToGraph_CollapsesUnaryAndProjects verifies the tree→graph
+// projection matches Python raptor_tree_to_graph: unary chains are collapsed
+// (descriptions concatenated), each node becomes an entity of type tree_node,
+// and each parent→child edge (skipping self-loops) becomes a child relation.
+func TestRaptorTreeToGraph_CollapsesUnaryAndProjects(t *testing.T) {
+ // Build a tree: root → [X, A → [B]]. A is a unary wrapper (only child B); the
+ // collapse merges A into B, while root (two children) and B (no children)
+ // stay. Entities: root, X, B. Relations: root→X, root→B.
+ root := &graphNode{
+ title: "root",
+ description: "root desc",
+ children: []*graphNode{
+ {title: "X", description: "X desc"},
+ {
+ title: "A",
+ description: "A desc",
+ children: []*graphNode{
+ {title: "B", description: "B desc", sourceChunkIDs: []string{"c1"}},
+ },
+ },
+ },
+ }
+ root = collapseUnary(root)
+ entities, relations := raptorTreeToGraph(root)
+
+ // The unary node A survives but the child B is folded into it (it keeps A's
+ // title, concatenated descriptions and B's source chunk ids), matching
+ // Python _collapse_unary. Entities: root, X, A.
+ var names []string
+ for _, e := range entities {
+ names = append(names, e["name"].(string))
+ }
+ if !reflect.DeepEqual(names, []string{"root", "X", "A"}) {
+ t.Fatalf("entities after unary collapse = %v, want [root X A]", names)
+ }
+ // A must have B folded in: concatenated descriptions + the source chunk id.
+ byName := map[string]map[string]any{}
+ for _, e := range entities {
+ byName[e["name"].(string)] = e
+ }
+ a := byName["A"]
+ if a["description"] != "A desc\n\nB desc" {
+ t.Errorf("collapsed A description = %q, want %q", a["description"], "A desc\n\nB desc")
+ }
+ if !reflect.DeepEqual(a["source_chunk_ids"], []string{"c1"}) {
+ t.Errorf("collapsed A source_chunk_ids = %v, want [c1]", a["source_chunk_ids"])
+ }
+ if a["type"] != "tree_node" {
+ t.Errorf("entity type = %v, want tree_node", a["type"])
+ }
+ // Two relations: root → X and root → A (A is root's child after collapse).
+ if len(relations) != 2 {
+ t.Fatalf("relations = %v, want exactly [root->X, root->A]", relations)
+ }
+ wantRels := []struct{ from, to string }{{"root", "X"}, {"root", "A"}}
+ gotRels := []struct{ from, to string }{
+ {relations[0]["from"].(string), relations[0]["to"].(string)},
+ {relations[1]["from"].(string), relations[1]["to"].(string)},
+ }
+ if !reflect.DeepEqual(gotRels, wantRels) {
+ t.Errorf("relations = %v, want %v", gotRels, wantRels)
+ }
+}
+
+// TestRaptorTreeToGraph_SkipsSelfLoop verifies a parent whose child has the same
+// title does not produce a self-loop relation (Python's guard).
+func TestRaptorTreeToGraph_SkipsSelfLoop(t *testing.T) {
+ root := &graphNode{
+ title: "same",
+ children: []*graphNode{
+ {title: "same", description: "child"},
+ },
+ }
+ _, relations := raptorTreeToGraph(root)
+ if len(relations) != 0 {
+ t.Fatalf("self-loop relation must be skipped, got %v", relations)
+ }
+}
+
+// TestReconstructTree_FromFlatProducts verifies the flat summary products
+// (root + ParentID chains) are reassembled into a nested tree.
+func TestReconstructTree_FromFlatProducts(t *testing.T) {
+ products := []common.Product{
+ {ID: "root", Meta: map[string]any{"kind": "root", "title": "root"}},
+ {ID: "n1", ParentID: "root", Meta: map[string]any{"kind": "summary", "title": "N1"}, Content: "n1 desc"},
+ {ID: "n2", ParentID: "n1", Meta: map[string]any{"kind": "summary", "title": "N2"}, Content: "n2 desc"},
+ }
+ root := reconstructTree(products)
+ if root == nil || root.title != "root" {
+ t.Fatalf("reconstructed root = %+v, want title root", root)
+ }
+ if len(root.children) != 1 || root.children[0].title != "N1" {
+ t.Fatalf("root children = %+v, want [N1]", root.children)
+ }
+ if len(root.children[0].children) != 1 || root.children[0].children[0].title != "N2" {
+ t.Fatalf("N1 children = %+v, want [N2]", root.children[0].children)
+ }
+}
diff --git a/internal/ingestion/component/knowledge_compiler/tree/raptor.go b/internal/ingestion/component/knowledge_compiler/tree/raptor.go
index 5813e2e460..486d2bdb2f 100644
--- a/internal/ingestion/component/knowledge_compiler/tree/raptor.go
+++ b/internal/ingestion/component/knowledge_compiler/tree/raptor.go
@@ -53,6 +53,18 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
return common.Outputs{}, err
}
+ // Project the RAPTOR tree onto the {entities, relations} structure-graph
+ // shape (Python raptor_tree_to_graph) and persist it as entity/relation rows
+ // plus a compact graph blob (knowledge_graph_kwd="graph"), so the
+ // document-structure /structure/graph endpoint can serve the tree. A failure
+ // here must not abort the whole tree compile — the summary nodes are already
+ // valid on their own — so it is best-effort and surfaced as a log.
+ if graphProds, err := buildTreeGraph(ctx, deps, docID, products); err != nil {
+ log.Printf("tree: graph projection failed (best-effort, continuing): %v", err)
+ } else {
+ products = append(products, graphProds...)
+ }
+
out := common.Outputs{
Products: products,
}
diff --git a/internal/service/agent.go b/internal/service/agent.go
index 1d9910ce5c..7e4dabe513 100644
--- a/internal/service/agent.go
+++ b/internal/service/agent.go
@@ -310,6 +310,7 @@ type AgentService struct {
versionDAO *dao.UserCanvasVersionDAO
api4ConversationDAO *dao.API4ConversationDAO
compilationTemplateGroupDAO *dao.CompilationTemplateGroupDAO
+ compilationTemplateDAO *dao.CompilationTemplateDAO
// driver is the per-process runner that drives canvas
// invocations and produces SSE events. V1 persistence is
@@ -383,6 +384,7 @@ func NewAgentServiceWithOptions(
versionDAO: dao.NewUserCanvasVersionDAO(),
api4ConversationDAO: dao.NewAPI4ConversationDAO(),
compilationTemplateGroupDAO: dao.NewCompilationTemplateGroupDAO(),
+ compilationTemplateDAO: dao.NewCompilationTemplateDAO(),
runner: canvas.NewRunner(),
activeSessions: make(map[string]*activeAgentRun),
checkpointStore: cp,
@@ -415,14 +417,28 @@ type AgentItem struct {
CreateTime *int64 `json:"create_time,omitempty"`
UpdateTime *int64 `json:"update_time,omitempty"`
ReleaseTime *int64 `json:"release_time,omitempty"`
+ // Type discriminates agent vs compilation-template-group items in the merged
+ // /agents response. It is set only for merged items (agent => "agent").
+ Type string `json:"type,omitempty"`
}
-// ListAgentsResponse is the response body for GET /api/v1/agents.
+// ListAgentsResponse is the response body for GET /api/v1/agents. Canvas holds
+// pre-marshalled JSON items because a canvas entry is either an agent (AgentItem
+// shape) or a compilation template group (group shape with a "type" of
+// "compilation_template_group"); a single typed slice cannot express both.
type ListAgentsResponse struct {
- Canvas []*AgentItem `json:"canvas"`
- Total int64 `json:"total"`
+ Canvas []json.RawMessage `json:"canvas"`
+ Total int64 `json:"total"`
}
+// AgentItemType discriminates the two kinds of canvas items in the merged
+// /agents response, mirroring Python _COMPILATION_TEMPLATE_GROUP_CATEGORY and
+// the frontend AgentListItem union.
+const (
+ AgentItemTypeAgent = "agent"
+ AgentItemTypeGroup = CompilationTemplateGroupCategory // "compilation_template_group"
+)
+
// CompilationTemplateGroupCategory is the synthetic canvas_category the
// frontend uses to filter compilation template groups through the merged
// /agents endpoint. Mirrors Python _COMPILATION_TEMPLATE_GROUP_CATEGORY.
@@ -592,17 +608,46 @@ func (s *AgentService) ListAgents(ctx context.Context, userID string, keywords s
}
}
+ // A canvas entry is either an agent (user_canvas) or a compilation template
+ // group. Python splits canvas_category on commas and, when the tenant is the
+ // sole effective owner, merges the caller's template groups into the list.
+ categories := splitCategoryList(canvasCategory)
+ wantsGroups := sliceContains(categories, CompilationTemplateGroupCategory)
+ agentCategories := filterCategory(categories, CompilationTemplateGroupCategory)
+ // Merge mode mirrors Python: no category, no canvas_type, no tags -> the
+ // caller's template groups are interleaved with agents by update_time.
+ mergeMode := len(categories) == 0 && canvasType == "" && len(tags) == 0
+
+ // Groups-only mode: canvas_category is exactly ["compilation_template_group"].
+ // Template groups are always the caller's own, so they are only visible when
+ // the caller is an effective owner; otherwise (e.g. owner_ids names another
+ // user) return an empty list (review Major).
+ if len(categories) == 1 && wantsGroups {
+ if !sliceContains(effectiveOwnerIDs, userID) {
+ return &ListAgentsResponse{Canvas: []json.RawMessage{}, Total: 0}, common.CodeSuccess, nil
+ }
+ return s.listAgentsGroupsOnly(ctx, userID, keywords, orderBy, desc, page, pageSize)
+ }
+
+ // Fetch agents. In merge/mixed modes we disable SQL pagination (page=0) and
+ // paginate in Go after interleaving with groups, matching Python.
+ listPage, listSize := page, pageSize
+ agentCategoryFilter := canvasCategory
+ if mergeMode || (wantsGroups && len(agentCategories) > 0) {
+ listPage, listSize = 0, 0
+ agentCategoryFilter = strings.Join(agentCategories, ",")
+ }
canvases, total, err := s.canvasDAO.ListByTenantIDs(
ctx,
dao.DB,
effectiveOwnerIDs,
userID,
- page,
- pageSize,
+ listPage,
+ listSize,
orderBy,
desc,
keywords,
- canvasCategory,
+ agentCategoryFilter,
canvasType,
tags,
)
@@ -610,31 +655,252 @@ func (s *AgentService) ListAgents(ctx context.Context, userID string, keywords s
return nil, common.CodeServerError, fmt.Errorf("failed to list agents: %w", err)
}
- items := make([]*AgentItem, len(canvases))
+ agentItems := make([]*AgentItem, len(canvases))
for i, c := range canvases {
- items[i] = toAgentItem(c)
+ agentItems[i] = toAgentItem(c)
+ }
+ s.attachReleaseTimes(ctx, agentItems)
+
+ // Groups are owner-only (no team sharing) and scoped to the caller, so they
+ // are merged only when the caller's own tenant is an effective owner
+ // (Python include_template_groups).
+ includeGroups := sliceContains(effectiveOwnerIDs, userID)
+ if includeGroups && (mergeMode || wantsGroups) {
+ return s.mergeAgentsAndGroups(ctx, userID, agentItems, keywords, orderBy, desc, page, pageSize)
+ }
+
+ raw := make([]json.RawMessage, len(agentItems))
+ for i, item := range agentItems {
+ item.Type = AgentItemTypeAgent
+ raw[i] = marshalAgentItem(item)
}
+ return &ListAgentsResponse{Canvas: raw, Total: total}, common.CodeSuccess, nil
+}
- // Attach the latest release time per canvas so agent cards can render
- // the "published at" line (Python UserCanvasService.get_list parity).
- if len(items) > 0 {
- canvasIDs := make([]string, 0, len(items))
- for _, item := range items {
- canvasIDs = append(canvasIDs, item.ID)
+// listAgentsGroupsOnly returns only the caller's compilation template groups
+// (Python canvas_category == ["compilation_template_group"] branch).
+func (s *AgentService) listAgentsGroupsOnly(ctx context.Context, userID, keywords, orderBy string, desc bool, page, pageSize int) (*ListAgentsResponse, common.ErrorCode, error) {
+ groups, err := s.compilationTemplateGroupDAO.ListOwnedSaved(ctx, dao.DB, userID, keywords, "", orderBy, desc)
+ if err != nil {
+ return nil, common.CodeServerError, fmt.Errorf("failed to list compilation template groups: %w", err)
+ }
+ total := int64(len(groups))
+ groups = slicePage(groups, page, pageSize)
+ raw := make([]json.RawMessage, 0, len(groups))
+ for _, g := range groups {
+ item, err := s.marshalMergeGroupItem(ctx, userID, g)
+ if err != nil {
+ return nil, common.CodeServerError, fmt.Errorf("failed to build group item: %w", err)
}
- var releaseTimes map[string]int64
- releaseTimes, err = s.versionDAO.GetLatestReleaseTimes(ctx, dao.DB, canvasIDs)
+ raw = append(raw, item)
+ }
+ return &ListAgentsResponse{Canvas: raw, Total: total}, common.CodeSuccess, nil
+}
+
+// mergeAgentsAndGroups combines agents and the caller's compilation template
+// groups into a single list ordered by (canvas_category, name) ascending, then
+// pages in Go. desc is accepted for API signature parity but the ordering is
+// intentionally category/name based, not chronological.
+func (s *AgentService) mergeAgentsAndGroups(ctx context.Context, userID string, agentItems []*AgentItem, keywords, orderBy string, desc bool, page, pageSize int) (*ListAgentsResponse, common.ErrorCode, error) {
+ groups, err := s.compilationTemplateGroupDAO.ListOwnedSaved(ctx, dao.DB, userID, keywords, "", orderBy, desc)
+ if err != nil {
+ return nil, common.CodeServerError, fmt.Errorf("failed to list compilation template groups: %w", err)
+ }
+ merged := make([]mergeCanvasItem, 0, len(agentItems)+len(groups))
+ for _, item := range agentItems {
+ item.Type = AgentItemTypeAgent
+ merged = append(merged, mergeCanvasItem{
+ item: item,
+ category: item.CanvasCategory,
+ name: derefString(item.Title),
+ })
+ }
+ for _, g := range groups {
+ raw, err := s.marshalMergeGroupItem(ctx, userID, g)
if err != nil {
- return nil, common.CodeServerError, fmt.Errorf("failed to get release times: %w", err)
+ return nil, common.CodeServerError, fmt.Errorf("failed to build group item: %w", err)
}
- for _, item := range items {
- if t, ok := releaseTimes[item.ID]; ok {
- item.ReleaseTime = &t
- }
+ merged = append(merged, mergeCanvasItem{
+ raw: raw,
+ category: CompilationTemplateGroupCategory,
+ name: g.Name,
+ })
+ }
+ // Order the merged list by (category, name) ascending (A-Z): agents and
+ // compilation template groups are grouped by flow category, then sorted by
+ // display name within each category. The category sort key uses the raw
+ // canvas_category string so the natural order is agent_canvas <
+ // compilation_template_group < dataflow_canvas.
+ sort.SliceStable(merged, func(i, j int) bool {
+ if merged[i].category != merged[j].category {
+ return merged[i].category < merged[j].category
+ }
+ return strings.ToLower(merged[i].name) < strings.ToLower(merged[j].name)
+ })
+ total := int64(len(merged))
+ merged = slicePage(merged, page, pageSize)
+ raw := make([]json.RawMessage, 0, len(merged))
+ for _, m := range merged {
+ if m.raw != nil {
+ raw = append(raw, m.raw)
+ } else if m.item != nil {
+ raw = append(raw, marshalAgentItem(m.item))
}
}
+ return &ListAgentsResponse{Canvas: raw, Total: total}, common.CodeSuccess, nil
+}
- return &ListAgentsResponse{Canvas: items, Total: total}, common.CodeSuccess, nil
+// marshalMergeGroupItem renders a compilation template group as a merged /agents
+// canvas item: the group shape (ICompilationTemplateGroup) plus the "type" and
+// "title" discriminators the frontend union expects (Python _group_to_dict).
+func (s *AgentService) marshalMergeGroupItem(ctx context.Context, userID string, g *entity.CompilationTemplateGroup) (json.RawMessage, error) {
+ children, err := s.compilationTemplateDAO.ListByGroup(ctx, dao.DB, g.ID)
+ if err != nil {
+ return nil, err
+ }
+ templates := make([]json.RawMessage, 0, len(children))
+ for _, c := range children {
+ templates = append(templates, marshalGroupTemplate(c))
+ }
+ item := map[string]interface{}{
+ "id": g.ID,
+ "name": g.Name,
+ "title": g.Name,
+ "description": derefString(g.Description),
+ "scope": g.Scope,
+ "create_time": intValuePtr(g.CreateTime),
+ "update_time": intValuePtr(g.UpdateTime),
+ "templates": templates,
+ "type": AgentItemTypeGroup,
+ }
+ b, err := json.Marshal(item)
+ if err != nil {
+ return nil, err
+ }
+ return b, nil
+}
+
+// mergeCanvasItem is a decoded entry in the merged /agents list: exactly one of
+// item (an agent) or raw (a marshalled group) is set. category is the flow
+// classification (canvas_category, or "compilation_template_group" for groups)
+// and name is the display title; the merged list is ordered by (category, name).
+type mergeCanvasItem struct {
+ item *AgentItem
+ raw json.RawMessage
+ category string
+ name string
+}
+
+// attachReleaseTimes populates ReleaseTime for each agent item from the latest
+// published version (Python UserCanvasService.get_list parity).
+func (s *AgentService) attachReleaseTimes(ctx context.Context, items []*AgentItem) {
+ if len(items) == 0 {
+ return
+ }
+ canvasIDs := make([]string, 0, len(items))
+ for _, item := range items {
+ canvasIDs = append(canvasIDs, item.ID)
+ }
+ releaseTimes, err := s.versionDAO.GetLatestReleaseTimes(ctx, dao.DB, canvasIDs)
+ if err != nil {
+ return
+ }
+ for _, item := range items {
+ if t, ok := releaseTimes[item.ID]; ok {
+ item.ReleaseTime = &t
+ }
+ }
+}
+
+// marshalAgentItem renders an agent item to its JSON representation.
+func marshalAgentItem(item *AgentItem) json.RawMessage {
+ b, err := json.Marshal(item)
+ if err != nil {
+ return json.RawMessage("null")
+ }
+ return b
+}
+
+// marshalGroupTemplate renders a compilation template child to the read-side
+// shape the frontend group card expects (mirrors Python _to_saved_dict).
+func marshalGroupTemplate(c *entity.CompilationTemplate) json.RawMessage {
+ item := map[string]interface{}{
+ "id": c.ID,
+ "name": c.Name,
+ "description": derefString(c.Description),
+ "kind": c.Kind,
+ "config": c.Config,
+ "create_time": intValuePtr(c.CreateTime),
+ "update_time": intValuePtr(c.UpdateTime),
+ }
+ b, err := json.Marshal(item)
+ if err != nil {
+ return json.RawMessage("null")
+ }
+ return b
+}
+
+// splitCategoryList splits a comma-separated canvas_category query into the
+// non-empty categories, mirroring Python
+// request.args.get("canvas_category", "").strip().split(",").
+func splitCategoryList(s string) []string {
+ var out []string
+ for _, part := range strings.Split(s, ",") {
+ if p := strings.TrimSpace(part); p != "" {
+ out = append(out, p)
+ }
+ }
+ return out
+}
+
+// filterCategory returns the categories in src that are not equal to drop.
+func filterCategory(src []string, drop string) []string {
+ var out []string
+ for _, c := range src {
+ if c != drop {
+ out = append(out, c)
+ }
+ }
+ return out
+}
+
+// sliceContains reports whether v is present in s.
+func sliceContains[T comparable](s []T, v T) bool {
+ for _, e := range s {
+ if e == v {
+ return true
+ }
+ }
+ return false
+}
+
+// slicePage returns a shallow copy of s bounded to the requested page window.
+// A page <= 0 or pageSize <= 0 returns s unchanged (caller did not ask to page).
+// The page bound is checked arithmetically before computing the offset so an
+// unbounded positive page/page_size can never overflow to a negative start and
+// panic (review Critical).
+func slicePage[T any](s []T, page, pageSize int) []T {
+ if page <= 0 || pageSize <= 0 || len(s) == 0 {
+ return s
+ }
+ // page-1 must be <= (len(s)-1)/pageSize, i.e. start must be < len(s).
+ if page-1 > (len(s)-1)/pageSize {
+ return nil
+ }
+ start := (page - 1) * pageSize
+ if pageSize >= len(s)-start {
+ return s[start:]
+ }
+ return s[start : start+pageSize]
+}
+
+// intValuePtr dereferences a *int64 to a plain int64 (0 when nil), used for the
+// group/template create_time & update_time epoch fields in merged items.
+func intValuePtr(p *int64) int64 {
+ if p == nil {
+ return 0
+ }
+ return *p
}
// CreateAgentRequest is the input shape for CreateAgent.
diff --git a/internal/service/agent_test.go b/internal/service/agent_test.go
index aea52e6cc5..e5f2a53f05 100644
--- a/internal/service/agent_test.go
+++ b/internal/service/agent_test.go
@@ -834,6 +834,10 @@ func setupAgentSessionServiceTest(t *testing.T) {
&entity.UserCanvasVersion{},
&entity.UserTenant{},
&entity.API4Conversation{},
+ // The merged /agents list reads compilation template groups (and their
+ // children) alongside canvases, so those tables must exist too.
+ &entity.CompilationTemplateGroup{},
+ &entity.CompilationTemplate{},
); err != nil {
t.Fatalf("failed to migrate: %v", err)
}
@@ -1886,16 +1890,20 @@ func TestListAgentsIncludesReleaseTime(t *testing.T) {
}
var released, draft *AgentItem
- for _, item := range resp.Canvas {
+ for _, raw := range resp.Canvas {
+ var item AgentItem
+ if err := json.Unmarshal(raw, &item); err != nil {
+ t.Fatalf("unmarshal canvas item: %v", err)
+ }
switch item.ID {
case "canvas-listed-released":
- released = item
+ released = &item
case "canvas-listed-draft":
- draft = item
+ draft = &item
}
}
if released == nil || draft == nil {
- t.Fatalf("expected both canvases in list, got %#v", resp.Canvas)
+ t.Fatalf("expected both canvases in list, got %s", resp.Canvas)
}
if released.ReleaseTime == nil || *released.ReleaseTime != releaseTime {
t.Fatalf("ReleaseTime = %v, want %d", released.ReleaseTime, releaseTime)
@@ -1905,6 +1913,151 @@ func TestListAgentsIncludesReleaseTime(t *testing.T) {
}
}
+// TestListAgents_MergesCompilationTemplateGroups verifies that a compilation
+// template group owned by the caller appears in the merged /agents list
+// (no canvas_category filter), carrying the "compilation_template_group" type
+// discriminator and its title = name. Built-in catalogue groups (empty tenant)
+// must NOT leak in.
+func TestListAgents_MergesCompilationTemplateGroups(t *testing.T) {
+ setupAgentSessionServiceTest(t)
+
+ base := time.Now().UnixMilli()
+ groupUpdate := base + 1000 // group updated most recently
+ canvasUpdate := base + 5000 // agent updated most recently
+ if err := dao.DB.Create(&entity.User{ID: "user-1", Nickname: "owner", Email: "[email protected]"}).Error; err != nil {
+ t.Fatalf("failed to seed user: %v", err)
+ }
+ // Agent updated most recently (canvasUpdate > groupUpdate).
+ if err := dao.DB.Create(&entity.UserCanvas{
+ ID: "canvas-1", UserID: "user-1", Title: sptr("Agent canvas"),
+ CanvasCategory: "agent_canvas",
+ BaseModel: entity.BaseModel{UpdateTime: &canvasUpdate},
+ }).Error; err != nil {
+ t.Fatalf("failed to seed canvas: %v", err)
+ }
+ // The caller's own group (must appear), updated before the canvas.
+ createAgentSessionTestCompilationGroup(t, "group-own", "user-1", groupUpdate)
+ // A built-in catalogue group with empty tenant_id (must NOT appear).
+ if err := dao.DB.Create(&entity.CompilationTemplateGroup{
+ ID: "group-builtin", TenantID: "", Name: "Built-in templates",
+ Scope: "file", BaseModel: entity.BaseModel{CreateTime: &base},
+ }).Error; err != nil {
+ t.Fatalf("failed to seed builtin group: %v", err)
+ }
+
+ resp, code, err := NewAgentService().ListAgents(t.Context(), "user-1", "", 1, 30, "create_time", true, nil, "", "", nil)
+ if err != nil || code != common.CodeSuccess {
+ t.Fatalf("ListAgents failed: code=%v err=%v", code, err)
+ }
+
+ var sawAgent, sawOwnGroup, sawBuiltin bool
+ var prevCat, prevName string
+ orderOK := true
+ for i, raw := range resp.Canvas {
+ var item map[string]interface{}
+ if err := json.Unmarshal(raw, &item); err != nil {
+ t.Fatalf("unmarshal canvas item: %v", err)
+ }
+ // Sort category: agents carry canvas_category; groups are classified by
+ // the "compilation_template_group" type discriminator (mirroring the
+ // frontend AgentListItem union).
+ cat := "agent"
+ if item["type"] == AgentItemTypeGroup {
+ cat = CompilationTemplateGroupCategory
+ } else if cc, ok := item["canvas_category"].(string); ok {
+ cat = cc
+ }
+ name, _ := item["title"].(string)
+ if i > 0 {
+ if cat < prevCat || (cat == prevCat && strings.ToLower(name) < strings.ToLower(prevName)) {
+ orderOK = false
+ }
+ }
+ prevCat, prevName = cat, name
+ switch item["type"] {
+ case AgentItemTypeAgent:
+ if item["id"] == "canvas-1" {
+ sawAgent = true
+ }
+ case AgentItemTypeGroup:
+ if item["id"] == "group-own" {
+ sawOwnGroup = true
+ if item["title"] != "own-group" || item["name"] != "own-group" {
+ t.Errorf("group title/name = %v/%v, want own-group", item["title"], item["name"])
+ }
+ if item["templates"] == nil {
+ t.Errorf("group item missing templates")
+ }
+ }
+ if item["id"] == "group-builtin" {
+ sawBuiltin = true
+ }
+ }
+ }
+ if !sawAgent {
+ t.Fatal("agent canvas-1 not in merged list")
+ }
+ if !sawOwnGroup {
+ t.Fatal("caller's compilation template group not in merged list")
+ }
+ if sawBuiltin {
+ t.Fatal("built-in catalogue group leaked into merged list")
+ }
+ if !orderOK {
+ t.Fatal("merged list is not ordered by (canvas_category, name) ascending")
+ }
+ // agent_canvas < compilation_template_group, so the canvas must be first.
+ var first map[string]interface{}
+ if err := json.Unmarshal(resp.Canvas[0], &first); err != nil {
+ t.Fatalf("unmarshal first item: %v", err)
+ }
+ if first["id"] != "canvas-1" {
+ t.Fatalf("first merged item = %v, want canvas-1 (agent_canvas category)", first["id"])
+ }
+}
+
+// TestSlicePage_NoOverflowPanic guards the review-Critical integer overflow:
+// an unbounded positive page/page_size must never overflow (page-1)*pageSize to
+// a negative start and panic; out-of-range pages return nil.
+func TestSlicePage_NoOverflowPanic(t *testing.T) {
+ s := []int{1, 2, 3, 4, 5}
+ // A huge page must return nil, not panic on an overflowing offset.
+ if got := slicePage(s, 1<<40, 1<<40); got != nil {
+ t.Fatalf("huge page/pageSize = %v, want nil", got)
+ }
+ if got := slicePage(s, 2, 3); len(got) != 2 || got[0] != 4 || got[1] != 5 {
+ t.Fatalf("page2/size3 = %v, want [4 5]", got)
+ }
+ if got := slicePage(s, 3, 2); len(got) != 1 || got[0] != 5 {
+ t.Fatalf("page3/size2 (start 4) = %v, want [5]", got)
+ }
+ if got := slicePage(s, 4, 2); got != nil {
+ t.Fatalf("page4/size2 (start 6 >= len 5) = %v, want nil", got)
+ }
+ // Empty/no-paging semantics preserved (empty input returns the empty slice,
+ // never a panic).
+ if got := slicePage([]int{}, 1, 10); len(got) != 0 {
+ t.Fatalf("empty slice = %v, want empty", got)
+ }
+}
+
+func createAgentSessionTestCompilationGroup(t *testing.T, id, tenantID string, now int64) {
+ t.Helper()
+ if err := dao.DB.Create(&entity.CompilationTemplateGroup{
+ ID: id, TenantID: tenantID, Name: "own-group", Scope: "file",
+ BaseModel: entity.BaseModel{CreateTime: &now, UpdateTime: &now},
+ }).Error; err != nil {
+ t.Fatalf("failed to seed group %s: %v", id, err)
+ }
+ if err := dao.DB.Create(&entity.CompilationTemplate{
+ ID: id + "-tpl", GroupID: &id, Name: "tree tpl", Kind: "tree",
+ Config: entity.JSONMap{},
+ BaseModel: entity.BaseModel{CreateTime: &now},
+ }).Error; err != nil {
+ t.Fatalf("failed to seed template for %s: %v", id, err)
+ }
+}
+
func TestPublishAgentUpdatesCanvasAndReleasedVersion(t *testing.T) {
setupAgentSessionServiceTest(t)
diff --git a/internal/service/compilation_template_service.go b/internal/service/compilation_template_service.go
index 8b55809c09..ecd02c66f3 100644
--- a/internal/service/compilation_template_service.go
+++ b/internal/service/compilation_template_service.go
@@ -210,8 +210,17 @@ func ValidateTemplatePayload(req map[string]interface{}, requireAll bool) error
}
config, hasConfig := req["config"]
if hasConfig {
- configMap, ok := config.(map[string]interface{})
- if !ok {
+ // config may arrive as map[string]interface{} (raw payloads) or as
+ // entity.JSONMap (payloads built from a typed GroupTemplate, whose Config
+ // field is JSONMap). A bare type assertion on the former would reject the
+ // latter because JSONMap is a distinct named type, so normalize both.
+ var configMap map[string]interface{}
+ switch c := config.(type) {
+ case map[string]interface{}:
+ configMap = c
+ case entity.JSONMap:
+ configMap = map[string]interface{}(c)
+ default:
return errors.New("invalid template config")
}
if len(fmt.Sprint(configMap["global_rules"])) > 4096 {
diff --git a/internal/service/compilation_template_service_test.go b/internal/service/compilation_template_service_test.go
new file mode 100644
index 0000000000..fb9392d810
--- /dev/null
+++ b/internal/service/compilation_template_service_test.go
@@ -0,0 +1,69 @@
+package service
+
+import (
+ "testing"
+
+ "ragflow/internal/entity"
+)
+
+// TestValidateTemplatePayload_AcceptsJSONMapConfig covers the create-from-UI
+// regression: GroupTemplate.Config is an entity.JSONMap (a named map type), and
+// ValidateTemplatePayload used a bare config.(map[string]interface{}) assertion
+// that silently rejected it, surfacing as "102 invalid template config". Both
+// the raw map and the JSONMap forms must validate identically.
+func TestValidateTemplatePayload_AcceptsJSONMapConfig(t *testing.T) {
+ cfg := map[string]interface{}{
+ "kind": "graph",
+ "entity": map[string]interface{}{
+ "fields": []interface{}{
+ map[string]interface{}{
+ "type": "organization",
+ "description": "company",
+ },
+ },
+ },
+ "relation": map[string]interface{}{
+ "fields": []interface{}{
+ map[string]interface{}{
+ "type": "acquisition",
+ "description": "took over",
+ },
+ },
+ },
+ }
+ base := map[string]interface{}{
+ "name": "tpl",
+ "kind": "graph",
+ }
+
+ // 1) Raw map[string]interface{} payload (older path) must pass.
+ raw := map[string]interface{}{}
+ for k, v := range base {
+ raw[k] = v
+ }
+ raw["config"] = cfg
+ if err := ValidateTemplatePayload(raw, true); err != nil {
+ t.Fatalf("raw map config rejected: %v", err)
+ }
+
+ // 2) entity.JSONMap payload (what the group create path builds from
+ // GroupTemplate.Config) must also pass — this is the regression.
+ jm := map[string]interface{}{}
+ for k, v := range base {
+ jm[k] = v
+ }
+ jm["config"] = entity.JSONMap(cfg)
+ if err := ValidateTemplatePayload(jm, true); err != nil {
+ t.Fatalf("entity.JSONMap config rejected: %v", err)
+ }
+
+ // 3) A non-map config is still an error.
+ bad := map[string]interface{}{}
+ for k, v := range base {
+ bad[k] = v
+ }
+ bad["config"] = "not-a-map"
+ if err := ValidateTemplatePayload(bad, true); err == nil {
+ t.Fatal("non-map config should be rejected")
+ }
+}
diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.go
index c43f925af5..0ef4ca3744 100644
--- a/internal/service/dataset_artifact_service.go
+++ b/internal/service/dataset_artifact_service.go
@@ -49,7 +49,6 @@ const (
FieldStructureIndexType = "structure_index_type"
FieldStructureKind = "structure_kind"
FieldPageID = "page_id"
- FieldGraphType = "graph_type"
)
// DatasetArtifactService reads knowledge-compilation artifacts (wiki pages,
@@ -620,38 +619,6 @@ func (s *DatasetArtifactService) DeleteStructures(ctx context.Context, tenantID,
return len(ids), nil
}
-// DocGraphItem is a single node/edge entry in a document's structure graph.
-type DocGraphItem struct {
- ID string `json:"id"`
- Content string `json:"content"`
- SourceID string `json:"source_id"`
-}
-
-// GetDocumentGraph returns the structure graph of a single document.
-func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, tenantID, datasetID, documentID, graphType string) ([]DocGraphItem, int64, error) {
- filter := map[string]interface{}{
- "doc_id": []string{documentID},
- "compiled_graph_kwd": []string{"graph"},
- }
- if graphType != "" {
- filter[FieldGraphType] = []string{graphType}
- }
- chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter,
- []string{"id", "content_with_weight", "source_id"}, 0, 10000, nil)
- if err != nil {
- return nil, 0, err
- }
- items := make([]DocGraphItem, 0, len(chunks))
- for _, c := range chunks {
- items = append(items, DocGraphItem{
- ID: firstStringValue(c["id"]),
- Content: firstStringValue(c["content_with_weight"]),
- SourceID: firstStringValue(c["source_id"]),
- })
- }
- return items, total, nil
-}
-
// DeleteDocumentGraph deletes the structure graph of a single document.
func (s *DatasetArtifactService) DeleteDocumentGraph(ctx context.Context, tenantID, datasetID, documentID string) (int, error) {
docEngine := engine.Get()
@@ -682,28 +649,12 @@ func (s *DatasetArtifactService) DeleteDocumentGraph(ctx context.Context, tenant
return len(ids), nil
}
-// NavigationItem is a single navigation cluster (REST response shape, kept
-// stable for frontend compatibility).
-type NavigationItem struct {
- Name string `json:"name"`
- Title string `json:"title"`
- Count int `json:"count"`
-}
-
-// NavChildItem is a single child entry under a navigation cluster (REST
-// response shape, kept stable for frontend compatibility).
-type NavChildItem struct {
- Name string `json:"name"`
- Title string `json:"title"`
- Count int `json:"count"`
-}
-
-// ListNavClusters returns the navigation clusters of a dataset. It is DEPRECATED
-// and now delegates to the ES-backed NavService (internal/service datasetnav):
-// the previous implementation queried nav_cluster_kwd/count_int fields that
-// Python never writes, so it could never read the real nav tree. Do not add
-// field-level patches here — route everything through NavService.
-func (s *DatasetArtifactService) ListNavClusters(ctx context.Context, tenantID, datasetID string) ([]NavigationItem, int64, error) {
+// ListNavClusters returns the navigation clusters of a dataset. It delegates to
+// the ES-backed NavService and returns the frontend DatasetNavNode shape
+// (snake_case NavNode JSON), matching Python GET /navigation exactly. The old
+// NavigationItem{name,title,count} shape did not match the frontend interface
+// and has been removed.
+func (s *DatasetArtifactService) ListNavClusters(ctx context.Context, tenantID, datasetID string) ([]nav.NavNode, int64, error) {
ns := nav.GetNavService()
if ns == nil {
return nil, 0, fmt.Errorf("datasetnav: NavService not initialized (SetNavService must be called at bootstrap)")
@@ -712,16 +663,12 @@ func (s *DatasetArtifactService) ListNavClusters(ctx context.Context, tenantID,
if err != nil {
return nil, 0, err
}
- items := make([]NavigationItem, 0, len(nodes))
- for _, n := range nodes {
- items = append(items, NavigationItem{Name: n.Name, Title: n.Description, Count: n.DocCount})
- }
- return items, total, nil
+ return nodes, total, nil
}
-// ListNavChildren returns the children of a navigation cluster. DEPRECATED —
-// delegates to NavService.ListChildren.
-func (s *DatasetArtifactService) ListNavChildren(ctx context.Context, tenantID, datasetID, name string) ([]NavChildItem, int64, error) {
+// ListNavChildren returns the children of a navigation cluster in the frontend
+// DatasetNavNode shape.
+func (s *DatasetArtifactService) ListNavChildren(ctx context.Context, tenantID, datasetID, name string) ([]nav.NavNode, int64, error) {
ns := nav.GetNavService()
if ns == nil {
return nil, 0, fmt.Errorf("datasetnav: NavService not initialized (SetNavService must be called at bootstrap)")
@@ -730,11 +677,7 @@ func (s *DatasetArtifactService) ListNavChildren(ctx context.Context, tenantID,
if err != nil {
return nil, 0, err
}
- items := make([]NavChildItem, 0, len(nodes))
- for _, n := range nodes {
- items = append(items, NavChildItem{Name: n.Name, Title: n.Description, Count: n.DocCount})
- }
- return items, total, nil
+ return nodes, total, nil
}
// DeleteNav removes the direct nav_doc children of every root cluster of a
diff --git a/internal/service/dataset_structure_graph.go b/internal/service/dataset_structure_graph.go
new file mode 100644
index 0000000000..77ed8f448a
--- /dev/null
+++ b/internal/service/dataset_structure_graph.go
@@ -0,0 +1,1190 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+//
+
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "regexp"
+ "sort"
+ "strings"
+
+ "ragflow/internal/dao"
+ "ragflow/internal/engine"
+ "ragflow/internal/engine/types"
+)
+
+// Structure-graph sampling constants (mirror structure_graph_common.py).
+const (
+ graphFullThreshold = 1024 // below this combined count, return all rows
+ graphTopEntities = 256 // seed set A size for large buckets
+ graphKeywordCandidates = 16 // keyword/KNN candidate rows
+ graphExpansionCap = 4096 // hub-node expansion cap
+)
+
+var graphEntityFields = []string{"id", "content_with_weight", "name_kwd", "mention_count_int", "source_chunk_ids", "doc_id", "doc_ids_kwd", "source_doc_ids"}
+var graphRelationFields = []string{"id", "content_with_weight", "from_entity_kwd", "to_entity_kwd", "doc_id", "doc_ids_kwd", "source_doc_ids"}
+var graphAllFields = []string{
+ "id", "content_with_weight", "name_kwd", "mention_count_int", "source_chunk_ids",
+ "from_entity_kwd", "to_entity_kwd", "knowledge_graph_kwd", "doc_id", "doc_ids_kwd", "source_doc_ids",
+}
+
+// StructureGraphNode is a projected entity in the structure graph response.
+type StructureGraphNode map[string]interface{}
+
+// StructureGraphRelation is a projected relation in the structure graph response.
+type StructureGraphRelation map[string]interface{}
+
+// DocumentStructureGraphTemplate is one per-template bucket in the response.
+type DocumentStructureGraphTemplate struct {
+ TemplateID string `json:"template_id"`
+ TemplateName string `json:"template_name"`
+ Kind string `json:"kind"`
+ Entities []StructureGraphNode `json:"entities"`
+ Relations []StructureGraphRelation `json:"relations"`
+}
+
+// DocumentStructureGraphResponse mirrors Python's {"templates": [...]}.
+type DocumentStructureGraphResponse struct {
+ Templates []DocumentStructureGraphTemplate `json:"templates"`
+}
+
+// graphRowSearch runs one raw-row search over the tenant's document index.
+func graphRowSearch(ctx context.Context, tenantID, datasetID string, selectFields []string, filter map[string]interface{}, orderBy *types.OrderByExpr, offset, limit int, matchExprs []interface{}) (map[string]map[string]interface{}, int64, error) {
+ docEngine := engine.Get()
+ if docEngine == nil {
+ return nil, 0, fmt.Errorf("document engine is not initialized")
+ }
+ merged := make(map[string]interface{}, len(filter)+1)
+ for k, v := range filter {
+ merged[k] = v
+ }
+ merged["kb_id"] = []string{datasetID}
+ if limit < 1 {
+ limit = 1
+ }
+ res, err := docEngine.Search(ctx, &types.SearchRequest{
+ IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},
+ KbIDs: []string{datasetID},
+ Offset: offset,
+ Limit: limit,
+ SelectFields: selectFields,
+ Filter: merged,
+ OrderBy: orderBy,
+ MatchExprs: matchExprs,
+ })
+ if err != nil {
+ return nil, 0, err
+ }
+ if res == nil {
+ return nil, 0, nil
+ }
+ byID := make(map[string]map[string]interface{}, len(res.Chunks))
+ for _, c := range res.Chunks {
+ id := firstStringValue(c["id"])
+ if id != "" {
+ byID[id] = c
+ }
+ }
+ return byID, res.Total, nil
+}
+
+// graphLoadPayload parses content_with_weight into a dict.
+func graphLoadPayload(row map[string]interface{}) map[string]interface{} {
+ raw := firstStringValue(row["content_with_weight"])
+ if raw == "" {
+ return nil
+ }
+ var m map[string]interface{}
+ if err := json.Unmarshal([]byte(raw), &m); err != nil {
+ return nil
+ }
+ return m
+}
+
+func graphIsInvalidSentinel(v string) bool {
+ if v == "" {
+ return true
+ }
+ lv := strings.ToLower(v)
+ for _, bad := range []string{"unknown", "none", "null", "nan", "n/a", "undefined", "other"} {
+ if lv == bad {
+ return true
+ }
+ }
+ return false
+}
+
+// projectEntity mirrors _struct_graph_entity + project_entity.
+func projectEntity(row map[string]interface{}) StructureGraphNode {
+ payload := graphLoadPayload(row)
+ if payload == nil {
+ return nil
+ }
+ name := ""
+ for _, k := range []string{"name", "text", "term", "title"} {
+ if v, ok := payload[k].(string); ok && strings.TrimSpace(v) != "" {
+ name = strings.TrimSpace(v)
+ break
+ }
+ }
+ if name == "" || graphIsInvalidSentinel(name) {
+ return nil
+ }
+ typ := "other"
+ if v, ok := payload["type"].(string); ok && strings.TrimSpace(v) != "" {
+ typ = strings.TrimSpace(v)
+ }
+ var aliases []string
+ switch a := payload["aliases"].(type) {
+ case string:
+ if strings.TrimSpace(a) != "" {
+ aliases = []string{strings.TrimSpace(a)}
+ }
+ case []interface{}:
+ for _, e := range a {
+ if s, ok := e.(string); ok && strings.TrimSpace(s) != "" {
+ aliases = append(aliases, strings.TrimSpace(s))
+ }
+ }
+ case []string:
+ for _, s := range a {
+ if strings.TrimSpace(s) != "" {
+ aliases = append(aliases, strings.TrimSpace(s))
+ }
+ }
+ }
+ desc := ""
+ for _, k := range []string{"description", "definition_excerpt"} {
+ if v, ok := payload[k].(string); ok {
+ desc = strings.TrimSpace(v)
+ if desc != "" {
+ break
+ }
+ }
+ }
+ chunkIDs := graphSourceChunkIDs(payload, row)
+ node := StructureGraphNode{
+ "aliases": aliases,
+ "mention_count": 1,
+ "name": name,
+ "source_chunk_ids": chunkIDs,
+ "type": typ,
+ "description": desc,
+ }
+ if mc, ok := graphMentionCount(row); ok {
+ node["mention_count"] = mc
+ }
+ return node
+}
+
+func graphSourceChunkIDs(payload, row map[string]interface{}) []string {
+ var out []string
+ add := func(v interface{}) {
+ switch x := v.(type) {
+ case []string:
+ for _, s := range x {
+ if s != "" {
+ out = append(out, s)
+ }
+ }
+ case []interface{}:
+ for _, e := range x {
+ if s, ok := e.(string); ok && s != "" {
+ out = append(out, s)
+ }
+ }
+ case string:
+ if x != "" {
+ out = append(out, x)
+ }
+ }
+ }
+ if raw, ok := payload["source_chunk_ids"]; ok {
+ add(raw)
+ } else {
+ add(row["source_chunk_ids"])
+ }
+ // dedup order-preserving
+ seen := map[string]bool{}
+ dedup := out[:0]
+ for _, s := range out {
+ if !seen[s] {
+ seen[s] = true
+ dedup = append(dedup, s)
+ }
+ }
+ return dedup
+}
+
+func graphMentionCount(row map[string]interface{}) (int, bool) {
+ v := row["mention_count_int"]
+ if l, ok := v.([]interface{}); ok {
+ if len(l) > 0 {
+ v = l[0]
+ } else {
+ return 0, false
+ }
+ }
+ switch n := v.(type) {
+ case int:
+ return n, true
+ case int64:
+ return int(n), true
+ case float64:
+ return int(n), true
+ case json.Number:
+ if i, err := n.Int64(); err == nil {
+ return int(i), true
+ }
+ }
+ return 0, false
+}
+
+// projectRelation mirrors _struct_graph_relation + project_relation.
+func projectRelation(row map[string]interface{}) StructureGraphRelation {
+ payload := graphLoadPayload(row)
+ src, tgt := "", ""
+ if payload != nil {
+ for _, k := range []string{"source", "src", "from"} {
+ if v, ok := payload[k].(string); ok {
+ src = strings.TrimSpace(v)
+ if src != "" {
+ break
+ }
+ }
+ }
+ for _, k := range []string{"target", "tgt", "to"} {
+ if v, ok := payload[k].(string); ok {
+ tgt = strings.TrimSpace(v)
+ if tgt != "" {
+ break
+ }
+ }
+ }
+ }
+ typ := "related"
+ if payload != nil {
+ if v, ok := payload["type"].(string); ok && strings.TrimSpace(v) != "" {
+ typ = strings.TrimSpace(v)
+ }
+ }
+ if src == "" || tgt == "" || graphIsInvalidSentinel(src) || graphIsInvalidSentinel(tgt) {
+ // Fall back to the authoritative *_entity_kwd columns.
+ fallbackSrc := strings.TrimSpace(firstStringValue(row["from_entity_kwd"]))
+ fallbackTgt := strings.TrimSpace(firstStringValue(row["to_entity_kwd"]))
+ if fallbackSrc != "" && fallbackTgt != "" {
+ return StructureGraphRelation{"from": fallbackSrc, "to": fallbackTgt, "type": typ}
+ }
+ return nil
+ }
+ return StructureGraphRelation{"from": src, "to": tgt, "type": typ}
+}
+
+// dedupEntities order-preserving by (lowercased name, type).
+func dedupEntities(entities []StructureGraphNode) []StructureGraphNode {
+ var out []StructureGraphNode
+ seen := map[string]bool{}
+ for _, e := range entities {
+ name := strings.ToLower(strings.TrimSpace(graphStr(e["name"])))
+ typ := strings.ToLower(strings.TrimSpace(graphStr(e["type"])))
+ key := name + "\x00" + typ
+ if name == "" || seen[key] {
+ continue
+ }
+ seen[key] = true
+ out = append(out, e)
+ }
+ return out
+}
+
+func entityResponseID(entity StructureGraphNode) string {
+ for _, f := range []string{"id", "name", "slug"} {
+ if v, ok := entity[f].(string); ok && strings.TrimSpace(v) != "" {
+ return strings.TrimSpace(v)
+ }
+ }
+ return ""
+}
+
+func endpointTerms(value string) []string {
+ value = strings.TrimSpace(value)
+ if value == "" {
+ return nil
+ }
+ return sortedUnique([]string{value, strings.ToLower(value)})
+}
+
+func sortedUnique(in []string) []string {
+ seen := map[string]bool{}
+ var out []string
+ for _, s := range in {
+ if s != "" && !seen[s] {
+ seen[s] = true
+ out = append(out, s)
+ }
+ }
+ sort.Strings(out)
+ return out
+}
+
+// normalizeRelationEndpoints aligns relation endpoints to returned entity ids.
+func normalizeRelationEndpoints(entities []StructureGraphNode, relations []StructureGraphRelation) []StructureGraphRelation {
+ if len(entities) == 0 || len(relations) == 0 {
+ return relations
+ }
+ lookup := map[string]string{}
+ ambiguous := map[string]bool{}
+ for _, entity := range entities {
+ respID := entityResponseID(entity)
+ if respID == "" {
+ continue
+ }
+ for _, f := range []string{"id", "name", "slug"} {
+ v, ok := entity[f].(string)
+ if !ok || strings.TrimSpace(v) == "" {
+ continue
+ }
+ key := strings.ToLower(strings.TrimSpace(v))
+ if cur, exists := lookup[key]; exists && cur != respID {
+ ambiguous[key] = true
+ continue
+ }
+ lookup[key] = respID
+ }
+ }
+ for k := range ambiguous {
+ delete(lookup, k)
+ }
+ normalized := make([]StructureGraphRelation, 0, len(relations))
+ for _, rel := range relations {
+ item := make(StructureGraphRelation, len(rel)+2)
+ for k, v := range rel {
+ item[k] = v
+ }
+ for _, f := range []string{"from", "to"} {
+ if v, ok := item[f].(string); ok {
+ if mapped, found := lookup[strings.ToLower(strings.TrimSpace(v))]; found {
+ item[f] = mapped
+ }
+ }
+ }
+ normalized = append(normalized, item)
+ }
+ return normalized
+}
+
+// rowHasEnabledSource mirrors _row_has_enabled_source.
+func rowHasEnabledSource(row map[string]interface{}, excludedDocIDs map[string]bool) bool {
+ if len(excludedDocIDs) == 0 {
+ return true
+ }
+ sourceIDs := map[string]bool{}
+ flatten := func(v interface{}) {
+ var walk func(interface{})
+ walk = func(x interface{}) {
+ switch val := x.(type) {
+ case string:
+ t := strings.TrimSpace(val)
+ if t == "" {
+ return
+ }
+ var parsed []interface{}
+ if json.Unmarshal([]byte(t), &parsed) == nil {
+ for _, p := range parsed {
+ walk(p)
+ }
+ return
+ }
+ sourceIDs[t] = true
+ case []interface{}:
+ for _, e := range val {
+ walk(e)
+ }
+ case []string:
+ for _, s := range val {
+ walk(s)
+ }
+ default:
+ if s, ok := x.(string); ok && s != "" {
+ sourceIDs[s] = true
+ }
+ }
+ }
+ walk(v)
+ }
+ for _, f := range []string{"doc_ids_kwd", "source_doc_ids"} {
+ if v, ok := row[f]; ok && v != nil {
+ flatten(v)
+ }
+ }
+ if len(sourceIDs) > 0 {
+ for id := range sourceIDs {
+ if !excludedDocIDs[id] {
+ return true
+ }
+ }
+ return false
+ }
+ flatten(row["doc_id"])
+ if len(sourceIDs) > 0 {
+ for id := range sourceIDs {
+ if !excludedDocIDs[id] {
+ return true
+ }
+ }
+ return false
+ }
+ return true
+}
+
+func graphStr(v interface{}) string {
+ if v == nil {
+ return ""
+ }
+ if s, ok := v.(string); ok {
+ return s
+ }
+ return fmt.Sprintf("%v", v)
+}
+
+// buildBucket mirrors sgc.build_bucket: small buckets whole, large sampled.
+func (s *DatasetArtifactService) buildBucket(ctx context.Context, tenantID, datasetID string, scope map[string]interface{}, excludedDocIDs map[string]bool) ([]StructureGraphNode, []StructureGraphRelation, error) {
+ excludedDocIDs = excludedDocIDsOrEmpty(excludedDocIDs)
+ bothCond := copyFilter(scope)
+ bothCond["knowledge_graph_kwd"] = []string{"entity", "relation"}
+ _, total, err := graphRowSearch(ctx, tenantID, datasetID, []string{"id"}, bothCond, nil, 0, 1, nil)
+ if err != nil {
+ return nil, nil, err
+ }
+ if total < graphFullThreshold {
+ fieldMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphAllFields, bothCond, nil, 0, int(total), nil)
+ if err != nil {
+ return nil, nil, err
+ }
+ var entities []StructureGraphNode
+ var relations []StructureGraphRelation
+ for _, row := range fieldMap {
+ if !rowHasEnabledSource(row, excludedDocIDs) {
+ continue
+ }
+ kg := firstStringValue(row["knowledge_graph_kwd"])
+ if kg == "relation" {
+ if edge := projectRelation(row); edge != nil {
+ relations = append(relations, edge)
+ }
+ } else {
+ if node := projectEntity(row); node != nil {
+ entities = append(entities, node)
+ }
+ }
+ }
+ entities = dedupEntities(entities)
+ return entities, normalizeRelationEndpoints(entities, relations), nil
+ }
+
+ // Large bucket: sample. A = top entities by mention_count_int desc.
+ orderBy := (&types.OrderByExpr{}).Desc("mention_count_int")
+ var setA []StructureGraphNode
+ entityOffset := 0
+ var entityTotal int64 = -1
+ for len(setA) < graphTopEntities && (entityTotal == -1 || int64(entityOffset) < entityTotal) {
+ cond := copyFilter(scope)
+ cond["knowledge_graph_kwd"] = []string{"entity"}
+ entAMap, entTotal, err := graphRowSearch(ctx, tenantID, datasetID, graphEntityFields, cond, orderBy, entityOffset, graphTopEntities, nil)
+ if err != nil {
+ return nil, nil, err
+ }
+ entityTotal = entTotal
+ if len(entAMap) == 0 {
+ break
+ }
+ for _, row := range entAMap {
+ if !rowHasEnabledSource(row, excludedDocIDs) {
+ continue
+ }
+ if n := projectEntity(row); n != nil {
+ setA = append(setA, n)
+ }
+ }
+ entityOffset += len(entAMap)
+ }
+ if len(setA) > graphTopEntities {
+ setA = setA[:graphTopEntities]
+ }
+ var aNames []string
+ for _, e := range setA {
+ if n := strings.TrimSpace(graphStr(e["name"])); n != "" {
+ aNames = append(aNames, n)
+ }
+ }
+ var aNameTerms []string
+ for _, name := range aNames {
+ aNameTerms = append(aNameTerms, endpointTerms(name)...)
+ }
+ aNameTerms = sortedUnique(aNameTerms)
+
+ var relations []StructureGraphRelation
+ targetNamesLower := map[string]bool{}
+ if len(aNameTerms) > 0 {
+ cond := copyFilter(scope)
+ cond["knowledge_graph_kwd"] = []string{"relation"}
+ cond["from_entity_kwd"] = aNameTerms
+ relMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphRelationFields, cond, nil, 0, graphExpansionCap, nil)
+ if err != nil {
+ return nil, nil, err
+ }
+ for _, row := range relMap {
+ if !rowHasEnabledSource(row, excludedDocIDs) {
+ continue
+ }
+ if edge := projectRelation(row); edge != nil {
+ relations = append(relations, edge)
+ if tgt := strings.ToLower(strings.TrimSpace(graphStr(edge["to"]))); tgt != "" {
+ targetNamesLower[tgt] = true
+ }
+ }
+ }
+ }
+ var setT []StructureGraphNode
+ if len(targetNamesLower) > 0 {
+ cond := copyFilter(scope)
+ cond["knowledge_graph_kwd"] = []string{"entity"}
+ cond["name_kwd"] = sortedKeys(targetNamesLower)
+ tgtMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphEntityFields, cond, nil, 0, graphExpansionCap, nil)
+ if err != nil {
+ return nil, nil, err
+ }
+ for _, row := range tgtMap {
+ if !rowHasEnabledSource(row, excludedDocIDs) {
+ continue
+ }
+ if n := projectEntity(row); n != nil {
+ setT = append(setT, n)
+ }
+ }
+ }
+ entities := dedupEntities(append(setA, setT...))
+ return entities, normalizeRelationEndpoints(entities, relations), nil
+}
+
+func copyFilter(in map[string]interface{}) map[string]interface{} {
+ out := make(map[string]interface{}, len(in))
+ for k, v := range in {
+ out[k] = v
+ }
+ return out
+}
+
+func excludedDocIDsOrEmpty(in map[string]bool) map[string]bool {
+ if in == nil {
+ return map[string]bool{}
+ }
+ return in
+}
+
+func sortedKeys(m map[string]bool) []string {
+ out := make([]string, 0, len(m))
+ for k := range m {
+ out = append(out, k)
+ }
+ sort.Strings(out)
+ return out
+}
+
+// compilationTemplateKind normalizes a template kind (mirror Python
+// _compilation_template_kind).
+func compilationTemplateKind(kind string) string {
+ k := strings.ToLower(strings.TrimSpace(kind))
+ switch k {
+ case "page_index", "pageindex":
+ return "page_index"
+ case "wiki", "wiki_page":
+ return "wiki"
+ case "raptor":
+ return "raptor"
+ }
+ return k
+}
+
+// structureKindForBucket returns the normalized kind used for tree/page_index
+// hierarchy handling.
+func structureKindForBucket(kind string) string {
+ k := compilationTemplateKind(kind)
+ if k == "page_index" {
+ return "page_index"
+ }
+ return strings.ReplaceAll(k, "-", "_")
+}
+
+// loadDocumentTemplateMeta resolves the doc's configured template group into
+// configured_ids + template_meta maps, mirroring the Python resolution.
+func (s *DatasetArtifactService) loadDocumentTemplateMeta(ctx context.Context, tenantID, documentID string) ([]string, map[string]map[string]interface{}, error) {
+ empty := map[string]map[string]interface{}{}
+ doc, err := dao.NewDocumentDAO().GetByID(ctx, dao.DB, documentID)
+ if err != nil || doc == nil {
+ return nil, empty, err
+ }
+ groupID := ""
+ if doc.ParserConfig != nil {
+ if gid, ok := doc.ParserConfig["compilation_template_group_id"].(string); ok && gid != "" {
+ groupID = gid
+ } else if ext, ok := doc.ParserConfig["ext"].(map[string]interface{}); ok {
+ if gid, ok := ext["compilation_template_group_id"].(string); ok && gid != "" {
+ groupID = gid
+ }
+ }
+ }
+ if groupID == "" {
+ return nil, empty, nil
+ }
+ group, err := NewCompilationTemplateGroupService().GetSaved(ctx, tenantID, groupID)
+ if err != nil {
+ return nil, empty, err
+ }
+ configuredIDs := []string{}
+ meta := map[string]map[string]interface{}{}
+ seen := map[string]bool{}
+ if group != nil {
+ for _, t := range group.Templates {
+ tid := t.ID
+ if tid == "" || seen[tid] {
+ continue
+ }
+ rawKind := ""
+ if t.Config != nil {
+ if k, ok := t.Config["kind"].(string); ok {
+ rawKind = k
+ }
+ }
+ if rawKind == "" {
+ rawKind = t.Kind
+ }
+ kindNorm := compilationTemplateKind(rawKind)
+ if kindNorm == "wiki" {
+ continue
+ }
+ seen[tid] = true
+ configuredIDs = append(configuredIDs, tid)
+ meta[tid] = map[string]interface{}{
+ "template_id": tid,
+ "template_name": t.Name,
+ "kind": rawKind,
+ }
+ }
+ }
+ return configuredIDs, meta, nil
+}
+
+// DocumentStructureGraphInput is the parsed request for GetDocumentGraph.
+type DocumentStructureGraphInput struct {
+ TenantID string
+ DatasetID string
+ DocumentID string
+ GraphType string
+ Keywords string
+}
+
+// GetDocumentGraph returns the per-template structure graph of a document,
+// mirroring Python get_document_structure_graph (normal + keywords modes).
+func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, in DocumentStructureGraphInput) (*DocumentStructureGraphResponse, error) {
+ configuredIDs, templateMeta, err := s.loadDocumentTemplateMeta(ctx, in.TenantID, in.DocumentID)
+ if err != nil {
+ return nil, err
+ }
+
+ resp := &DocumentStructureGraphResponse{Templates: []DocumentStructureGraphTemplate{}}
+
+ // keywords mode: name matching/KNN → matched entities' subgraph.
+ if in.Keywords != "" {
+ bucketMeta, entities, relations, err := s.keywordSubgraph(ctx, in.TenantID, in.DatasetID, in.DocumentID, in.Keywords, templateMeta)
+ if err != nil {
+ return nil, err
+ }
+ if bucketMeta == nil || (len(entities) == 0 && len(relations) == 0) {
+ return resp, nil
+ }
+ resp.Templates = append(resp.Templates, DocumentStructureGraphTemplate{
+ TemplateID: graphStr(bucketMeta["template_id"]),
+ TemplateName: graphStr(bucketMeta["template_name"]),
+ Kind: graphStr(bucketMeta["kind"]),
+ Entities: entities,
+ Relations: relations,
+ })
+ return resp, nil
+ }
+
+ // normal mode: discover buckets from per-doc graph blob rows.
+ metaFields := []string{"compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd"}
+ metaRows, _, err := graphRowSearch(ctx, in.TenantID, in.DatasetID, metaFields,
+ map[string]interface{}{"doc_id": []string{in.DocumentID}, "knowledge_graph_kwd": []string{"graph"}}, nil, 0, 1000, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ bucketMetas := map[string]map[string]interface{}{}
+ bucketScopes := map[string]map[string]interface{}{}
+ for _, row := range metaRows {
+ meta, scope := resolveGraphBucket(row, templateMeta, in.DocumentID)
+ bid := graphStr(meta["template_id"])
+ if _, ok := bucketMetas[bid]; !ok {
+ bucketMetas[bid] = meta
+ bucketScopes[bid] = scope
+ }
+ }
+
+ grouped := map[string]DocumentStructureGraphTemplate{}
+ for bid, meta := range bucketMetas {
+ entities, relations, err := s.buildBucket(ctx, in.TenantID, in.DatasetID, bucketScopes[bid], nil)
+ if err != nil {
+ return nil, err
+ }
+ if len(entities) == 0 && len(relations) == 0 {
+ continue
+ }
+ grouped[bid] = DocumentStructureGraphTemplate{
+ TemplateID: graphStr(meta["template_id"]),
+ TemplateName: graphStr(meta["template_name"]),
+ Kind: graphStr(meta["kind"]),
+ Entities: entities,
+ Relations: relations,
+ }
+ }
+
+ // RAPTOR summary graph blob (compile_kwd = raptor_graph).
+ s.appendRaptorBlob(ctx, in.TenantID, in.DatasetID, in.DocumentID, grouped)
+
+ // Order: configured templates first, then discovered.
+ orderedIDs := []string{}
+ for _, tid := range configuredIDs {
+ if _, ok := grouped[tid]; ok && !containsStr(orderedIDs, tid) {
+ orderedIDs = append(orderedIDs, tid)
+ }
+ }
+ for bid := range grouped {
+ if !containsStr(orderedIDs, bid) {
+ orderedIDs = append(orderedIDs, bid)
+ }
+ }
+ for _, bid := range orderedIDs {
+ if g, ok := grouped[bid]; ok && (len(g.Entities) > 0 || len(g.Relations) > 0) {
+ resp.Templates = append(resp.Templates, g)
+ }
+ }
+ return resp, nil
+}
+
+func containsStr(list []string, v string) bool {
+ for _, e := range list {
+ if e == v {
+ return true
+ }
+ }
+ return false
+}
+
+// resolveGraphBucket mirrors Python _resolve_bucket.
+func resolveGraphBucket(row map[string]interface{}, templateMeta map[string]map[string]interface{}, documentID string) (map[string]interface{}, map[string]interface{}) {
+ compileKwd := firstStringValue(row["compile_kwd"])
+ kindVal := firstStringValue(row["compilation_template_kind_kwd"])
+ if kindVal == "" {
+ kindVal = compileKwd
+ }
+ tid := rowTemplateID(row)
+ if tid != "" {
+ kindNorm := compilationTemplateKind(kindVal)
+ meta := templateMeta[tid]
+ // Only a unique kind match can substitute a missing template meta; an
+ // empty kindNorm must never match (review Major — the previous nil
+ // round-trip stored a nil entry that the kind-only loop then matched).
+ if meta == nil && kindNorm != "" {
+ kindMatches := []map[string]interface{}{}
+ for _, m := range templateMeta {
+ if compilationTemplateKind(graphStr(m["kind"])) == kindNorm {
+ kindMatches = append(kindMatches, m)
+ }
+ }
+ if len(kindMatches) == 1 {
+ meta = kindMatches[0]
+ }
+ }
+ bucketName := graphStr(meta["template_name"])
+ if bucketName == "" {
+ bucketName = tid
+ }
+ bucketKind := graphStr(meta["kind"])
+ if bucketKind == "" {
+ bucketKind = kindVal
+ }
+ return map[string]interface{}{
+ "template_id": tid,
+ "template_name": bucketName,
+ "kind": bucketKind,
+ }, map[string]interface{}{
+ "doc_id": []string{documentID},
+ "compilation_template_ids": []string{tid},
+ }
+ }
+ bucketID := "legacy:" + compileKwd
+ return map[string]interface{}{
+ "template_id": bucketID,
+ "template_name": "Legacy (" + compileKwd + ")",
+ "kind": kindVal,
+ }, map[string]interface{}{
+ "doc_id": []string{documentID},
+ "compile_kwd": []string{compileKwd},
+ "must_not": map[string]interface{}{"exists": "compilation_template_ids"},
+ }
+}
+
+func rowTemplateID(row map[string]interface{}) string {
+ switch v := row["compilation_template_ids"].(type) {
+ case []interface{}:
+ for _, e := range v {
+ if s, ok := e.(string); ok && strings.TrimSpace(s) != "" {
+ return strings.TrimSpace(s)
+ }
+ }
+ case []string:
+ for _, s := range v {
+ if strings.TrimSpace(s) != "" {
+ return strings.TrimSpace(s)
+ }
+ }
+ case string:
+ if strings.TrimSpace(v) != "" {
+ return strings.TrimSpace(v)
+ }
+ }
+ return ""
+}
+
+func (s *DatasetArtifactService) appendRaptorBlob(ctx context.Context, tenantID, datasetID, documentID string, grouped map[string]DocumentStructureGraphTemplate) {
+ rows, _, err := graphRowSearch(ctx, tenantID, datasetID, []string{"content_with_weight", "compile_kwd"},
+ map[string]interface{}{"doc_id": []string{documentID}, "compile_kwd": []string{"raptor_graph"}}, nil, 0, 16, nil)
+ if err != nil {
+ return
+ }
+ for _, row := range rows {
+ payload := graphLoadPayload(row)
+ if payload == nil {
+ continue
+ }
+ rEntities, _ := payload["entities"].([]interface{})
+ rRelations, _ := payload["relations"].([]interface{})
+ if len(rEntities) == 0 && len(rRelations) == 0 {
+ continue
+ }
+ rb, ok := grouped["raptor"]
+ if !ok {
+ rb = DocumentStructureGraphTemplate{TemplateID: "raptor", TemplateName: "RAPTOR Summary", Kind: "raptor"}
+ grouped["raptor"] = rb
+ }
+ rb.Entities = append(rb.Entities, toNodeSlice(rEntities)...)
+ rb.Relations = append(rb.Relations, toRelationSlice(rRelations)...)
+ grouped["raptor"] = rb
+ }
+}
+
+func toNodeSlice(in []interface{}) []StructureGraphNode {
+ var out []StructureGraphNode
+ for _, e := range in {
+ if m, ok := e.(map[string]interface{}); ok {
+ out = append(out, StructureGraphNode(m))
+ }
+ }
+ return out
+}
+
+func toRelationSlice(in []interface{}) []StructureGraphRelation {
+ var out []StructureGraphRelation
+ for _, e := range in {
+ if m, ok := e.(map[string]interface{}); ok {
+ out = append(out, StructureGraphRelation(m))
+ }
+ }
+ return out
+}
+
+var graphTextQueryRE = regexp.MustCompile(`[ :|\r\n\t,,。??/\x60!!&^%()\[\]{}<>*~'"\\=]+`)
+
+func nameMatchesQuery(node StructureGraphNode, query string) bool {
+ name := strings.ToLower(strings.TrimSpace(graphStr(node["name"])))
+ q := strings.ToLower(query)
+ if name == "" || q == "" {
+ return false
+ }
+ if strings.Contains(name, q) {
+ return true
+ }
+ terms := strings.Fields(q)
+ if len(terms) == 0 {
+ return false
+ }
+ for _, t := range terms {
+ if !strings.Contains(name, t) {
+ return false
+ }
+ }
+ return true
+}
+
+// keywordSubgraph mirrors sgc.keyword_subgraph (BM25 + KNN fallback + ancestor walk).
+func (s *DatasetArtifactService) keywordSubgraph(ctx context.Context, tenantID, datasetID, documentID, keywords string, templateMeta map[string]map[string]interface{}) (map[string]interface{}, []StructureGraphNode, []StructureGraphRelation, error) {
+ baseEntityCond := map[string]interface{}{
+ "doc_id": []string{documentID},
+ "knowledge_graph_kwd": []string{"entity"},
+ }
+ topFields := append(append([]string{}, graphEntityFields...), "compilation_template_ids", "compile_kwd", "compilation_template_kind_kwd")
+
+ textQuery := strings.TrimSpace(graphTextQueryRE.ReplaceAllString(keywords, " "))
+ var candidates []struct {
+ row map[string]interface{}
+ node StructureGraphNode
+ }
+ validTop := func(rows map[string]map[string]interface{}) []struct {
+ row map[string]interface{}
+ node StructureGraphNode
+ } {
+ var out []struct {
+ row map[string]interface{}
+ node StructureGraphNode
+ }
+ for _, row := range rows {
+ if !rowHasEnabledSource(row, map[string]bool{}) {
+ continue
+ }
+ if node := projectEntity(row); node != nil && strings.TrimSpace(graphStr(node["name"])) != "" {
+ out = append(out, struct {
+ row map[string]interface{}
+ node StructureGraphNode
+ }{row, node})
+ }
+ }
+ return out
+ }
+ if textQuery != "" {
+ textExpr := &types.MatchTextExpr{
+ Fields: []string{"content_ltks^10", "content_sm_ltks"},
+ MatchingText: textQuery,
+ TopN: graphKeywordCandidates,
+ ExtraOptions: map[string]interface{}{"original_query": keywords},
+ }
+ topMap, _, err := graphRowSearch(ctx, tenantID, datasetID, topFields, baseEntityCond, nil, 0, graphKeywordCandidates, []interface{}{textExpr})
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ for _, c := range validTop(topMap) {
+ if nameMatchesQuery(c.node, textQuery) {
+ candidates = append(candidates, c)
+ }
+ }
+ }
+ // Prefer detail entities over title-typed ancestors.
+ detailCandidates := candidates[:0]
+ for _, c := range candidates {
+ if strings.ToLower(strings.TrimSpace(graphStr(c.node["type"]))) != "title" {
+ detailCandidates = append(detailCandidates, c)
+ }
+ }
+ if len(detailCandidates) > 0 {
+ candidates = detailCandidates
+ }
+
+ // Semantic fallback via KNN.
+ if len(candidates) == 0 {
+ vec, err := embedQuery(ctx, tenantID, datasetID, keywords)
+ if err != nil || len(vec) == 0 {
+ return nil, nil, nil, nil
+ }
+ denseExpr := &types.MatchDenseExpr{
+ VectorColumnName: fmt.Sprintf("q_%d_vec", len(vec)),
+ EmbeddingData: vec,
+ EmbeddingDataType: "float",
+ DistanceType: "cosine",
+ TopN: graphKeywordCandidates,
+ ExtraOptions: map[string]interface{}{"similarity": 0.3},
+ }
+ topMap, _, err := graphRowSearch(ctx, tenantID, datasetID, topFields, baseEntityCond, nil, 0, graphKeywordCandidates, []interface{}{denseExpr})
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ candidates = validTop(topMap)
+ }
+ if len(candidates) == 0 {
+ return nil, nil, nil, nil
+ }
+
+ scopeForTemplate := func(row map[string]interface{}) (map[string]interface{}, map[string]interface{}) {
+ return resolveGraphBucket(row, templateMeta, documentID)
+ }
+ bucketMeta, scope := scopeForTemplate(candidates[0].row)
+ bucketID := graphStr(bucketMeta["template_id"])
+
+ matchedNodes := []StructureGraphNode{}
+ for _, c := range candidates {
+ cMeta, _ := scopeForTemplate(c.row)
+ if graphStr(cMeta["template_id"]) == bucketID {
+ matchedNodes = append(matchedNodes, c.node)
+ }
+ }
+ if len(matchedNodes) == 0 {
+ return nil, nil, nil, nil
+ }
+ structureKind := structureKindForBucket(graphStr(bucketMeta["kind"]))
+
+ var relations []StructureGraphRelation
+ seenRel := map[string]bool{}
+ neighborNamesLower := map[string]bool{}
+ matchedNames := map[string]bool{}
+ for _, n := range matchedNodes {
+ if name := strings.ToLower(strings.TrimSpace(graphStr(n["name"]))); name != "" {
+ matchedNames[name] = true
+ }
+ }
+ if structureKind != "tree" && structureKind != "page_index" {
+ for _, matchedNode := range matchedNodes {
+ matchedName := strings.TrimSpace(graphStr(matchedNode["name"]))
+ terms := endpointTerms(matchedName)
+ for _, field := range []string{"from_entity_kwd", "to_entity_kwd"} {
+ cond := copyFilter(scope)
+ cond["knowledge_graph_kwd"] = []string{"relation"}
+ cond[field] = terms
+ relMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphRelationFields, cond, nil, 0, graphExpansionCap, nil)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ for _, row := range relMap {
+ if !rowHasEnabledSource(row, map[string]bool{}) {
+ continue
+ }
+ if edge := projectRelation(row); edge != nil {
+ key := graphStr(edge["from"]) + "\x00" + graphStr(edge["to"]) + "\x00" + graphStr(edge["type"])
+ if seenRel[key] {
+ continue
+ }
+ seenRel[key] = true
+ relations = append(relations, edge)
+ for _, endpoint := range []string{graphStr(edge["from"]), graphStr(edge["to"])} {
+ if ep := strings.TrimSpace(endpoint); ep != "" && !matchedNames[strings.ToLower(ep)] {
+ neighborNamesLower[strings.ToLower(ep)] = true
+ }
+ }
+ }
+ if len(relations) >= graphExpansionCap {
+ break
+ }
+ }
+ if len(relations) >= graphExpansionCap {
+ break
+ }
+ }
+ if len(relations) >= graphExpansionCap {
+ break
+ }
+ }
+ }
+
+ // Ancestor walk for tree/page_index.
+ if (structureKind == "tree" || structureKind == "page_index") && len(relations) < graphExpansionCap {
+ ancestorFrontier := map[string]bool{}
+ seenAncestors := map[string]bool{}
+ for n := range matchedNames {
+ ancestorFrontier[n] = true
+ seenAncestors[n] = true
+ }
+ for len(ancestorFrontier) > 0 && len(relations) < graphExpansionCap {
+ nextFrontier := map[string]bool{}
+ cond := copyFilter(scope)
+ cond["knowledge_graph_kwd"] = []string{"relation"}
+ cond["to_entity_kwd"] = sortedKeys(ancestorFrontier)
+ relMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphRelationFields, cond, nil, 0, graphExpansionCap-len(relations), nil)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ for _, row := range relMap {
+ if !rowHasEnabledSource(row, map[string]bool{}) {
+ continue
+ }
+ if edge := projectRelation(row); edge != nil {
+ key := graphStr(edge["from"]) + "\x00" + graphStr(edge["to"]) + "\x00" + graphStr(edge["type"])
+ if seenRel[key] {
+ continue
+ }
+ seenRel[key] = true
+ relations = append(relations, edge)
+ parent := strings.ToLower(strings.TrimSpace(graphStr(edge["from"])))
+ if parent != "" && !seenAncestors[parent] {
+ seenAncestors[parent] = true
+ nextFrontier[parent] = true
+ }
+ if len(relations) >= graphExpansionCap {
+ break
+ }
+ }
+ }
+ ancestorFrontier = nextFrontier
+ for n := range nextFrontier {
+ neighborNamesLower[n] = true
+ }
+ }
+ }
+
+ entities := append([]StructureGraphNode{}, matchedNodes...)
+ if len(neighborNamesLower) > 0 {
+ cond := copyFilter(scope)
+ cond["knowledge_graph_kwd"] = []string{"entity"}
+ cond["name_kwd"] = sortedKeys(neighborNamesLower)
+ nbMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphEntityFields, cond, nil, 0, graphExpansionCap, nil)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ for _, row := range nbMap {
+ if !rowHasEnabledSource(row, map[string]bool{}) {
+ continue
+ }
+ if n := projectEntity(row); n != nil {
+ entities = append(entities, n)
+ }
+ }
+ }
+ entities = dedupEntities(entities)
+ if structureKind == "tree" || structureKind == "page_index" {
+ entityNames := map[string]bool{}
+ for _, e := range entities {
+ if n := strings.ToLower(strings.TrimSpace(graphStr(e["name"]))); n != "" {
+ entityNames[n] = true
+ }
+ }
+ var filtered []StructureGraphRelation
+ for _, r := range relations {
+ if entityNames[strings.ToLower(strings.TrimSpace(graphStr(r["from"])))] && entityNames[strings.ToLower(strings.TrimSpace(graphStr(r["to"])))] {
+ filtered = append(filtered, r)
+ }
+ }
+ relations = filtered
+ }
+ return bucketMeta, entities, normalizeRelationEndpoints(entities, relations), nil
+}
+
+// embedQuery embeds a query via the nav embedder (best-effort). The embedder
+// resolves the tenant's embedding model on demand; an empty result (or error) is
+// treated as "no semantic candidates", matching Python's fallback behavior.
+func embedQuery(ctx context.Context, tenantID, datasetID, query string) ([]float64, error) {
+ embedder := NewNavEmbedder(NewModelProviderService(), "")
+ vecs, err := embedder.Encode(ctx, tenantID, []string{query})
+ if err != nil || len(vecs) == 0 || len(vecs[0]) == 0 {
+ return nil, fmt.Errorf("embed query failed")
+ }
+ f := make([]float64, len(vecs[0]))
+ for i, v := range vecs[0] {
+ f[i] = float64(v)
+ }
+ return f, nil
+}
diff --git a/internal/service/dataset_structure_graph_test.go b/internal/service/dataset_structure_graph_test.go
new file mode 100644
index 0000000000..74a767b9e2
--- /dev/null
+++ b/internal/service/dataset_structure_graph_test.go
@@ -0,0 +1,107 @@
+package service
+
+import (
+ "reflect"
+ "testing"
+)
+
+// TestProjectEntity_FromPayload verifies projectEntity maps the tree-node
+// payload shape to the graph-node shape (mirroring _struct_graph_entity).
+func TestProjectEntity_FromPayload(t *testing.T) {
+ row := map[string]interface{}{
+ "content_with_weight": `{"name":"NVIDIA","type":"tree_node","description":"chip maker","source_chunk_ids":["c1","c2"]}`,
+ "source_chunk_ids": []string{"c1"},
+ "mention_count_int": 3,
+ }
+ n := projectEntity(row)
+ if n == nil {
+ t.Fatal("projectEntity returned nil")
+ }
+ if n["name"] != "NVIDIA" || n["type"] != "tree_node" {
+ t.Errorf("name/type = %v/%v, want NVIDIA/tree_node", n["name"], n["type"])
+ }
+ if n["mention_count"] != 3 {
+ t.Errorf("mention_count = %v, want 3", n["mention_count"])
+ }
+ chunks, _ := n["source_chunk_ids"].([]string)
+ if !reflect.DeepEqual(chunks, []string{"c1", "c2"}) {
+ t.Errorf("source_chunk_ids = %v, want [c1 c2]", chunks)
+ }
+}
+
+// TestProjectEntity_RejectsInvalidName verifies sentinel/empty names are dropped.
+func TestProjectEntity_RejectsInvalidName(t *testing.T) {
+ for _, payload := range []string{`{"name":""}`, `{"name":"unknown"}`} {
+ if n := projectEntity(map[string]interface{}{"content_with_weight": payload}); n != nil {
+ t.Errorf("expected nil for payload %s, got %v", payload, n)
+ }
+ }
+}
+
+// TestProjectRelation_FromPayload verifies the relation projection.
+func TestProjectRelation_FromPayload(t *testing.T) {
+ row := map[string]interface{}{"content_with_weight": `{"from":"NVIDIA","to":"GPU","type":"child"}`}
+ r := projectRelation(row)
+ if r == nil {
+ t.Fatal("projectRelation returned nil")
+ }
+ if r["from"] != "NVIDIA" || r["to"] != "GPU" || r["type"] != "child" {
+ t.Errorf("relation = %v, want {NVIDIA GPU child}", r)
+ }
+}
+
+// TestProjectRelation_FallsBackToKwdColumns verifies the authoritative *_entity_kwd
+// columns are used when the payload has no from/to.
+func TestProjectRelation_FallsBackToKwdColumns(t *testing.T) {
+ row := map[string]interface{}{
+ "content_with_weight": `{"type":"related"}`,
+ "from_entity_kwd": "NVIDIA",
+ "to_entity_kwd": "GPU",
+ }
+ r := projectRelation(row)
+ if r == nil || r["from"] != "NVIDIA" || r["to"] != "GPU" || r["type"] != "related" {
+ t.Errorf("relation = %v, want {NVIDIA GPU related}", r)
+ }
+}
+
+// TestDedupEntities_OrderPreserving verifies dedup by (lowercased name, type).
+func TestDedupEntities_OrderPreserving(t *testing.T) {
+ in := []StructureGraphNode{
+ {"name": "A", "type": "x"},
+ {"name": "a", "type": "x"}, // dup (case-insensitive)
+ {"name": "B", "type": "y"},
+ {"name": ""}, // dropped (empty name)
+ }
+ out := dedupEntities(in)
+ if len(out) != 2 || out[0]["name"] != "A" || out[1]["name"] != "B" {
+ t.Errorf("dedupEntities = %v, want [A B]", out)
+ }
+}
+
+// TestNormalizeRelationEndpoints aligns relation endpoints to entity ids/names.
+func TestNormalizeRelationEndpoints(t *testing.T) {
+ entities := []StructureGraphNode{{"name": "NVIDIA", "type": "x"}}
+ relations := []StructureGraphRelation{{"from": "nvidia", "to": "GPU", "type": "child"}}
+ out := normalizeRelationEndpoints(entities, relations)
+ if out[0]["from"] != "NVIDIA" {
+ t.Errorf("normalized from = %v, want NVIDIA (matched to entity name)", out[0]["from"])
+ }
+}
+
+// TestCompilationTemplateKind_Normalization covers kind normalization.
+func TestCompilationTemplateKind_Normalization(t *testing.T) {
+ if compilationTemplateKind("Page_Index") != "page_index" {
+ t.Errorf("Page_Index => %q, want page_index", compilationTemplateKind("Page_Index"))
+ }
+ if compilationTemplateKind("tree") != "tree" {
+ t.Errorf("tree => %q, want tree", compilationTemplateKind("tree"))
+ }
+}
+
+// TestRowTemplateID_FromList covers compilation_template_ids extraction.
+func TestRowTemplateID_FromList(t *testing.T) {
+ row := map[string]interface{}{"compilation_template_ids": []interface{}{"", "tid1", "tid2"}}
+ if got := rowTemplateID(row); got != "tid1" {
+ t.Errorf("rowTemplateID = %q, want tid1", got)
+ }
+}
diff --git a/internal/service/nav/nav.go b/internal/service/nav/nav.go
index a87fcfd8c9..243ea0ea0a 100644
--- a/internal/service/nav/nav.go
+++ b/internal/service/nav/nav.go
@@ -27,14 +27,16 @@ import (
"sync"
)
-// NavNode mirrors Python's _nav_item (dataset_api_service.py).
+// NavNode mirrors Python's _nav_item (dataset_api_service.py). The JSON field
+// names are snake_case to match the frontend DatasetNavNode contract and the
+// Python GET /navigation payload exactly.
type NavNode struct {
- Name string // name
- Description string // content_with_weight payload description
- DocCount int // doc_count_int (cluster) or 1 (leaf)
- Type string // "cluster" | "doc"
- DocID string // leaf doc_id; empty for cluster
- HasChildren bool // is_cluster
+ Name string `json:"name"`
+ Description string `json:"description"`
+ DocCount int `json:"doc_count"`
+ Type string `json:"type"`
+ DocID string `json:"doc_id,omitempty"`
+ HasChildren bool `json:"has_children"`
}
// NavHit is one KNN hit on a nav row.
diff --git a/internal/service/nlp/datasetnav.go b/internal/service/nlp/datasetnav.go
index cf057f8795..4959b1b4d2 100644
--- a/internal/service/nlp/datasetnav.go
+++ b/internal/service/nlp/datasetnav.go
@@ -177,8 +177,23 @@ func (s *NavService) ListChildren(ctx context.Context, tenantID, kbID, name stri
// nodeFromRow converts an engine row into a NavNode.
func (s *NavService) nodeFromRow(row map[string]interface{}, fallbackType string) nav.NavNode {
+ name := firstStringValue(row["title_kwd"])
+ // Prefer an explicit readable "name" column (Python rows carry one); fall
+ // back to title_kwd.
+ if n := firstStringValue(row["name"]); n != "" {
+ name = n
+ }
+ // A raw id (doc_id or "cluster_<hash>") is not a human-readable name; fall
+ // back to a title derived from the payload description. Cluster names are the
+ // child-lookup key (parent_kwd references them verbatim), so they must stay
+ // intact; only non-cluster (leaf) rows get the readable fallback, otherwise
+ // GET /navigation/{cluster}/children would no longer match (review Major).
+ isCluster := firstStringValue(row["type_kwd"]) == "nav_cluster"
+ if !isCluster && graphIsRawID(name) {
+ name = ""
+ }
node := nav.NavNode{
- Name: firstStringValue(row["title_kwd"]),
+ Name: name,
DocCount: intValue(row["doc_count_int"]),
Type: fallbackType,
DocID: firstStringValue(row["doc_id"]),
@@ -196,6 +211,13 @@ func (s *NavService) nodeFromRow(row map[string]interface{}, fallbackType string
if d, ok := m["description"].(string); ok {
node.Description = d
}
+ if node.Name == "" {
+ if t, ok := m["title"].(string); ok && strings.TrimSpace(t) != "" {
+ node.Name = cleanTitle(t)
+ } else if d, ok := m["description"].(string); ok && strings.TrimSpace(d) != "" {
+ node.Name = cleanTitle(fallbackTitle(d))
+ }
+ }
}
}
node.HasChildren = node.Type == "cluster"
@@ -205,6 +227,31 @@ func (s *NavService) nodeFromRow(row map[string]interface{}, fallbackType string
return node
}
+// graphIsRawID reports whether a name is a meaningless internal id (a doc id or
+// a "cluster_<hex>" key) rather than a human-readable title.
+func graphIsRawID(name string) bool {
+ name = strings.TrimSpace(name)
+ if name == "" {
+ return true
+ }
+ if strings.HasPrefix(name, "cluster_") && len(name) == len("cluster_")+8 {
+ return true
+ }
+ if len(name) == 32 && isHexString(name) {
+ return true
+ }
+ return false
+}
+
+func isHexString(s string) bool {
+ for _, r := range s {
+ if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')) {
+ return false
+ }
+ }
+ return true
+}
+
// Search runs query KNN over nav rows and returns routed doc ids.
func (s *NavService) Search(ctx context.Context, tenantID, kbID, query string, embd []float32, topK int) ([]nav.NavHit, error) {
if topK <= 0 {
@@ -343,8 +390,11 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
"compile_kwd": navCompileKwd,
"available_int": 0,
"type_kwd": "nav_doc",
- "title_kwd": in.DocID,
- "parent_kwd": parent,
+ // The nav_doc's display name is a readable title derived from the summary
+ // (Python _clean_title/_fallback_title) — NOT the raw doc id, which is
+ // meaningless in the UI. The doc_id is still stored for lookups.
+ "title_kwd": cleanTitle(fallbackTitle(in.Summary)),
+ "parent_kwd": parent,
// The nav_doc sits one level below its (possibly nested) parent
// cluster, so its depth is parentDepth+1 — not a hard-coded 1.
"depth_int": bestDepth + 1,
@@ -376,6 +426,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
}
_, err = de.InsertChunks(ctx, []map[string]interface{}{{
"id": navClusterID(in.TenantID, in.KbID, name),
+ "doc_id": in.KbID, // cluster rows carry the kb as doc_id (Python _build_nav_cluster_row), so ES InsertChunks does not skip them
"compile_kwd": navCompileKwd,
"available_int": 0,
"type_kwd": "nav_cluster",
@@ -387,6 +438,27 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error
"content_with_weight": payloadJSONNav(map[string]interface{}{"type": "nav_cluster", "description": summary}),
"q_" + fmt.Sprintf("%d", len(vec)) + "_vec": f32ToF64Slice(vec),
}}, idx, in.KbID)
+ if err != nil {
+ return err
+ }
+ // Always emit a nav_doc leaf under the (new) cluster, mirroring Python
+ // upsert_dataset_nav_doc (the merge branch does the same at its parent). A
+ // new root cluster that only folds the doc into doc_ids_kwd would leave the
+ // nav tree with a single cluster and no nav_doc child, so /children returns
+ // empty. The nav_doc carries a readable title + parent_kwd = the cluster name.
+ _, err = de.InsertChunks(ctx, []map[string]interface{}{{
+ "id": navDocID(in.TenantID, in.KbID, in.DocID),
+ "compile_kwd": navCompileKwd,
+ "available_int": 0,
+ "type_kwd": "nav_doc",
+ "title_kwd": cleanTitle(fallbackTitle(in.Summary)),
+ "parent_kwd": name,
+ "depth_int": depth + 1,
+ "doc_id": in.DocID,
+ "doc_count_int": 1,
+ "content_with_weight": payloadJSONNav(map[string]interface{}{"type": "nav_doc", "description": in.Summary}),
+ "q_" + fmt.Sprintf("%d", len(vec)) + "_vec": f32ToF64Slice(vec),
+ }}, idx, in.KbID)
return err
}
@@ -775,16 +847,54 @@ func (s *NavService) llmMergeDescription(ctx context.Context, tenantID string, t
return strings.Join(texts, "\n")
}
+// cleanTitle normalizes a raw title/summary into a one-line, length-capped
+// display name (mirroring Python dataset_nav._clean_title).
+func cleanTitle(title string) string {
+ return truncateString(strings.Join(strings.Fields(title), " "), 48)
+}
+
+// fallbackTitle derives a short readable title from a summary by taking the
+// first non-empty line and stripping Markdown emphasis/heading markers. The
+// tree-root summaries carry a one-line Markdown title ("**Title**" or
+// "## Title") on the first line, so this yields a clean, URL-friendly label —
+// far more readable than taking the first whitespace words of the body.
+func fallbackTitle(summary string) string {
+ line := summary
+ if idx := strings.IndexAny(line, "\n\r"); idx >= 0 {
+ line = line[:idx]
+ }
+ line = strings.TrimSpace(line)
+ // Strip Markdown emphasis and heading markers.
+ line = strings.Trim(line, "*# \t")
+ if line == "" {
+ return "Cluster"
+ }
+ return line
+}
+
+// readableClusterName returns a readable yet unique nav-cluster key of the form
+// "<title> <8-hex>" (mirroring Python dataset_nav._readable_cluster_name): the
+// title keeps the node name human-readable, the short hash of the seed keeps the
+// per-KB uniqueness that the tree keying relies on.
+func readableClusterName(title, seed string) string {
+ t := cleanTitle(title)
+ if t == "" {
+ t = "Cluster"
+ }
+ return t + " " + shortHash8(seed)
+}
+
// llmCreateSummary builds a short cluster name + summary for a source text via
// the optional LLM (mirroring Python _llm_create_summary); without an LLM it
-// falls back to a deterministic name from a content hash and the text itself.
+// falls back to a readable name derived from the summary's first words plus a
+// short hash, so nav clusters are human-readable instead of "cluster_<hash>".
func (s *NavService) llmCreateSummary(ctx context.Context, tenantID, text string) (name, summary string) {
if s.llm != nil {
if n, sm, err := s.llm.CreateSummary(ctx, tenantID, text); err == nil && n != "" {
return n, sm
}
}
- return "cluster_" + contentHash8(text), text
+ return readableClusterName(fallbackTitle(text), text), text
}
// appendDocToCluster appends a doc id to a cluster's doc_ids_kwd and bumps its
@@ -1015,6 +1125,20 @@ func payloadJSONNav(v map[string]interface{}) string {
return string(b)
}
+// truncateString caps s to n runes (not bytes).
+func truncateString(s string, n int) string {
+ runes := []rune(s)
+ if len(runes) <= n {
+ return s
+ }
+ return string(runes[:n])
+}
+
+// shortHash8 is a stable 8-char hash suffix for readable nav names.
+func shortHash8(s string) string {
+ return contentHash8(s)
+}
+
// contentHash8 is a stable 8-char hash.
func contentHash8(s string) string {
h := uint32(2166136261)
diff --git a/internal/service/nlp/datasetnav_test.go b/internal/service/nlp/datasetnav_test.go
index fbf4620704..19523e9733 100644
--- a/internal/service/nlp/datasetnav_test.go
+++ b/internal/service/nlp/datasetnav_test.go
@@ -2,8 +2,10 @@ package nlp
import (
"context"
+ "encoding/json"
"fmt"
"math"
+ "strings"
"testing"
"ragflow/internal/engine/types"
@@ -303,6 +305,101 @@ func TestNavService_ListClusters_FiltersRoot(t *testing.T) {
}
}
+// TestNavNode_JSONShape_SnakeCase locks the REST contract: the frontend
+// DatasetNavNode and Python GET /navigation both use snake_case keys. If the
+// Go field names leaked (Name/Description/DocCount...) the frontend tree would
+// read undefined fields and render empty.
+func TestNavNode_JSONShape_SnakeCase(t *testing.T) {
+ n := nav.NavNode{Name: "cluster_x", Description: "d", DocCount: 3, Type: "cluster", HasChildren: true}
+ b, err := json.Marshal(n)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var m map[string]interface{}
+ if err := json.Unmarshal(b, &m); err != nil {
+ t.Fatal(err)
+ }
+ for _, want := range []string{"name", "description", "doc_count", "type", "has_children"} {
+ if _, ok := m[want]; !ok {
+ t.Errorf("NavNode JSON missing snake_case key %q; got %s", want, b)
+ }
+ }
+ for _, bad := range []string{"Name", "Description", "DocCount", "HasChildren"} {
+ if _, ok := m[bad]; ok {
+ t.Errorf("NavNode JSON leaked PascalCase key %q; got %s", bad, b)
+ }
+ }
+}
+
+// TestNavNamingHelpers_Readable verifies the nav display-name helpers produce
+// human-readable names (mirroring Python _clean_title/_fallback_title/
+// _readable_cluster_name) instead of raw ids.
+func TestNavNamingHelpers_Readable(t *testing.T) {
+ if cleanTitle(" hello world ") != "hello world" {
+ t.Errorf("cleanTitle = %q, want %q", cleanTitle(" hello world "), "hello world")
+ }
+ // fallbackTitle takes the first non-empty line and strips Markdown markers.
+ if got := fallbackTitle("a b c d e f g h"); got != "a b c d e f g h" {
+ t.Errorf("fallbackTitle = %q, want the first line", got)
+ }
+ if got := fallbackTitle("**何进诛阉与董后之废**\n\nbody text"); got != "何进诛阉与董后之废" {
+ t.Errorf("fallbackTitle = %q, want the stripped markdown title", got)
+ }
+ if got := fallbackTitle(""); got != "Cluster" {
+ t.Errorf("fallbackTitle('') = %q, want Cluster", got)
+ }
+ name := readableClusterName("何进诛阉", "seed-text")
+ if !strings.HasPrefix(name, "何进诛阉 ") {
+ t.Errorf("readableClusterName = %q, want prefix %q", name, "何进诛阉 ")
+ }
+}
+
+// TestNavNamingHelpers_RawIDDetection verifies meaningless ids are detected so
+// nodeFromRow can fall back to a readable title.
+func TestNavNamingHelpers_RawIDDetection(t *testing.T) {
+ for _, raw := range []string{"d3778ef9c0f5495fa4bdadc00a5bf15c", "cluster_abc12345", ""} {
+ if !graphIsRawID(raw) {
+ t.Errorf("graphIsRawID(%q) = false, want true", raw)
+ }
+ }
+ for _, ok := range []string{"何进诛阉", "NVIDIA financial performance 8738e200"} {
+ if graphIsRawID(ok) {
+ t.Errorf("graphIsRawID(%q) = true, want false", ok)
+ }
+ }
+}
+
+// TestNodeFromRow_ReadableName verifies nodeFromRow falls back to a readable
+// title derived from the payload description when title_kwd is a raw id.
+func TestNodeFromRow_ReadableName(t *testing.T) {
+ ns := &NavService{}
+ row := map[string]interface{}{
+ "title_kwd": "d3778ef9c0f5495fa4bdadc00a5bf15c",
+ "doc_id": "d3778ef9c0f5495fa4bdadc00a5bf15c",
+ "type_kwd": "nav_doc",
+ "doc_count_int": 1,
+ "content_with_weight": `{"type":"nav_doc","description":"刘备三战黄巾军与何进诛阉之议\nsecond line"}`,
+ }
+ node := ns.nodeFromRow(row, "doc")
+ if node.Name == "" || node.Name == "d3778ef9c0f5495fa4bdadc00a5bf15c" {
+ t.Fatalf("nodeFromRow name = %q, want readable fallback", node.Name)
+ }
+ if node.DocID != "d3778ef9c0f5495fa4bdadc00a5bf15c" {
+ t.Errorf("nodeFromRow DocID = %q, want the raw doc id preserved", node.DocID)
+ }
+ // A cluster row keeps its stored key verbatim (it is the child-lookup
+ // parent_kwd), even when it looks like a raw id (review Major).
+ clusterRow := map[string]interface{}{
+ "title_kwd": "cluster_abc12345",
+ "type_kwd": "nav_cluster",
+ "doc_count_int": 2,
+ }
+ cluster := ns.nodeFromRow(clusterRow, "cluster")
+ if cluster.Name != "cluster_abc12345" {
+ t.Errorf("cluster key = %q, want it kept verbatim as the parent_kwd lookup key", cluster.Name)
+ }
+}
+
// TestNavService_Search_ReturnsHit asserts acceptance #5.
func TestNavService_Search_ReturnsHit(t *testing.T) {
eng := newMemNavEngine()
@@ -353,15 +450,21 @@ func TestNavService_Acceptance4_ListChildren(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- // Exactly one nav_doc (for d2) sits under the cluster; d1 is the cluster.
- if total != 1 || len(children) != 1 {
- t.Fatalf("expected 1 child under cluster, got total=%d len=%d", total, len(children))
- }
- if children[0].DocID != "d2" {
- t.Errorf("child doc_id = %q, want d2", children[0].DocID)
+ // Every doc upserted under the cluster emits a nav_doc leaf (Python
+ // upsert_dataset_nav_doc), so d1 (which created the cluster) and d2 (which
+ // merged in) both sit under it: two nav_docs total.
+ if total != 2 || len(children) != 2 {
+ t.Fatalf("expected 2 children under cluster, got total=%d len=%d", total, len(children))
+ }
+ gotIDs := map[string]bool{}
+ for _, c := range children {
+ gotIDs[c.DocID] = true
+ if c.Type != "doc" {
+ t.Errorf("child type = %q, want doc", c.Type)
+ }
}
- if children[0].Type != "doc" {
- t.Errorf("child type = %q, want doc", children[0].Type)
+ if !gotIDs["d1"] || !gotIDs["d2"] {
+ t.Errorf("expected nav_docs for both d1 and d2, got %v", gotIDs)
}
}