RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Port dataset nav and structure graph fixes to Go, merge agents list (#18183)
Zhichang Yu6677f14bdff0
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.goindex 75898b31f9..e7c6fab1db 100644--- a/cmd/ragflow_server.go+++ b/cmd/ragflow_server.go@@ -569,6 +569,14 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArgglobalConfig.GetDefaultChatModel().Name,globalConfig.GetDefaultEmbeddingModel().Name,)+ // The dataset-level knowledge-compile consumer (tree/structure products) upserts+ // into the dataset-nav tree, so the Ingestor must install the same ES-backed+ // NavService the API server installs. Without this, nav.GetNavService() returns+ // nil and tree/structure products are dropped (the consumer logs "nav service+ // unavailable, skipping dataset-nav upsert"), leaving the dataset tree empty.+ // The embedder resolves the tenant's embedding model on demand, so both+ // Search and UpsertDoc can embed queries/summaries automatically.+ nav.SetNavService(nlp.NewNavService(service.NewNavEmbedder(service.NewModelProviderService(), "")))// Memory extraction runs on the Ingestor's shared NATS consumer + worker// pool (task_type="memory" dispatched by processMessage -> executeMemoryTask),// so there is no longer a dedicated Redis memory consumer to start.diff --git a/internal/dao/compilation_template_group.go b/internal/dao/compilation_template_group.goindex 76826d6b9d..2075a292b6 100644--- a/internal/dao/compilation_template_group.go+++ b/internal/dao/compilation_template_group.go@@ -59,6 +59,33 @@ func (dao *CompilationTemplateGroupDAO) ListSaved(ctx context.Context, db *gorm.return groups, nil}+// ListOwnedSaved returns only the tenant's own valid groups (built-in groups+// with empty tenant_id are excluded), mirroring the Python list_saved() query+// (cls.model.tenant_id == tenant_id). The merged /agents list uses this so+// built-in catalogue groups do not leak into a tenant's canvas list.+func (dao *CompilationTemplateGroupDAO) ListOwnedSaved(ctx context.Context, db *gorm.DB, tenantID, keywords, scope, orderby string, desc bool) ([]*entity.CompilationTemplateGroup, error) {+ q := db.WithContext(ctx).+ Where("tenant_id = ? AND status = ?", tenantID, string(entity.StatusValid))+ if keywords != "" {+ q = q.Where("name LIKE ?", "%"+keywords+"%")+ }+ if scope != "" {+ q = q.Where("scope = ?", scope)+ }+ if orderby != "name" && orderby != "scope" && orderby != "create_time" && orderby != "update_time" {+ orderby = "create_time"+ }+ dir := "asc"+ if desc {+ dir = "desc"+ }+ var groups []*entity.CompilationTemplateGroup+ if err := q.Order(orderby + " " + dir).Find(&groups).Error; err != nil {+ return nil, err+ }+ return groups, nil+}+// CountSavedByTenant counts the tenant's own valid groups; built-in groups// (empty tenant_id) are excluded, mirroring the group_count query in Python// get_owner_filter / get_category_filter.diff --git a/internal/handler/agent_test.go b/internal/handler/agent_test.goindex da88e819c4..1873868473 100644--- a/internal/handler/agent_test.go+++ b/internal/handler/agent_test.go@@ -393,9 +393,12 @@ func setupAgentRouter(svc agentServiceIface) *gin.Engine {func TestListAgents_Success(t *testing.T) {title := "My Agent"+ agentJSON, _ := json.Marshal(service.AgentItem{+ ID: "canvas-1", Title: &title, Permission: "me", CanvasCategory: "agent_canvas",+ })svc := &fakeAgentService{result: &service.ListAgentsResponse{- Canvas: []*service.AgentItem{{ID: "canvas-1", Title: &title, Permission: "me", CanvasCategory: "agent_canvas"}},+ Canvas: []json.RawMessage{agentJSON},Total: 1,},code: common.CodeSuccess,diff --git a/internal/handler/dataset_artifact.go b/internal/handler/dataset_artifact.goindex dcddfddda7..3ac47a86f2 100644--- a/internal/handler/dataset_artifact.go+++ b/internal/handler/dataset_artifact.go@@ -339,7 +339,9 @@ func (h *DatasetArtifactHandler) ListNavigation(c *gin.Context) {common.ErrorWithCode(c, common.CodeDataError, err.Error())return}- common.SuccessWithData(c, gin.H{"total": total, "nav": items}, "success")+ // Response key is "items" (not "nav") — the frontend DatasetNavList reads+ // data.items and Python list_nav_clusters/_nav_search return {"total","items"}.+ common.SuccessWithData(c, gin.H{"total": total, "items": items}, "success")}// DeleteNavigation handles DELETE /navigation — delete all navigation clusters.@@ -381,7 +383,9 @@ func (h *DatasetArtifactHandler) ListNavigationChildren(c *gin.Context) {common.ErrorWithCode(c, common.CodeDataError, err.Error())return}- common.SuccessWithData(c, gin.H{"total": total, "children": items}, "success")+ // Same contract as the top-level nav list: Python list_nav_children returns+ // {"total","items"} and the frontend reads data.items for child expansion.+ common.SuccessWithData(c, gin.H{"total": total, "items": items}, "success")}// GetSkillTree handles GET /skills — skill tree.@@ -453,13 +457,20 @@ func (h *DatasetArtifactHandler) GetDocumentGraph(c *gin.Context) {}datasetID := c.Param("dataset_id")documentID := c.Param("document_id")- graphType := c.Query("graph_type")- items, total, err := h.svc.GetDocumentGraph(c.Request.Context(), tenantID, datasetID, documentID, graphType)+ resp, err := h.svc.GetDocumentGraph(c.Request.Context(), service.DocumentStructureGraphInput{+ TenantID: tenantID,+ DatasetID: datasetID,+ DocumentID: documentID,+ Keywords: c.Query("keywords"),+ })if err != nil {common.ErrorWithCode(c, common.CodeDataError, err.Error())return}- common.SuccessWithData(c, gin.H{"total": total, "graph": items}, "success")+ if resp == nil {+ resp = &service.DocumentStructureGraphResponse{Templates: []service.DocumentStructureGraphTemplate{}}+ }+ common.SuccessWithData(c, resp, "success")}// DeleteDocumentGraph handles DELETE /documents/<document_id>/structure/graph — delete document structure graph.diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.goindex aa0b23357b..8b29cbebb1 100644--- a/internal/ingestion/component/knowledge_compiler/component.go+++ b/internal/ingestion/component/knowledge_compiler/component.go@@ -449,44 +449,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {switch p.Variant {case common.VariantStructure:// knowledge_graph_kwd: "entity" | "relation" | "graph".- if kind != "" {- if err := doc.SetExtraValue("knowledge_graph_kwd", kind); err != nil {- return err- }- }- // Relations carry from/to entity endpoints (from_entity_kwd / to_entity_kwd).- if kind == "relation" {- if v := metaString(p.Meta, "from"); v != "" {- if err := doc.SetExtraValue("from_entity_kwd", v); err != nil {- return err- }- }- if v := metaString(p.Meta, "to"); v != "" {- if err := doc.SetExtraValue("to_entity_kwd", v); err != nil {- return err- }- }- }- // Entities carry their canonical name on name_kwd (lowercased, mirroring- // Python's _struct_to_doc_storage_doc; the structure-graph endpoints- // filter/sort on it) plus entity_type_kwd and mention_count_int.- if kind == "entity" {- if v := metaString(p.Meta, "name"); v != "" {- if err := doc.SetExtraValue("name_kwd", strings.ToLower(v)); err != nil {- return err- }- }- if v := metaString(p.Meta, "entity_type"); v != "" {- if err := doc.SetExtraValue("entity_type_kwd", v); err != nil {- return err- }- }- }- if v, ok := metaInt(p.Meta, "mention_count"); ok {- if err := doc.SetExtraValue("mention_count_int", v); err != nil {- return err- }- }+ return applyStructureGraphColumns(doc, p, kind)case common.VariantWiki:// One artifact_page row per wiki page; section rows reuse the same@@ -568,23 +531,35 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {}case common.VariantTree:- // raptor_kwd tags summary/root nodes; raptor_layer_int records tree depth.- if kind != "" {- if err := doc.SetExtraValue("raptor_kwd", kind); err != nil {- return err- }- }- if v, ok := metaInt(p.Meta, "level"); ok {- if err := doc.SetExtraValue("raptor_layer_int", v); err != nil {- return err+ switch kind {+ case "entity", "relation", "graph":+ // The tree is also projected onto the structure-graph shape (Python+ // raptor_tree_to_graph + _struct_upsert_tree_graph_rows): entity /+ // relation rows carry knowledge_graph_kwd and the compact graph blob+ // (kind "graph") is the /structure/graph discovery row. This is the+ // same storage contract as the structure variant, so both share+ // applyStructureGraphColumns.+ return applyStructureGraphColumns(doc, p, kind)+ default:+ // RAPTOR summary/root rows: raptor_kwd tags the node kind;+ // raptor_layer_int records tree depth.+ if kind != "" {+ if err := doc.SetExtraValue("raptor_kwd", kind); err != nil {+ return err+ }}- if err := doc.SetExtraValue("depth_int", v); err != nil {- return err+ if v, ok := metaInt(p.Meta, "level"); ok {+ if err := doc.SetExtraValue("raptor_layer_int", v); err != nil {+ return err+ }+ if err := doc.SetExtraValue("depth_int", v); err != nil {+ return err+ }}- }- if v := metaStringSlice(p.Meta, "children"); len(v) > 0 {- if err := doc.SetExtraValue("children_kwd", v); err != nil {- return err+ if v := metaStringSlice(p.Meta, "children"); len(v) > 0 {+ if err := doc.SetExtraValue("children_kwd", v); err != nil {+ return err+ }}}@@ -611,6 +586,53 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {return nil}+// applyStructureGraphColumns emits the structure-graph row columns shared by the+// structure and tree variants (Python _struct_to_doc_storage_doc contract):+// - knowledge_graph_kwd: "entity" | "relation" | "graph"+// - relations: from_entity_kwd / to_entity_kwd+// - entities: name_kwd (lowercased) / entity_type_kwd+// - mention_count_int+//+// Keeping this in one helper prevents the two variants' storage contracts from+// diverging (review Major).+func applyStructureGraphColumns(doc *schema.ChunkDoc, p common.Product, kind string) error {+ if kind != "" {+ if err := doc.SetExtraValue("knowledge_graph_kwd", kind); err != nil {+ return err+ }+ }+ if kind == "relation" {+ if v := metaString(p.Meta, "from"); v != "" {+ if err := doc.SetExtraValue("from_entity_kwd", v); err != nil {+ return err+ }+ }+ if v := metaString(p.Meta, "to"); v != "" {+ if err := doc.SetExtraValue("to_entity_kwd", v); err != nil {+ return err+ }+ }+ }+ if kind == "entity" {+ if v := metaString(p.Meta, "name"); v != "" {+ if err := doc.SetExtraValue("name_kwd", strings.ToLower(v)); err != nil {+ return err+ }+ }+ if v := metaString(p.Meta, "entity_type"); v != "" {+ if err := doc.SetExtraValue("entity_type_kwd", v); err != nil {+ return err+ }+ }+ }+ if v, ok := metaInt(p.Meta, "mention_count"); ok {+ if err := doc.SetExtraValue("mention_count_int", v); err != nil {+ return err+ }+ }+ return nil+}+// metaString reads a string-valued Product.Meta key.func metaString(m map[string]any, key string) string {v, _ := m[key].(string)diff --git a/internal/ingestion/component/knowledge_compiler/golden/metrics.go b/internal/ingestion/component/knowledge_compiler/golden/metrics.goindex 753cfa8ff2..27224bb628 100644--- a/internal/ingestion/component/knowledge_compiler/golden/metrics.go+++ b/internal/ingestion/component/knowledge_compiler/golden/metrics.go@@ -47,10 +47,18 @@ func AnalyzeTreeProducts(chunks []schema.ChunkDoc, validSourceIDs ...string) TrevalidSet[id] = true}checkValid := len(validSet) > 0- m := TreeMetrics{ProductCount: len(chunks), AllParented: true, VectorOK: true, SchemaOK: true, covered: make(map[string]bool)}+ m := TreeMetrics{AllParented: true, VectorOK: true, SchemaOK: true, covered: make(map[string]bool)}maxLevel := -1for _, c := range chunks {kind, _ := c.GetExtraString("kc_kind")+ // Auxiliary tree-graph rows (kind entity/relation/graph, from the+ // structure-graph projection) are not part of the RAPTOR tree structure,+ // so they are excluded from the structural metrics (ProductCount,+ // AllParented, SchemaOK, ...). The tree itself is the root/summary set.+ if kind != "root" && kind != "summary" {+ continue+ }+ m.ProductCount++level := 0if lf, ok := extraFloat(c, "kc_level"); ok {level = int(lf)diff --git a/internal/ingestion/component/knowledge_compiler/tree/graph.go b/internal/ingestion/component/knowledge_compiler/tree/graph.gonew file mode 100644index 0000000000..a6b3a678ac--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/tree/graph.go@@ -0,0 +1,356 @@+package tree++import (+ "bytes"+ "context"+ "encoding/json"+ "fmt"+ "sort"+ "strings"++ "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++// stringsContains reports whether s contains substring sub.+func stringsContains(s, sub string) bool {+ return strings.Contains(s, sub)+}++// strSliceContains reports whether s contains v.+func strSliceContains(s []string, v string) bool {+ for _, e := range s {+ if e == v {+ return true+ }+ }+ return false+}++// stringsJoinNonEmpty joins the non-empty parts with sep.+func stringsJoinNonEmpty(parts []string, sep string) string {+ var kept []string+ for _, p := range parts {+ if p != "" {+ kept = append(kept, p)+ }+ }+ return strings.Join(kept, sep)+}++// stringMetaSlice coerces a Product.Meta value ([]string or []any of strings)+// into a []string.+func stringMetaSlice(v any) []string {+ switch x := v.(type) {+ case []string:+ return x+ case []any:+ out := make([]string, 0, len(x))+ for _, e := range x {+ if s, ok := e.(string); ok && s != "" {+ out = append(out, s)+ }+ }+ return out+ }+ return nil+}++// payloadChunkIDs extracts the source_chunk_ids from a tree-graph payload.+func payloadChunkIDs(payload map[string]any) []string {+ switch v := payload["source_chunk_ids"].(type) {+ case []string:+ return v+ case []any:+ var out []string+ for _, e := range v {+ if s, ok := e.(string); ok && s != "" {+ out = append(out, s)+ }+ }+ return out+ }+ return nil+}++// payloadDescription is the embedding input for a tree-graph entity/relation:+// the concatenated string values of every field except description (lists+// flattened), matching Python _struct_payload_description.+func payloadDescription(payload map[string]any) string {+ keys := make([]string, 0, len(payload))+ for k := range payload {+ if k != "description" {+ keys = append(keys, k)+ }+ }+ sort.Strings(keys)+ var parts []string+ for _, k := range keys {+ switch v := payload[k].(type) {+ case string:+ if v != "" {+ parts = append(parts, v)+ }+ case []string:+ for _, e := range v {+ if e != "" {+ parts = append(parts, e)+ }+ }+ case []any:+ for _, e := range v {+ if s, ok := e.(string); ok && s != "" {+ parts = append(parts, s)+ }+ }+ }+ }+ return strings.Join(parts, " ")+}++// payloadJSON serialises a payload the way Python's json.dumps(ensure_ascii=+// False) does (no HTML escaping), with alphabetically sorted keys for a+// canonical, hash-stable form.+func payloadJSON(payload map[string]any) string {+ var b bytes.Buffer+ enc := json.NewEncoder(&b)+ enc.SetEscapeHTML(false)+ if err := enc.Encode(payload); err != nil {+ return "{}"+ }+ return strings.TrimSpace(b.String())+}++// graphNode mirrors Python's RAPTOR tree node dict (title/description/children/+// source_chunk_ids) reconstructed from the flat products emitted by buildTree,+// so the tree can be projected to a {entities, relations} graph exactly like+// Python's raptor_tree_to_graph (chunk_post_processor.py:470).+type graphNode struct {+ title string+ description string+ sourceChunkIDs []string+ children []*graphNode+}++// collapseUnary merges a node that wraps exactly one child into that child,+// mirroring Python raptor_tree_to_graph._collapse_unary: the parent's and the+// child's descriptions/source-chunk-ids are concatenated (dedup'd), then the+// collapsed node adopts the child's children.+func collapseUnary(node *graphNode) *graphNode {+ collapsed := &graphNode{+ title: node.title,+ description: node.description,+ sourceChunkIDs: node.sourceChunkIDs,+ }+ for _, c := range node.children {+ collapsed.children = append(collapsed.children, collapseUnary(c))+ }+ for len(collapsed.children) == 1 {+ child := collapsed.children[0]+ parentTitle := collapsed.title+ childTitle := child.title+ parentDesc := collapsed.description+ if parentDesc == "" {+ parentDesc = parentTitle+ }+ childDesc := child.description+ if childDesc == "" {+ childDesc = childTitle+ }++ var descriptions []string+ descriptions = append(descriptions, parentDesc)+ if childTitle != "" && childTitle != parentTitle &&+ !stringsContains(childDesc, childTitle) {+ descriptions = append(descriptions, childTitle)+ }+ if childDesc != "" && !strSliceContains(descriptions, childDesc) {+ descriptions = append(descriptions, childDesc)+ }++ sourceChunkIDs := append([]string{}, collapsed.sourceChunkIDs...)+ for _, id := range child.sourceChunkIDs {+ if id != "" && !strSliceContains(sourceChunkIDs, id) {+ sourceChunkIDs = append(sourceChunkIDs, id)+ }+ }++ collapsed.description = stringsJoinNonEmpty(descriptions, "\n\n")+ collapsed.sourceChunkIDs = sourceChunkIDs+ collapsed.children = child.children+ }+ return collapsed+}++// raptorTreeToGraph projects a RAPTOR tree onto {entities, relations}, matching+// Python raptor_tree_to_graph: every node becomes an entity of type "tree_node";+// every parent→child edge (that is not a self-loop) becomes a "child" relation.+func raptorTreeToGraph(root *graphNode) ([]map[string]any, []map[string]any) {+ var entities []map[string]any+ var relations []map[string]any+ var walk func(node *graphNode, parentTitle string)+ walk = func(node *graphNode, parentTitle string) {+ if node == nil {+ return+ }+ title := node.title+ ent := map[string]any{+ "name": title,+ "type": "tree_node",+ "description": firstNonEmpty(node.description, title),+ "mention_count": 1,+ }+ if len(node.sourceChunkIDs) > 0 {+ ent["source_chunk_ids"] = node.sourceChunkIDs+ }+ entities = append(entities, ent)+ if parentTitle != "" && parentTitle != title {+ relations = append(relations, map[string]any{+ "from": parentTitle,+ "to": title,+ "type": "child",+ })+ }+ for _, child := range node.children {+ walk(child, title)+ }+ }+ walk(root, "")+ return entities, relations+}++// buildTreeGraph reconstructs the tree from the flat summary products and+// produces the entity/relation/graph products Python writes for a tree variant+// (_struct_upsert_tree_graph_rows + _struct_upsert_graph_json):+// - one entity product per tree node (kind "entity", knowledge_graph_kwd via+// the writer), type "tree_node";+// - one relation product per parent→child edge (kind "relation");+// - one compact graph blob product (kind "graph") carrying the whole+// {entities, relations} projection, which is the /structure/graph discovery+// row (Python scans knowledge_graph_kwd="graph").+//+// templateID is stamped into each row so the document-structure endpoint can+// group by template id; compileKWD is "tree".+func buildTreeGraph(ctx context.Context, deps common.Deps, docID string, products []common.Product) ([]common.Product, error) {+ if deps.Embed == nil {+ return nil, fmt.Errorf("tree: embedder required to build the tree graph")+ }+ root := reconstructTree(products)+ if root == nil {+ // No root summary survived; there is no tree to project.+ return nil, nil+ }+ root = collapseUnary(root)+ entities, relations := raptorTreeToGraph(root)++ var out []common.Product+ var descs []string+ var payloads []map[string]any+ var kinds []string+ for _, ent := range entities {+ descs = append(descs, payloadDescription(ent))+ payloads = append(payloads, ent)+ kinds = append(kinds, "entity")+ }+ for _, rel := range relations {+ descs = append(descs, payloadDescription(rel))+ payloads = append(payloads, rel)+ kinds = append(kinds, "relation")+ }+ vecs, err := deps.Embed.Encode(ctx, descs)+ if err != nil {+ return nil, err+ }+ for i, payload := range payloads {+ kind := kinds[i]+ var vec []float32+ if i < len(vecs) {+ vec = vecs[i]+ }+ meta := map[string]any{+ "kind": kind,+ "compile_kwd": "tree",+ "source_chunk_ids": payloadChunkIDs(payload),+ "mention_count": 1,+ }+ if kind == "entity" {+ if name, ok := payload["name"].(string); ok && name != "" {+ meta["name"] = name+ }+ if typ, ok := payload["type"].(string); ok && typ != "" {+ meta["entity_type"] = typ+ } else {+ meta["entity_type"] = "other"+ }+ } else {+ if from, ok := payload["from"].(string); ok {+ meta["from"] = from+ }+ if to, ok := payload["to"].(string); ok {+ meta["to"] = to+ }+ }+ out = append(out, common.Product{+ ID: common.StableRowID(payloadJSON(payload), docID),+ DocID: docID,+ TenantID: deps.TenantID,+ Variant: common.VariantTree,+ Content: payloadJSON(payload),+ Vector: vec,+ Meta: meta,+ })+ }++ // Compact graph blob discovery row (knowledge_graph_kwd="graph").+ graph := map[string]any{"entities": entities, "relations": relations}+ graphContent := payloadJSON(graph)+ graphVecs, err := deps.Embed.Encode(ctx, []string{graphContent})+ if err != nil {+ return nil, err+ }+ var gv []float32+ if len(graphVecs) > 0 {+ gv = graphVecs[0]+ }+ out = append(out, common.Product{+ ID: common.StableRowID(docID, "tree", "structure_graph"),+ DocID: docID,+ TenantID: deps.TenantID,+ Variant: common.VariantTree,+ Content: graphContent,+ Vector: gv,+ Meta: map[string]any{+ "kind": "graph",+ "compile_kwd": "tree",+ },+ })+ return out, nil+}++// reconstructTree assembles a graphNode tree from the flat summary products:+// the root has Meta.kind=="root"; every other node's parent is the product with+// ID == node.ParentID. Node title comes from Meta.title, description from+// Content, source chunk ids from Meta.source_chunk_ids.+func reconstructTree(products []common.Product) *graphNode {+ byID := make(map[string]*graphNode, len(products))+ for _, p := range products {+ title, _ := p.Meta["title"].(string)+ byID[p.ID] = &graphNode{+ title: title,+ description: p.Content,+ sourceChunkIDs: stringMetaSlice(p.Meta["source_chunk_ids"]),+ }+ }+ var root *graphNode+ for _, p := range products {+ kind, _ := p.Meta["kind"].(string)+ node := byID[p.ID]+ if kind == "root" {+ root = node+ continue+ }+ if parent := byID[p.ParentID]; parent != nil {+ parent.children = append(parent.children, node)+ }+ }+ return root+}diff --git a/internal/ingestion/component/knowledge_compiler/tree/graph_test.go b/internal/ingestion/component/knowledge_compiler/tree/graph_test.gonew file mode 100644index 0000000000..5aa56797ac--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/tree/graph_test.go@@ -0,0 +1,107 @@+package tree++import (+ "reflect"+ "testing"++ "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++// TestRaptorTreeToGraph_CollapsesUnaryAndProjects verifies the tree→graph+// projection matches Python raptor_tree_to_graph: unary chains are collapsed+// (descriptions concatenated), each node becomes an entity of type tree_node,+// and each parent→child edge (skipping self-loops) becomes a child relation.+func TestRaptorTreeToGraph_CollapsesUnaryAndProjects(t *testing.T) {+ // Build a tree: root → [X, A → [B]]. A is a unary wrapper (only child B); the+ // collapse merges A into B, while root (two children) and B (no children)+ // stay. Entities: root, X, B. Relations: root→X, root→B.+ root := &graphNode{+ title: "root",+ description: "root desc",+ children: []*graphNode{+ {title: "X", description: "X desc"},+ {+ title: "A",+ description: "A desc",+ children: []*graphNode{+ {title: "B", description: "B desc", sourceChunkIDs: []string{"c1"}},+ },+ },+ },+ }+ root = collapseUnary(root)+ entities, relations := raptorTreeToGraph(root)++ // The unary node A survives but the child B is folded into it (it keeps A's+ // title, concatenated descriptions and B's source chunk ids), matching+ // Python _collapse_unary. Entities: root, X, A.+ var names []string+ for _, e := range entities {+ names = append(names, e["name"].(string))+ }+ if !reflect.DeepEqual(names, []string{"root", "X", "A"}) {+ t.Fatalf("entities after unary collapse = %v, want [root X A]", names)+ }+ // A must have B folded in: concatenated descriptions + the source chunk id.+ byName := map[string]map[string]any{}+ for _, e := range entities {+ byName[e["name"].(string)] = e+ }+ a := byName["A"]+ if a["description"] != "A desc\n\nB desc" {+ t.Errorf("collapsed A description = %q, want %q", a["description"], "A desc\n\nB desc")+ }+ if !reflect.DeepEqual(a["source_chunk_ids"], []string{"c1"}) {+ t.Errorf("collapsed A source_chunk_ids = %v, want [c1]", a["source_chunk_ids"])+ }+ if a["type"] != "tree_node" {+ t.Errorf("entity type = %v, want tree_node", a["type"])+ }+ // Two relations: root → X and root → A (A is root's child after collapse).+ if len(relations) != 2 {+ t.Fatalf("relations = %v, want exactly [root->X, root->A]", relations)+ }+ wantRels := []struct{ from, to string }{{"root", "X"}, {"root", "A"}}+ gotRels := []struct{ from, to string }{+ {relations[0]["from"].(string), relations[0]["to"].(string)},+ {relations[1]["from"].(string), relations[1]["to"].(string)},+ }+ if !reflect.DeepEqual(gotRels, wantRels) {+ t.Errorf("relations = %v, want %v", gotRels, wantRels)+ }+}++// TestRaptorTreeToGraph_SkipsSelfLoop verifies a parent whose child has the same+// title does not produce a self-loop relation (Python's guard).+func TestRaptorTreeToGraph_SkipsSelfLoop(t *testing.T) {+ root := &graphNode{+ title: "same",+ children: []*graphNode{+ {title: "same", description: "child"},+ },+ }+ _, relations := raptorTreeToGraph(root)+ if len(relations) != 0 {+ t.Fatalf("self-loop relation must be skipped, got %v", relations)+ }+}++// TestReconstructTree_FromFlatProducts verifies the flat summary products+// (root + ParentID chains) are reassembled into a nested tree.+func TestReconstructTree_FromFlatProducts(t *testing.T) {+ products := []common.Product{+ {ID: "root", Meta: map[string]any{"kind": "root", "title": "root"}},+ {ID: "n1", ParentID: "root", Meta: map[string]any{"kind": "summary", "title": "N1"}, Content: "n1 desc"},+ {ID: "n2", ParentID: "n1", Meta: map[string]any{"kind": "summary", "title": "N2"}, Content: "n2 desc"},+ }+ root := reconstructTree(products)+ if root == nil || root.title != "root" {+ t.Fatalf("reconstructed root = %+v, want title root", root)+ }+ if len(root.children) != 1 || root.children[0].title != "N1" {+ t.Fatalf("root children = %+v, want [N1]", root.children)+ }+ if len(root.children[0].children) != 1 || root.children[0].children[0].title != "N2" {+ t.Fatalf("N1 children = %+v, want [N2]", root.children[0].children)+ }+}diff --git a/internal/ingestion/component/knowledge_compiler/tree/raptor.go b/internal/ingestion/component/knowledge_compiler/tree/raptor.goindex 5813e2e460..486d2bdb2f 100644--- a/internal/ingestion/component/knowledge_compiler/tree/raptor.go+++ b/internal/ingestion/component/knowledge_compiler/tree/raptor.go@@ -53,6 +53,18 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commoreturn common.Outputs{}, err}+ // Project the RAPTOR tree onto the {entities, relations} structure-graph+ // shape (Python raptor_tree_to_graph) and persist it as entity/relation rows+ // plus a compact graph blob (knowledge_graph_kwd="graph"), so the+ // document-structure /structure/graph endpoint can serve the tree. A failure+ // here must not abort the whole tree compile — the summary nodes are already+ // valid on their own — so it is best-effort and surfaced as a log.+ if graphProds, err := buildTreeGraph(ctx, deps, docID, products); err != nil {+ log.Printf("tree: graph projection failed (best-effort, continuing): %v", err)+ } else {+ products = append(products, graphProds...)+ }+out := common.Outputs{Products: products,}diff --git a/internal/service/agent.go b/internal/service/agent.goindex 1d9910ce5c..7e4dabe513 100644--- a/internal/service/agent.go+++ b/internal/service/agent.go@@ -310,6 +310,7 @@ type AgentService struct {versionDAO *dao.UserCanvasVersionDAOapi4ConversationDAO *dao.API4ConversationDAOcompilationTemplateGroupDAO *dao.CompilationTemplateGroupDAO+ compilationTemplateDAO *dao.CompilationTemplateDAO// driver is the per-process runner that drives canvas// invocations and produces SSE events. V1 persistence is@@ -383,6 +384,7 @@ func NewAgentServiceWithOptions(versionDAO: dao.NewUserCanvasVersionDAO(),api4ConversationDAO: dao.NewAPI4ConversationDAO(),compilationTemplateGroupDAO: dao.NewCompilationTemplateGroupDAO(),+ compilationTemplateDAO: dao.NewCompilationTemplateDAO(),runner: canvas.NewRunner(),activeSessions: make(map[string]*activeAgentRun),checkpointStore: cp,@@ -415,14 +417,28 @@ type AgentItem struct {CreateTime *int64 `json:"create_time,omitempty"`UpdateTime *int64 `json:"update_time,omitempty"`ReleaseTime *int64 `json:"release_time,omitempty"`+ // Type discriminates agent vs compilation-template-group items in the merged+ // /agents response. It is set only for merged items (agent => "agent").+ Type string `json:"type,omitempty"`}-// ListAgentsResponse is the response body for GET /api/v1/agents.+// ListAgentsResponse is the response body for GET /api/v1/agents. Canvas holds+// pre-marshalled JSON items because a canvas entry is either an agent (AgentItem+// shape) or a compilation template group (group shape with a "type" of+// "compilation_template_group"); a single typed slice cannot express both.type ListAgentsResponse struct {- Canvas []*AgentItem `json:"canvas"`- Total int64 `json:"total"`+ Canvas []json.RawMessage `json:"canvas"`+ Total int64 `json:"total"`}+// AgentItemType discriminates the two kinds of canvas items in the merged+// /agents response, mirroring Python _COMPILATION_TEMPLATE_GROUP_CATEGORY and+// the frontend AgentListItem union.+const (+ AgentItemTypeAgent = "agent"+ AgentItemTypeGroup = CompilationTemplateGroupCategory // "compilation_template_group"+)+// CompilationTemplateGroupCategory is the synthetic canvas_category the// frontend uses to filter compilation template groups through the merged// /agents endpoint. Mirrors Python _COMPILATION_TEMPLATE_GROUP_CATEGORY.@@ -592,17 +608,46 @@ func (s *AgentService) ListAgents(ctx context.Context, userID string, keywords s}}+ // A canvas entry is either an agent (user_canvas) or a compilation template+ // group. Python splits canvas_category on commas and, when the tenant is the+ // sole effective owner, merges the caller's template groups into the list.+ categories := splitCategoryList(canvasCategory)+ wantsGroups := sliceContains(categories, CompilationTemplateGroupCategory)+ agentCategories := filterCategory(categories, CompilationTemplateGroupCategory)+ // Merge mode mirrors Python: no category, no canvas_type, no tags -> the+ // caller's template groups are interleaved with agents by update_time.+ mergeMode := len(categories) == 0 && canvasType == "" && len(tags) == 0++ // Groups-only mode: canvas_category is exactly ["compilation_template_group"].+ // Template groups are always the caller's own, so they are only visible when+ // the caller is an effective owner; otherwise (e.g. owner_ids names another+ // user) return an empty list (review Major).+ if len(categories) == 1 && wantsGroups {+ if !sliceContains(effectiveOwnerIDs, userID) {+ return &ListAgentsResponse{Canvas: []json.RawMessage{}, Total: 0}, common.CodeSuccess, nil+ }+ return s.listAgentsGroupsOnly(ctx, userID, keywords, orderBy, desc, page, pageSize)+ }++ // Fetch agents. In merge/mixed modes we disable SQL pagination (page=0) and+ // paginate in Go after interleaving with groups, matching Python.+ listPage, listSize := page, pageSize+ agentCategoryFilter := canvasCategory+ if mergeMode || (wantsGroups && len(agentCategories) > 0) {+ listPage, listSize = 0, 0+ agentCategoryFilter = strings.Join(agentCategories, ",")+ }canvases, total, err := s.canvasDAO.ListByTenantIDs(ctx,dao.DB,effectiveOwnerIDs,userID,- page,- pageSize,+ listPage,+ listSize,orderBy,desc,keywords,- canvasCategory,+ agentCategoryFilter,canvasType,tags,)@@ -610,31 +655,252 @@ func (s *AgentService) ListAgents(ctx context.Context, userID string, keywords sreturn nil, common.CodeServerError, fmt.Errorf("failed to list agents: %w", err)}- items := make([]*AgentItem, len(canvases))+ agentItems := make([]*AgentItem, len(canvases))for i, c := range canvases {- items[i] = toAgentItem(c)+ agentItems[i] = toAgentItem(c)+ }+ s.attachReleaseTimes(ctx, agentItems)++ // Groups are owner-only (no team sharing) and scoped to the caller, so they+ // are merged only when the caller's own tenant is an effective owner+ // (Python include_template_groups).+ includeGroups := sliceContains(effectiveOwnerIDs, userID)+ if includeGroups && (mergeMode || wantsGroups) {+ return s.mergeAgentsAndGroups(ctx, userID, agentItems, keywords, orderBy, desc, page, pageSize)+ }++ raw := make([]json.RawMessage, len(agentItems))+ for i, item := range agentItems {+ item.Type = AgentItemTypeAgent+ raw[i] = marshalAgentItem(item)}+ return &ListAgentsResponse{Canvas: raw, Total: total}, common.CodeSuccess, nil+}- // Attach the latest release time per canvas so agent cards can render- // the "published at" line (Python UserCanvasService.get_list parity).- if len(items) > 0 {- canvasIDs := make([]string, 0, len(items))- for _, item := range items {- canvasIDs = append(canvasIDs, item.ID)+// listAgentsGroupsOnly returns only the caller's compilation template groups+// (Python canvas_category == ["compilation_template_group"] branch).+func (s *AgentService) listAgentsGroupsOnly(ctx context.Context, userID, keywords, orderBy string, desc bool, page, pageSize int) (*ListAgentsResponse, common.ErrorCode, error) {+ groups, err := s.compilationTemplateGroupDAO.ListOwnedSaved(ctx, dao.DB, userID, keywords, "", orderBy, desc)+ if err != nil {+ return nil, common.CodeServerError, fmt.Errorf("failed to list compilation template groups: %w", err)+ }+ total := int64(len(groups))+ groups = slicePage(groups, page, pageSize)+ raw := make([]json.RawMessage, 0, len(groups))+ for _, g := range groups {+ item, err := s.marshalMergeGroupItem(ctx, userID, g)+ if err != nil {+ return nil, common.CodeServerError, fmt.Errorf("failed to build group item: %w", err)}- var releaseTimes map[string]int64- releaseTimes, err = s.versionDAO.GetLatestReleaseTimes(ctx, dao.DB, canvasIDs)+ raw = append(raw, item)+ }+ return &ListAgentsResponse{Canvas: raw, Total: total}, common.CodeSuccess, nil+}++// mergeAgentsAndGroups combines agents and the caller's compilation template+// groups into a single list ordered by (canvas_category, name) ascending, then+// pages in Go. desc is accepted for API signature parity but the ordering is+// intentionally category/name based, not chronological.+func (s *AgentService) mergeAgentsAndGroups(ctx context.Context, userID string, agentItems []*AgentItem, keywords, orderBy string, desc bool, page, pageSize int) (*ListAgentsResponse, common.ErrorCode, error) {+ groups, err := s.compilationTemplateGroupDAO.ListOwnedSaved(ctx, dao.DB, userID, keywords, "", orderBy, desc)+ if err != nil {+ return nil, common.CodeServerError, fmt.Errorf("failed to list compilation template groups: %w", err)+ }+ merged := make([]mergeCanvasItem, 0, len(agentItems)+len(groups))+ for _, item := range agentItems {+ item.Type = AgentItemTypeAgent+ merged = append(merged, mergeCanvasItem{+ item: item,+ category: item.CanvasCategory,+ name: derefString(item.Title),+ })+ }+ for _, g := range groups {+ raw, err := s.marshalMergeGroupItem(ctx, userID, g)if err != nil {- return nil, common.CodeServerError, fmt.Errorf("failed to get release times: %w", err)+ return nil, common.CodeServerError, fmt.Errorf("failed to build group item: %w", err)}- for _, item := range items {- if t, ok := releaseTimes[item.ID]; ok {- item.ReleaseTime = &t- }+ merged = append(merged, mergeCanvasItem{+ raw: raw,+ category: CompilationTemplateGroupCategory,+ name: g.Name,+ })+ }+ // Order the merged list by (category, name) ascending (A-Z): agents and+ // compilation template groups are grouped by flow category, then sorted by+ // display name within each category. The category sort key uses the raw+ // canvas_category string so the natural order is agent_canvas <+ // compilation_template_group < dataflow_canvas.+ sort.SliceStable(merged, func(i, j int) bool {+ if merged[i].category != merged[j].category {+ return merged[i].category < merged[j].category+ }+ return strings.ToLower(merged[i].name) < strings.ToLower(merged[j].name)+ })+ total := int64(len(merged))+ merged = slicePage(merged, page, pageSize)+ raw := make([]json.RawMessage, 0, len(merged))+ for _, m := range merged {+ if m.raw != nil {+ raw = append(raw, m.raw)+ } else if m.item != nil {+ raw = append(raw, marshalAgentItem(m.item))}}+ return &ListAgentsResponse{Canvas: raw, Total: total}, common.CodeSuccess, nil+}- return &ListAgentsResponse{Canvas: items, Total: total}, common.CodeSuccess, nil+// marshalMergeGroupItem renders a compilation template group as a merged /agents+// canvas item: the group shape (ICompilationTemplateGroup) plus the "type" and+// "title" discriminators the frontend union expects (Python _group_to_dict).+func (s *AgentService) marshalMergeGroupItem(ctx context.Context, userID string, g *entity.CompilationTemplateGroup) (json.RawMessage, error) {+ children, err := s.compilationTemplateDAO.ListByGroup(ctx, dao.DB, g.ID)+ if err != nil {+ return nil, err+ }+ templates := make([]json.RawMessage, 0, len(children))+ for _, c := range children {+ templates = append(templates, marshalGroupTemplate(c))+ }+ item := map[string]interface{}{+ "id": g.ID,+ "name": g.Name,+ "title": g.Name,+ "description": derefString(g.Description),+ "scope": g.Scope,+ "create_time": intValuePtr(g.CreateTime),+ "update_time": intValuePtr(g.UpdateTime),+ "templates": templates,+ "type": AgentItemTypeGroup,+ }+ b, err := json.Marshal(item)+ if err != nil {+ return nil, err+ }+ return b, nil+}++// mergeCanvasItem is a decoded entry in the merged /agents list: exactly one of+// item (an agent) or raw (a marshalled group) is set. category is the flow+// classification (canvas_category, or "compilation_template_group" for groups)+// and name is the display title; the merged list is ordered by (category, name).+type mergeCanvasItem struct {+ item *AgentItem+ raw json.RawMessage+ category string+ name string+}++// attachReleaseTimes populates ReleaseTime for each agent item from the latest+// published version (Python UserCanvasService.get_list parity).+func (s *AgentService) attachReleaseTimes(ctx context.Context, items []*AgentItem) {+ if len(items) == 0 {+ return+ }+ canvasIDs := make([]string, 0, len(items))+ for _, item := range items {+ canvasIDs = append(canvasIDs, item.ID)+ }+ releaseTimes, err := s.versionDAO.GetLatestReleaseTimes(ctx, dao.DB, canvasIDs)+ if err != nil {+ return+ }+ for _, item := range items {+ if t, ok := releaseTimes[item.ID]; ok {+ item.ReleaseTime = &t+ }+ }+}++// marshalAgentItem renders an agent item to its JSON representation.+func marshalAgentItem(item *AgentItem) json.RawMessage {+ b, err := json.Marshal(item)+ if err != nil {+ return json.RawMessage("null")+ }+ return b+}++// marshalGroupTemplate renders a compilation template child to the read-side+// shape the frontend group card expects (mirrors Python _to_saved_dict).+func marshalGroupTemplate(c *entity.CompilationTemplate) json.RawMessage {+ item := map[string]interface{}{+ "id": c.ID,+ "name": c.Name,+ "description": derefString(c.Description),+ "kind": c.Kind,+ "config": c.Config,+ "create_time": intValuePtr(c.CreateTime),+ "update_time": intValuePtr(c.UpdateTime),+ }+ b, err := json.Marshal(item)+ if err != nil {+ return json.RawMessage("null")+ }+ return b+}++// splitCategoryList splits a comma-separated canvas_category query into the+// non-empty categories, mirroring Python+// request.args.get("canvas_category", "").strip().split(",").+func splitCategoryList(s string) []string {+ var out []string+ for _, part := range strings.Split(s, ",") {+ if p := strings.TrimSpace(part); p != "" {+ out = append(out, p)+ }+ }+ return out+}++// filterCategory returns the categories in src that are not equal to drop.+func filterCategory(src []string, drop string) []string {+ var out []string+ for _, c := range src {+ if c != drop {+ out = append(out, c)+ }+ }+ return out+}++// sliceContains reports whether v is present in s.+func sliceContains[T comparable](s []T, v T) bool {+ for _, e := range s {+ if e == v {+ return true+ }+ }+ return false+}++// slicePage returns a shallow copy of s bounded to the requested page window.+// A page <= 0 or pageSize <= 0 returns s unchanged (caller did not ask to page).+// The page bound is checked arithmetically before computing the offset so an+// unbounded positive page/page_size can never overflow to a negative start and+// panic (review Critical).+func slicePage[T any](s []T, page, pageSize int) []T {+ if page <= 0 || pageSize <= 0 || len(s) == 0 {+ return s+ }+ // page-1 must be <= (len(s)-1)/pageSize, i.e. start must be < len(s).+ if page-1 > (len(s)-1)/pageSize {+ return nil+ }+ start := (page - 1) * pageSize+ if pageSize >= len(s)-start {+ return s[start:]+ }+ return s[start : start+pageSize]+}++// intValuePtr dereferences a *int64 to a plain int64 (0 when nil), used for the+// group/template create_time & update_time epoch fields in merged items.+func intValuePtr(p *int64) int64 {+ if p == nil {+ return 0+ }+ return *p}// CreateAgentRequest is the input shape for CreateAgent.diff --git a/internal/service/agent_test.go b/internal/service/agent_test.goindex aea52e6cc5..e5f2a53f05 100644--- a/internal/service/agent_test.go+++ b/internal/service/agent_test.go@@ -834,6 +834,10 @@ func setupAgentSessionServiceTest(t *testing.T) {&entity.UserCanvasVersion{},&entity.UserTenant{},&entity.API4Conversation{},+ // The merged /agents list reads compilation template groups (and their+ // children) alongside canvases, so those tables must exist too.+ &entity.CompilationTemplateGroup{},+ &entity.CompilationTemplate{},); err != nil {t.Fatalf("failed to migrate: %v", err)}@@ -1886,16 +1890,20 @@ func TestListAgentsIncludesReleaseTime(t *testing.T) {}var released, draft *AgentItem- for _, item := range resp.Canvas {+ for _, raw := range resp.Canvas {+ var item AgentItem+ if err := json.Unmarshal(raw, &item); err != nil {+ t.Fatalf("unmarshal canvas item: %v", err)+ }switch item.ID {case "canvas-listed-released":- released = item+ released = &itemcase "canvas-listed-draft":- draft = item+ draft = &item}}if released == nil || draft == nil {- t.Fatalf("expected both canvases in list, got %#v", resp.Canvas)+ t.Fatalf("expected both canvases in list, got %s", resp.Canvas)}if released.ReleaseTime == nil || *released.ReleaseTime != releaseTime {t.Fatalf("ReleaseTime = %v, want %d", released.ReleaseTime, releaseTime)@@ -1905,6 +1913,151 @@ func TestListAgentsIncludesReleaseTime(t *testing.T) {}}+// TestListAgents_MergesCompilationTemplateGroups verifies that a compilation+// template group owned by the caller appears in the merged /agents list+// (no canvas_category filter), carrying the "compilation_template_group" type+// discriminator and its title = name. Built-in catalogue groups (empty tenant)+// must NOT leak in.+func TestListAgents_MergesCompilationTemplateGroups(t *testing.T) {+ setupAgentSessionServiceTest(t)++ base := time.Now().UnixMilli()+ groupUpdate := base + 1000 // group updated most recently+ canvasUpdate := base + 5000 // agent updated most recently+ if err := dao.DB.Create(&entity.User{ID: "user-1", Nickname: "owner", Email: "[email protected]"}).Error; err != nil {+ t.Fatalf("failed to seed user: %v", err)+ }+ // Agent updated most recently (canvasUpdate > groupUpdate).+ if err := dao.DB.Create(&entity.UserCanvas{+ ID: "canvas-1", UserID: "user-1", Title: sptr("Agent canvas"),+ CanvasCategory: "agent_canvas",+ BaseModel: entity.BaseModel{UpdateTime: &canvasUpdate},+ }).Error; err != nil {+ t.Fatalf("failed to seed canvas: %v", err)+ }+ // The caller's own group (must appear), updated before the canvas.+ createAgentSessionTestCompilationGroup(t, "group-own", "user-1", groupUpdate)+ // A built-in catalogue group with empty tenant_id (must NOT appear).+ if err := dao.DB.Create(&entity.CompilationTemplateGroup{+ ID: "group-builtin", TenantID: "", Name: "Built-in templates",+ Scope: "file", BaseModel: entity.BaseModel{CreateTime: &base},+ }).Error; err != nil {+ t.Fatalf("failed to seed builtin group: %v", err)+ }++ resp, code, err := NewAgentService().ListAgents(t.Context(), "user-1", "", 1, 30, "create_time", true, nil, "", "", nil)+ if err != nil || code != common.CodeSuccess {+ t.Fatalf("ListAgents failed: code=%v err=%v", code, err)+ }++ var sawAgent, sawOwnGroup, sawBuiltin bool+ var prevCat, prevName string+ orderOK := true+ for i, raw := range resp.Canvas {+ var item map[string]interface{}+ if err := json.Unmarshal(raw, &item); err != nil {+ t.Fatalf("unmarshal canvas item: %v", err)+ }+ // Sort category: agents carry canvas_category; groups are classified by+ // the "compilation_template_group" type discriminator (mirroring the+ // frontend AgentListItem union).+ cat := "agent"+ if item["type"] == AgentItemTypeGroup {+ cat = CompilationTemplateGroupCategory+ } else if cc, ok := item["canvas_category"].(string); ok {+ cat = cc+ }+ name, _ := item["title"].(string)+ if i > 0 {+ if cat < prevCat || (cat == prevCat && strings.ToLower(name) < strings.ToLower(prevName)) {+ orderOK = false+ }+ }+ prevCat, prevName = cat, name+ switch item["type"] {+ case AgentItemTypeAgent:+ if item["id"] == "canvas-1" {+ sawAgent = true+ }+ case AgentItemTypeGroup:+ if item["id"] == "group-own" {+ sawOwnGroup = true+ if item["title"] != "own-group" || item["name"] != "own-group" {+ t.Errorf("group title/name = %v/%v, want own-group", item["title"], item["name"])+ }+ if item["templates"] == nil {+ t.Errorf("group item missing templates")+ }+ }+ if item["id"] == "group-builtin" {+ sawBuiltin = true+ }+ }+ }+ if !sawAgent {+ t.Fatal("agent canvas-1 not in merged list")+ }+ if !sawOwnGroup {+ t.Fatal("caller's compilation template group not in merged list")+ }+ if sawBuiltin {+ t.Fatal("built-in catalogue group leaked into merged list")+ }+ if !orderOK {+ t.Fatal("merged list is not ordered by (canvas_category, name) ascending")+ }+ // agent_canvas < compilation_template_group, so the canvas must be first.+ var first map[string]interface{}+ if err := json.Unmarshal(resp.Canvas[0], &first); err != nil {+ t.Fatalf("unmarshal first item: %v", err)+ }+ if first["id"] != "canvas-1" {+ t.Fatalf("first merged item = %v, want canvas-1 (agent_canvas category)", first["id"])+ }+}++// TestSlicePage_NoOverflowPanic guards the review-Critical integer overflow:+// an unbounded positive page/page_size must never overflow (page-1)*pageSize to+// a negative start and panic; out-of-range pages return nil.+func TestSlicePage_NoOverflowPanic(t *testing.T) {+ s := []int{1, 2, 3, 4, 5}+ // A huge page must return nil, not panic on an overflowing offset.+ if got := slicePage(s, 1<<40, 1<<40); got != nil {+ t.Fatalf("huge page/pageSize = %v, want nil", got)+ }+ if got := slicePage(s, 2, 3); len(got) != 2 || got[0] != 4 || got[1] != 5 {+ t.Fatalf("page2/size3 = %v, want [4 5]", got)+ }+ if got := slicePage(s, 3, 2); len(got) != 1 || got[0] != 5 {+ t.Fatalf("page3/size2 (start 4) = %v, want [5]", got)+ }+ if got := slicePage(s, 4, 2); got != nil {+ t.Fatalf("page4/size2 (start 6 >= len 5) = %v, want nil", got)+ }+ // Empty/no-paging semantics preserved (empty input returns the empty slice,+ // never a panic).+ if got := slicePage([]int{}, 1, 10); len(got) != 0 {+ t.Fatalf("empty slice = %v, want empty", got)+ }+}++func createAgentSessionTestCompilationGroup(t *testing.T, id, tenantID string, now int64) {+ t.Helper()+ if err := dao.DB.Create(&entity.CompilationTemplateGroup{+ ID: id, TenantID: tenantID, Name: "own-group", Scope: "file",+ BaseModel: entity.BaseModel{CreateTime: &now, UpdateTime: &now},+ }).Error; err != nil {+ t.Fatalf("failed to seed group %s: %v", id, err)+ }+ if err := dao.DB.Create(&entity.CompilationTemplate{+ ID: id + "-tpl", GroupID: &id, Name: "tree tpl", Kind: "tree",+ Config: entity.JSONMap{},+ BaseModel: entity.BaseModel{CreateTime: &now},+ }).Error; err != nil {+ t.Fatalf("failed to seed template for %s: %v", id, err)+ }+}+func TestPublishAgentUpdatesCanvasAndReleasedVersion(t *testing.T) {setupAgentSessionServiceTest(t)diff --git a/internal/service/compilation_template_service.go b/internal/service/compilation_template_service.goindex 8b55809c09..ecd02c66f3 100644--- a/internal/service/compilation_template_service.go+++ b/internal/service/compilation_template_service.go@@ -210,8 +210,17 @@ func ValidateTemplatePayload(req map[string]interface{}, requireAll bool) error}config, hasConfig := req["config"]if hasConfig {- configMap, ok := config.(map[string]interface{})- if !ok {+ // config may arrive as map[string]interface{} (raw payloads) or as+ // entity.JSONMap (payloads built from a typed GroupTemplate, whose Config+ // field is JSONMap). A bare type assertion on the former would reject the+ // latter because JSONMap is a distinct named type, so normalize both.+ var configMap map[string]interface{}+ switch c := config.(type) {+ case map[string]interface{}:+ configMap = c+ case entity.JSONMap:+ configMap = map[string]interface{}(c)+ default:return errors.New("invalid template config")}if len(fmt.Sprint(configMap["global_rules"])) > 4096 {diff --git a/internal/service/compilation_template_service_test.go b/internal/service/compilation_template_service_test.gonew file mode 100644index 0000000000..fb9392d810--- /dev/null+++ b/internal/service/compilation_template_service_test.go@@ -0,0 +1,69 @@+package service++import (+ "testing"++ "ragflow/internal/entity"+)++// TestValidateTemplatePayload_AcceptsJSONMapConfig covers the create-from-UI+// regression: GroupTemplate.Config is an entity.JSONMap (a named map type), and+// ValidateTemplatePayload used a bare config.(map[string]interface{}) assertion+// that silently rejected it, surfacing as "102 invalid template config". Both+// the raw map and the JSONMap forms must validate identically.+func TestValidateTemplatePayload_AcceptsJSONMapConfig(t *testing.T) {+ cfg := map[string]interface{}{+ "kind": "graph",+ "entity": map[string]interface{}{+ "fields": []interface{}{+ map[string]interface{}{+ "type": "organization",+ "description": "company",+ },+ },+ },+ "relation": map[string]interface{}{+ "fields": []interface{}{+ map[string]interface{}{+ "type": "acquisition",+ "description": "took over",+ },+ },+ },+ }+ base := map[string]interface{}{+ "name": "tpl",+ "kind": "graph",+ }++ // 1) Raw map[string]interface{} payload (older path) must pass.+ raw := map[string]interface{}{}+ for k, v := range base {+ raw[k] = v+ }+ raw["config"] = cfg+ if err := ValidateTemplatePayload(raw, true); err != nil {+ t.Fatalf("raw map config rejected: %v", err)+ }++ // 2) entity.JSONMap payload (what the group create path builds from+ // GroupTemplate.Config) must also pass — this is the regression.+ jm := map[string]interface{}{}+ for k, v := range base {+ jm[k] = v+ }+ jm["config"] = entity.JSONMap(cfg)+ if err := ValidateTemplatePayload(jm, true); err != nil {+ t.Fatalf("entity.JSONMap config rejected: %v", err)+ }++ // 3) A non-map config is still an error.+ bad := map[string]interface{}{}+ for k, v := range base {+ bad[k] = v+ }+ bad["config"] = "not-a-map"+ if err := ValidateTemplatePayload(bad, true); err == nil {+ t.Fatal("non-map config should be rejected")+ }+}diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.goindex c43f925af5..0ef4ca3744 100644--- a/internal/service/dataset_artifact_service.go+++ b/internal/service/dataset_artifact_service.go@@ -49,7 +49,6 @@ const (FieldStructureIndexType = "structure_index_type"FieldStructureKind = "structure_kind"FieldPageID = "page_id"- FieldGraphType = "graph_type")// DatasetArtifactService reads knowledge-compilation artifacts (wiki pages,@@ -620,38 +619,6 @@ func (s *DatasetArtifactService) DeleteStructures(ctx context.Context, tenantID,return len(ids), nil}-// DocGraphItem is a single node/edge entry in a document's structure graph.-type DocGraphItem struct {- ID string `json:"id"`- Content string `json:"content"`- SourceID string `json:"source_id"`-}--// GetDocumentGraph returns the structure graph of a single document.-func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, tenantID, datasetID, documentID, graphType string) ([]DocGraphItem, int64, error) {- filter := map[string]interface{}{- "doc_id": []string{documentID},- "compiled_graph_kwd": []string{"graph"},- }- if graphType != "" {- filter[FieldGraphType] = []string{graphType}- }- chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter,- []string{"id", "content_with_weight", "source_id"}, 0, 10000, nil)- if err != nil {- return nil, 0, err- }- items := make([]DocGraphItem, 0, len(chunks))- for _, c := range chunks {- items = append(items, DocGraphItem{- ID: firstStringValue(c["id"]),- Content: firstStringValue(c["content_with_weight"]),- SourceID: firstStringValue(c["source_id"]),- })- }- return items, total, nil-}-// DeleteDocumentGraph deletes the structure graph of a single document.func (s *DatasetArtifactService) DeleteDocumentGraph(ctx context.Context, tenantID, datasetID, documentID string) (int, error) {docEngine := engine.Get()@@ -682,28 +649,12 @@ func (s *DatasetArtifactService) DeleteDocumentGraph(ctx context.Context, tenantreturn len(ids), nil}-// NavigationItem is a single navigation cluster (REST response shape, kept-// stable for frontend compatibility).-type NavigationItem struct {- Name string `json:"name"`- Title string `json:"title"`- Count int `json:"count"`-}--// NavChildItem is a single child entry under a navigation cluster (REST-// response shape, kept stable for frontend compatibility).-type NavChildItem struct {- Name string `json:"name"`- Title string `json:"title"`- Count int `json:"count"`-}--// ListNavClusters returns the navigation clusters of a dataset. It is DEPRECATED-// and now delegates to the ES-backed NavService (internal/service datasetnav):-// the previous implementation queried nav_cluster_kwd/count_int fields that-// Python never writes, so it could never read the real nav tree. Do not add-// field-level patches here — route everything through NavService.-func (s *DatasetArtifactService) ListNavClusters(ctx context.Context, tenantID, datasetID string) ([]NavigationItem, int64, error) {+// ListNavClusters returns the navigation clusters of a dataset. It delegates to+// the ES-backed NavService and returns the frontend DatasetNavNode shape+// (snake_case NavNode JSON), matching Python GET /navigation exactly. The old+// NavigationItem{name,title,count} shape did not match the frontend interface+// and has been removed.+func (s *DatasetArtifactService) ListNavClusters(ctx context.Context, tenantID, datasetID string) ([]nav.NavNode, int64, error) {ns := nav.GetNavService()if ns == nil {return nil, 0, fmt.Errorf("datasetnav: NavService not initialized (SetNavService must be called at bootstrap)")@@ -712,16 +663,12 @@ func (s *DatasetArtifactService) ListNavClusters(ctx context.Context, tenantID,if err != nil {return nil, 0, err}- items := make([]NavigationItem, 0, len(nodes))- for _, n := range nodes {- items = append(items, NavigationItem{Name: n.Name, Title: n.Description, Count: n.DocCount})- }- return items, total, nil+ return nodes, total, nil}-// ListNavChildren returns the children of a navigation cluster. DEPRECATED —-// delegates to NavService.ListChildren.-func (s *DatasetArtifactService) ListNavChildren(ctx context.Context, tenantID, datasetID, name string) ([]NavChildItem, int64, error) {+// ListNavChildren returns the children of a navigation cluster in the frontend+// DatasetNavNode shape.+func (s *DatasetArtifactService) ListNavChildren(ctx context.Context, tenantID, datasetID, name string) ([]nav.NavNode, int64, error) {ns := nav.GetNavService()if ns == nil {return nil, 0, fmt.Errorf("datasetnav: NavService not initialized (SetNavService must be called at bootstrap)")@@ -730,11 +677,7 @@ func (s *DatasetArtifactService) ListNavChildren(ctx context.Context, tenantID,if err != nil {return nil, 0, err}- items := make([]NavChildItem, 0, len(nodes))- for _, n := range nodes {- items = append(items, NavChildItem{Name: n.Name, Title: n.Description, Count: n.DocCount})- }- return items, total, nil+ return nodes, total, nil}// DeleteNav removes the direct nav_doc children of every root cluster of adiff --git a/internal/service/dataset_structure_graph.go b/internal/service/dataset_structure_graph.gonew file mode 100644index 0000000000..77ed8f448a--- /dev/null+++ b/internal/service/dataset_structure_graph.go@@ -0,0 +1,1190 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+//++package service++import (+ "context"+ "encoding/json"+ "fmt"+ "regexp"+ "sort"+ "strings"++ "ragflow/internal/dao"+ "ragflow/internal/engine"+ "ragflow/internal/engine/types"+)++// Structure-graph sampling constants (mirror structure_graph_common.py).+const (+ graphFullThreshold = 1024 // below this combined count, return all rows+ graphTopEntities = 256 // seed set A size for large buckets+ graphKeywordCandidates = 16 // keyword/KNN candidate rows+ graphExpansionCap = 4096 // hub-node expansion cap+)++var graphEntityFields = []string{"id", "content_with_weight", "name_kwd", "mention_count_int", "source_chunk_ids", "doc_id", "doc_ids_kwd", "source_doc_ids"}+var graphRelationFields = []string{"id", "content_with_weight", "from_entity_kwd", "to_entity_kwd", "doc_id", "doc_ids_kwd", "source_doc_ids"}+var graphAllFields = []string{+ "id", "content_with_weight", "name_kwd", "mention_count_int", "source_chunk_ids",+ "from_entity_kwd", "to_entity_kwd", "knowledge_graph_kwd", "doc_id", "doc_ids_kwd", "source_doc_ids",+}++// StructureGraphNode is a projected entity in the structure graph response.+type StructureGraphNode map[string]interface{}++// StructureGraphRelation is a projected relation in the structure graph response.+type StructureGraphRelation map[string]interface{}++// DocumentStructureGraphTemplate is one per-template bucket in the response.+type DocumentStructureGraphTemplate struct {+ TemplateID string `json:"template_id"`+ TemplateName string `json:"template_name"`+ Kind string `json:"kind"`+ Entities []StructureGraphNode `json:"entities"`+ Relations []StructureGraphRelation `json:"relations"`+}++// DocumentStructureGraphResponse mirrors Python's {"templates": [...]}.+type DocumentStructureGraphResponse struct {+ Templates []DocumentStructureGraphTemplate `json:"templates"`+}++// graphRowSearch runs one raw-row search over the tenant's document index.+func graphRowSearch(ctx context.Context, tenantID, datasetID string, selectFields []string, filter map[string]interface{}, orderBy *types.OrderByExpr, offset, limit int, matchExprs []interface{}) (map[string]map[string]interface{}, int64, error) {+ docEngine := engine.Get()+ if docEngine == nil {+ return nil, 0, fmt.Errorf("document engine is not initialized")+ }+ merged := make(map[string]interface{}, len(filter)+1)+ for k, v := range filter {+ merged[k] = v+ }+ merged["kb_id"] = []string{datasetID}+ if limit < 1 {+ limit = 1+ }+ res, err := docEngine.Search(ctx, &types.SearchRequest{+ IndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},+ KbIDs: []string{datasetID},+ Offset: offset,+ Limit: limit,+ SelectFields: selectFields,+ Filter: merged,+ OrderBy: orderBy,+ MatchExprs: matchExprs,+ })+ if err != nil {+ return nil, 0, err+ }+ if res == nil {+ return nil, 0, nil+ }+ byID := make(map[string]map[string]interface{}, len(res.Chunks))+ for _, c := range res.Chunks {+ id := firstStringValue(c["id"])+ if id != "" {+ byID[id] = c+ }+ }+ return byID, res.Total, nil+}++// graphLoadPayload parses content_with_weight into a dict.+func graphLoadPayload(row map[string]interface{}) map[string]interface{} {+ raw := firstStringValue(row["content_with_weight"])+ if raw == "" {+ return nil+ }+ var m map[string]interface{}+ if err := json.Unmarshal([]byte(raw), &m); err != nil {+ return nil+ }+ return m+}++func graphIsInvalidSentinel(v string) bool {+ if v == "" {+ return true+ }+ lv := strings.ToLower(v)+ for _, bad := range []string{"unknown", "none", "null", "nan", "n/a", "undefined", "other"} {+ if lv == bad {+ return true+ }+ }+ return false+}++// projectEntity mirrors _struct_graph_entity + project_entity.+func projectEntity(row map[string]interface{}) StructureGraphNode {+ payload := graphLoadPayload(row)+ if payload == nil {+ return nil+ }+ name := ""+ for _, k := range []string{"name", "text", "term", "title"} {+ if v, ok := payload[k].(string); ok && strings.TrimSpace(v) != "" {+ name = strings.TrimSpace(v)+ break+ }+ }+ if name == "" || graphIsInvalidSentinel(name) {+ return nil+ }+ typ := "other"+ if v, ok := payload["type"].(string); ok && strings.TrimSpace(v) != "" {+ typ = strings.TrimSpace(v)+ }+ var aliases []string+ switch a := payload["aliases"].(type) {+ case string:+ if strings.TrimSpace(a) != "" {+ aliases = []string{strings.TrimSpace(a)}+ }+ case []interface{}:+ for _, e := range a {+ if s, ok := e.(string); ok && strings.TrimSpace(s) != "" {+ aliases = append(aliases, strings.TrimSpace(s))+ }+ }+ case []string:+ for _, s := range a {+ if strings.TrimSpace(s) != "" {+ aliases = append(aliases, strings.TrimSpace(s))+ }+ }+ }+ desc := ""+ for _, k := range []string{"description", "definition_excerpt"} {+ if v, ok := payload[k].(string); ok {+ desc = strings.TrimSpace(v)+ if desc != "" {+ break+ }+ }+ }+ chunkIDs := graphSourceChunkIDs(payload, row)+ node := StructureGraphNode{+ "aliases": aliases,+ "mention_count": 1,+ "name": name,+ "source_chunk_ids": chunkIDs,+ "type": typ,+ "description": desc,+ }+ if mc, ok := graphMentionCount(row); ok {+ node["mention_count"] = mc+ }+ return node+}++func graphSourceChunkIDs(payload, row map[string]interface{}) []string {+ var out []string+ add := func(v interface{}) {+ switch x := v.(type) {+ case []string:+ for _, s := range x {+ if s != "" {+ out = append(out, s)+ }+ }+ case []interface{}:+ for _, e := range x {+ if s, ok := e.(string); ok && s != "" {+ out = append(out, s)+ }+ }+ case string:+ if x != "" {+ out = append(out, x)+ }+ }+ }+ if raw, ok := payload["source_chunk_ids"]; ok {+ add(raw)+ } else {+ add(row["source_chunk_ids"])+ }+ // dedup order-preserving+ seen := map[string]bool{}+ dedup := out[:0]+ for _, s := range out {+ if !seen[s] {+ seen[s] = true+ dedup = append(dedup, s)+ }+ }+ return dedup+}++func graphMentionCount(row map[string]interface{}) (int, bool) {+ v := row["mention_count_int"]+ if l, ok := v.([]interface{}); ok {+ if len(l) > 0 {+ v = l[0]+ } else {+ return 0, false+ }+ }+ switch n := v.(type) {+ case int:+ return n, true+ case int64:+ return int(n), true+ case float64:+ return int(n), true+ case json.Number:+ if i, err := n.Int64(); err == nil {+ return int(i), true+ }+ }+ return 0, false+}++// projectRelation mirrors _struct_graph_relation + project_relation.+func projectRelation(row map[string]interface{}) StructureGraphRelation {+ payload := graphLoadPayload(row)+ src, tgt := "", ""+ if payload != nil {+ for _, k := range []string{"source", "src", "from"} {+ if v, ok := payload[k].(string); ok {+ src = strings.TrimSpace(v)+ if src != "" {+ break+ }+ }+ }+ for _, k := range []string{"target", "tgt", "to"} {+ if v, ok := payload[k].(string); ok {+ tgt = strings.TrimSpace(v)+ if tgt != "" {+ break+ }+ }+ }+ }+ typ := "related"+ if payload != nil {+ if v, ok := payload["type"].(string); ok && strings.TrimSpace(v) != "" {+ typ = strings.TrimSpace(v)+ }+ }+ if src == "" || tgt == "" || graphIsInvalidSentinel(src) || graphIsInvalidSentinel(tgt) {+ // Fall back to the authoritative *_entity_kwd columns.+ fallbackSrc := strings.TrimSpace(firstStringValue(row["from_entity_kwd"]))+ fallbackTgt := strings.TrimSpace(firstStringValue(row["to_entity_kwd"]))+ if fallbackSrc != "" && fallbackTgt != "" {+ return StructureGraphRelation{"from": fallbackSrc, "to": fallbackTgt, "type": typ}+ }+ return nil+ }+ return StructureGraphRelation{"from": src, "to": tgt, "type": typ}+}++// dedupEntities order-preserving by (lowercased name, type).+func dedupEntities(entities []StructureGraphNode) []StructureGraphNode {+ var out []StructureGraphNode+ seen := map[string]bool{}+ for _, e := range entities {+ name := strings.ToLower(strings.TrimSpace(graphStr(e["name"])))+ typ := strings.ToLower(strings.TrimSpace(graphStr(e["type"])))+ key := name + "\x00" + typ+ if name == "" || seen[key] {+ continue+ }+ seen[key] = true+ out = append(out, e)+ }+ return out+}++func entityResponseID(entity StructureGraphNode) string {+ for _, f := range []string{"id", "name", "slug"} {+ if v, ok := entity[f].(string); ok && strings.TrimSpace(v) != "" {+ return strings.TrimSpace(v)+ }+ }+ return ""+}++func endpointTerms(value string) []string {+ value = strings.TrimSpace(value)+ if value == "" {+ return nil+ }+ return sortedUnique([]string{value, strings.ToLower(value)})+}++func sortedUnique(in []string) []string {+ seen := map[string]bool{}+ var out []string+ for _, s := range in {+ if s != "" && !seen[s] {+ seen[s] = true+ out = append(out, s)+ }+ }+ sort.Strings(out)+ return out+}++// normalizeRelationEndpoints aligns relation endpoints to returned entity ids.+func normalizeRelationEndpoints(entities []StructureGraphNode, relations []StructureGraphRelation) []StructureGraphRelation {+ if len(entities) == 0 || len(relations) == 0 {+ return relations+ }+ lookup := map[string]string{}+ ambiguous := map[string]bool{}+ for _, entity := range entities {+ respID := entityResponseID(entity)+ if respID == "" {+ continue+ }+ for _, f := range []string{"id", "name", "slug"} {+ v, ok := entity[f].(string)+ if !ok || strings.TrimSpace(v) == "" {+ continue+ }+ key := strings.ToLower(strings.TrimSpace(v))+ if cur, exists := lookup[key]; exists && cur != respID {+ ambiguous[key] = true+ continue+ }+ lookup[key] = respID+ }+ }+ for k := range ambiguous {+ delete(lookup, k)+ }+ normalized := make([]StructureGraphRelation, 0, len(relations))+ for _, rel := range relations {+ item := make(StructureGraphRelation, len(rel)+2)+ for k, v := range rel {+ item[k] = v+ }+ for _, f := range []string{"from", "to"} {+ if v, ok := item[f].(string); ok {+ if mapped, found := lookup[strings.ToLower(strings.TrimSpace(v))]; found {+ item[f] = mapped+ }+ }+ }+ normalized = append(normalized, item)+ }+ return normalized+}++// rowHasEnabledSource mirrors _row_has_enabled_source.+func rowHasEnabledSource(row map[string]interface{}, excludedDocIDs map[string]bool) bool {+ if len(excludedDocIDs) == 0 {+ return true+ }+ sourceIDs := map[string]bool{}+ flatten := func(v interface{}) {+ var walk func(interface{})+ walk = func(x interface{}) {+ switch val := x.(type) {+ case string:+ t := strings.TrimSpace(val)+ if t == "" {+ return+ }+ var parsed []interface{}+ if json.Unmarshal([]byte(t), &parsed) == nil {+ for _, p := range parsed {+ walk(p)+ }+ return+ }+ sourceIDs[t] = true+ case []interface{}:+ for _, e := range val {+ walk(e)+ }+ case []string:+ for _, s := range val {+ walk(s)+ }+ default:+ if s, ok := x.(string); ok && s != "" {+ sourceIDs[s] = true+ }+ }+ }+ walk(v)+ }+ for _, f := range []string{"doc_ids_kwd", "source_doc_ids"} {+ if v, ok := row[f]; ok && v != nil {+ flatten(v)+ }+ }+ if len(sourceIDs) > 0 {+ for id := range sourceIDs {+ if !excludedDocIDs[id] {+ return true+ }+ }+ return false+ }+ flatten(row["doc_id"])+ if len(sourceIDs) > 0 {+ for id := range sourceIDs {+ if !excludedDocIDs[id] {+ return true+ }+ }+ return false+ }+ return true+}++func graphStr(v interface{}) string {+ if v == nil {+ return ""+ }+ if s, ok := v.(string); ok {+ return s+ }+ return fmt.Sprintf("%v", v)+}++// buildBucket mirrors sgc.build_bucket: small buckets whole, large sampled.+func (s *DatasetArtifactService) buildBucket(ctx context.Context, tenantID, datasetID string, scope map[string]interface{}, excludedDocIDs map[string]bool) ([]StructureGraphNode, []StructureGraphRelation, error) {+ excludedDocIDs = excludedDocIDsOrEmpty(excludedDocIDs)+ bothCond := copyFilter(scope)+ bothCond["knowledge_graph_kwd"] = []string{"entity", "relation"}+ _, total, err := graphRowSearch(ctx, tenantID, datasetID, []string{"id"}, bothCond, nil, 0, 1, nil)+ if err != nil {+ return nil, nil, err+ }+ if total < graphFullThreshold {+ fieldMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphAllFields, bothCond, nil, 0, int(total), nil)+ if err != nil {+ return nil, nil, err+ }+ var entities []StructureGraphNode+ var relations []StructureGraphRelation+ for _, row := range fieldMap {+ if !rowHasEnabledSource(row, excludedDocIDs) {+ continue+ }+ kg := firstStringValue(row["knowledge_graph_kwd"])+ if kg == "relation" {+ if edge := projectRelation(row); edge != nil {+ relations = append(relations, edge)+ }+ } else {+ if node := projectEntity(row); node != nil {+ entities = append(entities, node)+ }+ }+ }+ entities = dedupEntities(entities)+ return entities, normalizeRelationEndpoints(entities, relations), nil+ }++ // Large bucket: sample. A = top entities by mention_count_int desc.+ orderBy := (&types.OrderByExpr{}).Desc("mention_count_int")+ var setA []StructureGraphNode+ entityOffset := 0+ var entityTotal int64 = -1+ for len(setA) < graphTopEntities && (entityTotal == -1 || int64(entityOffset) < entityTotal) {+ cond := copyFilter(scope)+ cond["knowledge_graph_kwd"] = []string{"entity"}+ entAMap, entTotal, err := graphRowSearch(ctx, tenantID, datasetID, graphEntityFields, cond, orderBy, entityOffset, graphTopEntities, nil)+ if err != nil {+ return nil, nil, err+ }+ entityTotal = entTotal+ if len(entAMap) == 0 {+ break+ }+ for _, row := range entAMap {+ if !rowHasEnabledSource(row, excludedDocIDs) {+ continue+ }+ if n := projectEntity(row); n != nil {+ setA = append(setA, n)+ }+ }+ entityOffset += len(entAMap)+ }+ if len(setA) > graphTopEntities {+ setA = setA[:graphTopEntities]+ }+ var aNames []string+ for _, e := range setA {+ if n := strings.TrimSpace(graphStr(e["name"])); n != "" {+ aNames = append(aNames, n)+ }+ }+ var aNameTerms []string+ for _, name := range aNames {+ aNameTerms = append(aNameTerms, endpointTerms(name)...)+ }+ aNameTerms = sortedUnique(aNameTerms)++ var relations []StructureGraphRelation+ targetNamesLower := map[string]bool{}+ if len(aNameTerms) > 0 {+ cond := copyFilter(scope)+ cond["knowledge_graph_kwd"] = []string{"relation"}+ cond["from_entity_kwd"] = aNameTerms+ relMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphRelationFields, cond, nil, 0, graphExpansionCap, nil)+ if err != nil {+ return nil, nil, err+ }+ for _, row := range relMap {+ if !rowHasEnabledSource(row, excludedDocIDs) {+ continue+ }+ if edge := projectRelation(row); edge != nil {+ relations = append(relations, edge)+ if tgt := strings.ToLower(strings.TrimSpace(graphStr(edge["to"]))); tgt != "" {+ targetNamesLower[tgt] = true+ }+ }+ }+ }+ var setT []StructureGraphNode+ if len(targetNamesLower) > 0 {+ cond := copyFilter(scope)+ cond["knowledge_graph_kwd"] = []string{"entity"}+ cond["name_kwd"] = sortedKeys(targetNamesLower)+ tgtMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphEntityFields, cond, nil, 0, graphExpansionCap, nil)+ if err != nil {+ return nil, nil, err+ }+ for _, row := range tgtMap {+ if !rowHasEnabledSource(row, excludedDocIDs) {+ continue+ }+ if n := projectEntity(row); n != nil {+ setT = append(setT, n)+ }+ }+ }+ entities := dedupEntities(append(setA, setT...))+ return entities, normalizeRelationEndpoints(entities, relations), nil+}++func copyFilter(in map[string]interface{}) map[string]interface{} {+ out := make(map[string]interface{}, len(in))+ for k, v := range in {+ out[k] = v+ }+ return out+}++func excludedDocIDsOrEmpty(in map[string]bool) map[string]bool {+ if in == nil {+ return map[string]bool{}+ }+ return in+}++func sortedKeys(m map[string]bool) []string {+ out := make([]string, 0, len(m))+ for k := range m {+ out = append(out, k)+ }+ sort.Strings(out)+ return out+}++// compilationTemplateKind normalizes a template kind (mirror Python+// _compilation_template_kind).+func compilationTemplateKind(kind string) string {+ k := strings.ToLower(strings.TrimSpace(kind))+ switch k {+ case "page_index", "pageindex":+ return "page_index"+ case "wiki", "wiki_page":+ return "wiki"+ case "raptor":+ return "raptor"+ }+ return k+}++// structureKindForBucket returns the normalized kind used for tree/page_index+// hierarchy handling.+func structureKindForBucket(kind string) string {+ k := compilationTemplateKind(kind)+ if k == "page_index" {+ return "page_index"+ }+ return strings.ReplaceAll(k, "-", "_")+}++// loadDocumentTemplateMeta resolves the doc's configured template group into+// configured_ids + template_meta maps, mirroring the Python resolution.+func (s *DatasetArtifactService) loadDocumentTemplateMeta(ctx context.Context, tenantID, documentID string) ([]string, map[string]map[string]interface{}, error) {+ empty := map[string]map[string]interface{}{}+ doc, err := dao.NewDocumentDAO().GetByID(ctx, dao.DB, documentID)+ if err != nil || doc == nil {+ return nil, empty, err+ }+ groupID := ""+ if doc.ParserConfig != nil {+ if gid, ok := doc.ParserConfig["compilation_template_group_id"].(string); ok && gid != "" {+ groupID = gid+ } else if ext, ok := doc.ParserConfig["ext"].(map[string]interface{}); ok {+ if gid, ok := ext["compilation_template_group_id"].(string); ok && gid != "" {+ groupID = gid+ }+ }+ }+ if groupID == "" {+ return nil, empty, nil+ }+ group, err := NewCompilationTemplateGroupService().GetSaved(ctx, tenantID, groupID)+ if err != nil {+ return nil, empty, err+ }+ configuredIDs := []string{}+ meta := map[string]map[string]interface{}{}+ seen := map[string]bool{}+ if group != nil {+ for _, t := range group.Templates {+ tid := t.ID+ if tid == "" || seen[tid] {+ continue+ }+ rawKind := ""+ if t.Config != nil {+ if k, ok := t.Config["kind"].(string); ok {+ rawKind = k+ }+ }+ if rawKind == "" {+ rawKind = t.Kind+ }+ kindNorm := compilationTemplateKind(rawKind)+ if kindNorm == "wiki" {+ continue+ }+ seen[tid] = true+ configuredIDs = append(configuredIDs, tid)+ meta[tid] = map[string]interface{}{+ "template_id": tid,+ "template_name": t.Name,+ "kind": rawKind,+ }+ }+ }+ return configuredIDs, meta, nil+}++// DocumentStructureGraphInput is the parsed request for GetDocumentGraph.+type DocumentStructureGraphInput struct {+ TenantID string+ DatasetID string+ DocumentID string+ GraphType string+ Keywords string+}++// GetDocumentGraph returns the per-template structure graph of a document,+// mirroring Python get_document_structure_graph (normal + keywords modes).+func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, in DocumentStructureGraphInput) (*DocumentStructureGraphResponse, error) {+ configuredIDs, templateMeta, err := s.loadDocumentTemplateMeta(ctx, in.TenantID, in.DocumentID)+ if err != nil {+ return nil, err+ }++ resp := &DocumentStructureGraphResponse{Templates: []DocumentStructureGraphTemplate{}}++ // keywords mode: name matching/KNN → matched entities' subgraph.+ if in.Keywords != "" {+ bucketMeta, entities, relations, err := s.keywordSubgraph(ctx, in.TenantID, in.DatasetID, in.DocumentID, in.Keywords, templateMeta)+ if err != nil {+ return nil, err+ }+ if bucketMeta == nil || (len(entities) == 0 && len(relations) == 0) {+ return resp, nil+ }+ resp.Templates = append(resp.Templates, DocumentStructureGraphTemplate{+ TemplateID: graphStr(bucketMeta["template_id"]),+ TemplateName: graphStr(bucketMeta["template_name"]),+ Kind: graphStr(bucketMeta["kind"]),+ Entities: entities,+ Relations: relations,+ })+ return resp, nil+ }++ // normal mode: discover buckets from per-doc graph blob rows.+ metaFields := []string{"compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd"}+ metaRows, _, err := graphRowSearch(ctx, in.TenantID, in.DatasetID, metaFields,+ map[string]interface{}{"doc_id": []string{in.DocumentID}, "knowledge_graph_kwd": []string{"graph"}}, nil, 0, 1000, nil)+ if err != nil {+ return nil, err+ }++ bucketMetas := map[string]map[string]interface{}{}+ bucketScopes := map[string]map[string]interface{}{}+ for _, row := range metaRows {+ meta, scope := resolveGraphBucket(row, templateMeta, in.DocumentID)+ bid := graphStr(meta["template_id"])+ if _, ok := bucketMetas[bid]; !ok {+ bucketMetas[bid] = meta+ bucketScopes[bid] = scope+ }+ }++ grouped := map[string]DocumentStructureGraphTemplate{}+ for bid, meta := range bucketMetas {+ entities, relations, err := s.buildBucket(ctx, in.TenantID, in.DatasetID, bucketScopes[bid], nil)+ if err != nil {+ return nil, err+ }+ if len(entities) == 0 && len(relations) == 0 {+ continue+ }+ grouped[bid] = DocumentStructureGraphTemplate{+ TemplateID: graphStr(meta["template_id"]),+ TemplateName: graphStr(meta["template_name"]),+ Kind: graphStr(meta["kind"]),+ Entities: entities,+ Relations: relations,+ }+ }++ // RAPTOR summary graph blob (compile_kwd = raptor_graph).+ s.appendRaptorBlob(ctx, in.TenantID, in.DatasetID, in.DocumentID, grouped)++ // Order: configured templates first, then discovered.+ orderedIDs := []string{}+ for _, tid := range configuredIDs {+ if _, ok := grouped[tid]; ok && !containsStr(orderedIDs, tid) {+ orderedIDs = append(orderedIDs, tid)+ }+ }+ for bid := range grouped {+ if !containsStr(orderedIDs, bid) {+ orderedIDs = append(orderedIDs, bid)+ }+ }+ for _, bid := range orderedIDs {+ if g, ok := grouped[bid]; ok && (len(g.Entities) > 0 || len(g.Relations) > 0) {+ resp.Templates = append(resp.Templates, g)+ }+ }+ return resp, nil+}++func containsStr(list []string, v string) bool {+ for _, e := range list {+ if e == v {+ return true+ }+ }+ return false+}++// resolveGraphBucket mirrors Python _resolve_bucket.+func resolveGraphBucket(row map[string]interface{}, templateMeta map[string]map[string]interface{}, documentID string) (map[string]interface{}, map[string]interface{}) {+ compileKwd := firstStringValue(row["compile_kwd"])+ kindVal := firstStringValue(row["compilation_template_kind_kwd"])+ if kindVal == "" {+ kindVal = compileKwd+ }+ tid := rowTemplateID(row)+ if tid != "" {+ kindNorm := compilationTemplateKind(kindVal)+ meta := templateMeta[tid]+ // Only a unique kind match can substitute a missing template meta; an+ // empty kindNorm must never match (review Major — the previous nil+ // round-trip stored a nil entry that the kind-only loop then matched).+ if meta == nil && kindNorm != "" {+ kindMatches := []map[string]interface{}{}+ for _, m := range templateMeta {+ if compilationTemplateKind(graphStr(m["kind"])) == kindNorm {+ kindMatches = append(kindMatches, m)+ }+ }+ if len(kindMatches) == 1 {+ meta = kindMatches[0]+ }+ }+ bucketName := graphStr(meta["template_name"])+ if bucketName == "" {+ bucketName = tid+ }+ bucketKind := graphStr(meta["kind"])+ if bucketKind == "" {+ bucketKind = kindVal+ }+ return map[string]interface{}{+ "template_id": tid,+ "template_name": bucketName,+ "kind": bucketKind,+ }, map[string]interface{}{+ "doc_id": []string{documentID},+ "compilation_template_ids": []string{tid},+ }+ }+ bucketID := "legacy:" + compileKwd+ return map[string]interface{}{+ "template_id": bucketID,+ "template_name": "Legacy (" + compileKwd + ")",+ "kind": kindVal,+ }, map[string]interface{}{+ "doc_id": []string{documentID},+ "compile_kwd": []string{compileKwd},+ "must_not": map[string]interface{}{"exists": "compilation_template_ids"},+ }+}++func rowTemplateID(row map[string]interface{}) string {+ switch v := row["compilation_template_ids"].(type) {+ case []interface{}:+ for _, e := range v {+ if s, ok := e.(string); ok && strings.TrimSpace(s) != "" {+ return strings.TrimSpace(s)+ }+ }+ case []string:+ for _, s := range v {+ if strings.TrimSpace(s) != "" {+ return strings.TrimSpace(s)+ }+ }+ case string:+ if strings.TrimSpace(v) != "" {+ return strings.TrimSpace(v)+ }+ }+ return ""+}++func (s *DatasetArtifactService) appendRaptorBlob(ctx context.Context, tenantID, datasetID, documentID string, grouped map[string]DocumentStructureGraphTemplate) {+ rows, _, err := graphRowSearch(ctx, tenantID, datasetID, []string{"content_with_weight", "compile_kwd"},+ map[string]interface{}{"doc_id": []string{documentID}, "compile_kwd": []string{"raptor_graph"}}, nil, 0, 16, nil)+ if err != nil {+ return+ }+ for _, row := range rows {+ payload := graphLoadPayload(row)+ if payload == nil {+ continue+ }+ rEntities, _ := payload["entities"].([]interface{})+ rRelations, _ := payload["relations"].([]interface{})+ if len(rEntities) == 0 && len(rRelations) == 0 {+ continue+ }+ rb, ok := grouped["raptor"]+ if !ok {+ rb = DocumentStructureGraphTemplate{TemplateID: "raptor", TemplateName: "RAPTOR Summary", Kind: "raptor"}+ grouped["raptor"] = rb+ }+ rb.Entities = append(rb.Entities, toNodeSlice(rEntities)...)+ rb.Relations = append(rb.Relations, toRelationSlice(rRelations)...)+ grouped["raptor"] = rb+ }+}++func toNodeSlice(in []interface{}) []StructureGraphNode {+ var out []StructureGraphNode+ for _, e := range in {+ if m, ok := e.(map[string]interface{}); ok {+ out = append(out, StructureGraphNode(m))+ }+ }+ return out+}++func toRelationSlice(in []interface{}) []StructureGraphRelation {+ var out []StructureGraphRelation+ for _, e := range in {+ if m, ok := e.(map[string]interface{}); ok {+ out = append(out, StructureGraphRelation(m))+ }+ }+ return out+}++var graphTextQueryRE = regexp.MustCompile(`[ :|\r\n\t,,。??/\x60!!&^%()\[\]{}<>*~'"\\=]+`)++func nameMatchesQuery(node StructureGraphNode, query string) bool {+ name := strings.ToLower(strings.TrimSpace(graphStr(node["name"])))+ q := strings.ToLower(query)+ if name == "" || q == "" {+ return false+ }+ if strings.Contains(name, q) {+ return true+ }+ terms := strings.Fields(q)+ if len(terms) == 0 {+ return false+ }+ for _, t := range terms {+ if !strings.Contains(name, t) {+ return false+ }+ }+ return true+}++// keywordSubgraph mirrors sgc.keyword_subgraph (BM25 + KNN fallback + ancestor walk).+func (s *DatasetArtifactService) keywordSubgraph(ctx context.Context, tenantID, datasetID, documentID, keywords string, templateMeta map[string]map[string]interface{}) (map[string]interface{}, []StructureGraphNode, []StructureGraphRelation, error) {+ baseEntityCond := map[string]interface{}{+ "doc_id": []string{documentID},+ "knowledge_graph_kwd": []string{"entity"},+ }+ topFields := append(append([]string{}, graphEntityFields...), "compilation_template_ids", "compile_kwd", "compilation_template_kind_kwd")++ textQuery := strings.TrimSpace(graphTextQueryRE.ReplaceAllString(keywords, " "))+ var candidates []struct {+ row map[string]interface{}+ node StructureGraphNode+ }+ validTop := func(rows map[string]map[string]interface{}) []struct {+ row map[string]interface{}+ node StructureGraphNode+ } {+ var out []struct {+ row map[string]interface{}+ node StructureGraphNode+ }+ for _, row := range rows {+ if !rowHasEnabledSource(row, map[string]bool{}) {+ continue+ }+ if node := projectEntity(row); node != nil && strings.TrimSpace(graphStr(node["name"])) != "" {+ out = append(out, struct {+ row map[string]interface{}+ node StructureGraphNode+ }{row, node})+ }+ }+ return out+ }+ if textQuery != "" {+ textExpr := &types.MatchTextExpr{+ Fields: []string{"content_ltks^10", "content_sm_ltks"},+ MatchingText: textQuery,+ TopN: graphKeywordCandidates,+ ExtraOptions: map[string]interface{}{"original_query": keywords},+ }+ topMap, _, err := graphRowSearch(ctx, tenantID, datasetID, topFields, baseEntityCond, nil, 0, graphKeywordCandidates, []interface{}{textExpr})+ if err != nil {+ return nil, nil, nil, err+ }+ for _, c := range validTop(topMap) {+ if nameMatchesQuery(c.node, textQuery) {+ candidates = append(candidates, c)+ }+ }+ }+ // Prefer detail entities over title-typed ancestors.+ detailCandidates := candidates[:0]+ for _, c := range candidates {+ if strings.ToLower(strings.TrimSpace(graphStr(c.node["type"]))) != "title" {+ detailCandidates = append(detailCandidates, c)+ }+ }+ if len(detailCandidates) > 0 {+ candidates = detailCandidates+ }++ // Semantic fallback via KNN.+ if len(candidates) == 0 {+ vec, err := embedQuery(ctx, tenantID, datasetID, keywords)+ if err != nil || len(vec) == 0 {+ return nil, nil, nil, nil+ }+ denseExpr := &types.MatchDenseExpr{+ VectorColumnName: fmt.Sprintf("q_%d_vec", len(vec)),+ EmbeddingData: vec,+ EmbeddingDataType: "float",+ DistanceType: "cosine",+ TopN: graphKeywordCandidates,+ ExtraOptions: map[string]interface{}{"similarity": 0.3},+ }+ topMap, _, err := graphRowSearch(ctx, tenantID, datasetID, topFields, baseEntityCond, nil, 0, graphKeywordCandidates, []interface{}{denseExpr})+ if err != nil {+ return nil, nil, nil, err+ }+ candidates = validTop(topMap)+ }+ if len(candidates) == 0 {+ return nil, nil, nil, nil+ }++ scopeForTemplate := func(row map[string]interface{}) (map[string]interface{}, map[string]interface{}) {+ return resolveGraphBucket(row, templateMeta, documentID)+ }+ bucketMeta, scope := scopeForTemplate(candidates[0].row)+ bucketID := graphStr(bucketMeta["template_id"])++ matchedNodes := []StructureGraphNode{}+ for _, c := range candidates {+ cMeta, _ := scopeForTemplate(c.row)+ if graphStr(cMeta["template_id"]) == bucketID {+ matchedNodes = append(matchedNodes, c.node)+ }+ }+ if len(matchedNodes) == 0 {+ return nil, nil, nil, nil+ }+ structureKind := structureKindForBucket(graphStr(bucketMeta["kind"]))++ var relations []StructureGraphRelation+ seenRel := map[string]bool{}+ neighborNamesLower := map[string]bool{}+ matchedNames := map[string]bool{}+ for _, n := range matchedNodes {+ if name := strings.ToLower(strings.TrimSpace(graphStr(n["name"]))); name != "" {+ matchedNames[name] = true+ }+ }+ if structureKind != "tree" && structureKind != "page_index" {+ for _, matchedNode := range matchedNodes {+ matchedName := strings.TrimSpace(graphStr(matchedNode["name"]))+ terms := endpointTerms(matchedName)+ for _, field := range []string{"from_entity_kwd", "to_entity_kwd"} {+ cond := copyFilter(scope)+ cond["knowledge_graph_kwd"] = []string{"relation"}+ cond[field] = terms+ relMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphRelationFields, cond, nil, 0, graphExpansionCap, nil)+ if err != nil {+ return nil, nil, nil, err+ }+ for _, row := range relMap {+ if !rowHasEnabledSource(row, map[string]bool{}) {+ continue+ }+ if edge := projectRelation(row); edge != nil {+ key := graphStr(edge["from"]) + "\x00" + graphStr(edge["to"]) + "\x00" + graphStr(edge["type"])+ if seenRel[key] {+ continue+ }+ seenRel[key] = true+ relations = append(relations, edge)+ for _, endpoint := range []string{graphStr(edge["from"]), graphStr(edge["to"])} {+ if ep := strings.TrimSpace(endpoint); ep != "" && !matchedNames[strings.ToLower(ep)] {+ neighborNamesLower[strings.ToLower(ep)] = true+ }+ }+ }+ if len(relations) >= graphExpansionCap {+ break+ }+ }+ if len(relations) >= graphExpansionCap {+ break+ }+ }+ if len(relations) >= graphExpansionCap {+ break+ }+ }+ }++ // Ancestor walk for tree/page_index.+ if (structureKind == "tree" || structureKind == "page_index") && len(relations) < graphExpansionCap {+ ancestorFrontier := map[string]bool{}+ seenAncestors := map[string]bool{}+ for n := range matchedNames {+ ancestorFrontier[n] = true+ seenAncestors[n] = true+ }+ for len(ancestorFrontier) > 0 && len(relations) < graphExpansionCap {+ nextFrontier := map[string]bool{}+ cond := copyFilter(scope)+ cond["knowledge_graph_kwd"] = []string{"relation"}+ cond["to_entity_kwd"] = sortedKeys(ancestorFrontier)+ relMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphRelationFields, cond, nil, 0, graphExpansionCap-len(relations), nil)+ if err != nil {+ return nil, nil, nil, err+ }+ for _, row := range relMap {+ if !rowHasEnabledSource(row, map[string]bool{}) {+ continue+ }+ if edge := projectRelation(row); edge != nil {+ key := graphStr(edge["from"]) + "\x00" + graphStr(edge["to"]) + "\x00" + graphStr(edge["type"])+ if seenRel[key] {+ continue+ }+ seenRel[key] = true+ relations = append(relations, edge)+ parent := strings.ToLower(strings.TrimSpace(graphStr(edge["from"])))+ if parent != "" && !seenAncestors[parent] {+ seenAncestors[parent] = true+ nextFrontier[parent] = true+ }+ if len(relations) >= graphExpansionCap {+ break+ }+ }+ }+ ancestorFrontier = nextFrontier+ for n := range nextFrontier {+ neighborNamesLower[n] = true+ }+ }+ }++ entities := append([]StructureGraphNode{}, matchedNodes...)+ if len(neighborNamesLower) > 0 {+ cond := copyFilter(scope)+ cond["knowledge_graph_kwd"] = []string{"entity"}+ cond["name_kwd"] = sortedKeys(neighborNamesLower)+ nbMap, _, err := graphRowSearch(ctx, tenantID, datasetID, graphEntityFields, cond, nil, 0, graphExpansionCap, nil)+ if err != nil {+ return nil, nil, nil, err+ }+ for _, row := range nbMap {+ if !rowHasEnabledSource(row, map[string]bool{}) {+ continue+ }+ if n := projectEntity(row); n != nil {+ entities = append(entities, n)+ }+ }+ }+ entities = dedupEntities(entities)+ if structureKind == "tree" || structureKind == "page_index" {+ entityNames := map[string]bool{}+ for _, e := range entities {+ if n := strings.ToLower(strings.TrimSpace(graphStr(e["name"]))); n != "" {+ entityNames[n] = true+ }+ }+ var filtered []StructureGraphRelation+ for _, r := range relations {+ if entityNames[strings.ToLower(strings.TrimSpace(graphStr(r["from"])))] && entityNames[strings.ToLower(strings.TrimSpace(graphStr(r["to"])))] {+ filtered = append(filtered, r)+ }+ }+ relations = filtered+ }+ return bucketMeta, entities, normalizeRelationEndpoints(entities, relations), nil+}++// embedQuery embeds a query via the nav embedder (best-effort). The embedder+// resolves the tenant's embedding model on demand; an empty result (or error) is+// treated as "no semantic candidates", matching Python's fallback behavior.+func embedQuery(ctx context.Context, tenantID, datasetID, query string) ([]float64, error) {+ embedder := NewNavEmbedder(NewModelProviderService(), "")+ vecs, err := embedder.Encode(ctx, tenantID, []string{query})+ if err != nil || len(vecs) == 0 || len(vecs[0]) == 0 {+ return nil, fmt.Errorf("embed query failed")+ }+ f := make([]float64, len(vecs[0]))+ for i, v := range vecs[0] {+ f[i] = float64(v)+ }+ return f, nil+}diff --git a/internal/service/dataset_structure_graph_test.go b/internal/service/dataset_structure_graph_test.gonew file mode 100644index 0000000000..74a767b9e2--- /dev/null+++ b/internal/service/dataset_structure_graph_test.go@@ -0,0 +1,107 @@+package service++import (+ "reflect"+ "testing"+)++// TestProjectEntity_FromPayload verifies projectEntity maps the tree-node+// payload shape to the graph-node shape (mirroring _struct_graph_entity).+func TestProjectEntity_FromPayload(t *testing.T) {+ row := map[string]interface{}{+ "content_with_weight": `{"name":"NVIDIA","type":"tree_node","description":"chip maker","source_chunk_ids":["c1","c2"]}`,+ "source_chunk_ids": []string{"c1"},+ "mention_count_int": 3,+ }+ n := projectEntity(row)+ if n == nil {+ t.Fatal("projectEntity returned nil")+ }+ if n["name"] != "NVIDIA" || n["type"] != "tree_node" {+ t.Errorf("name/type = %v/%v, want NVIDIA/tree_node", n["name"], n["type"])+ }+ if n["mention_count"] != 3 {+ t.Errorf("mention_count = %v, want 3", n["mention_count"])+ }+ chunks, _ := n["source_chunk_ids"].([]string)+ if !reflect.DeepEqual(chunks, []string{"c1", "c2"}) {+ t.Errorf("source_chunk_ids = %v, want [c1 c2]", chunks)+ }+}++// TestProjectEntity_RejectsInvalidName verifies sentinel/empty names are dropped.+func TestProjectEntity_RejectsInvalidName(t *testing.T) {+ for _, payload := range []string{`{"name":""}`, `{"name":"unknown"}`} {+ if n := projectEntity(map[string]interface{}{"content_with_weight": payload}); n != nil {+ t.Errorf("expected nil for payload %s, got %v", payload, n)+ }+ }+}++// TestProjectRelation_FromPayload verifies the relation projection.+func TestProjectRelation_FromPayload(t *testing.T) {+ row := map[string]interface{}{"content_with_weight": `{"from":"NVIDIA","to":"GPU","type":"child"}`}+ r := projectRelation(row)+ if r == nil {+ t.Fatal("projectRelation returned nil")+ }+ if r["from"] != "NVIDIA" || r["to"] != "GPU" || r["type"] != "child" {+ t.Errorf("relation = %v, want {NVIDIA GPU child}", r)+ }+}++// TestProjectRelation_FallsBackToKwdColumns verifies the authoritative *_entity_kwd+// columns are used when the payload has no from/to.+func TestProjectRelation_FallsBackToKwdColumns(t *testing.T) {+ row := map[string]interface{}{+ "content_with_weight": `{"type":"related"}`,+ "from_entity_kwd": "NVIDIA",+ "to_entity_kwd": "GPU",+ }+ r := projectRelation(row)+ if r == nil || r["from"] != "NVIDIA" || r["to"] != "GPU" || r["type"] != "related" {+ t.Errorf("relation = %v, want {NVIDIA GPU related}", r)+ }+}++// TestDedupEntities_OrderPreserving verifies dedup by (lowercased name, type).+func TestDedupEntities_OrderPreserving(t *testing.T) {+ in := []StructureGraphNode{+ {"name": "A", "type": "x"},+ {"name": "a", "type": "x"}, // dup (case-insensitive)+ {"name": "B", "type": "y"},+ {"name": ""}, // dropped (empty name)+ }+ out := dedupEntities(in)+ if len(out) != 2 || out[0]["name"] != "A" || out[1]["name"] != "B" {+ t.Errorf("dedupEntities = %v, want [A B]", out)+ }+}++// TestNormalizeRelationEndpoints aligns relation endpoints to entity ids/names.+func TestNormalizeRelationEndpoints(t *testing.T) {+ entities := []StructureGraphNode{{"name": "NVIDIA", "type": "x"}}+ relations := []StructureGraphRelation{{"from": "nvidia", "to": "GPU", "type": "child"}}+ out := normalizeRelationEndpoints(entities, relations)+ if out[0]["from"] != "NVIDIA" {+ t.Errorf("normalized from = %v, want NVIDIA (matched to entity name)", out[0]["from"])+ }+}++// TestCompilationTemplateKind_Normalization covers kind normalization.+func TestCompilationTemplateKind_Normalization(t *testing.T) {+ if compilationTemplateKind("Page_Index") != "page_index" {+ t.Errorf("Page_Index => %q, want page_index", compilationTemplateKind("Page_Index"))+ }+ if compilationTemplateKind("tree") != "tree" {+ t.Errorf("tree => %q, want tree", compilationTemplateKind("tree"))+ }+}++// TestRowTemplateID_FromList covers compilation_template_ids extraction.+func TestRowTemplateID_FromList(t *testing.T) {+ row := map[string]interface{}{"compilation_template_ids": []interface{}{"", "tid1", "tid2"}}+ if got := rowTemplateID(row); got != "tid1" {+ t.Errorf("rowTemplateID = %q, want tid1", got)+ }+}diff --git a/internal/service/nav/nav.go b/internal/service/nav/nav.goindex a87fcfd8c9..243ea0ea0a 100644--- a/internal/service/nav/nav.go+++ b/internal/service/nav/nav.go@@ -27,14 +27,16 @@ import ("sync")-// NavNode mirrors Python's _nav_item (dataset_api_service.py).+// NavNode mirrors Python's _nav_item (dataset_api_service.py). The JSON field+// names are snake_case to match the frontend DatasetNavNode contract and the+// Python GET /navigation payload exactly.type NavNode struct {- Name string // name- Description string // content_with_weight payload description- DocCount int // doc_count_int (cluster) or 1 (leaf)- Type string // "cluster" | "doc"- DocID string // leaf doc_id; empty for cluster- HasChildren bool // is_cluster+ Name string `json:"name"`+ Description string `json:"description"`+ DocCount int `json:"doc_count"`+ Type string `json:"type"`+ DocID string `json:"doc_id,omitempty"`+ HasChildren bool `json:"has_children"`}// NavHit is one KNN hit on a nav row.diff --git a/internal/service/nlp/datasetnav.go b/internal/service/nlp/datasetnav.goindex cf057f8795..4959b1b4d2 100644--- a/internal/service/nlp/datasetnav.go+++ b/internal/service/nlp/datasetnav.go@@ -177,8 +177,23 @@ func (s *NavService) ListChildren(ctx context.Context, tenantID, kbID, name stri// nodeFromRow converts an engine row into a NavNode.func (s *NavService) nodeFromRow(row map[string]interface{}, fallbackType string) nav.NavNode {+ name := firstStringValue(row["title_kwd"])+ // Prefer an explicit readable "name" column (Python rows carry one); fall+ // back to title_kwd.+ if n := firstStringValue(row["name"]); n != "" {+ name = n+ }+ // A raw id (doc_id or "cluster_<hash>") is not a human-readable name; fall+ // back to a title derived from the payload description. Cluster names are the+ // child-lookup key (parent_kwd references them verbatim), so they must stay+ // intact; only non-cluster (leaf) rows get the readable fallback, otherwise+ // GET /navigation/{cluster}/children would no longer match (review Major).+ isCluster := firstStringValue(row["type_kwd"]) == "nav_cluster"+ if !isCluster && graphIsRawID(name) {+ name = ""+ }node := nav.NavNode{- Name: firstStringValue(row["title_kwd"]),+ Name: name,DocCount: intValue(row["doc_count_int"]),Type: fallbackType,DocID: firstStringValue(row["doc_id"]),@@ -196,6 +211,13 @@ func (s *NavService) nodeFromRow(row map[string]interface{}, fallbackType stringif d, ok := m["description"].(string); ok {node.Description = d}+ if node.Name == "" {+ if t, ok := m["title"].(string); ok && strings.TrimSpace(t) != "" {+ node.Name = cleanTitle(t)+ } else if d, ok := m["description"].(string); ok && strings.TrimSpace(d) != "" {+ node.Name = cleanTitle(fallbackTitle(d))+ }+ }}}node.HasChildren = node.Type == "cluster"@@ -205,6 +227,31 @@ func (s *NavService) nodeFromRow(row map[string]interface{}, fallbackType stringreturn node}+// graphIsRawID reports whether a name is a meaningless internal id (a doc id or+// a "cluster_<hex>" key) rather than a human-readable title.+func graphIsRawID(name string) bool {+ name = strings.TrimSpace(name)+ if name == "" {+ return true+ }+ if strings.HasPrefix(name, "cluster_") && len(name) == len("cluster_")+8 {+ return true+ }+ if len(name) == 32 && isHexString(name) {+ return true+ }+ return false+}++func isHexString(s string) bool {+ for _, r := range s {+ if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')) {+ return false+ }+ }+ return true+}+// Search runs query KNN over nav rows and returns routed doc ids.func (s *NavService) Search(ctx context.Context, tenantID, kbID, query string, embd []float32, topK int) ([]nav.NavHit, error) {if topK <= 0 {@@ -343,8 +390,11 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error"compile_kwd": navCompileKwd,"available_int": 0,"type_kwd": "nav_doc",- "title_kwd": in.DocID,- "parent_kwd": parent,+ // The nav_doc's display name is a readable title derived from the summary+ // (Python _clean_title/_fallback_title) — NOT the raw doc id, which is+ // meaningless in the UI. The doc_id is still stored for lookups.+ "title_kwd": cleanTitle(fallbackTitle(in.Summary)),+ "parent_kwd": parent,// The nav_doc sits one level below its (possibly nested) parent// cluster, so its depth is parentDepth+1 — not a hard-coded 1."depth_int": bestDepth + 1,@@ -376,6 +426,7 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error}_, err = de.InsertChunks(ctx, []map[string]interface{}{{"id": navClusterID(in.TenantID, in.KbID, name),+ "doc_id": in.KbID, // cluster rows carry the kb as doc_id (Python _build_nav_cluster_row), so ES InsertChunks does not skip them"compile_kwd": navCompileKwd,"available_int": 0,"type_kwd": "nav_cluster",@@ -387,6 +438,27 @@ func (s *NavService) UpsertDoc(ctx context.Context, in nav.UpsertDocInput) error"content_with_weight": payloadJSONNav(map[string]interface{}{"type": "nav_cluster", "description": summary}),"q_" + fmt.Sprintf("%d", len(vec)) + "_vec": f32ToF64Slice(vec),}}, idx, in.KbID)+ if err != nil {+ return err+ }+ // Always emit a nav_doc leaf under the (new) cluster, mirroring Python+ // upsert_dataset_nav_doc (the merge branch does the same at its parent). A+ // new root cluster that only folds the doc into doc_ids_kwd would leave the+ // nav tree with a single cluster and no nav_doc child, so /children returns+ // empty. The nav_doc carries a readable title + parent_kwd = the cluster name.+ _, err = de.InsertChunks(ctx, []map[string]interface{}{{+ "id": navDocID(in.TenantID, in.KbID, in.DocID),+ "compile_kwd": navCompileKwd,+ "available_int": 0,+ "type_kwd": "nav_doc",+ "title_kwd": cleanTitle(fallbackTitle(in.Summary)),+ "parent_kwd": name,+ "depth_int": depth + 1,+ "doc_id": in.DocID,+ "doc_count_int": 1,+ "content_with_weight": payloadJSONNav(map[string]interface{}{"type": "nav_doc", "description": in.Summary}),+ "q_" + fmt.Sprintf("%d", len(vec)) + "_vec": f32ToF64Slice(vec),+ }}, idx, in.KbID)return err}@@ -775,16 +847,54 @@ func (s *NavService) llmMergeDescription(ctx context.Context, tenantID string, treturn strings.Join(texts, "\n")}+// cleanTitle normalizes a raw title/summary into a one-line, length-capped+// display name (mirroring Python dataset_nav._clean_title).+func cleanTitle(title string) string {+ return truncateString(strings.Join(strings.Fields(title), " "), 48)+}++// fallbackTitle derives a short readable title from a summary by taking the+// first non-empty line and stripping Markdown emphasis/heading markers. The+// tree-root summaries carry a one-line Markdown title ("**Title**" or+// "## Title") on the first line, so this yields a clean, URL-friendly label —+// far more readable than taking the first whitespace words of the body.+func fallbackTitle(summary string) string {+ line := summary+ if idx := strings.IndexAny(line, "\n\r"); idx >= 0 {+ line = line[:idx]+ }+ line = strings.TrimSpace(line)+ // Strip Markdown emphasis and heading markers.+ line = strings.Trim(line, "*# \t")+ if line == "" {+ return "Cluster"+ }+ return line+}++// readableClusterName returns a readable yet unique nav-cluster key of the form+// "<title> <8-hex>" (mirroring Python dataset_nav._readable_cluster_name): the+// title keeps the node name human-readable, the short hash of the seed keeps the+// per-KB uniqueness that the tree keying relies on.+func readableClusterName(title, seed string) string {+ t := cleanTitle(title)+ if t == "" {+ t = "Cluster"+ }+ return t + " " + shortHash8(seed)+}+// llmCreateSummary builds a short cluster name + summary for a source text via// the optional LLM (mirroring Python _llm_create_summary); without an LLM it-// falls back to a deterministic name from a content hash and the text itself.+// falls back to a readable name derived from the summary's first words plus a+// short hash, so nav clusters are human-readable instead of "cluster_<hash>".func (s *NavService) llmCreateSummary(ctx context.Context, tenantID, text string) (name, summary string) {if s.llm != nil {if n, sm, err := s.llm.CreateSummary(ctx, tenantID, text); err == nil && n != "" {return n, sm}}- return "cluster_" + contentHash8(text), text+ return readableClusterName(fallbackTitle(text), text), text}// appendDocToCluster appends a doc id to a cluster's doc_ids_kwd and bumps its@@ -1015,6 +1125,20 @@ func payloadJSONNav(v map[string]interface{}) string {return string(b)}+// truncateString caps s to n runes (not bytes).+func truncateString(s string, n int) string {+ runes := []rune(s)+ if len(runes) <= n {+ return s+ }+ return string(runes[:n])+}++// shortHash8 is a stable 8-char hash suffix for readable nav names.+func shortHash8(s string) string {+ return contentHash8(s)+}+// contentHash8 is a stable 8-char hash.func contentHash8(s string) string {h := uint32(2166136261)diff --git a/internal/service/nlp/datasetnav_test.go b/internal/service/nlp/datasetnav_test.goindex fbf4620704..19523e9733 100644--- a/internal/service/nlp/datasetnav_test.go+++ b/internal/service/nlp/datasetnav_test.go@@ -2,8 +2,10 @@ package nlpimport ("context"+ "encoding/json""fmt""math"+ "strings""testing""ragflow/internal/engine/types"@@ -303,6 +305,101 @@ func TestNavService_ListClusters_FiltersRoot(t *testing.T) {}}+// TestNavNode_JSONShape_SnakeCase locks the REST contract: the frontend+// DatasetNavNode and Python GET /navigation both use snake_case keys. If the+// Go field names leaked (Name/Description/DocCount...) the frontend tree would+// read undefined fields and render empty.+func TestNavNode_JSONShape_SnakeCase(t *testing.T) {+ n := nav.NavNode{Name: "cluster_x", Description: "d", DocCount: 3, Type: "cluster", HasChildren: true}+ b, err := json.Marshal(n)+ if err != nil {+ t.Fatal(err)+ }+ var m map[string]interface{}+ if err := json.Unmarshal(b, &m); err != nil {+ t.Fatal(err)+ }+ for _, want := range []string{"name", "description", "doc_count", "type", "has_children"} {+ if _, ok := m[want]; !ok {+ t.Errorf("NavNode JSON missing snake_case key %q; got %s", want, b)+ }+ }+ for _, bad := range []string{"Name", "Description", "DocCount", "HasChildren"} {+ if _, ok := m[bad]; ok {+ t.Errorf("NavNode JSON leaked PascalCase key %q; got %s", bad, b)+ }+ }+}++// TestNavNamingHelpers_Readable verifies the nav display-name helpers produce+// human-readable names (mirroring Python _clean_title/_fallback_title/+// _readable_cluster_name) instead of raw ids.+func TestNavNamingHelpers_Readable(t *testing.T) {+ if cleanTitle(" hello world ") != "hello world" {+ t.Errorf("cleanTitle = %q, want %q", cleanTitle(" hello world "), "hello world")+ }+ // fallbackTitle takes the first non-empty line and strips Markdown markers.+ if got := fallbackTitle("a b c d e f g h"); got != "a b c d e f g h" {+ t.Errorf("fallbackTitle = %q, want the first line", got)+ }+ if got := fallbackTitle("**何进诛阉与董后之废**\n\nbody text"); got != "何进诛阉与董后之废" {+ t.Errorf("fallbackTitle = %q, want the stripped markdown title", got)+ }+ if got := fallbackTitle(""); got != "Cluster" {+ t.Errorf("fallbackTitle('') = %q, want Cluster", got)+ }+ name := readableClusterName("何进诛阉", "seed-text")+ if !strings.HasPrefix(name, "何进诛阉 ") {+ t.Errorf("readableClusterName = %q, want prefix %q", name, "何进诛阉 ")+ }+}++// TestNavNamingHelpers_RawIDDetection verifies meaningless ids are detected so+// nodeFromRow can fall back to a readable title.+func TestNavNamingHelpers_RawIDDetection(t *testing.T) {+ for _, raw := range []string{"d3778ef9c0f5495fa4bdadc00a5bf15c", "cluster_abc12345", ""} {+ if !graphIsRawID(raw) {+ t.Errorf("graphIsRawID(%q) = false, want true", raw)+ }+ }+ for _, ok := range []string{"何进诛阉", "NVIDIA financial performance 8738e200"} {+ if graphIsRawID(ok) {+ t.Errorf("graphIsRawID(%q) = true, want false", ok)+ }+ }+}++// TestNodeFromRow_ReadableName verifies nodeFromRow falls back to a readable+// title derived from the payload description when title_kwd is a raw id.+func TestNodeFromRow_ReadableName(t *testing.T) {+ ns := &NavService{}+ row := map[string]interface{}{+ "title_kwd": "d3778ef9c0f5495fa4bdadc00a5bf15c",+ "doc_id": "d3778ef9c0f5495fa4bdadc00a5bf15c",+ "type_kwd": "nav_doc",+ "doc_count_int": 1,+ "content_with_weight": `{"type":"nav_doc","description":"刘备三战黄巾军与何进诛阉之议\nsecond line"}`,+ }+ node := ns.nodeFromRow(row, "doc")+ if node.Name == "" || node.Name == "d3778ef9c0f5495fa4bdadc00a5bf15c" {+ t.Fatalf("nodeFromRow name = %q, want readable fallback", node.Name)+ }+ if node.DocID != "d3778ef9c0f5495fa4bdadc00a5bf15c" {+ t.Errorf("nodeFromRow DocID = %q, want the raw doc id preserved", node.DocID)+ }+ // A cluster row keeps its stored key verbatim (it is the child-lookup+ // parent_kwd), even when it looks like a raw id (review Major).+ clusterRow := map[string]interface{}{+ "title_kwd": "cluster_abc12345",+ "type_kwd": "nav_cluster",+ "doc_count_int": 2,+ }+ cluster := ns.nodeFromRow(clusterRow, "cluster")+ if cluster.Name != "cluster_abc12345" {+ t.Errorf("cluster key = %q, want it kept verbatim as the parent_kwd lookup key", cluster.Name)+ }+}+// TestNavService_Search_ReturnsHit asserts acceptance #5.func TestNavService_Search_ReturnsHit(t *testing.T) {eng := newMemNavEngine()@@ -353,15 +450,21 @@ func TestNavService_Acceptance4_ListChildren(t *testing.T) {if err != nil {t.Fatal(err)}- // Exactly one nav_doc (for d2) sits under the cluster; d1 is the cluster.- if total != 1 || len(children) != 1 {- t.Fatalf("expected 1 child under cluster, got total=%d len=%d", total, len(children))- }- if children[0].DocID != "d2" {- t.Errorf("child doc_id = %q, want d2", children[0].DocID)+ // Every doc upserted under the cluster emits a nav_doc leaf (Python+ // upsert_dataset_nav_doc), so d1 (which created the cluster) and d2 (which+ // merged in) both sit under it: two nav_docs total.+ if total != 2 || len(children) != 2 {+ t.Fatalf("expected 2 children under cluster, got total=%d len=%d", total, len(children))+ }+ gotIDs := map[string]bool{}+ for _, c := range children {+ gotIDs[c.DocID] = true+ if c.Type != "doc" {+ t.Errorf("child type = %q, want doc", c.Type)+ }}- if children[0].Type != "doc" {- t.Errorf("child type = %q, want doc", children[0].Type)+ if !gotIDs["d1"] || !gotIDs["d2"] {+ t.Errorf("expected nav_docs for both d1 and d2, got %v", gotIDs)}}