RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Port dataset-level structure merge for timeline/graph/mindmap (#18201)
Zhichang Yu0784bef5b076
diff --git a/AGENTS.md b/AGENTS.mdindex 37f9f28a51..06eb1683e6 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -76,8 +76,9 @@ Rules:- `unit` (no tag) must stay free of external-service dependencies so `go test ./...` passes without MySQL/MinIO/ES/Infinity/LLM. The native CGO static libraries (`office_oxide`/`pdfium`/`pdf_oxide`) are still required at build time and are wired automatically by `build.sh --test`; that is expected, not an external service.## Working Rules-- Before editing, inspect the nearest code path that actually owns the behavior.+- When reviewing documentation or code, inspect the full affected path and report all verifiable findings in one review; do not return after only a few findings and expose further issues in later rounds.- When handling review comments, independently verify each substantive claim against the current code or tests before accepting, rejecting, or acting on it.+- Before editing, inspect the nearest code path that actually owns the behavior.- Keep changes small and local unless the task is explicitly a broader refactor.- Prefer one implementation path instead of preserving old and new versions side by side.- Preserve behavior with focused tests when the behavior is still valid; do not keep tests that protect obsolete behavior.diff --git a/internal/entity/dataset.go b/internal/entity/dataset.goindex 25392cdf9b..1c1bad0ddf 100644--- a/internal/entity/dataset.go+++ b/internal/entity/dataset.go@@ -124,7 +124,19 @@ type Knowledgebase struct {WikiTaskFinishAt *time.Time `gorm:"column:wiki_task_finish_at" json:"wiki_task_finish_at,omitempty"`SkillTaskID *string `gorm:"column:skill_task_id;size:32;index" json:"skill_task_id,omitempty"`SkillTaskFinishAt *time.Time `gorm:"column:skill_task_finish_at" json:"skill_task_finish_at,omitempty"`- Status *string `gorm:"column:status;size:1;index;default:'1'" json:"status,omitempty"`+ // Dataset-level structure merge task state (plan §8, mirror Python+ // _INDEX_TYPE_TO_TASK_ID_FIELD's per-structure-index entries).+ StructureGraphTaskID *string `gorm:"column:structure_graph_task_id;size:32;index" json:"structure_graph_task_id,omitempty"`+ StructureGraphTaskFinishAt *time.Time `gorm:"column:structure_graph_task_finish_at" json:"structure_graph_task_finish_at,omitempty"`+ StructureMindmapTaskID *string `gorm:"column:structure_mindmap_task_id;size:32;index" json:"structure_mindmap_task_id,omitempty"`+ StructureMindmapTaskFinishAt *time.Time `gorm:"column:structure_mindmap_task_finish_at" json:"structure_mindmap_task_finish_at,omitempty"`+ TimelineTaskID *string `gorm:"column:timeline_task_id;size:32;index" json:"timeline_task_id,omitempty"`+ TimelineTaskFinishAt *time.Time `gorm:"column:timeline_task_finish_at" json:"timeline_task_finish_at,omitempty"`+ SessionGraphTaskID *string `gorm:"column:session_graph_task_id;size:32;index" json:"session_graph_task_id,omitempty"`+ SessionGraphTaskFinishAt *time.Time `gorm:"column:session_graph_task_finish_at" json:"session_graph_task_finish_at,omitempty"`+ SessionEssenceTaskID *string `gorm:"column:session_essence_task_id;size:32;index" json:"session_essence_task_id,omitempty"`+ SessionEssenceTaskFinishAt *time.Time `gorm:"column:session_essence_task_finish_at" json:"session_essence_task_finish_at,omitempty"`+ Status *string `gorm:"column:status;size:1;index;default:'1'" json:"status,omitempty"`BaseModel}diff --git a/internal/handler/dataset_artifact.go b/internal/handler/dataset_artifact.goindex 3ac47a86f2..2d091b192f 100644--- a/internal/handler/dataset_artifact.go+++ b/internal/handler/dataset_artifact.go@@ -14,6 +14,7 @@package handlerimport (+ "errors""net/http""strconv"@@ -276,35 +277,56 @@ func (h *DatasetArtifactHandler) GetArtifactGraph(c *gin.Context) {common.SuccessWithData(c, graph, "success")}-// ListStructures handles GET /artifacts/structure — list compiled structures of a dataset.+// ListStructures handles GET /artifacts/structure?kind=<kind> — the dataset-scope+// structure graph for a resolved kind (mirrors Python get_dataset_structure).+// kind is REQUIRED: missing or invalid → 400 ARGUMENT_ERROR.func (h *DatasetArtifactHandler) ListStructures(c *gin.Context) {_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))if tenantID == "" {return}datasetID := c.Param("dataset_id")- structureKind := c.Query("structure_kind")- structureIndexType := c.Query("structure_index_type")- items, total, err := h.svc.ListStructures(c.Request.Context(), tenantID, datasetID, structureKind, structureIndexType)+ kind := c.Query("kind")+ if kind == "" {+ common.ErrorWithCode(c, common.CodeArgumentError, "kind is required")+ return+ }+ in := service.DatasetStructureGraphInput{TenantID: tenantID, DatasetID: datasetID, Kind: kind}+ resp, err := h.svc.GetDatasetStructure(c.Request.Context(), in)if err != nil {- common.ErrorWithCode(c, common.CodeDataError, err.Error())+ if errors.Is(err, service.ErrInvalidStructureKind) {+ common.ErrorWithCode(c, common.CodeArgumentError, err.Error())+ } else {+ common.ErrorWithCode(c, common.CodeServerError, err.Error())+ }return}- common.SuccessWithData(c, gin.H{"total": total, "structures": items}, "success")+ common.SuccessWithData(c, resp, "success")}-// DeleteStructures handles DELETE /artifacts/structure — delete compiled structures of a dataset.+// DeleteStructures handles DELETE /artifacts/structure?kind=<kind>&wipe=<bool> —+// cancel the kind's task (wipe=false) or delete its dataset rows (wipe=true).+// kind is REQUIRED.func (h *DatasetArtifactHandler) DeleteStructures(c *gin.Context) {_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))if tenantID == "" {return}datasetID := c.Param("dataset_id")- structureKind := c.Query("structure_kind")- structureIndexType := c.Query("structure_index_type")- n, err := h.svc.DeleteStructures(c.Request.Context(), tenantID, datasetID, structureKind, structureIndexType)+ kind := c.Query("kind")+ if kind == "" {+ common.ErrorWithCode(c, common.CodeArgumentError, "kind is required")+ return+ }+ wipe := c.Query("wipe") == "true"+ in := service.DatasetStructureGraphInput{TenantID: tenantID, DatasetID: datasetID, Kind: kind, Wipe: wipe}+ n, err := h.svc.DeleteDatasetStructure(c.Request.Context(), in)if err != nil {- common.ErrorWithCode(c, common.CodeDataError, err.Error())+ if errors.Is(err, service.ErrInvalidStructureKind) {+ common.ErrorWithCode(c, common.CodeArgumentError, err.Error())+ } else {+ common.ErrorWithCode(c, common.CodeServerError, err.Error())+ }return}common.SuccessWithData(c, gin.H{"deleted": n}, "success")diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.goindex 8b29cbebb1..29696603cd 100644--- a/internal/ingestion/component/knowledge_compiler/component.go+++ b/internal/ingestion/component/knowledge_compiler/component.go@@ -157,11 +157,31 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, indeps.TenantID = tenantIDdeps.DatasetID = datasetID+ // Per-spec Inputs copy: each spec must get its own VariantSpecific map,+ // otherwise specIn.VariantSpecific below would mutate the shared map and+ // let a later template inherit the previous template's parser_config+ // (a template with an empty config would then run with the wrong parser+ // behavior). Copy the map (and preserve an empty map when nil).specIn := in- if specIn.VariantSpecific == nil {- specIn.VariantSpecific = map[string]any{}+ specIn.VariantSpecific = make(map[string]any, len(in.VariantSpecific)+1)+ for k, v := range in.VariantSpecific {+ specIn.VariantSpecific[k] = v+ }+ // The template config (flat: kind/entity/relation/plan/…) is delivered to+ // the structure and wiki variants under the "parser_config" key — the SAME+ // key those variants read (structure.Run / wikiPipeline.mapBatch do+ // VariantSpecific["parser_config"]). Storing it as "config" left the+ // variants with a nil config, so InferType saw no "kind" and fell back to+ // "list" (breaking timeline: its compile_kwd became "list" instead of+ // "timeline", so dropIsolatedTimelineEntities never ran and the timeline+ // rendered every entity isolated).+ //+ // Only overwrite when the template actually carries a config: an empty+ // template config (e.g. a resolver stub) must not clobber a parser_config+ // the caller already supplied on the inputs.+ if len(spec.Config) > 0 {+ specIn.VariantSpecific["parser_config"] = spec.Config}- specIn.VariantSpecific["config"] = spec.Configvar o common.Outputsswitch variant {@@ -564,23 +584,14 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {}case common.VariantMindmap:- // Tree nodes: depth_int records the outline level.- if v, ok := metaInt(p.Meta, "level"); ok {- if err := doc.SetExtraValue("depth_int", v); err != nil {- return err- }- }- if v := metaString(p.Meta, "name"); v != "" {- if err := doc.SetExtraValue("title_kwd", v); err != nil {- return err- }- setTitleTokens(doc, v)- }- if v := metaStringSlice(p.Meta, "children"); len(v) > 0 {- if err := doc.SetExtraValue("children_kwd", v); err != nil {- return err- }- }+ // Mindmap now emits entity/relation rows (plan §1.2) so it participates in+ // dataset-level merge exactly like graph/timeline: each node is an entity,+ // each parent→child edge is a relation. Reuse the shared structure-graph+ // column contract (knowledge_graph_kwd + from/to_entity_kwd + name_kwd ++ // entity_type_kwd + mention_count_int). The relation type lives in the+ // content_with_weight payload ({"from","to","type"}), matching Python —+ // NOT a dedicated relation_type_kwd column.+ return applyStructureGraphColumns(doc, p, kind)}return nildiff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.goindex dc055d1bcd..21420a5805 100644--- a/internal/ingestion/component/knowledge_compiler/component_test.go+++ b/internal/ingestion/component/knowledge_compiler/component_test.go@@ -388,21 +388,27 @@ func TestKnowledgeCompiler_Tree_EndToEnd(t *testing.T) {func TestKnowledgeCompiler_Mindmap_EndToEnd(t *testing.T) {installProseDeps(t)- // The proseChat reply is flat text; parseOutline still yields a root + the- // reply as a child node, so chunks are non-empty and parent-linked.+ // The proseChat reply is flat text; ShapeTree yields a bare root (no parsed+ // children). Mindmap now emits entity/relation rows (plan §1.2): every node+ // is an entity, every parent→child edge a relation. With a flat reply there+ // is at least one entity (the root) and it must carry name_kwd ++ // knowledge_graph_kwd="entity" (the structure-graph storage contract).chunks := runVariant(t, "mindmap", nil)- // Root chunk must have empty parent_kwd and kind "root".- var root map[string]any+ entityCount := 0for _, c := range chunks {- if kind, _ := c["kc_kind"].(string); kind == "root" {- root = c+ kind, _ := c["kc_kind"].(string)+ if kind == "entity" {+ entityCount+++ if _, ok := c["name_kwd"]; !ok {+ t.Fatalf("mindmap entity chunk missing name_kwd: %+v", c)+ }+ if kg, _ := c["knowledge_graph_kwd"].(string); kg != "entity" {+ t.Fatalf("mindmap entity chunk knowledge_graph_kwd = %q, want entity", kg)+ }}}- if root == nil {- t.Fatalf("mindmap: no root chunk")- }- if pid, _ := root["parent_kwd"].(string); pid != "" {- t.Fatalf("mindmap: root parent_kwd = %q, want empty", pid)+ if entityCount == 0 {+ t.Fatalf("mindmap: no entity chunks; got %d chunks", len(chunks))}}diff --git a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.goindex b643f03f53..61460d84b3 100644--- a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go+++ b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go@@ -12,7 +12,6 @@ package mindmapimport ("context"- "encoding/json""fmt""strings"@@ -132,38 +131,47 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commoreturn out, nil}-// treeToProducts flattens the shaped mind-map tree into Products. The root-// becomes a "root" product whose content is the serialized {"id","children"}-// tree (Python's MindMapResult.output shape); each inner node becomes a-// "node" product linked via parent_id.+// treeToProducts flattens the shaped mind-map tree into entity/relation Products+// so mindmap participates in dataset-level merge exactly like graph (plan §1.2,+// aligning with Python's dataset_structure_merger which merges+// knowledge_graph_kwd IN {entity,relation} rows for structure_mindmap too).//-// NOTE: the per-node "node" products (content = the node title, plus a-// meta["children"] list of immediate child titles) are a Go component-layer-// contract for streaming the tree into the upstream chunk list. They go-// beyond Python's nested dict output and exist so downstream code can index-// each node independently with parent links.+// Mapping:+// - each node (including the root) → an entity product (kind="entity",+// name = node id, type = "mindmap").+// - each parent→child edge → a relation product (kind="relation",+// from = parent id, to = child id, type = "related" — Python's default).+//+// The entity/relation discriminator is carried in Meta["kind"] so the consumer's+// mergeStructureDataset buckets entities by (name,type) and relations by+// (from,type,to), matching graph/timeline.func treeToProducts(tenantID, docID string, root *utility.Node) []common.Product {var out []common.Product- rootID := common.StableRowID(tenantID, docID, string(common.VariantMindmap), "root")+ if root == nil || root.ID == "" {+ return out+ }+ seen := map[string]bool{}+ // Entity: root node.out = append(out, common.Product{- ID: rootID,+ ID: common.StableRowID(tenantID, docID, string(common.VariantMindmap), "entity", root.ID),DocID: docID,TenantID: tenantID,Variant: common.VariantMindmap,- Content: serializeNode(root),+ Content: root.ID,Meta: map[string]any{- "kind": "root",- "level": 0,- "name": root.ID,+ "kind": "entity",+ "name": root.ID,+ "entity_type": "mindmap",+ "compile_kwd": "mindmap",},})+ seen[root.ID] = truetype pending struct {- node *utility.Node- parentID string- level int+ node *utility.Node+ parent string}- queue := []pending{{root, rootID, 0}}+ queue := []pending{{root, root.ID}}for len(queue) > 0 {p := queue[0]queue = queue[1:]@@ -171,64 +179,45 @@ func treeToProducts(tenantID, docID string, root *utility.Node) []common.Productif child.ID == "" {continue}- level := p.level + 1- id := common.StableRowID(tenantID, docID, string(common.VariantMindmap), "node", child.ID)- childTitles := make([]string, 0, len(child.Children))- for _, gc := range child.Children {- if gc.ID != "" {- childTitles = append(childTitles, gc.ID)- }- }- meta := map[string]any{- "kind": "node",- "level": level,- "name": child.ID,- }- if len(childTitles) > 0 {- meta["children"] = childTitles+ // Entity: child node (dedup by id so a DAG-shaped tree does not emit+ // the same node twice).+ if !seen[child.ID] {+ seen[child.ID] = true+ out = append(out, common.Product{+ ID: common.StableRowID(tenantID, docID, string(common.VariantMindmap), "entity", child.ID),+ DocID: docID,+ TenantID: tenantID,+ Variant: common.VariantMindmap,+ Content: child.ID,+ Meta: map[string]any{+ "kind": "entity",+ "name": child.ID,+ "entity_type": "mindmap",+ "compile_kwd": "mindmap",+ },+ })}+ // Relation: parent → child edge (type = "related", Python default).out = append(out, common.Product{- ID: id,+ ID: common.StableRowID(tenantID, docID, string(common.VariantMindmap), "relation", p.parent, child.ID),DocID: docID,TenantID: tenantID,Variant: common.VariantMindmap,- Content: child.ID,- ParentID: p.parentID,- Meta: meta,+ Content: p.parent + " related " + child.ID,+ Meta: map[string]any{+ "kind": "relation",+ "from": p.parent,+ "to": child.ID,+ "relation_type": "related",+ "compile_kwd": "mindmap",+ },})- queue = append(queue, pending{child, id, level})+ queue = append(queue, pending{child, child.ID})}}return out}-// jsonNode mirrors the {"id","children"} mind-map node shape (Python's-// MindMapResult.output) for JSON serialization.-type jsonNode struct {- ID string `json:"id"`- Children []*jsonNode `json:"children"`-}--func toJSONNode(n *utility.Node) *jsonNode {- jn := &jsonNode{ID: n.ID}- for _, c := range n.Children {- jn.Children = append(jn.Children, toJSONNode(c))- }- return jn-}--// serializeNode renders the tree in Python's MindMapResult.output shape:-// {"id": ..., "children": [...]}. Uses encoding/json so every control-// character (including \r, \b, \f, etc.) is escaped correctly; the previous-// hand-rolled serializer only escaped ", \, \n and \t.-func serializeNode(node *utility.Node) string {- b, err := json.Marshal(toJSONNode(node))- if err != nil {- return `{"id":"` + node.ID + `","children":[]}`- }- return string(b)-}-func chunkTexts(chunks []common.Chunk) []string {var out []stringfor _, c := range chunks {diff --git a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.go b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.goindex 32e276ca99..a354ee2a25 100644--- a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.go+++ b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.go@@ -1,7 +1,6 @@package mindmapimport (- "encoding/json""strings""testing"@@ -48,57 +47,50 @@ func TestTreeToProducts_ParentLinks(t *testing.T) {{ID: "B"},}}products := treeToProducts("t1", "d1", root)- if len(products) != 5 {- t.Fatalf("products = %d, want 5 (root+A+A1+A2+B)", len(products))+ // 5 entities (root, A, A1, A2, B) + 4 relations (root→A, A→A1, A→A2, root→B).+ if len(products) != 9 {+ t.Fatalf("products = %d, want 9 (5 entities + 4 relations)", len(products))}- if products[0].Meta["kind"] != "root" || products[0].ParentID != "" {- t.Errorf("root product malformed: %+v", products[0].Meta)- }- if products[1].ParentID != products[0].ID {- t.Errorf("A parent link = %q, want root id", products[1].ParentID)- }- var aID string+ entCount, relCount := 0, 0+ fromTo := map[string]bool{}for _, p := range products {- if p.Meta["name"] == "A" {- aID = p.ID- }- }- for _, p := range products {- if n, _ := p.Meta["name"].(string); n == "A1" || n == "A2" {- if p.ParentID != aID {- t.Errorf("%s parent = %q, want A id", n, p.ParentID)+ kind, _ := p.Meta["kind"].(string)+ switch kind {+ case "entity":+ entCount+++ if p.Meta["entity_type"] != "mindmap" {+ t.Errorf("entity %v type = %v, want mindmap", p.Meta["name"], p.Meta["entity_type"])+ }+ if p.Meta["compile_kwd"] != "mindmap" {+ t.Errorf("entity %v compile_kwd = %v, want mindmap", p.Meta["name"], p.Meta["compile_kwd"])+ }+ case "relation":+ relCount+++ from, _ := p.Meta["from"].(string)+ to, _ := p.Meta["to"].(string)+ fromTo[from+"->"+to] = true+ if p.Meta["relation_type"] != "related" {+ t.Errorf("relation %v->%v type = %v, want related", from, to, p.Meta["relation_type"])}+ default:+ t.Errorf("unexpected kind %q", kind)}}-}--func TestSerializeNode_Shape(t *testing.T) {- root := &utility.Node{ID: "Top \"quoted\"", Children: []*utility.Node{{ID: "child"}}}- js := serializeNode(root)- if !strings.HasPrefix(js, `{"id":"Top \"quoted\""`) {- t.Errorf("serializeNode = %q", js)+ if entCount != 5 || relCount != 4 {+ t.Errorf("entities=%d relations=%d, want 5/4", entCount, relCount)+ }+ for _, edge := range []string{"root->A", "A->A1", "A->A2", "root->B"} {+ if !fromTo[edge] {+ t.Errorf("missing relation %s", edge)+ }}}-// TestSerializeNode_EscapesControlChars locks that serializeNode uses-// encoding/json: control characters such as CR/LF/TAB/FF are escaped (not-// embedded raw), so the output is always valid JSON.-func TestSerializeNode_EscapesControlChars(t *testing.T) {- root := &utility.Node{ID: "a\rb\tc\fd", Children: []*utility.Node{{ID: "x\ny"}}}- js := serializeNode(root)- var dec struct {- ID string `json:"id"`- Children []struct {- ID string `json:"id"`- } `json:"children"`- }- if err := json.Unmarshal([]byte(js), &dec); err != nil {- t.Fatalf("serializeNode produced invalid JSON: %v (%q)", err, js)- }- if dec.ID != "a\rb\tc\fd" {- t.Errorf("root id round-trip = %q, want %q", dec.ID, "a\rb\tc\fd")+func TestTreeToProducts_EmptyAndNil(t *testing.T) {+ if got := treeToProducts("t1", "d1", nil); len(got) != 0 {+ t.Errorf("nil root produced %d products", len(got))}- if len(dec.Children) != 1 || dec.Children[0].ID != "x\ny" {- t.Errorf("child id round-trip = %+v", dec.Children)+ if got := treeToProducts("t1", "d1", &utility.Node{ID: ""}); len(got) != 0 {+ t.Errorf("empty root id produced %d products", len(got))}}diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex dd1c0b5cd8..f4acf69110 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -1026,17 +1026,40 @@ func pageIndexSummary(graphJSON string) string {// dataset_structure_merger._merge_bucket but is a self-contained from-scratch// path (B3: the legacy unified merge never aggregated structure by name/type).func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string, products []kccommon.Product) error {- // Bucket by (lower(name), type) for entities and (lower(from), lower(to))- // for relations — relations have no "name" and must not be silently dropped- // (review issue 3).- type ekey struct{ name, typ string }- type rkey struct{ from, to string }+ common.Info("knowledge_compile: mergeStructureDataset entry",+ zap.String("kb_id", kb),+ zap.Int("products", len(products)))+ // Bucket by (lower(name), type, compile kind) for entities and+ // (lower(from), lower(type), lower(to), compile kind) for relations —+ // relations have no "name" and must not be silently dropped (review issue 3),+ // and the relation type + compile kind keep distinct structure kinds and+ // distinct relation types from collapsing into one dataset row.+ type ekey struct{ name, typ, ckwd string }+ type rkey struct{ from, typ, to, ckwd string }entByKey := make(map[ekey]*StructureBucket, 16)relByKey := make(map[rkey]*StructureBucket, 16)+ // The authoritative Python (dataset_structure_merger._do_build) merges EVERY+ // structure-kind doc row unconditionally — dataset scope is driven by the+ // task type (structure_graph/timeline/structure_mindmap), NOT by a template+ // Config["dataset_merge"] flag (that was a stale runner.py concept). So+ // structure/mindmap entity/relation products always enter the dataset merge.for _, p := range products {- if p.Variant != kccommon.VariantStructure {+ if p.Variant != kccommon.VariantStructure && p.Variant != kccommon.VariantMindmap {continue}+ // Dataset rows carry the SAME compile_kwd as the doc rows (the inferred+ // compile type / autotype: "hypergraph"/"timeline"/"mindmap"/"list"/…),+ // mirroring Python dataset_structure_merger._do_build which passes the doc+ // row's compile_kwd through verbatim. The template kind (p.Kind, restored+ // from compilation_template_kind_kwd) is stored on the SEPARATE+ // compilation_template_kind_kwd field and is what read/delete paths match+ // on — NOT compile_kwd. Do not rewrite compile_kwd to the template kind:+ // that would diverge from Python (which never does) and split doc vs+ // dataset rows.+ ckwd := metaString(p.Meta, "compile_kwd")+ if ckwd == "" {+ ckwd = compileKwdForVariant(p.Variant)+ }kind := metaString(p.Meta, "kind")from := metaString(p.Meta, "from")to := metaString(p.Meta, "to")@@ -1047,10 +1070,17 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,if from == "" || to == "" {continue}- k := rkey{from: strings.ToLower(from), to: strings.ToLower(to)}+ relType := metaString(p.Meta, "relation_type")+ if relType == "" {+ relType = metaString(p.Meta, "type")+ }+ if relType == "" {+ relType = "related"+ }+ k := rkey{from: strings.ToLower(from), typ: strings.ToLower(relType), to: strings.ToLower(to), ckwd: ckwd}b := relByKey[k]if b == nil {- b = &StructureBucket{Name: from + " -> " + to, Type: "relation", FromEntity: from, ToEntity: to}+ b = &StructureBucket{Name: from + " -> " + to, Type: "relation", FromEntity: from, ToEntity: to, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind, RelationType: relType}relByKey[k] = b}appendBucket(b, p)@@ -1065,10 +1095,10 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,if name == "" {continue}- k := ekey{name: strings.ToLower(name), typ: typ}+ k := ekey{name: strings.ToLower(name), typ: typ, ckwd: ckwd}b := entByKey[k]if b == nil {- b = &StructureBucket{Name: name, Type: typ}+ b = &StructureBucket{Name: name, Type: typ, CompileKwd: ckwd, TemplateID: p.TemplateID, TemplateKind: p.Kind}entByKey[k] = b}appendBucket(b, p)@@ -1080,6 +1110,11 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,for _, b := range relByKey {buckets = append(buckets, *b)}+ common.Info("knowledge_compile: mergeStructureDataset buckets",+ zap.String("kb_id", kb),+ zap.Int("entities", len(entByKey)),+ zap.Int("relations", len(relByKey)),+ zap.Int("total_buckets", len(buckets)))if len(buckets) == 0 {return nil}@@ -1097,14 +1132,28 @@ func (c *Consumer) mergeStructureDataset(ctx context.Context, tenant, kb string,// element-wise running mean (review issue 13). VecCount tracks how many vectors// have been folded so the mean is order-independent.func appendBucket(b *StructureBucket, p kccommon.Product) {- if strings.TrimSpace(p.Content) != "" {+ // The product's Content is the doc row's full content_with_weight JSON+ // (e.g. {"category":"Person","description":"…","name":"…","type":"entity"}),+ // NOT the plain-text description. Extract the "description" field so the+ // dataset row's description stays a plain-text string (mirror Python+ // _struct_merge_graph_entities, which folds the entity description, not the+ // whole payload). Fall back to Content only when it is not a JSON object.+ if desc := structureProductDescription(p.Content); desc != "" {if b.Description != "" {b.Description += "\n"}- b.Description += strings.TrimSpace(p.Content)+ b.Description += desc}b.SourceDocIDs = appendUnique(b.SourceDocIDs, []string{p.DocID})b.SourceChunkIDs = appendUnique(b.SourceChunkIDs, metaStringSlice(p.Meta, "source_chunk_ids"))+ // mention_count_int: sum the per-entity mention counts (Python+ // _struct_merge_graph_entities sums mention_count across the merged entities).+ // Each entity product carries mention_count ≥ 1 (structure compile.go L231);+ // mindmap entities carry none, so they contribute a default of 0 and are+ // simply not stamped (mention_count_int is omitted when 0).+ if mc, ok := metaInt(p.Meta, "mention_count"); ok && mc > 0 {+ b.MentionCount += int(mc)+ }if len(p.Vector) > 0 {b.VecCount++if len(b.Vector) == 0 {@@ -1122,6 +1171,33 @@ func appendBucket(b *StructureBucket, p kccommon.Product) {}}+// structureProductDescription extracts the plain-text description from a structure+// product's Content. In production Content is the doc row's content_with_weight+// JSON ({"description":"…","name":"…","type":"…"}), so we parse it and return the+// "description" field — the whole payload must NOT leak into the dataset row's+// description (that is the bug: projectEntity then renders the raw JSON object).+// When Content is NOT a JSON object (plain text, legacy/unit-test rows), it is+// returned verbatim so existing folded-description behavior is preserved.+func structureProductDescription(content string) string {+ content = strings.TrimSpace(content)+ if content == "" {+ return ""+ }+ if content[0] == '{' {+ var m map[string]any+ if err := json.Unmarshal([]byte(content), &m); err == nil {+ if d, ok := m["description"].(string); ok {+ if s := strings.TrimSpace(d); s != "" {+ return s+ }+ }+ // JSON object without a plain-text description: nothing to fold.+ return ""+ }+ }+ return content+}+// appendUnique appends only values not already present, preserving order.func appendUnique(dst, values []string) []string {seen := make(map[string]bool, len(dst)+len(values))diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.goindex 6861ba35db..7a75e52c96 100644--- a/internal/ingestion/knowledge_compile/reader.go+++ b/internal/ingestion/knowledge_compile/reader.go@@ -79,12 +79,16 @@ var compiledSelectFields = []string{"name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd","slug_kwd", "type","kc_kind", "create_timestamp_flt", "create_time",- "compilation_template_kind_kwd",+ "compilation_template_kind_kwd", "compilation_template_ids",// The product kind discriminator for structure/tree: the component stores it// under knowledge_graph_kwd (structure: graph/entity/relation) / raptor_kwd// (tree: root/summary). Without these the reader cannot restore Meta["kind"]// and the dataset-nav dispatch would skip structure/tree products (B2)."knowledge_graph_kwd", "raptor_kwd",+ // mention_count_int round-trips the entity mention count for reprojection.+ // (relation type lives in the content_with_weight payload, matching Python —+ // there is NO relation_type_kwd column.)+ "mention_count_int",}// wikiSelectFields are the additional columns a wiki page carries (beyond@@ -208,9 +212,21 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommoif err != nil || mapped != expect {return kccommon.Product{}, false}+ // Round-trip the raw compile_kwd (the inferred compile type / autotype:+ // list/set/hypergraph/timeline/mindmap/…) so the dataset-level merge can+ // stamp the SAME value on the dataset row as the doc row (Python _do_build+ // carries the doc row's compile_kwd through verbatim). Without this, the+ // merge falls back to compileKwdForVariant ("structure"/"mindmap"), which+ // diverges from the doc row's autotype ("hypergraph"/"timeline").merged := isAvailable(c["available_int"])meta := map[string]any{}+ // Preserve the raw compile_kwd (autotype) for the dataset merge (see the+ // round-trip note above). A non-string scalar from the engine is normalized+ // via asString, matching the variant reverse-map at the top of this func.+ if v := asString(c["compile_kwd"]); v != "" {+ meta["compile_kwd"] = v+ }if v, ok := c["name_kwd"].(string); ok && v != "" {meta["name"] = v}@@ -225,6 +241,9 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommometa["to"] = vmeta["kind"] = "relation"}+ if v, ok := metaInt(c, "mention_count_int"); ok {+ meta["mention_count"] = v+ }if v, ok := c["slug_kwd"].(string); ok && v != "" {// slug_kwd is the full "<page_type>/<slug>" form (Python writer// contract); reconstruct it verbatim so the round-trip stays full-form.@@ -310,16 +329,26 @@ func productFromChunkMap(c map[string]interface{}, tenant string, expect kccommo// so callers (e.g. RebuildDataset's variant recovery, B1a) can map it back// via KindToVariant instead of re-deriving from the ambiguous compile_kwd.kind := asString(c["compilation_template_kind_kwd"])+ // Restore the compilation template id (from compilation_template_ids) so the+ // dataset merge can bucket structure rows per template and read/delete paths+ // can filter by it. A row should carry exactly one template id; if it carries+ // more than one the first is used (multi-template rows are a config error+ // surfaced elsewhere).+ templateID := ""+ if ids := metaStringSlice(c, "compilation_template_ids"); len(ids) > 0 {+ templateID = ids[0]+ }return kccommon.Product{- ID: id,- DocID: docID,- TenantID: tenant,- Variant: expect,- Kind: kind,- Content: content,- Vector: vec,- Meta: meta,- Merged: merged,+ ID: id,+ DocID: docID,+ TenantID: tenant,+ Variant: expect,+ Kind: kind,+ TemplateID: templateID,+ Content: content,+ Vector: vec,+ Meta: meta,+ Merged: merged,}, true}diff --git a/internal/ingestion/knowledge_compile/structure_merge_test.go b/internal/ingestion/knowledge_compile/structure_merge_test.goindex 764a9b8b9c..ce1f5ada00 100644--- a/internal/ingestion/knowledge_compile/structure_merge_test.go+++ b/internal/ingestion/knowledge_compile/structure_merge_test.go@@ -114,17 +114,102 @@ func TestMergeStructureDataset_RelationsNotDropped(t *testing.T) {}}+// TestMergeStructureDataset_DescriptionIsPlainText covers the description bug:+// the product Content is the doc row's content_with_weight JSON, so the dataset+// row's folded description must be the plain-text "description" field, NOT the raw+// JSON object (which would leak the whole entity payload into the rendered graph).+func TestMergeStructureDataset_DescriptionIsPlainText(t *testing.T) {+ c := &Consumer{writer: &fakeWriter{}}+ products := []kccommon.Product{+ {Variant: kccommon.VariantStructure, DocID: "d1",+ Content: `{"category":"Person","description":"汉桓帝,禁锢善类","name":"桓帝","type":"entity"}`,+ Meta: map[string]any{"name": "桓帝", "entity_type": "Person", "compile_kwd": "list", "source_chunk_ids": []string{"c1"}}},+ }+ if err := c.mergeStructureDataset(context.Background(), "t1", "kb1", products); err != nil {+ t.Fatalf("mergeStructureDataset: %v", err)+ }+ fw := c.writer.(*fakeWriter)+ if len(fw.buckets) != 1 {+ t.Fatalf("want 1 bucket, got %d", len(fw.buckets))+ }+ got := fw.buckets[0].Description+ if got != "汉桓帝,禁锢善类" {+ t.Errorf("description = %q, want plain-text \"汉桓帝,禁锢善类\" (not the raw JSON payload)", got)+ }+ if strings.Contains(got, "{") || strings.Contains(got, "category") {+ t.Errorf("description leaked the JSON payload: %q", got)+ }+}++// TestMergeStructureDataset_CompileKwdIsAutotypeNotTemplateKind covers the+// option-A alignment: the dataset row's compile_kwd is the doc row's inferred+// compile type (autotype, e.g. "hypergraph"/"mindmap"), NOT the template kind.+// The template kind travels on the separate TemplateKind field (stamped to+// compilation_template_kind_kwd by WriteMergedStructure), which is what read/+// delete paths match on. This mirrors Python _do_build (compile_kwd passes+// through verbatim) + get_dataset_structure (matches template kind).+func TestMergeStructureDataset_CompileKwdIsAutotypeNotTemplateKind(t *testing.T) {+ c := &Consumer{writer: &fakeWriter{}}+ products := []kccommon.Product{+ {Variant: kccommon.VariantStructure, DocID: "d1", Kind: "knowledge_graph", TemplateID: "tpl-graph",+ Content: "Engine desc",+ Meta: map[string]any{"name": "Engine", "entity_type": "component", "compile_kwd": "hypergraph", "source_chunk_ids": []string{"c1"}}},+ {Variant: kccommon.VariantMindmap, DocID: "d2", Kind: "mind_map", TemplateID: "tpl-mindmap",+ Content: "Fuel desc",+ Meta: map[string]any{"name": "Fuel", "entity_type": "substance", "compile_kwd": "mindmap", "source_chunk_ids": []string{"c2"}}},+ }+ if err := c.mergeStructureDataset(context.Background(), "t1", "kb1", products); err != nil {+ t.Fatalf("mergeStructureDataset: %v", err)+ }+ fw := c.writer.(*fakeWriter)+ if len(fw.buckets) != 2 {+ t.Fatalf("want 2 buckets, got %d", len(fw.buckets))+ }+ for i := range fw.buckets {+ b := fw.buckets[i]+ switch b.Name {+ case "Engine":+ if b.CompileKwd != "hypergraph" {+ t.Errorf("Engine compile_kwd = %q, want autotype \"hypergraph\" (not template kind)", b.CompileKwd)+ }+ if b.TemplateKind != "knowledge_graph" {+ t.Errorf("Engine TemplateKind = %q, want \"knowledge_graph\"", b.TemplateKind)+ }+ if b.TemplateID != "tpl-graph" {+ t.Errorf("Engine TemplateID = %q, want \"tpl-graph\"", b.TemplateID)+ }+ case "Fuel":+ if b.CompileKwd != "mindmap" {+ t.Errorf("Fuel compile_kwd = %q, want autotype \"mindmap\" (not template kind)", b.CompileKwd)+ }+ if b.TemplateKind != "mind_map" {+ t.Errorf("Fuel TemplateKind = %q, want \"mind_map\"", b.TemplateKind)+ }+ if b.TemplateID != "tpl-mindmap" {+ t.Errorf("Fuel TemplateID = %q, want \"tpl-mindmap\"", b.TemplateID)+ }+ }+ }+}+// TestDatasetLevelStructureID_StableAndCaseInsensitive covers G1: the id is// deterministic and case-insensitive on name so merges hit the same row.func TestDatasetLevelStructureID_StableAndCaseInsensitive(t *testing.T) {- a := datasetLevelStructureID("t1", "kb1", "Engine", "component")- b := datasetLevelStructureID("t1", "kb1", "engine", "component")+ a := datasetLevelStructureID("t1", "kb1", "Engine", "component", "timeline", "")+ b := datasetLevelStructureID("t1", "kb1", "engine", "component", "timeline", "")if a != b {t.Errorf("structure id should be case-insensitive on name: %q vs %q", a, b)}- if a == datasetLevelStructureID("t1", "kb1", "Fuel", "component") {+ if a == datasetLevelStructureID("t1", "kb1", "Fuel", "component", "timeline", "") {t.Errorf("different names must yield different ids")}+ if a == datasetLevelStructureID("t1", "kb1", "Engine", "component", "mindmap", "") {+ t.Errorf("different compile kinds must yield different ids")+ }+ if datasetLevelStructureID("t1", "kb1", "A -> B", "relation", "graph", "causes") ==+ datasetLevelStructureID("t1", "kb1", "A -> B", "relation", "graph", "contradicts") {+ t.Errorf("different relation types between the same endpoints must yield different ids")+ }}// TestKwdToVariant_MapsStructureSubKinds covers B1a: structure doc-level products@@ -182,6 +267,11 @@ func TestProductFromChunkMap_RestoresStructureTreeKind(t *testing.T) {if kind, _ := sp.Meta["kind"].(string); kind != "graph" {t.Errorf("structure Meta.kind = %q, want graph (so nav dispatch can pick it)", kind)}+ // Round-trip: the raw compile_kwd (autotype) must be preserved in Meta so the+ // dataset merge stamps the SAME value on the dataset row as the doc row.+ if ckwd, _ := sp.Meta["compile_kwd"].(string); ckwd != "list" {+ t.Errorf("structure Meta.compile_kwd = %q, want autotype \"list\" (not \"structure\")", ckwd)+ }// tree root product: kind stored in raptor_kwd.treeRow := map[string]interface{}{diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.goindex 88cc9514ef..1d2721e568 100644--- a/internal/ingestion/knowledge_compile/writer.go+++ b/internal/ingestion/knowledge_compile/writer.go@@ -101,6 +101,28 @@ type StructureBucket struct {VecCount int // number of vectors folded into Vector (for true mean)FromEntity string // relation onlyToEntity string // relation only+ // CompileKwd is the raw compile keyword (the inferred compile type / autotype,+ // e.g. "hypergraph", "timeline", "mindmap", "list") that produced this bucket.+ // It is stamped on the stored row's compile_kwd so distinct structure kinds+ // never collide in the same dataset namespace (plan §1.1). It matches the doc+ // row's own compile_kwd — Python _do_build carries the doc row's compile_kwd+ // verbatim onto the dataset row, it does NOT rewrite it to the template kind.+ CompileKwd string+ // TemplateID is the compilation template id (compilation_template_ids[0]) that+ // produced this bucket. Stamped on the dataset row so read/delete paths can+ // filter per template (mirror Python get_dataset_structure).+ TemplateID string+ // TemplateKind is the authoritative template kind (compilation_template_kind_kwd,+ // e.g. "knowledge_graph", "mind_map", "timeline"). This — NOT compile_kwd — is+ // the field read/delete paths match on for the dataset-structure kind+ // (mirror Python get_dataset_structure._discover_scope_templates, which+ // matches _resolve_dataset_structure_kind against compilation_template_kind_kwd).+ TemplateKind string+ // RelationType is the relation type (Python default "related"), persisted as+ // relation_type_kwd and part of the relation bucket/ID identity.+ RelationType string+ // MentionCount is the entity mention count (mention_count_int); 0 when unset.+ MentionCount int}// engineWriter persists dataset-level merged products through the global@@ -111,6 +133,33 @@ type engineWriter struct {eng engine.DocEngine}+// datasetStructureSupported reports whether the running doc engine can filter+// the dataset-structure fields (knowledge_graph_kwd + scope_kwd + raw+// compile_kwd) this feature writes/deletes. Only infinity and elasticsearch+// support them today; OceanBase/SeekDB/SereneDB have explicit schemas that lack+// these filter keys and would reject the query or silently drop unknown fields.+// The guard is checked at every dataset-structure write/delete/rebuild entry so+// those engines never leave partial state or hit an unknown-filter error (plan §5).+func datasetStructureSupported() bool {+ switch engine.GetEngineType() {+ case "infinity", "elasticsearch":+ return true+ case "":+ // Engine not initialized (unit tests inject a fake engine directly via+ // engineWriter.eng). The guard is only meaningful against a real engine+ // type, so an empty type is treated as supported.+ return true+ default:+ return false+ }+}++// errDatasetStructureUnsupported returns an error (not a silent no-op) naming+// the engine when a dataset-structure operation runs on an unsupported engine.+func errDatasetStructureUnsupported() error {+ return fmt.Errorf("dataset structure graph unsupported on doc engine %q", engine.GetEngineType())+}+// writeMergedBatchSize bounds how many rows each parallel InsertChunks call// carries, so the DocEngine write fan-out stays granular under the shared pool.const writeMergedBatchSize = 200@@ -202,13 +251,18 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, produc// WriteMergedStructure writes the dataset-level structure merged rows for a KB// (G1/G4). Each StructureBucket is a scope_kwd="dataset" row with a stable-// dataset-level id keyed on (name, type), the folded description, the union of-// source docs/chunks, and the bucket vector. Rows are available_int=1 so the-// dataset-level structure index is searchable, and compile_kwd="structure".+// dataset-level id keyed on (name, type, raw compile kind), the folded+// description, the union of source docs/chunks, and the bucket vector. Rows are+// available_int=1 so the dataset-level structure index is searchable; the raw+// compile kind (timeline/graph/mindmap) is stamped on compile_kwd and the+// entity/relation discriminator on knowledge_graph_kwd.func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb string, buckets []StructureBucket) error {if len(buckets) == 0 {return nil}+ if !datasetStructureSupported() {+ return errDatasetStructureUnsupported()+ }eng := w.engif eng == nil {eng = engine.Get()@@ -229,7 +283,7 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strinif b.Name == "" {continue}- ids = append(ids, datasetLevelStructureID(tenant, kb, b.Name, b.Type))+ ids = append(ids, datasetLevelStructureID(tenant, kb, b.Name, b.Type, b.CompileKwd, b.RelationType))}if len(ids) > 0 {res, err := eng.Search(ctx, &types.SearchRequest{@@ -263,43 +317,128 @@ func (w engineWriter) WriteMergedStructure(ctx context.Context, tenant, kb strinif b.Name == "" {continue}- bid := datasetLevelStructureID(tenant, kb, b.Name, b.Type)+ bid := datasetLevelStructureID(tenant, kb, b.Name, b.Type, b.CompileKwd, b.RelationType)// Union the current batch's sources with any already-accumulated ones.if prev, ok := existing[bid]; ok {b.SourceDocIDs = appendUnique(b.SourceDocIDs, prev.SourceDocIDs)b.SourceChunkIDs = appendUnique(b.SourceChunkIDs, prev.SourceChunkIDs)}+ ckwd := b.CompileKwd+ if ckwd == "" {+ ckwd = compileKwdStructure+ }+ // content_with_weight is a JSON payload, matching Python (and the existing+ // wiki projection writer below): the structured fields (from/to/type for+ // relations, name/type for entities) live INSIDE the payload, while+ // from/to are also copied to from_entity_kwd/to_entity_kwd top-level+ // columns for filtering. There is NO relation_type_kwd column — the+ // relation type is carried only in the payload, exactly as Python does.+ var payloadMap map[string]any+ if b.FromEntity != "" || b.ToEntity != "" {+ relType := b.RelationType+ if relType == "" {+ relType = "related"+ }+ payloadMap = map[string]any{+ "from": b.FromEntity,+ "to": b.ToEntity,+ "type": relType,+ "description": desc,+ }+ } else {+ typ := b.Type+ if typ == "" {+ typ = "other"+ }+ payloadMap = map[string]any{+ "name": b.Name,+ "type": typ,+ "description": desc,+ }+ }+ payloadBytes, err := json.Marshal(payloadMap)+ if err != nil {+ return err+ }+ payload := string(payloadBytes)row := map[string]interface{}{- // Stable dataset-level id keyed on the (name, type) or (from, to) bucket.+ // Stable dataset-level id keyed on the (name, type) or (from, to)+ // bucket plus the raw compile kind."id": bid,"doc_id": kb,"tenant_id": tenant,"kb_id": kb,"available_int": 1,- "compile_kwd": compileKwdStructure,+ "compile_kwd": ckwd,"scope_kwd": "dataset",- "content_with_weight": desc,- "kc_payload": desc,+ "content_with_weight": payload,+ "kc_payload": payload,"source_doc_ids": b.SourceDocIDs,"source_chunk_ids": b.SourceChunkIDs,"create_time": now.Format("2006-01-02 15:04:05"),"create_timestamp_flt": float64(now.Unix()),}+ // Stamp the authoritative template identity so read/delete paths can match+ // the dataset-structure kind by compilation_template_kind_kwd (mirror Python+ // get_dataset_structure._discover_scope_templates), independent of the+ // autotype-valued compile_kwd above.+ if b.TemplateKind != "" {+ row["compilation_template_kind_kwd"] = b.TemplateKind+ }+ if b.TemplateID != "" {+ row["compilation_template_ids"] = []string{b.TemplateID}+ }if b.FromEntity != "" || b.ToEntity != "" {// relation row: carries from/to entities; kind=relation, no name_kwd.+ row["knowledge_graph_kwd"] = "relation"row["type_kwd"] = "relation"row["from_entity_kwd"] = b.FromEntityrow["to_entity_kwd"] = b.ToEntity} else {+ row["knowledge_graph_kwd"] = "entity"row["type_kwd"] = "entity"row["name_kwd"] = b.Namerow["entity_type_kwd"] = b.Type+ if b.MentionCount > 0 {+ row["mention_count_int"] = b.MentionCount+ }}if len(b.Vector) > 0 {row["q_"+fmt.Sprintf("%d", len(b.Vector))+"_vec"] = f32ToF64Slice(b.Vector)}rows = append(rows, row)}+ // Write one kg_build_meta build-marker row per raw compile kind (the bucket+ // model is keyed by raw kind; template-id granularity is a follow-up when the+ // bucket key gains a template dimension). The marker is available_int=0 and+ // carries create_timestamp_flt as its timestamp (build_timestamp_flt is a+ // Python _META_ROW_KWD legacy field NOT present in the Infinity/ES/OS schemas,+ // so writing it would fail with an undefined-column error — see review). It is+ // a write/delete-side marker — GET discovery does NOT read it (it scans+ // knowledge_graph_kwd=["entity"] rows instead, per §6).+ seenKwd := map[string]bool{}+ for _, b := range buckets {+ ckwd := b.CompileKwd+ if ckwd == "" {+ ckwd = compileKwdStructure+ }+ if seenKwd[ckwd] {+ continue+ }+ seenKwd[ckwd] = true+ rows = append(rows, map[string]interface{}{+ "id": "dataset_build_meta_" + hashStr(tenant+"\x00"+kb+"\x00"+ckwd),+ "doc_id": kb,+ "tenant_id": tenant,+ "kb_id": kb,+ "available_int": 0,+ "compile_kwd": ckwd,+ "scope_kwd": "dataset",+ "knowledge_graph_kwd": "kg_build_meta",+ "create_time": now.Format("2006-01-02 15:04:05"),+ "create_timestamp_flt": float64(now.Unix()),+ })+ }if len(rows) == 0 {return nil}@@ -329,6 +468,9 @@ func (w engineWriter) DeleteStructureForDocs(ctx context.Context, tenant, kb strif len(deletedDocIDs) == 0 {return nil}+ if !datasetStructureSupported() {+ return errDatasetStructureUnsupported()+ }eng := w.engif eng == nil {eng = engine.Get()@@ -346,14 +488,25 @@ func (w engineWriter) DeleteStructureForDocs(ctx context.Context, tenant, kb str// surviving silently (review Minor).const pageSize = 500var ghostIDs []string+ // Raw compile kinds whose entity/relation rows were declared ghosts; used to+ // drop a now-empty kind's kg_build_meta build marker after cleanup.+ affectedKinds := map[string]bool{}for offset := 0; ; offset += pageSize {res, err := eng.Search(ctx, &types.SearchRequest{IndexNames: []string{baseName},KbIDs: []string{kb},- SelectFields: []string{"id", "source_doc_ids"},- Filter: map[string]interface{}{"kb_id": kb, "scope_kwd": "dataset", "compile_kwd": compileKwdStructure},- Offset: offset,- Limit: pageSize,+ SelectFields: []string{"id", "source_doc_ids", "compile_kwd"},+ // Match dataset-scope entity/relation rows across ALL structure kinds+ // (timeline/graph/session_graph/mindmap), which now each stamp their+ // raw compile_kwd; knowledge_graph_kwd ∈ {entity,relation} + scope_kwd+ // =dataset is the kind-agnostic predicate (plan §1, §4.2).+ Filter: map[string]interface{}{+ "kb_id": kb,+ "scope_kwd": "dataset",+ "knowledge_graph_kwd": []string{"entity", "relation"},+ },+ Offset: offset,+ Limit: pageSize,})if err != nil {return fmt.Errorf("structure ghost scan: %w", err)@@ -379,6 +532,9 @@ func (w engineWriter) DeleteStructureForDocs(ctx context.Context, tenant, kb str}if allGone {ghostIDs = append(ghostIDs, id)+ if ckwd, _ := c["compile_kwd"].(string); ckwd != "" {+ affectedKinds[ckwd] = true+ }}}if len(res.Chunks) < pageSize {@@ -388,18 +544,51 @@ func (w engineWriter) DeleteStructureForDocs(ctx context.Context, tenant, kb strif len(ghostIDs) == 0 {return nil}- _, err := eng.DeleteChunks(ctx, map[string]interface{}{"id": ghostIDs, "kb_id": kb}, baseName, kb)- if err != nil {+ if _, err := eng.DeleteChunks(ctx, map[string]interface{}{"id": ghostIDs, "kb_id": kb}, baseName, kb); err != nil {return fmt.Errorf("structure ghost cleanup: %w", err)}+ // Drop the kg_build_meta build marker for any raw kind that no longer has an+ // entity/relation row (plan §3.1.1 step 4).+ for ckwd := range affectedKinds {+ res, err := eng.Search(ctx, &types.SearchRequest{+ IndexNames: []string{baseName},+ KbIDs: []string{kb},+ SelectFields: []string{"id"},+ Filter: map[string]interface{}{+ "kb_id": kb,+ "scope_kwd": "dataset",+ "compile_kwd": ckwd,+ "knowledge_graph_kwd": []string{"entity", "relation"},+ },+ Limit: 1,+ })+ if err != nil {+ return fmt.Errorf("structure meta check (kind %s): %w", ckwd, err)+ }+ if len(res.Chunks) > 0 {+ continue // still has entity/relation rows; keep the marker+ }+ if _, err := eng.DeleteChunks(ctx, map[string]interface{}{+ "kb_id": kb,+ "scope_kwd": "dataset",+ "compile_kwd": ckwd,+ "knowledge_graph_kwd": "kg_build_meta",+ }, baseName, kb); err != nil {+ return fmt.Errorf("structure meta cleanup (kind %s): %w", ckwd, err)+ }+ }return nil}// datasetLevelStructureID builds the stable dataset-level id for a structure-// bucket, keyed on (name, type). It must be deterministic so an incremental-// merge read-modify-writes the same row (and a rebuild clean removes it).-func datasetLevelStructureID(tenant, kb, name, typ string) string {- return "dataset_structure_" + hashStr(tenant+"\x00"+kb+"\x00"+strings.ToLower(name)+"\x00"+typ)+// bucket, keyed on (name, type, compile kind, relation type). It must be+// deterministic so an incremental merge read-modify-writes the same row (and a+// rebuild clean removes it). Including the raw compile kind keeps+// timeline/graph/mindmap buckets from colliding in the same dataset namespace;+// including the relation type keeps two relation types between the same endpoints+// (e.g. "causes" vs "contradicts") from colliding into one id (review fix).+func datasetLevelStructureID(tenant, kb, name, typ, compileKwd, relationType string) string {+ return "dataset_structure_" + hashStr(tenant+"\x00"+kb+"\x00"+strings.ToLower(name)+"\x00"+typ+"\x00"+compileKwd+"\x00"+strings.ToLower(relationType))}// f32ToF64Slice converts a float32 vector to float64 for the engine's dense@@ -937,7 +1126,13 @@ func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb str// Full-set clean (B1b): everything the consumer manages. Structure// dataset rows are tagged scope_kwd="dataset" (not a fixed compile_kwd),// so the full clean deletes by kb_id + compile_kwd IN (the fixed-kwd- // buckets) OR scope_kwd="dataset".+ // buckets) OR scope_kwd="dataset". The scope_kwd="dataset" sweep requires+ // the dataset-structure filter keys, so it is guarded like the per-variant+ // structure branch (review fix: this is the highest-impact destructive+ // path and must fail loudly on unsupported engines).+ if !datasetStructureSupported() {+ return errDatasetStructureUnsupported()+ }_, err := eng.DeleteChunks(ctx, map[string]interface{}{"kb_id": kb,"compile_kwd": []string{@@ -979,17 +1174,23 @@ func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb str}, baseName, kb); err != nil {return fmt.Errorf("delete merged (wiki): %w", err)}- case kccommon.VariantStructure:- // structure dataset rows are scope_kwd="dataset" + compile_kwd=- // "structure". The compile_kwd is required: wiki merged rows ALSO carry- // scope_kwd="dataset" (W5), so a scope-only sweep would wrongly delete- // wiki merged rows too (review Major).+ case kccommon.VariantStructure, kccommon.VariantMindmap:+ if !datasetStructureSupported() {+ return errDatasetStructureUnsupported()+ }+ // structure/mindmap dataset rows are scope_kwd="dataset" ++ // knowledge_graph_kwd ∈ {entity,relation,kg_build_meta} with a raw+ // compile_kwd (timeline/graph/session_graph/mindmap). knowledge_graph_kwd+ // is required: wiki merged rows ALSO carry scope_kwd="dataset" (W5), so+ // a scope-only sweep would wrongly delete wiki merged rows too (review+ // Major). kg_build_meta (the build marker) is deleted together with the+ // entity/relation rows.if _, err := eng.DeleteChunks(ctx, map[string]interface{}{- "kb_id": kb,- "scope_kwd": "dataset",- "compile_kwd": []string{compileKwdStructure},+ "kb_id": kb,+ "scope_kwd": "dataset",+ "knowledge_graph_kwd": []string{"entity", "relation", "kg_build_meta"},}, baseName, kb); err != nil {- return fmt.Errorf("delete merged (structure): %w", err)+ return fmt.Errorf("delete merged (structure/mindmap): %w", err)}case kccommon.VariantTree:// tree/nav rows are compile_kwd="dataset_nav" with available_int=0;@@ -1000,8 +1201,6 @@ func (w engineWriter) DeleteMergedForVariant(ctx context.Context, tenant, kb str}, baseName, kb); err != nil {return fmt.Errorf("delete merged (nav): %w", err)}- case kccommon.VariantMindmap:- // mindmap has no dataset-level merged rows in the consumer path}}return nildiff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.goindex 0ef4ca3744..37ad7cd953 100644--- a/internal/service/dataset_artifact_service.go+++ b/internal/service/dataset_artifact_service.go@@ -39,16 +39,6 @@ const (CompileKwdSkillAll = "skill_all"CompileKwdDatasetNav = "dataset_nav"CompileKwdRaptorGraph = "raptor_graph"-- CompileKwdStructure = "structure"- CompileKwdStructureIndex = "structureIndex"- CompileKwdStructureEntity = "structureEntity"- CompileKwdStructureRelation = "structureRelation"- CompileKwdStructureCommunity = "structureCommunity"-- FieldStructureIndexType = "structure_index_type"- FieldStructureKind = "structure_kind"- FieldPageID = "page_id")// DatasetArtifactService reads knowledge-compilation artifacts (wiki pages,@@ -550,75 +540,6 @@ func (s *DatasetArtifactService) ClearWiki(ctx context.Context, tenantID, datasereturn deleted, nil}-// StructureItem is a single compiled structure entry for a dataset.-type StructureItem struct {- PageID string `json:"page_id"`- StructureKind string `json:"structure_kind"`- StructureIndexType string `json:"structure_index_type"`- Data string `json:"data"`-}--// ListStructures returns the compiled structures of a dataset, filtered by-// optional structure_kind and structure_index_type.-func (s *DatasetArtifactService) ListStructures(ctx context.Context, tenantID, datasetID, structureKind, structureIndexType string) ([]StructureItem, int64, error) {- filter := map[string]interface{}{"compile_kwd": []string{CompileKwdStructure}}- if structureKind != "" {- filter[FieldStructureKind] = []string{structureKind}- }- if structureIndexType != "" {- filter[FieldStructureIndexType] = []string{structureIndexType}- }- chunks, total, err := s.searchCompiled(ctx, tenantID, datasetID, filter,- []string{FieldPageID, FieldStructureKind, FieldStructureIndexType, "content_with_weight"}, 0, 10000, nil)- if err != nil {- return nil, 0, err- }- items := make([]StructureItem, 0, len(chunks))- for _, c := range chunks {- items = append(items, StructureItem{- PageID: firstStringValue(c[FieldPageID]),- StructureKind: firstStringValue(c[FieldStructureKind]),- StructureIndexType: firstStringValue(c[FieldStructureIndexType]),- Data: firstStringValue(c["content_with_weight"]),- })- }- return items, total, nil-}--// DeleteStructures deletes the compiled structures of a dataset, optionally-// scoped by structure_kind and structure_index_type.-func (s *DatasetArtifactService) DeleteStructures(ctx context.Context, tenantID, datasetID, structureKind, structureIndexType string) (int, error) {- docEngine := engine.Get()- if docEngine == nil {- return 0, fmt.Errorf("document engine is not initialized")- }- filter := map[string]interface{}{"compile_kwd": []string{CompileKwdStructure}}- if structureKind != "" {- filter[FieldStructureKind] = []string{structureKind}- }- if structureIndexType != "" {- filter[FieldStructureIndexType] = []string{structureIndexType}- }- chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, []string{"id"}, 0, 10000, nil)- if err != nil {- return 0, err- }- if len(chunks) == 0 {- return 0, nil- }- ids := make([]string, 0, len(chunks))- for _, c := range chunks {- if id, ok := c["id"].(string); ok {- ids = append(ids, id)- }- }- cond := map[string]interface{}{"id": ids, "kb_id": datasetID}- if _, err := docEngine.DeleteChunks(ctx, cond, wikiIndexName(tenantID), datasetID); err != nil {- return 0, err- }- return len(ids), nil-}-// DeleteDocumentGraph deletes the structure graph of a single document.func (s *DatasetArtifactService) DeleteDocumentGraph(ctx context.Context, tenantID, datasetID, documentID string) (int, error) {docEngine := engine.Get()diff --git a/internal/service/dataset_structure_graph.go b/internal/service/dataset_structure_graph.goindex 77ed8f448a..caa557f38a 100644--- a/internal/service/dataset_structure_graph.go+++ b/internal/service/dataset_structure_graph.go@@ -10,11 +10,13 @@ package serviceimport ("context""encoding/json"+ "errors""fmt""regexp""sort""strings"+ "gorm.io/gorm""ragflow/internal/dao""ragflow/internal/engine""ragflow/internal/engine/types"@@ -468,8 +470,12 @@ func (s *DatasetArtifactService) buildBucket(ctx context.Context, tenantID, dataif err != nil {return nil, nil, err}- var entities []StructureGraphNode- var relations []StructureGraphRelation+ // Initialize as empty (non-nil) slices so a bucket with entities but no+ // relations serializes "relations": [] instead of null — the frontend+ // adapters (adaptKnowledgeGraphToForceGraph et al.) call .filter() on+ // relations without a null guard (mirror Python, which always returns []).+ entities := make([]StructureGraphNode, 0)+ relations := make([]StructureGraphRelation, 0)for _, row := range fieldMap {if !rowHasEnabledSource(row, excludedDocIDs) {continue@@ -530,7 +536,7 @@ func (s *DatasetArtifactService) buildBucket(ctx context.Context, tenantID, data}aNameTerms = sortedUnique(aNameTerms)- var relations []StructureGraphRelation+ relations := make([]StructureGraphRelation, 0)targetNamesLower := map[string]bool{}if len(aNameTerms) > 0 {cond := copyFilter(scope)@@ -613,6 +619,59 @@ func compilationTemplateKind(kind string) string {return k}+// ErrInvalidStructureKind is returned when the request kind does not resolve to a+// supported dataset-structure kind. Handlers use errors.Is to map it to+// CodeArgumentError (400) while treating every other service error as a server+// fault (500).+var ErrInvalidStructureKind = errors.New("invalid structure kind")++// datasetStructureSupported mirrors the writer's engine guard: only Infinity and+// Elasticsearch resolve the dataset-structure filter keys (scope_kwd ++// knowledge_graph_kwd). The writer keeps a private copy in the knowledge_compile+// package; this service-side copy keeps the API delete path in agreement without+// exporting an ingestion-internal helper across package boundaries.+func datasetStructureSupported() bool {+ switch engine.GetEngineType() {+ case "infinity", "elasticsearch", "":+ return true+ default:+ return false+ }+}++// errDatasetStructureUnsupported returns a visible error naming the engine.+func errDatasetStructureUnsupported() error {+ return fmt.Errorf("dataset structure graph unsupported on doc engine %q", engine.GetEngineType())+}++// resolveDatasetStructureKind maps a user-facing dataset-structure kind to the+// stored top-level kind (mirror Python _DATASET_STRUCTURE_KIND_ALIASES). It is+// DELIBERATELY separate from compilationTemplateKind: that helper folds+// knowledge_graph→timeline, which would merge distinct dataset kinds, and it+// lacks the graph→knowledge_graph / mindmap→mind_map aliases (Python's+// dataset_api_service.py L1803-1805 documents exactly why it does not reuse the+// general normalizer). Returns "" for an invalid kind (caller maps that to+// 400 ARGUMENT_ERROR, never "return all").+func resolveDatasetStructureKind(kind string) string {+ k := strings.ToLower(strings.TrimSpace(strings.ReplaceAll(kind, "-", "_")))+ switch k {+ case "graph", "knowledge_graph":+ return "knowledge_graph"+ case "mindmap", "mind_map":+ // Align with Python _DATASET_STRUCTURE_KIND_ALIASES: resolve the+ // user-facing "mindmap" to the stored compilation_template_kind_kwd value+ // "mind_map" (the template kind, which is what read/delete paths match on).+ return "mind_map"+ case "timeline":+ return "timeline"+ case "session_essence":+ return "session_essence"+ case "session_graph":+ return "session_graph"+ }+ return ""+}+// structureKindForBucket returns the normalized kind used for tree/page_index// hierarchy handling.func structureKindForBucket(kind string) string {@@ -720,8 +779,10 @@ func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, in Documereturn resp, nil}- // normal mode: discover buckets from per-doc graph blob rows.- metaFields := []string{"compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd"}+ // normal mode: discover buckets from per-doc graph blob rows. "id" is+ // required for the same reason as dataset discovery (Infinity only projects+ // listed fields; graphRowSearch keys by id).+ metaFields := []string{"id", "compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd"}metaRows, _, err := graphRowSearch(ctx, in.TenantID, in.DatasetID, metaFields,map[string]interface{}{"doc_id": []string{in.DocumentID}, "knowledge_graph_kwd": []string{"graph"}}, nil, 0, 1000, nil)if err != nil {@@ -789,6 +850,197 @@ func containsStr(list []string, v string) bool {return false}+// DatasetStructureGraphInput is the parsed request for the dataset-scope+// structure graph endpoint (GET/DELETE /datasets/:id/artifacts/structure).+// Kind is REQUIRED (mirrors Python dataset_api.py:715): missing or invalid →+// 400 ARGUMENT_ERROR. Wipe applies to DELETE: true deletes the dataset rows,+// false only cancels the task (rows are left for the next rebuild to clean).+type DatasetStructureGraphInput struct {+ TenantID string+ DatasetID string+ Kind string+ Wipe bool+}++// DatasetStructureGraphResponse mirrors Python get_dataset_structure's+// {"kind": ..., "templates": [...]}.+type DatasetStructureGraphResponse struct {+ Kind string `json:"kind"`+ Templates []DocumentStructureGraphTemplate `json:"templates"`+}++// GetDatasetStructure returns the dataset-scope structure graph for a resolved+// kind, mirroring Python get_dataset_structure (dataset_api_service.py). Discovery+// scans knowledge_graph_kwd=["entity"] dataset rows (scope_kwd="dataset") and+// matches the resolved kind against the stamped compilation_template_kind_kwd —+// NOT compile_kwd (compile_kwd holds the autotype "hypergraph"/"list"/"mindmap",+// which is never the kind discriminator; Python _discover_scope_templates matches+// _resolve_dataset_structure_kind against compilation_template_kind_kwd the same+// way). It collects distinct template ids, then reads each template's dataset+// entity/relation rows via buildBucket. It does NOT read kg_build_meta (write/+// delete-side only).+func (s *DatasetArtifactService) GetDatasetStructure(ctx context.Context, in DatasetStructureGraphInput) (*DatasetStructureGraphResponse, error) {+ resolved := resolveDatasetStructureKind(in.Kind)+ if resolved == "" {+ return nil, fmt.Errorf("%w: %q", ErrInvalidStructureKind, in.Kind)+ }+ if !datasetStructureSupported() {+ return nil, errDatasetStructureUnsupported()+ }+ resp := &DatasetStructureGraphResponse{Kind: resolved, Templates: []DocumentStructureGraphTemplate{}}++ // Discover distinct template ids whose stamped template kind resolves to the+ // requested kind, scanning dataset-scope entity rows only. scope_kwd="dataset"+ // is required here (unlike the legacy doc_graph fallback) because dataset rows+ // are the only ones carrying compilation_template_kind_kwd we can trust for the+ // dataset-scope kind match.+ templateIDs := map[string]struct{}{}+ // "id" must be projected: graphRowSearch keys its result map by the row id,+ // and Infinity only returns fields listed in SelectFields (it does not+ // synthesize id), so omitting it silently drops every row (review Major).+ // The resolved kind is pushed into the filter so the engine applies the+ // predicate instead of scanning all entity rows and discarding them in Go.+ metaFields := []string{"id", "compilation_template_kind_kwd", "compilation_template_ids"}+ for offset := 0; ; offset += 1000 {+ rows, total, err := graphRowSearch(ctx, in.TenantID, in.DatasetID, metaFields,+ map[string]interface{}{+ "knowledge_graph_kwd": []string{"entity"},+ "scope_kwd": []string{"dataset"},+ "compilation_template_kind_kwd": []string{resolved},+ }, nil, offset, 1000, nil)+ if err != nil {+ return nil, err+ }+ for _, row := range rows {+ tkind := firstStringValue(row["compilation_template_kind_kwd"])+ if tkind == "" || resolveDatasetStructureKind(tkind) != resolved {+ continue+ }+ tid := rowTemplateID(row)+ if tid != "" {+ templateIDs[tid] = struct{}{}+ }+ }+ if int64(offset+1000) >= total || len(rows) == 0 {+ break+ }+ }++ // Read each template's dataset entity/relation rows.+ for tid := range templateIDs {+ scope := map[string]interface{}{+ "scope_kwd": []string{"dataset"},+ "compilation_template_ids": []string{tid},+ "compilation_template_kind_kwd": []string{resolved},+ }+ entities, relations, err := s.buildBucket(ctx, in.TenantID, in.DatasetID, scope, nil)+ if err != nil {+ return nil, err+ }+ if len(entities) == 0 && len(relations) == 0 {+ continue+ }+ resp.Templates = append(resp.Templates, DocumentStructureGraphTemplate{+ TemplateID: tid,+ TemplateName: tid,+ Kind: resolved,+ Entities: entities,+ Relations: relations,+ })+ }+ return resp, nil+}++// DeleteDatasetStructure handles DELETE /datasets/:id/artifacts/structure?kind=&wipe=.+// It validates kind like GET. wipe=false cancels the kind's task (via the task-id+// field) without deleting rows; wipe=true deletes the kind's kg_build_meta marker+// + dataset entity/relation rows (document-scope rows are never touched).+func (s *DatasetArtifactService) DeleteDatasetStructure(ctx context.Context, in DatasetStructureGraphInput) (int, error) {+ resolved := resolveDatasetStructureKind(in.Kind)+ if resolved == "" {+ return 0, fmt.Errorf("%w: %q", ErrInvalidStructureKind, in.Kind)+ }+ if !in.Wipe {+ // Cancel the task without deleting rows. Task cancellation is task-id+ // granular (mirrors Python delete_index REDIS set "{task_id}-cancel"); the+ // actual row cleanup happens on the next rebuild. Rows are preserved.+ return s.cancelDatasetStructureTask(ctx, in.TenantID, in.DatasetID, resolved)+ }+ if !datasetStructureSupported() {+ return 0, errDatasetStructureUnsupported()+ }+ docEngine := engine.Get()+ if docEngine == nil {+ return 0, fmt.Errorf("document engine is not initialized")+ }+ indexName := fmt.Sprintf("ragflow_%s", in.TenantID)+ cond := map[string]interface{}{+ "kb_id": in.DatasetID,+ "scope_kwd": "dataset",+ "compilation_template_kind_kwd": resolved,+ "knowledge_graph_kwd": []string{"entity", "relation", "kg_build_meta"},+ }+ n, err := docEngine.DeleteChunks(ctx, cond, indexName, in.DatasetID)+ if err != nil {+ return 0, err+ }+ return int(n), nil+}++// cancelDatasetStructureTask cancels the running dataset-structure task for a+// resolved kind by clearing both its per-index task-id and finish-at fields.+// This mirrors Python delete_index (dataset_api_service.py), which clears+// {task_id_field: "", task_finish_at_field: None} so the task state is fully+// reset (a stale finish-at would otherwise leave the kind looking "done" after+// cancel). The row cleanup itself is driven by the next rebuild; Go executes the+// merge synchronously inside the ingestor (no independent task/marker), so there+// is no Redis "{task_id}-cancel" marker to publish here.+func (s *DatasetArtifactService) cancelDatasetStructureTask(ctx context.Context, tenantID, datasetID, resolvedKind string) (int, error) {+ field := datasetStructureTaskIDField(resolvedKind)+ if field == "" {+ return 0, nil+ }+ updates := map[string]interface{}{+ field: "",+ // gorm.Updates ignores nil values, so use an explicit NULL expression to+ // actually clear the finish-at timestamp (mirrors Python None).+ datasetStructureTaskFinishAtField(field): gorm.Expr("NULL"),+ }+ if err := dao.NewKnowledgebaseDAO().UpdateByID(ctx, dao.DB, datasetID, updates); err != nil {+ return 0, err+ }+ return 0, nil+}++// datasetStructureTaskFinishAtField maps a task-id field to its sibling finish-at+// field, mirroring Python f"{task_id_field.replace('_task_id', '_task_finish_at')}".+func datasetStructureTaskFinishAtField(taskIDField string) string {+ return strings.Replace(taskIDField, "_task_id", "_task_finish_at", 1)+}++// datasetStructureTaskIDField maps a resolved dataset-structure kind to its kb+// task-id field name. It mirrors Python _INDEX_TYPE_TO_TASK_ID_FIELD: each+// dataset-merge kind carries its own "<index_type>_task_id" field (structure_graph,+// structure_mindmap, timeline, session_graph, session_essence), NOT the legacy+// doc-level graphrag_task_id/mindmap_task_id. Empty means "no task-id field".+func datasetStructureTaskIDField(resolvedKind string) string {+ switch resolvedKind {+ case "knowledge_graph":+ // "graph" is already normalized to "knowledge_graph" by+ // resolveDatasetStructureKind, so no separate "graph" case is needed.+ return "structure_graph_task_id"+ case "mindmap", "mind_map":+ return "structure_mindmap_task_id"+ case "timeline":+ return "timeline_task_id"+ case "session_graph":+ return "session_graph_task_id"+ case "session_essence":+ return "session_essence_task_id"+ }+ return ""+}+// resolveGraphBucket mirrors Python _resolve_bucket.func resolveGraphBucket(row map[string]interface{}, templateMeta map[string]map[string]interface{}, documentID string) (map[string]interface{}, map[string]interface{}) {compileKwd := firstStringValue(row["compile_kwd"])@@ -866,7 +1118,7 @@ func rowTemplateID(row map[string]interface{}) string {}func (s *DatasetArtifactService) appendRaptorBlob(ctx context.Context, tenantID, datasetID, documentID string, grouped map[string]DocumentStructureGraphTemplate) {- rows, _, err := graphRowSearch(ctx, tenantID, datasetID, []string{"content_with_weight", "compile_kwd"},+ rows, _, err := graphRowSearch(ctx, tenantID, datasetID, []string{"id", "content_with_weight", "compile_kwd"},map[string]interface{}{"doc_id": []string{documentID}, "compile_kwd": []string{"raptor_graph"}}, nil, 0, 16, nil)if err != nil {returndiff --git a/internal/service/ingestion_task_service_test.go b/internal/service/ingestion_task_service_test.goindex 53c8d1f217..e8a05d4ec6 100644--- a/internal/service/ingestion_task_service_test.go+++ b/internal/service/ingestion_task_service_test.go@@ -875,6 +875,45 @@ func TestIngestionTaskServiceMarkStoppedIdempotentOnAlreadyStopped(t *testing.T)}}+// TestIngestionTaskServiceRequestStopNeverReopensTerminalTask locks the invariant+// that a terminal task (COMPLETED/STOPPED/FAILED) can never be moved back to+// STOPPING. A terminal task whose message was already acked must not regress to+// the in-flight STOPPING state (which has no settled worker and would be stuck+// forever). RequestStop is a no-op for terminal states.+func TestIngestionTaskServiceRequestStopNeverReopensTerminalTask(t *testing.T) {+ for _, status := range []string{common.COMPLETED, common.STOPPED, common.FAILED} {+ t.Run(status, func(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestIngestionTaskWithStatus(t, "task-1", "user-1", "doc-1", "kb-1", status)+ ctx := t.Context()++ svc := NewIngestionTaskService()+ task, err := svc.RequestStop(ctx, "task-1")+ if err != nil {+ t.Fatalf("RequestStop on %s task should be a no-op, got: %v", status, err)+ }+ if task.Status != status {+ t.Fatalf("RequestStop moved %s task to %q, must stay %s", status, task.Status, status)+ }+ })+ }+}++// TestValidateTransitionRejectsTerminalToStopping locks the state-machine+// invariant directly: STOPPING is only reachable from RUNNING, never from a+// terminal state (COMPLETED/STOPPED/FAILED) or from STOPPING itself.+func TestValidateTransitionRejectsTerminalToStopping(t *testing.T) {+ for _, from := range []string{common.COMPLETED, common.STOPPED, common.FAILED, common.STOPPING, common.CREATED} {+ if err := validateTransition(from, common.STOPPING); err == nil {+ t.Errorf("validateTransition(%s -> STOPPING) = nil, want error", from)+ }+ }+ if err := validateTransition(common.RUNNING, common.STOPPING); err != nil {+ t.Errorf("validateTransition(RUNNING -> STOPPING) = %v, want nil", err)+ }+}+func TestIngestionTaskServiceMarkFailedIdempotentOnAlreadyTerminal(t *testing.T) {db := setupServiceTestDB(t)pushServiceDB(t, db)diff --git a/internal/service/nav_embedder.go b/internal/service/nav_embedder.goindex 6598695291..891e79bff0 100644--- a/internal/service/nav_embedder.go+++ b/internal/service/nav_embedder.go@@ -20,6 +20,8 @@ import ("context""fmt""strings"++ "ragflow/internal/entity")// NavEmbedder is the production implementation of nlp.NavEmbedder. It resolves@@ -45,7 +47,17 @@ func (e *NavEmbedder) Encode(ctx context.Context, tenantID string, texts []strin}name := e.embdModelNameif name == "" {- name = tenantID // composite name falls back to tenant default resolution+ // Resolve the tenant's default embedding model composite reference+ // ("<model>@<instance>@<provider>") — NOT the tenant id. Passing the+ // tenant id as the model ref makes ResolveModelConfig parse it as a+ // "model@provider" key, which fails with "provider name missing in model+ // name: <tenant_id>". Mirrors knowledge_compiler_wiring.go's chat-model+ // default resolution.+ ref, err := e.modelSvc.GetTenantDefaultModelRef(ctx, tenantID, entity.ModelTypeEmbedding)+ if err != nil {+ return nil, fmt.Errorf("datasetnav: resolve embedding model for tenant %s: %w", tenantID, err)+ }+ name = ref}model, err := e.modelSvc.GetEmbeddingModel(ctx, tenantID, name)if err != nil {diff --git a/web/src/components/structure-graph/adapters.ts b/web/src/components/structure-graph/adapters.tsindex 0d19a2a75a..e64c8e6029 100644--- a/web/src/components/structure-graph/adapters.ts+++ b/web/src/components/structure-graph/adapters.ts@@ -53,7 +53,7 @@ function buildTreeDataItems(relationTypes: string[],showEntityType = false,): TreeDataItem[] {- const normalized = entities+ const normalized = (entities ?? []).map(normalizeEntity).filter((entity) => entity.id);const map = new Map<string, TreeDataItem>(@@ -69,7 +69,7 @@ function buildTreeDataItems();const childIds = new Set<string>();- for (const relation of relations) {+ for (const relation of relations ?? []) {if (!relationTypes.includes(relation.type ?? '')) continue;// Self-referencing relation (same entity as its own child) creates// an infinite recursion in the tree renderer. This is a backend@@ -99,7 +99,7 @@ function buildUniqueTreeDataItems(): TreeDataItem[] {const normalized = [...new Map(- entities+ (entities ?? []).map(normalizeEntity).filter((entity) => entity.id).map((entity) => [entity.id, entity]),@@ -119,7 +119,7 @@ function buildUniqueTreeDataItems(const childIds = new Set<string>();const parentMap = new Map<string, string>();- for (const relation of relations) {+ for (const relation of relations ?? []) {if (!relationTypes.includes(relation.type ?? '')) continue;const parent = map.get(relation.from);@@ -172,7 +172,7 @@ export function adaptTreeToTreeData(export function adaptKnowledgeGraphToForceGraph(template: IStructureGraphTemplate,): IArtifactGraph {- const entities: IArtifactGraphEntity[] = template.entities.map((entity) => {+ const entities: IArtifactGraphEntity[] = (template.entities ?? []).map((entity) => {const normalized = normalizeEntity(entity);return {slug: normalized.id,@@ -189,7 +189,7 @@ export function adaptKnowledgeGraphToForceGraph(return {entities,- relations: template.relations+ relations: (template.relations ?? []).filter((relation) =>// Only keep relations whose source and target entities both exist in the graph.@@ -284,7 +284,7 @@ export function adaptTimelineToX6Data(template: IStructureGraphTemplate): {};});- const edges = template.relations+ const edges = (template.relations ?? []).filter((relation) => entityIds.has(relation.from) && entityIds.has(relation.to),)