RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: align graph keyword search with Python (#19778)
buua436df016eb5ea51
diff --git a/internal/handler/dataset_artifact.go b/internal/handler/dataset_artifact.goindex 9e0d02bb96..16ea8d1d9a 100644--- a/internal/handler/dataset_artifact.go+++ b/internal/handler/dataset_artifact.go@@ -255,13 +255,25 @@ func (h *DatasetArtifactHandler) GetArtifactAlteration(c *gin.Context) {}// GetArtifactGraph handles GET /artifacts/graph — wiki entity/relation graph.+// keywords seeds the graph with BM25 matches; top_n/topN controls the entity cap.func (h *DatasetArtifactHandler) GetArtifactGraph(c *gin.Context) {_, tenantID, _ := h.datasetOwner(c, c.Param("dataset_id"))if tenantID == "" {return}datasetID := c.Param("dataset_id")- graph, err := h.svc.GetWikiGraph(c.Request.Context(), tenantID, datasetID)+ keywords := strings.TrimSpace(c.Query("keywords"))+ topNValue := c.Query("top_n")+ if topNValue == "" {+ topNValue = c.Query("topN")+ }+ var topN *int+ if topNValue != "" {+ if value, parseErr := strconv.Atoi(topNValue); parseErr == nil {+ topN = &value+ }+ }+ graph, err := h.svc.GetWikiGraph(c.Request.Context(), tenantID, datasetID, keywords, topN)if err != nil {common.ErrorWithCode(c, common.CodeDataError, err.Error())returndiff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.goindex 8b53367891..ce0054b767 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go@@ -714,12 +714,108 @@ func (p *wikiPipeline) mapBatch(batch []common.Chunk) (wikiExtract, error) {}func (p *wikiPipeline) runPlan() (wikiPlan, error) {+ if p.wikiMode() == "entity" {+ return p.runEntityPlan()+ }if p.wikiMode() == "topic" && p.deps.Embed != nil {return p.runTopicPlan()}return p.runLegacyPlan()}+// runEntityPlan builds one page for every reduced entity or concept. Entity+// mode already has a canonical identity from MAP/REDUCE, so a PLAN LLM call+// would only repeat fields that the compiler can derive deterministically.+func (p *wikiPipeline) runEntityPlan() (wikiPlan, error) {+ if err := p.ctx.Err(); err != nil {+ return wikiPlan{}, err+ }++ plan := wikiPlan{Pages: buildWikiEntityPlanPages(p.reduced)}+ plan.Pages = assembleWikiPlanRelatedPages(plan.Pages, p.reduced.Relations)+ plan.Pages = normalizeWikiPlanPages(plan.Pages, p.reduced)+ plan.Pages = normalizeWikiPlanPageLinks(plan.Pages)+ reconciled, err := p.reconcilePlan(plan)+ if err != nil {+ return wikiPlan{}, err+ }+ reconciled.Pages = normalizeWikiPlanPageLinks(reconciled.Pages)+ return reconciled, nil+}++// buildWikiEntityPlanPages projects MAP/REDUCE identities into deterministic+// page metadata. Topic assignment uses source-chunk overlap with MAP topics;+// the page itself remains one-to-one with the extracted entity or concept.+func buildWikiEntityPlanPages(reduced wikiExtract) []wikiPlanPage {+ pages := make([]wikiPlanPage, 0, len(reduced.Entities)+len(reduced.Concepts))+ for _, entity := range reduced.Entities {+ name := strings.TrimSpace(entity.Name)+ if name == "" {+ continue+ }+ pages = append(pages, wikiPlanPage{+ Action: "CREATE",+ Slug: entityPageSlug(name, entity.Type),+ Title: name,+ PageType: "entity",+ Topic: selectWikiIdentityTopic(name, "entity", sourceChunkIDsForWikiIdentity(name, entity.SourceChunkIDs, reduced.Claims), reduced.Topics),+ EntityNames: uniqueStrings(append([]string{name}, entity.Aliases...)),+ Priority: len(pages) + 1,+ })+ }+ for _, concept := range reduced.Concepts {+ term := strings.TrimSpace(concept.Term)+ if term == "" {+ continue+ }+ pages = append(pages, wikiPlanPage{+ Action: "CREATE",+ Slug: "concept/" + slugify(term),+ Title: term,+ PageType: "concept",+ Topic: selectWikiIdentityTopic(term, "concept", sourceChunkIDsForWikiIdentity(term, concept.SourceChunkIDs, reduced.Claims), reduced.Topics),+ EntityNames: []string{term},+ Priority: len(pages) + 1,+ })+ }+ return pages+}++func sourceChunkIDsForWikiIdentity(name string, sourceChunkIDs []string, claims []wikiClaim) []string {+ ids := append([]string(nil), sourceChunkIDs...)+ for _, claim := range claims {+ if normKey(claim.Subject) == normKey(name) {+ ids = append(ids, claim.SourceChunkIDs...)+ }+ }+ return uniqueStrings(ids)+}++func selectWikiIdentityTopic(name, pageType string, sourceChunkIDs []string, topics []wikiTopic) string {+ identityChunks := normalizedStringSet(sourceChunkIDs)+ bestPath := ""+ bestOverlap := 0+ for _, topic := range topics {+ path := common.NormalizeWikiTopicPath(topic.Path)+ if path == "" || (pageType != "topic" && normKey(common.WikiTopicLeaf(path)) == normKey(name)) {+ continue+ }+ overlap := normalizedOverlapCount(identityChunks, topic.SourceChunkIDs)+ if overlap == 0 || overlap < bestOverlap {+ continue+ }+ if overlap == bestOverlap && bestPath != "" && normKey(path) >= normKey(bestPath) {+ continue+ }+ bestPath = path+ bestOverlap = overlap+ }+ if bestPath == "" {+ return common.GeneralWikiTopic+ }+ return bestPath+}+func (p *wikiPipeline) runLegacyPlan() (wikiPlan, error) {batches := packWikiPlanBatches(p.reduced, wikiPlanTokenBudget)if len(batches) == 0 {diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget_test.goindex f67d2423ca..d3054f1250 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget_test.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget_test.go@@ -235,7 +235,8 @@ func TestRunPlan_PromptMaxPagesNeverExceedsCap(t *testing.T) {entities = append(entities, wikiEntity{Name: "Ent " + itoa(i) + big})}p := &wikiPipeline{- ctx: context.Background(),+ ctx: context.Background(),+ param: topicModeParam(),deps: common.Deps{ModelContextLen: 1024,Chat: chatFunc(func(_ context.Context, req common.ChatRequest) (*common.ChatResponse, error) {@@ -352,7 +353,8 @@ func TestRunPlan_TruncatesToGlobalHardCap(t *testing.T) {}payload := map[string]any{"pages": pages}p := &wikiPipeline{- ctx: context.Background(),+ ctx: context.Background(),+ param: topicModeParam(),deps: common.Deps{Chat: reconcileChatStub{resp: mustJSON(payload)},},@@ -440,7 +442,8 @@ func TestRunPlan_ParallelBatchesMergeInOrder(t *testing.T) {// batch2 (token budget 3500).big := strings.Repeat("x", 7000)p := &wikiPipeline{- ctx: context.Background(),+ ctx: context.Background(),+ param: topicModeParam(),deps: common.Deps{Chat: batchPlanChatStub{},},@@ -499,7 +502,8 @@ func TestRunPlan_ParallelBatchesFirstError(t *testing.T) {big := strings.Repeat("x", 7000)boom := errors.New("planning failed")p := &wikiPipeline{- ctx: context.Background(),+ ctx: context.Background(),+ param: topicModeParam(),deps: common.Deps{Chat: failPlanChatStub{err: boom},},@@ -548,7 +552,8 @@ func TestRunPlan_CancelledCtxAborts(t *testing.T) {ctx, cancel := context.WithCancel(context.Background())cancel()p := &wikiPipeline{- ctx: ctx,+ ctx: ctx,+ param: topicModeParam(),deps: common.Deps{Chat: batchPlanChatStub{},},@@ -565,3 +570,8 @@ func TestRunPlan_CancelledCtxAborts(t *testing.T) {t.Fatalf("runPlan err = nil, want context cancelled")}}++func topicModeParam() common.Param {+ enabled := true+ return common.Param{Plan: &enabled}+}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.goindex 1430c5abc5..d1a0ea5c27 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go@@ -34,6 +34,69 @@ func TestReduceExtracts_MergesProvenance(t *testing.T) {}}+func TestRunPlan_EntityModeBuildsPagesFromReducedExtract(t *testing.T) {+ p := &wikiPipeline{+ ctx: context.Background(),+ reduced: wikiExtract{+ Entities: []wikiEntity{{+ Name: "刘备",+ Type: "person",+ Aliases: []string{"玄德"},+ SourceChunkIDs: []string{"c1"},+ }},+ Concepts: []wikiConcept{{+ Term: "三国演义",+ Definition: "历史小说",+ SourceChunkIDs: []string{"c2"},+ }},+ Claims: []wikiClaim{+ {Subject: "刘备", SourceChunkIDs: []string{"c1"}},+ {Subject: "三国演义", SourceChunkIDs: []string{"c2"}},+ },+ Relations: []wikiRelation{{From: "刘备", To: "三国演义", Type: "appears_in"}},+ Topics: []wikiTopic{+ {Path: "历史/人物", SourceChunkIDs: []string{"c1"}},+ {Path: "历史/作品", SourceChunkIDs: []string{"c2"}},+ },+ },+ docID: "doc-1",+ }++ plan, err := p.runPlan()+ if err != nil {+ t.Fatalf("runPlan err = %v", err)+ }+ if len(plan.Pages) != 2 {+ t.Fatalf("plan pages = %d, want 2", len(plan.Pages))+ }+ bySlug := make(map[string]wikiPlanPage, len(plan.Pages))+ for _, page := range plan.Pages {+ bySlug[page.Slug] = page+ }++ entityPage, ok := bySlug["entity/person/刘备"]+ if !ok {+ t.Fatalf("entity page missing: %#v", plan.Pages)+ }+ if entityPage.Title != "刘备" || entityPage.PageType != "entity" || entityPage.Topic != "历史/人物" {+ t.Fatalf("entity page metadata = %#v", entityPage)+ }+ if !slices.Equal(entityPage.EntityNames, []string{"刘备", "玄德"}) {+ t.Fatalf("entity page names = %#v", entityPage.EntityNames)+ }+ if !slices.Equal(entityPage.RelatedKB, []string{"concept/三国演义"}) {+ t.Fatalf("entity related pages = %#v", entityPage.RelatedKB)+ }++ conceptPage, ok := bySlug["concept/三国演义"]+ if !ok {+ t.Fatalf("concept page missing: %#v", plan.Pages)+ }+ if conceptPage.Topic != "历史/作品" || !slices.Equal(conceptPage.RelatedKB, []string{"entity/person/刘备"}) {+ t.Fatalf("concept page metadata = %#v", conceptPage)+ }+}+func TestParseWikiExtractNormalizesTopicPathAndProvenance(t *testing.T) {extract := parseWikiExtract(map[string]any{"topics": []any{map[string]any{diff --git a/internal/ingestion/knowledge_compile/wiki_graph_test.go b/internal/ingestion/knowledge_compile/wiki_graph_test.goindex a0e39e3c82..deb3c537e8 100644--- a/internal/ingestion/knowledge_compile/wiki_graph_test.go+++ b/internal/ingestion/knowledge_compile/wiki_graph_test.go@@ -18,6 +18,7 @@ package knowledge_compileimport ("context""encoding/json"+ "strings""testing")@@ -51,6 +52,27 @@ func findRelation(rows []map[string]interface{}, from, to string) map[string]intconst kbForTest = "kb1"+func TestProjectWikiGraphRowsWritesSearchContent(t *testing.T) {+ w := engineWriter{}+ p := page("entity", "刘备")+ p.Summary = "三国演义中的人物"++ rows, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, []wikiPageProjection{p})+ if err != nil {+ t.Fatalf("project: %v", err)+ }+ entity := findEntity(rows, "entity/刘备")+ if entity == nil {+ t.Fatal("missing entity")+ }+ for _, field := range []string{"title_tks", "title_sm_tks", "content_ltks", "content_sm_ltks"} {+ value, ok := entity[field].(string)+ if !ok || strings.TrimSpace(value) == "" {+ t.Fatalf("entity[%q] = %#v, want non-empty search content", field, entity[field])+ }+ }+}+// TestProjectWikiGraphRowsCrossBatchEdgeSurvives verifies a relation whose two// endpoints land in different "batches" (here modeled as two pages) is still// materialized: the projection is global over the full page set, so edges neverdiff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.goindex de2154d7c1..d7f1aa3084 100644--- a/internal/ingestion/knowledge_compile/writer.go+++ b/internal/ingestion/knowledge_compile/writer.go@@ -34,6 +34,7 @@ import ("ragflow/internal/entity"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common""ragflow/internal/service/file"+ "ragflow/internal/tokenizer""ragflow/internal/utility")@@ -1385,6 +1386,35 @@ func wikiGraphBareKey(slug string) string {return strings.TrimSpace(s)}+// tokenizeWikiGraphContent prepares the lexical fields used by wiki graph+// keyword search. Keep the raw text as a fallback so graph rows remain+// searchable when the tokenizer pool is unavailable during startup or tests.+func tokenizeWikiGraphContent(title, slug, description string) (string, string, string, string) {+ title = strings.TrimSpace(title)+ content := strings.TrimSpace(strings.Join([]string{slug, description}, " "))+ if title == "" && content == "" {+ return "", "", "", ""+ }++ titleLTKS, err := tokenizer.Tokenize(title)+ if err != nil || titleLTKS == "" {+ titleLTKS = title+ }+ titleSMLTKS, err := tokenizer.FineGrainedTokenize(titleLTKS)+ if err != nil || titleSMLTKS == "" {+ titleSMLTKS = titleLTKS+ }+ contentLTKS, err := tokenizer.Tokenize(content)+ if err != nil || contentLTKS == "" {+ contentLTKS = content+ }+ contentSMLTKS, err := tokenizer.FineGrainedTokenize(contentLTKS)+ if err != nil || contentSMLTKS == "" {+ contentSMLTKS = contentLTKS+ }+ return titleLTKS, titleSMLTKS, contentLTKS, contentSMLTKS+}+// wikiPageProjection is the subset of a merged wiki_page row that the graph// projection needs. It is reconstructed from the stored display columns (the// same fields GetWikiGraph reads back), not from the JSON payload.@@ -1799,6 +1829,7 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string,if err != nil {return nil, err}+ titleLTKS, titleSMLTKS, contentLTKS, contentSMLTKS := tokenizeWikiGraphContent(p.Title, p.Slug, p.Summary)rows = append(rows, map[string]interface{}{"id": entityID,"doc_id": kb,@@ -1815,6 +1846,10 @@ func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string,"weight_int": weight,"source_chunk_ids": p.SourceChunkIDs,"source_doc_ids": capSourceDocs(p.SourceDocIDs),+ "title_tks": titleLTKS,+ "title_sm_tks": titleSMLTKS,+ "content_ltks": contentLTKS,+ "content_sm_ltks": contentSMLTKS,"content_with_weight": string(content),})diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.goindex 12cb35c739..ef20e5161b 100644--- a/internal/service/dataset_artifact_service.go+++ b/internal/service/dataset_artifact_service.go@@ -42,6 +42,8 @@ const (CompileKwdSkill = "skill"CompileKwdSkillAll = "skill_all"CompileKwdDatasetNav = "dataset_nav"+ defaultWikiGraphTopN = 512+ maxWikiGraphTopN = 1024)// DatasetArtifactService reads knowledge-compilation artifacts (wiki pages,@@ -61,6 +63,10 @@ func wikiIndexName(tenantID string) string {// searchCompiled runs a filtered search over the tenant's document index for// the given dataset, returning the matching chunks.func (s *DatasetArtifactService) searchCompiled(ctx context.Context, tenantID, datasetID string, filter map[string]interface{}, selectFields []string, offset, limit int, orderBy *types.OrderByExpr) ([]map[string]interface{}, int64, error) {+ return s.searchCompiledWithMatch(ctx, tenantID, datasetID, filter, selectFields, offset, limit, orderBy, nil)+}++func (s *DatasetArtifactService) searchCompiledWithMatch(ctx context.Context, tenantID, datasetID string, filter map[string]interface{}, selectFields []string, offset, limit int, orderBy *types.OrderByExpr, matchExprs []interface{}) ([]map[string]interface{}, int64, error) {docEngine := engine.Get()if docEngine == nil {return nil, 0, fmt.Errorf("document engine is not initialized")@@ -80,6 +86,7 @@ func (s *DatasetArtifactService) searchCompiled(ctx context.Context, tenantID, dSelectFields: selectFields,Filter: merged,OrderBy: orderBy,+ MatchExprs: matchExprs,})if err != nil {return nil, 0, err@@ -544,8 +551,12 @@ var wikiTopicCollator = collate.New(language.Chinese)// WikiGraph is the entity/relation graph for a dataset's wiki artifacts.type WikiGraph struct {- Entities []WikiGraphEntity `json:"entities"`- Relations []WikiGraphRelation `json:"relations"`+ Entities []WikiGraphEntity `json:"entities"`+ Relations []WikiGraphRelation `json:"relations"`+ TotalEntities int `json:"total_entities"`+ TotalRelations int `json:"total_relations"`+ ReturnedEntities int `json:"returned_entities"`+ ReturnedRelations int `json:"returned_relations"`}// WikiGraphEntity is a single wiki graph entity.@@ -565,12 +576,47 @@ type WikiGraphRelation struct {To string `json:"to"`}-// GetWikiGraph returns the wiki entity/relation graph for a dataset.-func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, datasetID string) (*WikiGraph, error) {- entityChunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,- map[string]interface{}{"compile_kwd": []string{CompileKwdWikiEntity}, "available_int": 1},- []string{"slug_kwd", "title_kwd", "aliases_kwd", "description_with_weight", "entity_type_kwd", "weight_int", "source_chunk_ids"},- 0, 1000, (&types.OrderByExpr{}).Desc("weight_int"))+// GetWikiGraph returns an incremental wiki graph slice for a dataset. keywords+// seeds the overview with BM25 entity matches; topN overrides the entity cap.+func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, datasetID, keywords string, topN *int) (*WikiGraph, error) {+ entityFilter := map[string]interface{}{"compile_kwd": []string{CompileKwdWikiEntity}, "available_int": 1}+ relationFilter := map[string]interface{}{"compile_kwd": []string{CompileKwdWikiRelation}, "available_int": 1}+ _, totalEntities, err := s.searchCompiled(ctx, tenantID, datasetID, entityFilter, []string{"id"}, 0, 1, nil)+ if err != nil {+ return nil, err+ }+ _, totalRelations, err := s.searchCompiled(ctx, tenantID, datasetID, relationFilter, []string{"id"}, 0, 1, nil)+ if err != nil {+ return nil, err+ }++ limit := defaultWikiGraphTopN+ if topN != nil {+ limit = *topN+ if limit < 1 {+ limit = 1+ }+ if limit > maxWikiGraphTopN {+ limit = maxWikiGraphTopN+ }+ }++ keywords = strings.TrimSpace(keywords)+ var matchExprs []interface{}+ var orderBy *types.OrderByExpr+ if keywords != "" {+ matchExprs = []interface{}{&types.MatchTextExpr{+ Fields: []string{"title_tks^10", "title_sm_tks^5", "content_ltks^2", "content_sm_ltks"},+ MatchingText: keywords,+ TopN: limit,+ ExtraOptions: map[string]interface{}{"original_query": keywords},+ }}+ } else {+ orderBy = (&types.OrderByExpr{}).Desc("weight_int")+ }+ entityChunks, _, err := s.searchCompiledWithMatch(ctx, tenantID, datasetID, entityFilter,+ []string{"id", "slug_kwd", "title_kwd", "aliases_kwd", "description_with_weight", "entity_type_kwd", "weight_int", "source_chunk_ids"},+ 0, limit, orderBy, matchExprs)if err != nil {return nil, err}@@ -578,7 +624,12 @@ func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, datfor _, c := range entityChunks {fromKwds = append(fromKwds, firstStringValue(c["slug_kwd"]))}- graph := &WikiGraph{Entities: []WikiGraphEntity{}, Relations: []WikiGraphRelation{}}+ graph := &WikiGraph{+ Entities: []WikiGraphEntity{},+ Relations: []WikiGraphRelation{},+ TotalEntities: int(totalEntities),+ TotalRelations: int(totalRelations),+ }for _, c := range entityChunks {w := intValue(c["weight_int"])// Match the GetWikiPage / ListWikiPages contract: expose the bare slug@@ -605,8 +656,9 @@ func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, dat})}if len(fromKwds) > 0 {+ relationFilter["from_kwd"] = fromKwdsrelChunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,- map[string]interface{}{"compile_kwd": []string{CompileKwdWikiRelation}, "available_int": 1, "from_kwd": fromKwds},+ relationFilter,[]string{"from_kwd", "to_kwd"}, 0, 10000, nil)if err != nil {return nil, err@@ -621,6 +673,8 @@ func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, dat})}}+ graph.ReturnedEntities = len(graph.Entities)+ graph.ReturnedRelations = len(graph.Relations)return graph, nil}diff --git a/internal/service/dataset_structure_graph.go b/internal/service/dataset_structure_graph.goindex aa58a3639f..dd686cfb96 100644--- a/internal/service/dataset_structure_graph.go+++ b/internal/service/dataset_structure_graph.go@@ -25,7 +25,7 @@ import (const (graphFullThreshold = 1024 // below this combined count, return all rowsgraphTopEntities = 256 // seed set A size for large buckets- graphKeywordCandidates = 16 // keyword/KNN candidate rows+ graphKeywordCandidates = 16 // keyword candidate rowsgraphExpansionCap = 4096 // hub-node expansion cap)@@ -684,65 +684,6 @@ func structureKindForBucket(kind string) string {return strings.ReplaceAll(k, "-", "_")}-// loadDocumentTemplateMeta resolves the doc's configured template group into-// configured_ids + template_meta maps, mirroring the Python resolution.-func (s *DatasetArtifactService) loadDocumentTemplateMeta(ctx context.Context, tenantID, documentID string) ([]string, map[string]map[string]interface{}, error) {- empty := map[string]map[string]interface{}{}- doc, err := dao.NewDocumentDAO().GetByID(ctx, dao.DB, documentID)- if err != nil || doc == nil {- return nil, empty, err- }- groupID := ""- if doc.ParserConfig != nil {- if gid, ok := doc.ParserConfig["compilation_template_group_id"].(string); ok && gid != "" {- groupID = gid- } else if ext, ok := doc.ParserConfig["ext"].(map[string]interface{}); ok {- if gid, ok := ext["compilation_template_group_id"].(string); ok && gid != "" {- groupID = gid- }- }- }- if groupID == "" {- return nil, empty, nil- }- group, err := NewCompilationTemplateGroupService().GetSaved(ctx, tenantID, groupID)- if err != nil {- return nil, empty, err- }- configuredIDs := []string{}- meta := map[string]map[string]interface{}{}- seen := map[string]bool{}- if group != nil {- for _, t := range group.Templates {- tid := t.ID- if tid == "" || seen[tid] {- continue- }- rawKind := ""- if t.Config != nil {- if k, ok := t.Config["kind"].(string); ok {- rawKind = k- }- }- if rawKind == "" {- rawKind = t.Kind- }- kindNorm := compilationTemplateKind(rawKind)- if kindNorm == "wiki" {- continue- }- seen[tid] = true- configuredIDs = append(configuredIDs, tid)- meta[tid] = map[string]interface{}{- "template_id": tid,- "template_name": t.Name,- "kind": rawKind,- }- }- }- return configuredIDs, meta, nil-}-// DocumentStructureGraphInput is the parsed request for GetDocumentGraph.type DocumentStructureGraphInput struct {TenantID string@@ -755,10 +696,8 @@ type DocumentStructureGraphInput struct {// GetDocumentGraph returns the per-template structure graph of a document,// mirroring Python get_document_structure_graph (normal + keywords modes).func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, in DocumentStructureGraphInput) (*DocumentStructureGraphResponse, error) {- configuredIDs, templateMeta, err := s.loadDocumentTemplateMeta(ctx, in.TenantID, in.DocumentID)- if err != nil {- return nil, err- }+ templateIDs := []string{}+ templateMeta := map[string]map[string]interface{}{}resp := &DocumentStructureGraphResponse{Templates: []DocumentStructureGraphTemplate{}}entityCountFilter := map[string]interface{}{@@ -771,13 +710,17 @@ func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, in Docume}resp.TotalEntities = int(entityTotal)- // keywords mode: name matching/KNN → matched entities' subgraph.+ // keywords mode: name/BM25 matching → matched entities' subgraph.if in.Keywords != "" {bucketMeta, entities, relations, err := s.keywordSubgraph(ctx, in.TenantID, in.DatasetID, in.DocumentID, in.Keywords, templateMeta, nil)if err != nil {return nil, err}if bucketMeta == nil || (len(entities) == 0 && len(relations) == 0) {+ if err := s.discoverDocumentGraphTemplateMeta(ctx, in.TenantID, in.DatasetID, in.DocumentID, &templateIDs, templateMeta); err != nil {+ return nil, err+ }+ resp.Templates = buildDocumentGraphTemplateShells(templateIDs, templateMeta)return resp, nil}resp.Templates = append(resp.Templates, DocumentStructureGraphTemplate{@@ -817,10 +760,9 @@ func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, in DocumebucketScopes := map[string]map[string]interface{}{}templateDAO := dao.NewCompilationTemplateDAO()for _, row := range metaRows {- // Pipeline-produced rows may contain a template id even when the- // document parser config does not contain the corresponding template- // group. Resolve that id directly so template_name is the user-visible- // name instead of the opaque id.+ // Resolve template metadata from the persisted graph row. The template+ // name is loaded only as display metadata; the row remains authoritative+ // for whether the template exists in this document.templateID := rowTemplateID(row)if templateID != "" {if _, known := templateMeta[templateID]; !known {@@ -836,6 +778,9 @@ func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, in Docume}meta, scope := resolveGraphBucket(row, templateMeta, in.DocumentID)bid := graphStr(meta["template_id"])+ if bid != "" && !containsStr(templateIDs, bid) {+ templateIDs = append(templateIDs, bid)+ }if _, ok := bucketMetas[bid]; !ok {bucketMetas[bid] = metabucketScopes[bid] = scope@@ -859,9 +804,9 @@ func (s *DatasetArtifactService) GetDocumentGraph(ctx context.Context, in Docume}}- // Order: configured templates first, then discovered.+ // Order: discovered templates first, then any buckets not seen in the scan.orderedIDs := []string{}- for _, tid := range configuredIDs {+ for _, tid := range templateIDs {if _, ok := grouped[tid]; ok && !containsStr(orderedIDs, tid) {orderedIDs = append(orderedIDs, tid)}@@ -889,6 +834,93 @@ func containsStr(list []string, v string) bool {return false}+func buildDocumentGraphTemplateShells(templateIDs []string, templateMeta map[string]map[string]interface{}) []DocumentStructureGraphTemplate {+ templates := make([]DocumentStructureGraphTemplate, 0, len(templateIDs))+ for _, templateID := range templateIDs {+ meta := templateMeta[templateID]+ if meta == nil {+ continue+ }+ templateName := graphStr(meta["template_name"])+ if templateName == "" {+ templateName = templateID+ }+ templates = append(templates, DocumentStructureGraphTemplate{+ TemplateID: templateID,+ TemplateName: templateName,+ Kind: graphStr(meta["kind"]),+ Entities: []StructureGraphNode{},+ Relations: []StructureGraphRelation{},+ })+ }+ return templates+}++func (s *DatasetArtifactService) discoverDocumentGraphTemplateMeta(ctx context.Context, tenantID, datasetID, documentID string, templateIDs *[]string, templateMeta map[string]map[string]interface{}) error {+ if templateIDs == nil || templateMeta == nil {+ return nil+ }+ seen := make(map[string]bool, len(*templateIDs))+ for _, templateID := range *templateIDs {+ seen[templateID] = true+ }+ templateDAO := dao.NewCompilationTemplateDAO()+ fields := []string{"id", "compile_kwd", "compilation_template_ids", "compilation_template_kind_kwd"}+ filter := map[string]interface{}{+ "doc_id": []string{documentID},+ "knowledge_graph_kwd": []string{"entity", "relation"},+ }+ for offset := 0; ; offset += 1000 {+ rows, total, err := graphRowSearch(ctx, tenantID, datasetID, fields, filter, nil, offset, 1000, nil)+ if err != nil {+ return err+ }+ for _, row := range rows {+ templateID := rowTemplateID(row)+ kind := firstStringValue(row["compilation_template_kind_kwd"])+ if kind == "" {+ kind = firstStringValue(row["compile_kwd"])+ }+ if templateID == "" {+ compileKind := firstStringValue(row["compile_kwd"])+ if compileKind == "" {+ continue+ }+ templateID = "legacy:" + compileKind+ }+ if seen[templateID] {+ continue+ }++ templateName := templateID+ if !strings.HasPrefix(templateID, "legacy:") {+ if template, loadErr := templateDAO.GetTemplate(ctx, dao.DB, tenantID, templateID); loadErr == nil && template != nil {+ templateName = template.Name+ if kind == "" {+ kind = template.Kind+ }+ }+ } else {+ templateName = "Legacy (" + strings.TrimPrefix(templateID, "legacy:") + ")"+ }+ if templateName == "" {+ templateName = templateID+ }+ templateMeta[templateID] = map[string]interface{}{+ "template_id": templateID,+ "template_name": templateName,+ "kind": kind,+ }+ *templateIDs = append(*templateIDs, templateID)+ seen[templateID] = true+ }+ if int64(offset+1000) >= total || len(rows) == 0 {+ break+ }+ }+ return nil+}+// DatasetStructureGraphInput is the parsed request for the dataset-scope// structure graph endpoint (GET /datasets/:id/artifacts/structure).// Kind is REQUIRED (mirrors Python dataset_api.py:715): missing or invalid →@@ -922,8 +954,7 @@ type DatasetStructureGraphResponse struct {// way). It collects distinct template ids, then reads each template's dataset// entity/relation rows via buildBucket. It does NOT read kg_build_meta (write/// delete-side only). When Keywords is set, it returns the keyword-matched-// entity subgraph using the same BM25/KNN and relation expansion path as the-// document-level endpoint.+// entity subgraph using the name/BM25 and relation expansion path.func (s *DatasetArtifactService) GetDatasetStructure(ctx context.Context, in DatasetStructureGraphInput) (*DatasetStructureGraphResponse, error) {resolved := resolveDatasetStructureKind(in.Kind)if resolved == "" {@@ -1189,7 +1220,8 @@ func nameMatchesQuery(node StructureGraphNode, query string) bool {return true}-// keywordSubgraph mirrors sgc.keyword_subgraph (BM25 + KNN fallback + ancestor walk).+// keywordSubgraph mirrors sgc.keyword_subgraph (lexical lookup and ancestor+// walk).func (s *DatasetArtifactService) keywordSubgraph(ctx context.Context, tenantID, datasetID, documentID, keywords string, templateMeta map[string]map[string]interface{}, baseEntityCond map[string]interface{}) (map[string]interface{}, []StructureGraphNode, []StructureGraphRelation, error) {if baseEntityCond == nil {baseEntityCond = map[string]interface{}{@@ -1208,10 +1240,8 @@ func (s *DatasetArtifactService) keywordSubgraph(ctx context.Context, tenantID,resolveBucket := func(row map[string]interface{}) (map[string]interface{}, map[string]interface{}) {templateID := rowTemplateID(row)if templateID != "" && !templateMetaLoaded[templateID] {- // Pipeline-produced rows can carry a template id even when the- // document parser config does not contain the corresponding group.- // Resolve it here as well as in the normal path so keyword searches- // expose the user-visible template name instead of the opaque id.+ // Resolve the display metadata from the persisted template id so+ // keyword searches expose the user-visible name instead of the id.templateMetaLoaded[templateID] = trueif template, err := templateDAO.GetTemplate(ctx, dao.DB, tenantID, templateID); err == nil && template != nil {templateMeta[templateID] = map[string]interface{}{@@ -1323,26 +1353,6 @@ func (s *DatasetArtifactService) keywordSubgraph(ctx context.Context, tenantID,candidates = detailCandidates}- // Semantic fallback via KNN.- if len(candidates) == 0 {- vec, err := embedQuery(ctx, tenantID, datasetID, keywords)- if err != nil || len(vec) == 0 {- return nil, nil, nil, nil- }- denseExpr := &types.MatchDenseExpr{- VectorColumnName: fmt.Sprintf("q_%d_vec", len(vec)),- EmbeddingData: vec,- EmbeddingDataType: "float",- DistanceType: "cosine",- TopN: graphKeywordCandidates,- ExtraOptions: map[string]interface{}{"similarity": 0.3},- }- topMap, _, err := graphRowSearch(ctx, tenantID, datasetID, topFields, baseEntityCond, nil, 0, graphKeywordCandidates, []interface{}{denseExpr})- if err != nil {- return nil, nil, nil, err- }- candidates = validTop(topMap)- }if len(candidates) == 0 {return nil, nil, nil, nil}@@ -1498,19 +1508,3 @@ func (s *DatasetArtifactService) keywordSubgraph(ctx context.Context, tenantID,}return bucketMeta, entities, normalizeRelationEndpoints(entities, relations), nil}--// embedQuery embeds a query via the nav embedder (best-effort). The embedder-// resolves the tenant's embedding model on demand; an empty result (or error) is-// treated as "no semantic candidates", matching Python's fallback behavior.-func embedQuery(ctx context.Context, tenantID, datasetID, query string) ([]float64, error) {- embedder := NewNavEmbedder(NewModelProviderService(), "")- vecs, err := embedder.Encode(ctx, tenantID, []string{query})- if err != nil || len(vecs) == 0 || len(vecs[0]) == 0 {- return nil, fmt.Errorf("embed query failed")- }- f := make([]float64, len(vecs[0]))- for i, v := range vecs[0] {- f[i] = float64(v)- }- return f, nil-}diff --git a/internal/service/dataset_structure_graph_test.go b/internal/service/dataset_structure_graph_test.goindex 0ed32ccbe3..07a3379e55 100644--- a/internal/service/dataset_structure_graph_test.go+++ b/internal/service/dataset_structure_graph_test.go@@ -126,3 +126,18 @@ func TestResolveGraphBucket_UsesRawStructureMetadata(t *testing.T) {t.Fatalf("scope = %#v, want template-scoped raw-row filter", scope)}}++func TestBuildDocumentGraphTemplateShells_PreservesTemplateMetadata(t *testing.T) {+ templates := buildDocumentGraphTemplateShells([]string{"tree-1"}, map[string]map[string]interface{}{+ "tree-1": {"template_name": "Tree", "kind": "tree"},+ })+ if len(templates) != 1 {+ t.Fatalf("got %d templates, want 1", len(templates))+ }+ if templates[0].TemplateID != "tree-1" || templates[0].TemplateName != "Tree" || templates[0].Kind != "tree" {+ t.Fatalf("template = %#v, want metadata for tree-1", templates[0])+ }+ if templates[0].Entities == nil || templates[0].Relations == nil {+ t.Fatal("empty template collections must be non-nil")+ }+}