Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(knowledge_compiler): derive variant from template kind via single template id (#17630)

Zhichang Yu29287ef74b1d
diff --git a/.gitignore b/.gitignore
index 6be65faf9d..1d3ab6baa0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -245,6 +245,7 @@ bin/*
# Local agent tooling state (per-developer; not for commit)
.omc/
.marscode/
+.omx/
# Parser test fixtures and python tools
internal/deepdoc/parser/pdf/testdata/
diff --git a/agent/templates/compiler.json b/agent/templates/compiler.json
new file mode 100644
index 0000000000..f38526883c
--- /dev/null
+++ b/agent/templates/compiler.json
@@ -0,0 +1,443 @@
+{
+ "components": {
+ "Compiler:CurvySunsLove": {
+ "downstream": [],
+ "obj": {
+ "component_name": "Compiler",
+ "params": {
+ "compilation_template_group_id": "c3aa748c8b2111f191f3047c16ec874f",
+ "outputs": {
+ "chunks": {
+ "type": "Array<Object>",
+ "value": []
+ }
+ }
+ }
+ },
+ "upstream": [
+ "TokenChunker:LovelyPillowsOpen"
+ ]
+ },
+ "File": {
+ "downstream": [
+ "Parser:HipSignsRhyme"
+ ],
+ "obj": {
+ "component_name": "File",
+ "params": {}
+ },
+ "upstream": []
+ },
+ "Parser:HipSignsRhyme": {
+ "downstream": [
+ "TokenChunker:LovelyPillowsOpen"
+ ],
+ "obj": {
+ "component_name": "Parser",
+ "params": {
+ "outputs": {
+ "html": {
+ "type": "string",
+ "value": ""
+ },
+ "json": {
+ "type": "Array<object>",
+ "value": []
+ },
+ "markdown": {
+ "type": "string",
+ "value": ""
+ },
+ "text": {
+ "type": "string",
+ "value": ""
+ }
+ },
+ "setups": {
+ "doc": {
+ "flatten_media_to_text": false,
+ "order_index": 7,
+ "output_format": "json",
+ "remove_header_footer": false,
+ "suffix": [
+ "doc"
+ ],
+ "vlm": {}
+ },
+ "docx": {
+ "flatten_media_to_text": false,
+ "order_index": 8,
+ "output_format": "json",
+ "remove_header_footer": false,
+ "suffix": [
+ "docx"
+ ],
+ "vlm": {}
+ },
+ "email": {
+ "fields": [
+ "from",
+ "to",
+ "cc",
+ "bcc",
+ "date",
+ "subject",
+ "body",
+ "attachments"
+ ],
+ "order_index": 3,
+ "output_format": "text",
+ "suffix": [
+ "eml",
+ "msg"
+ ]
+ },
+ "html": {
+ "order_index": 6,
+ "output_format": "json",
+ "remove_header_footer": false,
+ "suffix": [
+ "htm",
+ "html"
+ ]
+ },
+ "image": {
+ "order_index": 2,
+ "output_format": "text",
+ "parse_method": "ocr",
+ "suffix": [
+ "jpg",
+ "jpeg",
+ "png",
+ "gif"
+ ]
+ },
+ "markdown": {
+ "flatten_media_to_text": false,
+ "order_index": 4,
+ "output_format": "json",
+ "suffix": [
+ "md",
+ "markdown",
+ "mdx"
+ ],
+ "vlm": {}
+ },
+ "pdf": {
+ "flatten_media_to_text": false,
+ "order_index": 0,
+ "output_format": "json",
+ "parse_method": "DeepDOC",
+ "remove_header_footer": false,
+ "suffix": [
+ "pdf"
+ ],
+ "vlm": {}
+ },
+ "slides": {
+ "order_index": 9,
+ "output_format": "json",
+ "parse_method": "DeepDOC",
+ "suffix": [
+ "pptx",
+ "ppt"
+ ]
+ },
+ "spreadsheet": {
+ "flatten_media_to_text": false,
+ "order_index": 1,
+ "output_format": "html",
+ "parse_method": "DeepDOC",
+ "suffix": [
+ "xls",
+ "xlsx",
+ "csv"
+ ],
+ "vlm": {}
+ },
+ "text&code": {
+ "order_index": 5,
+ "output_format": "json",
+ "suffix": [
+ "txt",
+ "py",
+ "js",
+ "java",
+ "c",
+ "cpp",
+ "h",
+ "php",
+ "go",
+ "ts",
+ "sh",
+ "cs",
+ "kt",
+ "sql"
+ ]
+ }
+ }
+ }
+ },
+ "upstream": [
+ "File"
+ ]
+ },
+ "TokenChunker:LovelyPillowsOpen": {
+ "downstream": [
+ "Compiler:CurvySunsLove"
+ ],
+ "obj": {
+ "component_name": "TokenChunker",
+ "params": {
+ "children_delimiters": [],
+ "chunk_token_size": 512,
+ "delimiter_mode": "token_size",
+ "delimiters": [],
+ "image_context_size": 0,
+ "outputs": {
+ "chunks": {
+ "type": "Array<Object>",
+ "value": []
+ }
+ },
+ "overlapped_percent": 0,
+ "table_context_size": 0
+ }
+ },
+ "upstream": [
+ "Parser:HipSignsRhyme"
+ ]
+ }
+ },
+ "globals": {
+ "sys.history": []
+ },
+ "graph": {
+ "edges": [
+ {
+ "id": "xy-edge__Filestart-Parser:HipSignsRhymeend",
+ "source": "File",
+ "sourceHandle": "start",
+ "target": "Parser:HipSignsRhyme",
+ "targetHandle": "end"
+ },
+ {
+ "data": {
+ "isHovered": false
+ },
+ "id": "xy-edge__Parser:HipSignsRhymestart-TokenChunker:LovelyPillowsOpenend",
+ "source": "Parser:HipSignsRhyme",
+ "sourceHandle": "start",
+ "target": "TokenChunker:LovelyPillowsOpen",
+ "targetHandle": "end"
+ },
+ {
+ "id": "xy-edge__TokenChunker:LovelyPillowsOpenstart-Compiler:CurvySunsLoveend",
+ "source": "TokenChunker:LovelyPillowsOpen",
+ "sourceHandle": "start",
+ "target": "Compiler:CurvySunsLove",
+ "targetHandle": "end"
+ }
+ ],
+ "nodes": [
+ {
+ "data": {
+ "label": "File",
+ "name": "File"
+ },
+ "id": "File",
+ "measured": {
+ "height": 49,
+ "width": 200
+ },
+ "position": {
+ "x": 50,
+ "y": 200
+ },
+ "sourcePosition": "left",
+ "targetPosition": "right",
+ "type": "beginNode"
+ },
+ {
+ "data": {
+ "form": {
+ "outputs": {
+ "html": {
+ "type": "string",
+ "value": ""
+ },
+ "json": {
+ "type": "Array<object>",
+ "value": []
+ },
+ "markdown": {
+ "type": "string",
+ "value": ""
+ },
+ "text": {
+ "type": "string",
+ "value": ""
+ }
+ },
+ "setups": [
+ {
+ "fileFormat": "pdf",
+ "flatten_media_to_text": false,
+ "output_format": "json",
+ "parse_method": "DeepDOC",
+ "remove_header_footer": false,
+ "vlm": {}
+ },
+ {
+ "fileFormat": "spreadsheet",
+ "flatten_media_to_text": false,
+ "output_format": "html",
+ "parse_method": "DeepDOC",
+ "vlm": {}
+ },
+ {
+ "fileFormat": "image",
+ "output_format": "text",
+ "parse_method": "ocr"
+ },
+ {
+ "fields": [
+ "from",
+ "to",
+ "cc",
+ "bcc",
+ "date",
+ "subject",
+ "body",
+ "attachments"
+ ],
+ "fileFormat": "email",
+ "output_format": "text"
+ },
+ {
+ "fileFormat": "markdown",
+ "flatten_media_to_text": false,
+ "output_format": "json",
+ "vlm": {}
+ },
+ {
+ "fileFormat": "text&code",
+ "output_format": "json"
+ },
+ {
+ "fileFormat": "html",
+ "output_format": "json",
+ "remove_header_footer": false
+ },
+ {
+ "fileFormat": "doc",
+ "flatten_media_to_text": false,
+ "output_format": "json",
+ "remove_header_footer": false,
+ "vlm": {}
+ },
+ {
+ "fileFormat": "docx",
+ "flatten_media_to_text": false,
+ "output_format": "json",
+ "remove_header_footer": false,
+ "vlm": {}
+ },
+ {
+ "fileFormat": "slides",
+ "output_format": "json",
+ "parse_method": "DeepDOC"
+ }
+ ]
+ },
+ "label": "Parser",
+ "name": "Parser_0"
+ },
+ "dragging": false,
+ "id": "Parser:HipSignsRhyme",
+ "measured": {
+ "height": 197,
+ "width": 200
+ },
+ "position": {
+ "x": 316.99524094206413,
+ "y": 195.39629819663406
+ },
+ "selected": true,
+ "sourcePosition": "right",
+ "targetPosition": "left",
+ "type": "parserNode"
+ },
+ {
+ "data": {
+ "form": {
+ "children_delimiters": [],
+ "chunk_token_size": 512,
+ "delimiter_mode": "token_size",
+ "delimiters": [
+ {
+ "value": "\n"
+ }
+ ],
+ "image_table_context_window": 0,
+ "outputs": {
+ "chunks": {
+ "type": "Array<Object>",
+ "value": []
+ }
+ },
+ "overlapped_percent": 0
+ },
+ "label": "TokenChunker",
+ "name": "Token Chunker_0"
+ },
+ "id": "TokenChunker:LovelyPillowsOpen",
+ "measured": {
+ "height": 73,
+ "width": 200
+ },
+ "position": {
+ "x": 616.9952409420641,
+ "y": 195.39629819663406
+ },
+ "selected": false,
+ "sourcePosition": "right",
+ "targetPosition": "left",
+ "type": "chunkerNode"
+ },
+ {
+ "data": {
+ "form": {
+ "compilation_template_group_id": "c3aa748c8b2111f191f3047c16ec874f",
+ "outputs": {
+ "chunks": {
+ "type": "Array<Object>",
+ "value": []
+ }
+ }
+ },
+ "label": "Compiler",
+ "name": "Compiler_0"
+ },
+ "id": "Compiler:CurvySunsLove",
+ "measured": {
+ "height": 93,
+ "width": 200
+ },
+ "position": {
+ "x": 916.9952409420641,
+ "y": 195.39629819663406
+ },
+ "selected": false,
+ "sourcePosition": "right",
+ "targetPosition": "left",
+ "type": "compilationNode"
+ }
+ ]
+ },
+ "history": [],
+ "messages": [],
+ "path": [],
+ "retrieval": [],
+ "variables": []
+}
diff --git a/internal/dao/compilation_template.go b/internal/dao/compilation_template.go
index 6791c7d002..6db5c3613f 100644
--- a/internal/dao/compilation_template.go
+++ b/internal/dao/compilation_template.go
@@ -49,21 +49,38 @@ func (dao *CompilationTemplateDAO) ResolveGroupTemplateIDs(ctx context.Context,
db := DB.WithContext(ctx)
- // Verify the requested groups exist, are valid, and belong to the tenant.
+ // Verify the requested groups exist and are valid. The built-in group
+ // (compiler.json's default) is a global, tenant-agnostic catalogue, so it
+ // resolves for every tenant; other groups are scoped to the caller's
+ // tenant.
+ isBuiltin := make(map[string]bool, len(groupIDs))
+ for _, gid := range groupIDs {
+ if gid == BuiltinCompilationTemplateGroupID {
+ isBuiltin[gid] = true
+ }
+ }
var validGroups []entity.CompilationTemplateGroup
if err := db.
- Where("id IN ? AND tenant_id = ? AND status = ?", groupIDs, tenantID, string(entity.StatusValid)).
+ Where("id IN ? AND status = ?", groupIDs, string(entity.StatusValid)).
Find(&validGroups).Error; err != nil {
return nil, fmt.Errorf("resolve compilation template groups: %w", err)
}
valid := make(map[string]struct{}, len(validGroups))
+ byID := make(map[string]entity.CompilationTemplateGroup, len(validGroups))
for _, g := range validGroups {
valid[g.ID] = struct{}{}
+ byID[g.ID] = g
}
for _, gid := range groupIDs {
if _, ok := valid[gid]; !ok {
return nil, fmt.Errorf("compilation_template_group %q not found for tenant %q", gid, tenantID)
}
+ // Non-built-in groups must belong to the requesting tenant.
+ if !isBuiltin[gid] {
+ if byID[gid].TenantID != tenantID {
+ return nil, fmt.Errorf("compilation_template_group %q not found for tenant %q", gid, tenantID)
+ }
+ }
}
var templates []entity.CompilationTemplate
@@ -92,3 +109,17 @@ func (dao *CompilationTemplateDAO) ResolveGroupTemplateIDs(ctx context.Context,
}
return out, nil
}
+
+// GetTemplate loads a single, valid compilation template by id for the tenant.
+// It backs the KnowledgeCompiler TemplateResolver: the template's kind selects
+// the Go compile variant (see common.KindToVariant) and its config is the
+// template "content" plumbed to the variant.
+func (dao *CompilationTemplateDAO) GetTemplate(ctx context.Context, tenantID, templateID string) (*entity.CompilationTemplate, error) {
+ var t entity.CompilationTemplate
+ if err := DB.WithContext(ctx).
+ Where("id = ? AND tenant_id = ? AND status = ?", templateID, tenantID, string(entity.StatusValid)).
+ First(&t).Error; err != nil {
+ return nil, fmt.Errorf("load compilation template %q: %w", templateID, err)
+ }
+ return &t, nil
+}
diff --git a/internal/dao/compilation_template_seed.go b/internal/dao/compilation_template_seed.go
new file mode 100644
index 0000000000..653a1f8e31
--- /dev/null
+++ b/internal/dao/compilation_template_seed.go
@@ -0,0 +1,110 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package dao
+
+import (
+ "context"
+ "ragflow/internal/common"
+ "ragflow/internal/entity"
+
+ "go.uber.org/zap"
+ "gorm.io/gorm"
+ "gorm.io/gorm/clause"
+)
+
+// BuiltinCompilationTemplateGroupID is the id of the system-provided
+// compilation template group that compiler.json references as its default
+// "create from template" group. It is a global (tenant-agnostic) catalogue so
+// any tenant can resolve it without depending on Python-side initialization.
+const BuiltinCompilationTemplateGroupID = "c3aa748c8b2111f191f3047c16ec874f"
+
+// builtinCompilationTemplateKinds is the set of variants shipped in the
+// built-in group. The Go KnowledgeCompiler derives the variant from each
+// template's kind via common.KindToVariant.
+var builtinCompilationTemplateKinds = []struct {
+ Kind string
+ Name string
+}{
+ {Kind: "tree", Name: "Tree — RAPTOR-based document tree"},
+ {Kind: "mind_map", Name: "Mind map - Radial concept hierarchy"},
+ {Kind: "wiki", Name: "Wiki — Graph-based wiki"},
+ {Kind: "knowledge_graph", Name: "Knowledge graph"},
+ {Kind: "datasetnav", Name: "Dataset nav — top-down drill-down"},
+ {Kind: "page_index", Name: "Page index"},
+ {Kind: "session_essence", Name: "Session essence"},
+ {Kind: "session_graph", Name: "Session graph"},
+ {Kind: "timeline", Name: "Timeline"},
+}
+
+func strptr(s string) *string { return &s }
+
+// SeedBuiltinCompilationTemplates provisions the built-in compilation template
+// group (and its child templates) as a global, tenant-agnostic catalogue, so
+// the Go compiler (and the frontend "create from template" catalogue) can
+// resolve compiler.json's default group id out of the box. The group row uses
+// an empty tenant_id and ResolveGroupTemplateIDs treats this built-in id as
+// resolvable for every tenant.
+func SeedBuiltinCompilationTemplates(ctx context.Context, db *gorm.DB) error {
+ if err := SeedBuiltinCompilationTemplatesForTenant(ctx, db, ""); err != nil {
+ common.Warn("failed to seed built-in compilation templates", zap.Error(err))
+ return err
+ }
+ return nil
+}
+
+// SeedBuiltinCompilationTemplatesForTenant upserts the built-in compilation
+// template group and its child templates as a global, tenant-agnostic
+// catalogue. The tenantID argument is accepted for API symmetry but the
+// built-in entries always use an empty tenant_id, because ResolveGroupTemplateIDs
+// resolves the built-in group id for every tenant. It is idempotent
+// (OnConflict DO NOTHING on id) so it is safe to call repeatedly.
+func SeedBuiltinCompilationTemplatesForTenant(ctx context.Context, db *gorm.DB, tenantID string) error {
+ valid := string(entity.StatusValid)
+ return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ group := &entity.CompilationTemplateGroup{
+ ID: BuiltinCompilationTemplateGroupID,
+ TenantID: "", // global built-in catalogue
+ Name: "Built-in templates",
+ Status: strptr(valid),
+ }
+ if err := tx.WithContext(ctx).Clauses(clause.OnConflict{
+ Columns: []clause.Column{{Name: "id"}},
+ DoNothing: true,
+ }).Create(group).Error; err != nil {
+ return err
+ }
+
+ for _, t := range builtinCompilationTemplateKinds {
+ tmpl := &entity.CompilationTemplate{
+ ID: BuiltinCompilationTemplateGroupID + "-" + t.Kind,
+ TenantID: nil, // global built-in catalogue
+ GroupID: strptr(BuiltinCompilationTemplateGroupID),
+ Name: t.Name,
+ Kind: t.Kind,
+ Config: entity.JSONMap{"kind": t.Kind},
+ Status: strptr(valid),
+ }
+ if err := tx.WithContext(ctx).Clauses(clause.OnConflict{
+ Columns: []clause.Column{{Name: "id"}},
+ DoNothing: true,
+ }).Create(tmpl).Error; err != nil {
+ return err
+ }
+ }
+ return nil
+ })
+}
diff --git a/internal/dao/compilation_template_seed_test.go b/internal/dao/compilation_template_seed_test.go
new file mode 100644
index 0000000000..d12fb76cd1
--- /dev/null
+++ b/internal/dao/compilation_template_seed_test.go
@@ -0,0 +1,112 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package dao
+
+import (
+ "testing"
+
+ "ragflow/internal/entity"
+
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+)
+
+func TestSeedBuiltinCompilationTemplatesForTenant(t *testing.T) {
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+ if err != nil {
+ t.Fatalf("open sqlite: %v", err)
+ }
+ if err = db.AutoMigrate(&entity.CompilationTemplateGroup{}, &entity.CompilationTemplate{}); err != nil {
+ t.Fatalf("migrate: %v", err)
+ }
+ // ResolveGroupTemplateIDs reads the package-global DB, so point it at our
+ // sqlite instance for the duration of the test.
+ orig := DB
+ DB = db
+ t.Cleanup(func() { DB = orig })
+
+ ctx := t.Context()
+ const tenantID = "tenant-1"
+
+ // Idempotent across repeated calls, and the built-in group is always a
+ // global (tenant_id = "") catalogue regardless of the tenantID argument.
+ for i := 0; i < 2; i++ {
+ if err = SeedBuiltinCompilationTemplatesForTenant(ctx, db, tenantID); err != nil {
+ t.Fatalf("seed (iter %d): %v", i, err)
+ }
+ }
+
+ var groups []entity.CompilationTemplateGroup
+ if err = db.Find(&groups).Error; err != nil {
+ t.Fatalf("read groups: %v", err)
+ }
+ if len(groups) != 1 {
+ t.Fatalf("got %d groups, want 1", len(groups))
+ }
+ if groups[0].ID != BuiltinCompilationTemplateGroupID || groups[0].TenantID != "" {
+ t.Fatalf("unexpected group: %#v", groups[0])
+ }
+
+ var templates []entity.CompilationTemplate
+ if err = db.Find(&templates).Error; err != nil {
+ t.Fatalf("read templates: %v", err)
+ }
+ if len(templates) != len(builtinCompilationTemplateKinds) {
+ t.Fatalf("got %d templates, want %d", len(templates), len(builtinCompilationTemplateKinds))
+ }
+ for _, tmpl := range templates {
+ // Built-in templates are a global catalogue (tenant_id nil).
+ if tmpl.TenantID != nil {
+ t.Errorf("template %s: tenant_id=%v, want nil (global catalogue)", tmpl.ID, tmpl.TenantID)
+ }
+ if tmpl.GroupID == nil || *tmpl.GroupID != BuiltinCompilationTemplateGroupID {
+ t.Errorf("template %s: group_id=%v, want %q", tmpl.ID, tmpl.GroupID, BuiltinCompilationTemplateGroupID)
+ }
+ if tmpl.Kind == "" {
+ t.Errorf("template %s: empty kind", tmpl.ID)
+ }
+ if tmpl.Config["kind"] != tmpl.Kind {
+ t.Errorf("template %s: config.kind=%v, want %q", tmpl.ID, tmpl.Config["kind"], tmpl.Kind)
+ }
+ }
+
+ // Re-seeding with a different tenant is idempotent for the global group:
+ // the single built-in group row stays (id is the PK), and templates are
+ // not duplicated.
+ if err = SeedBuiltinCompilationTemplatesForTenant(ctx, db, "tenant-2"); err != nil {
+ t.Fatalf("seed tenant-2: %v", err)
+ }
+ var allGroups []entity.CompilationTemplateGroup
+ if err = db.Find(&allGroups).Error; err != nil {
+ t.Fatalf("read all groups: %v", err)
+ }
+ if len(allGroups) != 1 {
+ t.Fatalf("got %d groups, want 1 (built-in group is global, single row)", len(allGroups))
+ }
+ if allGroups[0].TenantID != "" {
+ t.Errorf("built-in group tenant_id = %q, want empty (global catalogue)", allGroups[0].TenantID)
+ }
+
+ // ResolveGroupTemplateIDs resolves the built-in group for any tenant.
+ resolved, err := NewCompilationTemplateDAO().ResolveGroupTemplateIDs(ctx, "some-other-tenant", []string{BuiltinCompilationTemplateGroupID})
+ if err != nil {
+ t.Fatalf("resolve built-in group: %v", err)
+ }
+ if len(resolved) != len(builtinCompilationTemplateKinds) {
+ t.Fatalf("resolved %d template ids, want %d", len(resolved), len(builtinCompilationTemplateKinds))
+ }
+}
diff --git a/internal/dao/database.go b/internal/dao/database.go
index 6dfc8b1c3e..303eb70b8c 100644
--- a/internal/dao/database.go
+++ b/internal/dao/database.go
@@ -158,6 +158,11 @@ func InitDB(ctx context.Context, migrateDB bool) error {
&entity.FileCommit{},
&entity.FileCommitItem{},
&entity.KnowledgeCompileDoc{},
+ // Knowledge-compile compilation templates and their groups. The Go
+ // KnowledgeCompilerComponent resolves a compilation_template (or group)
+ // from these tables at runtime, so the Go side must guarantee they exist.
+ &entity.CompilationTemplate{},
+ &entity.CompilationTemplateGroup{},
}
if migrateDB {
@@ -180,6 +185,11 @@ func InitDB(ctx context.Context, migrateDB bool) error {
if err = SeedCanvasTemplates(ctx, DB); err != nil {
common.Warn("Failed to seed canvas templates", zap.Error(err))
}
+ // Seed the built-in compilation template group (c3aa748c...) for every
+ // tenant so compiler.json's default group resolves out of the box.
+ if err = SeedBuiltinCompilationTemplates(ctx, DB); err != nil {
+ common.Warn("Failed to seed built-in compilation templates", zap.Error(err))
+ }
common.Info("Database connected and migrated successfully")
diff --git a/internal/dao/tenant.go b/internal/dao/tenant.go
index 657b7f8987..39990d8311 100644
--- a/internal/dao/tenant.go
+++ b/internal/dao/tenant.go
@@ -105,6 +105,14 @@ func (dao *TenantDAO) Create(ctx context.Context, db *gorm.DB, tenant *entity.Te
return db.WithContext(ctx).Create(tenant).Error
}
+// GetAllIDs returns the IDs of all active tenants.
+func (dao *TenantDAO) GetAllIDs(ctx context.Context, db *gorm.DB) ([]string, error) {
+ var ids []string
+ err := db.WithContext(ctx).Model(&entity.Tenant{}).Where("status = ?", string(entity.StatusValid)).
+ Pluck("id", &ids).Error
+ return ids, err
+}
+
// Delete deletes a tenant by ID (soft delete)
func (dao *TenantDAO) Delete(ctx context.Context, db *gorm.DB, id string) error {
return db.WithContext(ctx).Model(&entity.Tenant{}).Where("id = ?", id).Update("status", "0").Error
diff --git a/internal/ingestion/component/knowledge_compiler/common/deps.go b/internal/ingestion/component/knowledge_compiler/common/deps.go
index b730f74975..de39b473ad 100644
--- a/internal/ingestion/component/knowledge_compiler/common/deps.go
+++ b/internal/ingestion/component/knowledge_compiler/common/deps.go
@@ -148,7 +148,7 @@ func ResolveDeps(tenantID, llmID, embeddingModel string) (Deps, error) {
// compilation_template ids they contain. It is a DB-backed seam installed by
// production wiring in internal/ingestion/task; the knowledge_compiler package
// itself stays DB-independent. When a component is configured with
-// compilation_template_group_ids but no GroupResolver is installed, the
+// compilation_template_group_id but no GroupResolver is installed, the
// component fails loudly instead of silently emitting rows that miss
// compilation_template_ids (a data-loss path).
type GroupResolver func(ctx context.Context, tenantID string, groupIDs []string) ([]string, error)
@@ -180,7 +180,48 @@ func ResolveGroupTemplateIDs(ctx context.Context, tenantID string, groupIDs []st
r := groupResolver
groupResolverMu.RUnlock()
if r == nil {
- return nil, fmt.Errorf("knowledge_compiler: compilation_template_group_ids provided but no GroupResolver installed (production wiring must call common.SetGroupResolver)")
+ return nil, fmt.Errorf("knowledge_compiler: compilation_template_group_id provided but no GroupResolver installed (production wiring must call common.SetGroupResolver)")
}
return r(ctx, tenantID, groupIDs)
}
+
+// TemplateResolver loads a single compilation template by id for the tenant.
+type TemplateResolver func(ctx context.Context, tenantID, templateID string) (TemplateInfo, error)
+
+// TemplateInfo is a resolved compilation template: its id, the kind that
+// selects the Go compiler variant (see KindToVariant), and its config blob (the
+// template "content"). It is dependency-light so common can return it without
+// importing entity/gorm — production wiring (internal/ingestion/task) converts
+// the entity row into this shape.
+type TemplateInfo struct {
+ ID string
+ Kind string
+ Config map[string]any
+}
+
+var (
+ templateResolverMu sync.RWMutex
+ templateResolver TemplateResolver
+)
+
+// SetTemplateResolver installs the production (or test) TemplateResolver.
+// Production wiring calls this from an init() in internal/ingestion/task; tests
+// may inject a stub.
+func SetTemplateResolver(r TemplateResolver) {
+ templateResolverMu.Lock()
+ defer templateResolverMu.Unlock()
+ templateResolver = r
+}
+
+// ResolveTemplate loads a single compilation template by id via the installed
+// resolver. Returns an error when no resolver is installed (compilation_template_id
+// provided but unwired) rather than silently compiling with no template config.
+func ResolveTemplate(ctx context.Context, tenantID, templateID string) (TemplateInfo, error) {
+ templateResolverMu.RLock()
+ r := templateResolver
+ templateResolverMu.RUnlock()
+ if r == nil {
+ return TemplateInfo{}, fmt.Errorf("knowledge_compiler: compilation_template_id provided but no TemplateResolver installed (production wiring must call common.SetTemplateResolver)")
+ }
+ return r(ctx, tenantID, templateID)
+}
diff --git a/internal/ingestion/component/knowledge_compiler/common/types.go b/internal/ingestion/component/knowledge_compiler/common/types.go
index cb8fe3bf4d..397516ba48 100644
--- a/internal/ingestion/component/knowledge_compiler/common/types.go
+++ b/internal/ingestion/component/knowledge_compiler/common/types.go
@@ -11,7 +11,6 @@ package common
import (
"errors"
"fmt"
- "log"
"strings"
)
@@ -21,7 +20,7 @@ type Variant string
const (
VariantStructure Variant = "structure"
VariantWiki Variant = "wiki"
- VariantRaptor Variant = "raptor"
+ VariantTree Variant = "tree"
VariantMindmap Variant = "mindmap"
VariantDatasetnav Variant = "datasetnav"
)
@@ -32,19 +31,42 @@ var (
ErrNotImplemented = errors.New("knowledge_compiler: variant not yet implemented")
)
-// Param is built from the DSL params map. Missing keys fall back to
-// Defaults(); variant-name aliases are normalised (see ParseParam).
+// Param is built from the DSL params map. The component is driven by a
+// compilation template (or template group); the variant is NOT taken from the
+// DSL — it is derived from the resolved template's kind at Invoke time (see
+// KindToVariant).
type Param struct {
- Variant Variant
- LLMID string
- EmbeddingModel string
- Language string
- TemplateIDs []string
- GroupIDs []string
+ // CompilationTemplateID selects a single compilation template directly.
+ // CompilationTemplateGroupID selects a compilation-template group, which
+ // resolves to one or more templates. When both are non-empty,
+ // CompilationTemplateID wins (priority: id > group_id).
+ CompilationTemplateID string
+ CompilationTemplateGroupID string
+
+ LLMID string
+ EmbeddingModel string
+ Language string
+ // SimilarityThreshold defaults to 0.99 when not provided (see Defaults).
SimilarityThreshold float64
MaxWorkers int
EnableHistoricalDedup bool
- Extra map[string]any
+ // Extra carries arbitrary caller-provided overrides merged into the
+ // resolved template config.
+ Extra map[string]any
+
+ // The following fields are resolved at runtime (not part of the DSL
+ // surface) and are set by the component before dispatching to a variant:
+ //
+ // Variant is derived from the resolved template's kind (see KindToVariant),
+ // not from the DSL.
+ Variant Variant
+ // TemplateID is the resolved compilation template id for the current spec
+ // (the id path uses it directly; the group path resolves each child). It is
+ // plumbed to the variant for stamping compilation_template_ids.
+ TemplateID string
+ // TemplateConfig is the resolved template's config blob (the template
+ // "content"), plumbed to the variant for prompt/structure extraction.
+ TemplateConfig map[string]any
}
// Defaults returns a Param populated with safe production fallbacks.
@@ -83,21 +105,23 @@ type Inputs struct {
HistoricalCandidates []Candidate // optional override path (test/offline)
}
-// LogDeprecated emits a one-line deprecation notice for legacy param names.
-func LogDeprecated(old, replacement string) {
- log.Printf("[knowledge_compiler] deprecated variant name %q; use %q", old, replacement)
-}
-
// Product is one compiled output row (schema_version=1).
type Product struct {
ID string
DocID string
TenantID string
Variant Variant
- Content string
- Vector []float32
- ParentID string
- Meta map[string]any
+ // Kind is the original compilation_template.kind (e.g. "page_index"),
+ // distinct from Variant (the collapsed Go strategy, e.g. "structure"). It
+ // is stamped onto compilation_template_kind_kwd so the document-structure
+ // endpoint can group rows by the true template kind.
+ Kind string
+ // TemplateID is the compilation template that produced this row.
+ TemplateID string
+ Content string
+ Vector []float32
+ ParentID string
+ Meta map[string]any
}
// Outputs is the result of a variant Run. All compiled products are buffered
@@ -107,15 +131,23 @@ type Outputs struct {
DuplicatesDropped int
}
-// ParseParam builds a Param from the DSL params map, applying variant-name
-// aliases (mind_map=>mindmap, dataset_nav=>datasetnav) and deprecation logs.
+// ParseParam builds a Param from the DSL params map. The variant is NOT taken
+// from the DSL — it is derived from the resolved template's kind at Invoke time
+// via KindToVariant. At least one of compilation_template_id /
+// compilation_template_group_id is required.
func ParseParam(m map[string]any) (Param, error) {
p := Param{}.Defaults()
if m == nil {
- return p, fmt.Errorf("knowledge_compiler: params required (variant missing)")
+ return p, fmt.Errorf("knowledge_compiler: params required (compilation_template_id missing)")
+ }
+ if v, ok := m["compilation_template_id"].(string); ok && strings.TrimSpace(v) != "" {
+ p.CompilationTemplateID = strings.TrimSpace(v)
+ }
+ if v, ok := m["compilation_template_group_id"].(string); ok && strings.TrimSpace(v) != "" {
+ p.CompilationTemplateGroupID = strings.TrimSpace(v)
}
- if v, ok := m["variant"].(string); ok && v != "" {
- p.Variant = normalizeVariant(v)
+ if p.CompilationTemplateID == "" && p.CompilationTemplateGroupID == "" {
+ return p, fmt.Errorf("knowledge_compiler: one of 'compilation_template_id' or 'compilation_template_group_id' is required")
}
if v, ok := m["llm_id"].(string); ok {
p.LLMID = v
@@ -132,46 +164,45 @@ func ParseParam(m map[string]any) (Param, error) {
if v, ok := m["enable_historical_dedup"].(bool); ok {
p.EnableHistoricalDedup = v
}
- // compilation_template_ids / compilation_template_group_ids mirror the
- // Python parser_config keys (see api/utils/validation_utils.py). Template
- // ids are stamped onto every compiled row so the document-structure
- // endpoint can group rows by template and the UI renders one tab per
- // template. Group ids are resolved to concrete template ids by the caller
- // (production wiring resolves them via the compilation-template-group
- // service); when passed through raw they are carried as-is on GroupIDs.
- p.TemplateIDs = parseStringList(m, "compilation_template_ids", "template_ids")
- p.GroupIDs = parseStringList(m, "compilation_template_group_ids", "template_group_ids", "group_ids")
if raw, ok := m["extra"].(map[string]any); ok {
p.Extra = raw
}
- if p.Variant == "" {
- return p, fmt.Errorf("knowledge_compiler: 'variant' is required")
- }
return p, nil
}
-// normalizeVariant maps a DSL variant name to the canonical Variant,
-// logging a deprecation note for the legacy underscore forms.
-func normalizeVariant(s string) Variant {
- switch strings.ToLower(strings.TrimSpace(s)) {
- case "structure":
- return VariantStructure
+// KindToVariant maps a compilation_template.kind to the Go compiler Variant.
+//
+// The Python model uses richer kind values (mind_map, page_index,
+// session_essence, session_graph, timeline, knowledge_graph, tree, wiki); the
+// Go port collapses them onto its variants. Per the agreed mapping:
+//
+// tree -> tree
+// mind_map -> mindmap
+// wiki -> wiki
+// page_index / session_essence / -> structure (the graph/knowledge-graph path)
+// session_graph / timeline / ->
+// knowledge_graph ->
+//
+// The canonical variant names are also accepted as identity (so a template kind
+// may equal the variant, e.g. "datasetnav" which has no Python kind, and the
+// internal unit tests can drive each variant through a template id). Unknown
+// kinds return ErrUnknownVariant; the caller turns that into a hard failure
+// rather than silently emitting uncompiled rows.
+func KindToVariant(kind string) (Variant, error) {
+ switch strings.ToLower(strings.TrimSpace(kind)) {
+ case "tree":
+ return VariantTree, nil
+ case "mind_map", "mindmap":
+ return VariantMindmap, nil
case "wiki":
- return VariantWiki
- case "raptor":
- return VariantRaptor
- case "mindmap", "mind_map":
- if strings.Contains(s, "_") {
- LogDeprecated("mind_map", "mindmap")
- }
- return VariantMindmap
+ return VariantWiki, nil
case "datasetnav", "dataset_nav":
- if strings.Contains(s, "_") {
- LogDeprecated("dataset_nav", "datasetnav")
- }
- return VariantDatasetnav
+ return VariantDatasetnav, nil
+ case "page_index", "session_essence", "session_graph", "timeline",
+ "knowledge_graph", "structure", "knowledgegraph", "graph":
+ return VariantStructure, nil
default:
- return Variant(s)
+ return "", fmt.Errorf("%w: %q", ErrUnknownVariant, kind)
}
}
@@ -185,55 +216,6 @@ func FirstNonEmpty(vals ...string) string {
return ""
}
-// parseStringList reads a []string from the params map under any of the given
-// keys (first hit wins). Accepts []string, []any (of strings), or a single
-// whitespace/comma-separated string. Returns nil when no key is present.
-func parseStringList(m map[string]any, keys ...string) []string {
- for _, k := range keys {
- raw, ok := m[k]
- if !ok {
- continue
- }
- switch v := raw.(type) {
- case []string:
- if len(v) > 0 {
- return v
- }
- case []any:
- out := make([]string, 0, len(v))
- for _, e := range v {
- if s, ok := e.(string); ok {
- s = strings.TrimSpace(s)
- if s != "" {
- out = append(out, s)
- }
- }
- }
- if len(out) > 0 {
- return out
- }
- case string:
- s := strings.TrimSpace(v)
- if s == "" {
- continue
- }
- // Tolerate comma- or whitespace-separated lists.
- parts := strings.FieldsFunc(s, func(r rune) bool { return r == ',' || r == ' ' || r == '\t' || r == '\n' })
- var out []string
- for _, p := range parts {
- p = strings.TrimSpace(p)
- if p != "" {
- out = append(out, p)
- }
- }
- if len(out) > 0 {
- return out
- }
- }
- }
- return nil
-}
-
// VectorFromChunkMap extracts a pre-computed embedding from a chunk map. The
// upstream pipeline (and the knowledge-compile store) carry the vector under a
// dimension-specific column key "q_<dim>_vec" — set by the tokenizer via
diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.go
index 668908a276..17724b98ba 100644
--- a/internal/ingestion/component/knowledge_compiler/component.go
+++ b/internal/ingestion/component/knowledge_compiler/component.go
@@ -1,6 +1,6 @@
// Package knowledge_compiler implements the KnowledgeCompiler ingestion
// component: a single runtime.Component that dispatches to one of the
-// knowledge-compile variants (structure / wiki / raptor / mindmap / datasetnav)
+// knowledge-compile variants (structure / wiki / tree / mindmap / datasetnav)
// based on the `variant` param. See PORT_PLAN.md for the full design.
package knowledge_compiler
@@ -13,8 +13,8 @@ import (
"ragflow/internal/ingestion/component/knowledge_compiler/common"
"ragflow/internal/ingestion/component/knowledge_compiler/datasetnav"
"ragflow/internal/ingestion/component/knowledge_compiler/mindmap"
- "ragflow/internal/ingestion/component/knowledge_compiler/raptor"
"ragflow/internal/ingestion/component/knowledge_compiler/structure"
+ "ragflow/internal/ingestion/component/knowledge_compiler/tree"
"ragflow/internal/ingestion/component/knowledge_compiler/wiki"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/tokenizer"
@@ -92,73 +92,83 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in
tenantID, _ := inputs["tenant_id"].(string)
datasetID, _ := inputs["dataset_id"].(string)
- // Validate the variant before resolving deps so a bad variant fails fast
- // with ErrUnknownVariant rather than a deps-resolution error.
- switch param.Variant {
- case common.VariantStructure, common.VariantWiki, common.VariantRaptor,
- common.VariantMindmap, common.VariantDatasetnav:
- // recognised; dispatch below
- default:
- return nil, fmt.Errorf("%w: %q", common.ErrUnknownVariant, param.Variant)
- }
-
- deps, err := common.ResolveDeps(tenantID, param.LLMID, param.EmbeddingModel)
+ // Resolve the compilation template spec(s). Priority:
+ // compilation_template_id > compilation_template_group_id. The variant is
+ // derived from each template's kind (see common.KindToVariant), not from
+ // the DSL.
+ specs, err := resolveTemplateSpecs(ctx, tenantID, param)
if err != nil {
return nil, err
}
- deps.TenantID = tenantID
- deps.DatasetID = datasetID
-
- // Resolve compilation-template-group ids to concrete template ids and merge
- // them with any explicitly-passed template ids. Group-only configs that
- // cannot be resolved fail loudly rather than silently emitting rows missing
- // compilation_template_ids (a data-loss path).
- resolvedGroups, err := common.ResolveGroupTemplateIDs(ctx, tenantID, param.GroupIDs)
- if err != nil {
- return nil, err
- }
- templateIDs := append(append([]string{}, param.TemplateIDs...), resolvedGroups...)
in, err := buildInputs(inputs, param)
if err != nil {
return nil, err
}
- // Stamp the resolved template ids onto every compiled product after the
- // variant returns. All products are buffered in out.Products (there is no
- // streaming sink path), so the post-Run loop below covers every row (M1).
+ // Each spec compiles independently; products are stamped with the producing
+ // template's id and kind. All products are buffered (no streaming sink), so
+ // the post-run loop below covers every row (M1).
var out common.Outputs
- switch param.Variant {
- case common.VariantStructure:
- out, err = structure.Run(ctx, deps, param, in)
- case common.VariantWiki:
- out, err = wiki.Run(ctx, deps, param, in)
- case common.VariantRaptor:
- out, err = raptor.Run(ctx, deps, param, in)
- case common.VariantMindmap:
- out, err = mindmap.Run(ctx, deps, param, in)
- case common.VariantDatasetnav:
- out, err = datasetnav.Run(ctx, deps, param, in)
- default:
- return nil, fmt.Errorf("%w: %q", common.ErrUnknownVariant, param.Variant)
- }
- if err != nil {
- return nil, err
- }
+ for _, spec := range specs {
+ variant, err := common.KindToVariant(spec.Kind)
+ if err != nil {
+ return nil, err
+ }
+
+ // Per-spec Param: variant + resolved template config, with scalar config
+ // overrides (language, similarity, workers, dedup, llm/embedding) layered
+ // on top of the DSL params and Extra. This is how the template's stored
+ // "content" drives the compile without the caller passing variant-specific
+ // params inline.
+ specParam := param
+ specParam.Variant = variant
+ specParam.TemplateID = spec.ID
+ specParam.TemplateConfig = spec.Config
+ overlayTemplateConfig(&specParam, spec.Config)
+
+ deps, err := common.ResolveDeps(tenantID, specParam.LLMID, specParam.EmbeddingModel)
+ if err != nil {
+ return nil, err
+ }
+ deps.TenantID = tenantID
+ deps.DatasetID = datasetID
+
+ specIn := in
+ if specIn.VariantSpecific == nil {
+ specIn.VariantSpecific = map[string]any{}
+ }
+ specIn.VariantSpecific["config"] = spec.Config
+
+ var o common.Outputs
+ switch variant {
+ case common.VariantStructure:
+ o, err = structure.Run(ctx, deps, specParam, specIn)
+ case common.VariantWiki:
+ o, err = wiki.Run(ctx, deps, specParam, specIn)
+ case common.VariantTree:
+ o, err = tree.Run(ctx, deps, specParam, specIn)
+ case common.VariantMindmap:
+ o, err = mindmap.Run(ctx, deps, specParam, specIn)
+ case common.VariantDatasetnav:
+ o, err = datasetnav.Run(ctx, deps, specParam, specIn)
+ default:
+ return nil, fmt.Errorf("%w: %q", common.ErrUnknownVariant, variant)
+ }
+ if err != nil {
+ return nil, err
+ }
- // Stamp the compilation template ids onto every product so the chunk
- // converter can emit compilation_template_ids (Python stamps one
- // template_id per row; here we carry the full resolved list since the Go
- // component runs one variant per Invoke and the caller may pass multiple
- // template ids from a compilation-template-group). The list is the union of
- // explicitly-passed template ids and any resolved from group ids.
- if len(templateIDs) > 0 {
- for i := range out.Products {
- if out.Products[i].Meta == nil {
- out.Products[i].Meta = map[string]any{}
+ for i := range o.Products {
+ if o.Products[i].Meta == nil {
+ o.Products[i].Meta = map[string]any{}
}
- out.Products[i].Meta["compilation_template_ids"] = templateIDs
+ o.Products[i].Meta["compilation_template_ids"] = []string{spec.ID}
+ o.Products[i].Kind = spec.Kind
+ o.Products[i].TemplateID = spec.ID
+ o.Products[i].Variant = variant
}
+ out.Products = append(out.Products, o.Products...)
}
// Convert the compiled products into chunk-aligned docs (matching
@@ -172,6 +182,79 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in
return mergeChunks(inputs, compiled), nil
}
+// resolveTemplateSpecs resolves the configured compilation template spec(s) to
+// their TemplateInfo rows. Priority: compilation_template_id >
+// compilation_template_group_id. The group path resolves the group to its
+// child template ids and then loads each template.
+func resolveTemplateSpecs(ctx context.Context, tenantID string, param common.Param) ([]common.TemplateInfo, error) {
+ if param.CompilationTemplateID != "" {
+ info, err := common.ResolveTemplate(ctx, tenantID, param.CompilationTemplateID)
+ if err != nil {
+ return nil, err
+ }
+ return []common.TemplateInfo{info}, nil
+ }
+ if param.CompilationTemplateGroupID != "" {
+ ids, err := common.ResolveGroupTemplateIDs(ctx, tenantID, []string{param.CompilationTemplateGroupID})
+ if err != nil {
+ return nil, err
+ }
+ specs := make([]common.TemplateInfo, 0, len(ids))
+ for _, id := range ids {
+ info, err := common.ResolveTemplate(ctx, tenantID, id)
+ if err != nil {
+ return nil, err
+ }
+ specs = append(specs, info)
+ }
+ return specs, nil
+ }
+ return nil, fmt.Errorf("knowledge_compiler: one of compilation_template_id or compilation_template_group_id is required")
+}
+
+// overlayTemplateConfig layers scalar fields from the resolved template config
+// (the template "content") onto the param. For self-documenting defaults
+// (language, similarity_threshold, max_workers, enable_historical_dedup) the
+// config value wins when the param is unset. For llm_id / embedding_model, the
+// caller's per-call override always wins; the config only fills them in when
+// the caller left them empty.
+func overlayTemplateConfig(param *common.Param, cfg map[string]any) {
+ if cfg == nil {
+ return
+ }
+ if v, ok := cfg["language"].(string); ok && v != "" {
+ param.Language = v
+ }
+ if v, ok := cfg["similarity_threshold"].(float64); ok && v > 0 {
+ param.SimilarityThreshold = v
+ }
+ if v, ok := cfg["max_workers"].(float64); ok && int(v) > 0 {
+ param.MaxWorkers = int(v)
+ }
+ if v, ok := cfg["enable_historical_dedup"].(bool); ok {
+ param.EnableHistoricalDedup = v
+ }
+ // llm_id / embedding_model are optional per-call overrides documented on
+ // Invoke. The template config supplies defaults, so only apply them when
+ // the caller has not already provided an explicit value (the caller wins).
+ if v, ok := cfg["llm_id"].(string); ok && v != "" && param.LLMID == "" {
+ param.LLMID = v
+ }
+ if v, ok := cfg["embedding_model"].(string); ok && v != "" && param.EmbeddingModel == "" {
+ param.EmbeddingModel = v
+ }
+}
+
+// kindOrVariant returns the original template kind when present (the true
+// compilation_template.kind, e.g. "page_index"), otherwise the collapsed Go
+// variant. It drives the compilation_template_kind_kwd stamp.
+func kindOrVariant(p common.Product) string {
+ if p.Kind != "" {
+ return p.Kind
+ }
+ return string(p.Variant)
+}
+
// variantCompileKWD maps each Go variant to the compile_kwd discriminator value
// Python writes into ES (rag/advanced_rag/knowlege_compile). It is the primary
// key that distinguishes compiled knowledge units from ordinary chunks and
@@ -179,7 +262,7 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in
var variantCompileKWD = map[common.Variant]string{
common.VariantStructure: "structure",
common.VariantWiki: "artifact_page",
- common.VariantRaptor: "raptor",
+ common.VariantTree: "tree",
common.VariantMindmap: "mindmap",
common.VariantDatasetnav: "dataset_nav",
}
@@ -237,7 +320,7 @@ func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {
if err := doc.SetExtraValue("compile_kwd", compileKWD); err != nil {
return nil, err
}
- if err := doc.SetExtraValue("compilation_template_kind_kwd", string(p.Variant)); err != nil {
+ if err := doc.SetExtraValue("compilation_template_kind_kwd", kindOrVariant(p)); err != nil {
return nil, err
}
if p.ParentID != "" {
@@ -398,7 +481,7 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {
}
}
- case common.VariantRaptor:
+ case common.VariantTree:
// raptor_kwd tags summary/root nodes; raptor_layer_int records tree depth.
if kind != "" {
if err := doc.SetExtraValue("raptor_kwd", kind); err != nil {
diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.go
index f7107cf6d0..93ff663315 100644
--- a/internal/ingestion/component/knowledge_compiler/component_test.go
+++ b/internal/ingestion/component/knowledge_compiler/component_test.go
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"hash/fnv"
+ "os"
"regexp"
"strings"
"sync"
@@ -109,7 +110,7 @@ func (mockChat) Chat(_ context.Context, req common.ChatRequest) (*common.ChatRes
}
// proseChat returns generic, non-empty prose for the non-structure variants
-// (wiki/raptor/mindmap/datasetnav), which all just need a summary/outline text.
+// (wiki/tree/mindmap/datasetnav), which all just need a summary/outline text.
type proseChat struct{}
func (proseChat) Chat(_ context.Context, req common.ChatRequest) (*common.ChatResponse, error) {
@@ -167,9 +168,10 @@ func installMockDeps(t *testing.T) {
func TestKnowledgeCompiler_Structure_EndToEnd(t *testing.T) {
installMockDeps(t)
+ installVariantTemplateResolver(t, "structure")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
- "variant": "structure", "llm_id": "llm1", "embedding_model": "emb1",
+ "compilation_template_id": "tpl-structure", "llm_id": "llm1", "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -227,7 +229,7 @@ func TestKnowledgeCompiler_Structure_EndToEnd(t *testing.T) {
}
func TestKnowledgeCompiler_UnknownVariant(t *testing.T) {
- c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{"variant": "nope"})
+ c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{"compilation_template_id": "nope"})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
}
@@ -242,7 +244,7 @@ func TestKnowledgeCompiler_Alias_Mindmap(t *testing.T) {
// "mind_map" is the deprecated alias for "mindmap"; both resolve to the
// implemented mindmap variant and must run (not ErrUnknownVariant / stub).
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
- "variant": "mind_map", "llm_id": "llm1", "embedding_model": "emb1",
+ "compilation_template_id": "mind_map", "llm_id": "llm1", "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -266,7 +268,8 @@ func TestKnowledgeCompiler_Alias_Mindmap(t *testing.T) {
// chunk list (upstream input chunks + compiled knowledge-unit chunks).
func runVariant(t *testing.T, variant string, extra map[string]any) []map[string]any {
t.Helper()
- params := map[string]any{"variant": variant, "llm_id": "llm1", "embedding_model": "emb1"}
+ installVariantTemplateResolver(t, variant)
+ params := map[string]any{"compilation_template_id": "tpl-" + variant, "llm_id": "llm1", "embedding_model": "emb1"}
for k, v := range extra {
params[k] = v
}
@@ -334,10 +337,10 @@ func TestKnowledgeCompiler_Wiki_EndToEnd(t *testing.T) {
}
}
-func TestKnowledgeCompiler_Raptor_EndToEnd(t *testing.T) {
+func TestKnowledgeCompiler_Tree_EndToEnd(t *testing.T) {
installProseDeps(t)
// watershed (default tree_order): zero external clustering dependency.
- chunks := runVariant(t, "raptor", nil)
+ chunks := runVariant(t, "tree", nil)
foundRoot := false
for _, c := range chunks {
if kind, _ := c["kc_kind"].(string); kind == "root" {
@@ -345,14 +348,14 @@ func TestKnowledgeCompiler_Raptor_EndToEnd(t *testing.T) {
}
}
if !foundRoot {
- t.Fatalf("raptor(default): no 'root' chunk; got %d chunks", len(chunks))
+ t.Fatalf("tree(default): no 'root' chunk; got %d chunks", len(chunks))
}
// A smaller tree_order (more, smaller clusters) must also run and still
// produce a well-formed tree (root present, chunks non-empty).
- chunksCoarse := runVariant(t, "raptor", map[string]any{"extra": map[string]any{"tree_order": 2}})
+ chunksCoarse := runVariant(t, "tree", map[string]any{"extra": map[string]any{"tree_order": 2}})
if len(chunksCoarse) == 0 {
- t.Fatalf("raptor(tree_order=2): produced no chunks")
+ t.Fatalf("tree(tree_order=2): produced no chunks")
}
foundRootCoarse := false
for _, c := range chunksCoarse {
@@ -361,7 +364,7 @@ func TestKnowledgeCompiler_Raptor_EndToEnd(t *testing.T) {
}
}
if !foundRootCoarse {
- t.Fatalf("raptor(tree_order=2): no 'root' chunk; got %d chunks", len(chunksCoarse))
+ t.Fatalf("tree(tree_order=2): no 'root' chunk; got %d chunks", len(chunksCoarse))
}
}
@@ -405,9 +408,10 @@ func TestKnowledgeCompiler_Datasetnav_EndToEnd(t *testing.T) {
// {output_format:"chunks", chunks:[...]}.
func TestKnowledgeCompiler_EmitsChunks(t *testing.T) {
installMockDeps(t)
+ installVariantTemplateResolver(t, "structure")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
- "variant": "structure", "llm_id": "llm1", "embedding_model": "emb1",
+ "compilation_template_id": "tpl-structure", "llm_id": "llm1", "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -461,12 +465,12 @@ func TestKnowledgeCompiler_EmitsChunks(t *testing.T) {
// pointing back at their originating input chunks.
func TestKnowledgeCompiler_TemplateIDsAndProvenance(t *testing.T) {
installMockDeps(t)
+ installVariantTemplateResolver(t, "structure")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
- "variant": "structure",
- "llm_id": "llm1",
- "embedding_model": "emb1",
- "compilation_template_ids": []any{"tpl-A", "tpl-B"},
+ "compilation_template_id": "tpl-structure",
+ "llm_id": "llm1",
+ "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -488,7 +492,8 @@ func TestKnowledgeCompiler_TemplateIDsAndProvenance(t *testing.T) {
if ck, _ := cm["compile_kwd"].(string); ck != "hypergraph" {
continue
}
- // Every compiled chunk must carry the resolved template ids.
+ // Every compiled chunk must carry the resolved template id (one per
+ // template; the component stamps the producing template's id).
var tidsCount int
switch tids := cm["compilation_template_ids"].(type) {
case []any:
@@ -496,8 +501,8 @@ func TestKnowledgeCompiler_TemplateIDsAndProvenance(t *testing.T) {
case []string:
tidsCount = len(tids)
}
- if tidsCount != 2 {
- t.Fatalf("compiled chunk %v: compilation_template_ids = %v, want 2", cm["id"], cm["compilation_template_ids"])
+ if tidsCount != 1 {
+ t.Fatalf("compiled chunk %v: compilation_template_ids = %v, want 1 (the resolved template id)", cm["id"], cm["compilation_template_ids"])
}
// Entity rows must carry source_chunk_ids.
if kg, _ := cm["knowledge_graph_kwd"].(string); kg == "entity" {
@@ -534,8 +539,8 @@ func (m constEmbedder) Encode(_ context.Context, texts []string) ([][]float32, e
return out, nil
}
-// TestKnowledgeCompiler_Raptor_DegenerateNoInfiniteLoop is a regression for the
-// High-1 bug: RAPTOR could recurse forever when a re-cluster returned a single
+// TestKnowledgeCompiler_Tree_DegenerateNoInfiniteLoop is a regression for the
+// High-1 bug: the tree builder could recurse forever when a re-cluster returned a single
// label covering all points. The default mockEmbedder emits non-negative
// vectors, so under the watershed default (tree_order=4, ratio 25) every adjacent pair
// has cosine >= 0 and a pathological input can collapse into one cluster; with
@@ -543,10 +548,11 @@ func (m constEmbedder) Encode(_ context.Context, texts []string) ([][]float32, e
// work item at level+1 and hung. This test uses 6 chunks and asserts the run
// terminates with a well-formed root. (If the guard regresses, go test's timeout
// turns the hang into a failure.)
-func TestKnowledgeCompiler_Raptor_DegenerateNoInfiniteLoop(t *testing.T) {
+func TestKnowledgeCompiler_Tree_DegenerateNoInfiniteLoop(t *testing.T) {
installProseDeps(t)
+ installVariantTemplateResolver(t, "tree")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
- "variant": "raptor", "llm_id": "llm1", "embedding_model": "emb1",
+ "compilation_template_id": "tpl-tree", "llm_id": "llm1", "embedding_model": "emb1",
"extra": map[string]any{"tree_order": 4},
})
if err != nil {
@@ -565,7 +571,7 @@ func TestKnowledgeCompiler_Raptor_DegenerateNoInfiniteLoop(t *testing.T) {
"tenant_id": "t1",
})
if err != nil {
- t.Fatalf("Invoke hung or errored (RAPTOR infinite recursion?): %v", err)
+ t.Fatalf("Invoke hung or errored (tree infinite recursion?): %v", err)
}
raw, ok := out["chunks"].([]any)
if !ok {
@@ -580,7 +586,7 @@ func TestKnowledgeCompiler_Raptor_DegenerateNoInfiniteLoop(t *testing.T) {
}
}
if !foundRoot {
- t.Fatalf("raptor(degenerate): no root chunk; got %d chunks", len(raw))
+ t.Fatalf("tree(degenerate): no root chunk; got %d chunks", len(raw))
}
}
@@ -599,8 +605,9 @@ func TestKnowledgeCompiler_Wiki_HistoricalDedupDropsDuplicates(t *testing.T) {
})
t.Cleanup(func() { common.SetDepsResolver(nil) })
+ installVariantTemplateResolver(t, "wiki")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
- "variant": "wiki", "llm_id": "llm1", "embedding_model": "emb1",
+ "compilation_template_id": "tpl-wiki", "llm_id": "llm1", "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -657,8 +664,9 @@ func TestKnowledgeCompiler_Wiki_UpdateMergesExistingPage(t *testing.T) {
})
t.Cleanup(func() { common.SetDepsResolver(nil) })
+ installVariantTemplateResolver(t, "wiki")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
- "variant": "wiki", "llm_id": "llm1", "embedding_model": "emb1",
+ "compilation_template_id": "tpl-wiki", "llm_id": "llm1", "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -737,8 +745,9 @@ func TestKnowledgeCompiler_Wiki_HistoricalDedupScopedByDataset(t *testing.T) {
})
t.Cleanup(func() { common.SetDepsResolver(nil) })
+ installVariantTemplateResolver(t, "wiki")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
- "variant": "wiki", "llm_id": "llm1", "embedding_model": "emb1",
+ "compilation_template_id": "tpl-wiki", "llm_id": "llm1", "embedding_model": "emb1",
"enable_historical_dedup": true,
})
if err != nil {
@@ -809,8 +818,9 @@ func TestKnowledgeCompiler_Datasetnav_LockKeyedByDataset(t *testing.T) {
})
t.Cleanup(func() { common.SetDepsResolver(nil) })
+ installVariantTemplateResolver(t, "datasetnav")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
- "variant": "datasetnav", "llm_id": "llm1", "embedding_model": "emb1",
+ "compilation_template_id": "tpl-datasetnav", "llm_id": "llm1", "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -873,8 +883,9 @@ func TestKnowledgeCompiler_Datasetnav_RootIncludesAllSummaries(t *testing.T) {
})
t.Cleanup(func() { common.SetDepsResolver(nil) })
+ installVariantTemplateResolver(t, "datasetnav")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
- "variant": "datasetnav", "llm_id": "llm1", "embedding_model": "emb1",
+ "compilation_template_id": "tpl-datasetnav", "llm_id": "llm1", "embedding_model": "emb1",
// nav_radius > 1 guarantees no two chunks group together (cosine <= 1),
// so each chunk becomes its own nav node.
"extra": map[string]any{"nav_radius": 1.1},
@@ -983,8 +994,9 @@ func TestKnowledgeCompiler_Structure_FencedJSONNotDropped(t *testing.T) {
})
t.Cleanup(func() { common.SetDepsResolver(nil) })
+ installVariantTemplateResolver(t, "structure")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
- "variant": "structure", "llm_id": "llm1", "embedding_model": "emb1",
+ "compilation_template_id": "tpl-structure", "llm_id": "llm1", "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -1022,8 +1034,9 @@ func TestKnowledgeCompiler_Structure_MalformedJSONFailsLoud(t *testing.T) {
})
t.Cleanup(func() { common.SetDepsResolver(nil) })
+ installVariantTemplateResolver(t, "structure")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
- "variant": "structure", "llm_id": "llm1", "embedding_model": "emb1",
+ "compilation_template_id": "tpl-structure", "llm_id": "llm1", "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -1046,9 +1059,10 @@ func TestKnowledgeCompiler_Structure_MalformedJSONFailsLoud(t *testing.T) {
// pipeline), so they must be forwarded when present.
func TestKnowledgeCompiler_PassThroughEnvelope(t *testing.T) {
installMockDeps(t)
+ installVariantTemplateResolver(t, "structure")
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
- "variant": "structure", "llm_id": "llm1", "embedding_model": "emb1",
+ "compilation_template_id": "tpl-structure", "llm_id": "llm1", "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -1091,21 +1105,34 @@ func groupResolverStub(_ context.Context, _ string, groupIDs []string) ([]string
return out, nil
}
-// TestKnowledgeCompiler_GroupIDsResolvedToTemplateIDs is a regression for the
-// open question: compilation_template_group_ids were parsed into Param.GroupIDs
-// but never resolved or stamped, so group-only configs silently missed
-// compilation_template_ids. The component now resolves groups via the installed
-// GroupResolver and merges them into the stamped template-id list.
+// TestKnowledgeCompiler_GroupIDsResolvedToTemplateIDs is a regression: a
+// compilation_template_group_id is resolved via the installed GroupResolver to
+// its concrete template ids, and each template is compiled as its own spec and
+// stamped with its producing template id. Group-only configs must not silently
+// miss compilation_template_ids.
func TestKnowledgeCompiler_GroupIDsResolvedToTemplateIDs(t *testing.T) {
installMockDeps(t)
+ // The group resolves to two concrete template ids; each is compiled as its
+ // own spec, so the TemplateResolver must return a valid kind for them.
+ // Override the package stub and restore it afterwards.
+ common.SetTemplateResolver(func(ctx context.Context, tenantID, templateID string) (common.TemplateInfo, error) {
+ kind := templateID
+ if strings.HasPrefix(templateID, "tpl-grp1") {
+ kind = "structure"
+ }
+ return common.TemplateInfo{ID: templateID, Kind: kind, Config: map[string]any{}}, nil
+ })
+ t.Cleanup(func() { common.SetTemplateResolver(testTemplateResolver) })
+
common.SetGroupResolver(groupResolverStub)
- t.Cleanup(func() { common.SetGroupResolver(nil) })
+ t.Cleanup(func() { common.SetGroupResolver(testGroupResolver) })
+ // compilation_template_group_id (not the obsolete plural list) selects the
+ // group; compilation_template_id is absent so the group path is taken.
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
- "variant": "structure",
- "llm_id": "llm1",
- "embedding_model": "emb1",
- "compilation_template_group_ids": []any{"grp1"},
+ "compilation_template_group_id": "grp1",
+ "llm_id": "llm1",
+ "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -1119,6 +1146,7 @@ func TestKnowledgeCompiler_GroupIDsResolvedToTemplateIDs(t *testing.T) {
t.Fatalf("Invoke: %v", err)
}
want := map[string]bool{"tpl-grp1-a": true, "tpl-grp1-b": true}
+ seen := map[string]bool{}
checked := 0
for _, r := range out["chunks"].([]any) {
cm := r.(map[string]any)
@@ -1137,13 +1165,21 @@ func TestKnowledgeCompiler_GroupIDsResolvedToTemplateIDs(t *testing.T) {
case []string:
got = v
}
- if len(got) != 2 || !want[got[0]] || !want[got[1]] {
- t.Fatalf("compiled chunk %v: compilation_template_ids = %v, want %v (group ids must resolve to template ids)", cm["id"], got, want)
+ // Each product is stamped with the single template id that produced it.
+ if len(got) != 1 || !want[got[0]] {
+ t.Fatalf("compiled chunk %v: compilation_template_ids = %v, want exactly one of %v", cm["id"], got, want)
}
+ seen[got[0]] = true
}
if checked == 0 {
t.Fatalf("no compiled chunks inspected (compile_kwd=list expected); assertion was vacuous")
}
+ // Both resolved template ids must appear across the products.
+ for id := range want {
+ if !seen[id] {
+ t.Fatalf("resolved template id %q was never stamped on any product", id)
+ }
+ }
}
// TestKnowledgeCompiler_GroupIDsWithoutResolverFailsLoud is the companion
@@ -1155,10 +1191,9 @@ func TestKnowledgeCompiler_GroupIDsWithoutResolverFailsLoud(t *testing.T) {
common.SetGroupResolver(nil) // ensure no resolver is installed
c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{
- "variant": "structure",
- "llm_id": "llm1",
- "embedding_model": "emb1",
- "compilation_template_group_ids": []any{"grp1"},
+ "compilation_template_group_id": "grp1",
+ "llm_id": "llm1",
+ "embedding_model": "emb1",
})
if err != nil {
t.Fatalf("NewKnowledgeCompilerComponent: %v", err)
@@ -1175,3 +1210,44 @@ func TestKnowledgeCompiler_GroupIDsWithoutResolverFailsLoud(t *testing.T) {
t.Fatalf("error %q does not mention GroupResolver", err.Error())
}
}
+
+// testTemplateResolver / testGroupResolver are the package-wide stub resolvers
+// installed by TestMain. The stub maps the template id 1:1 onto the template
+// kind, which lets tests resolve arbitrary synthetic ids (e.g. "tpl-structure")
+// to a kind. Tests must pass a TEMPLATE ID, never a variant name, as
+// compilation_template_id; use installVariantTemplateResolver(t, variant) to
+// register a synthetic "tpl-<variant>" id that maps to the desired kind.
+// Production wiring installs the real DB-backed resolvers (see
+// internal/ingestion/task/knowledge_compiler_wiring.go).
+var testTemplateResolver common.TemplateResolver = func(ctx context.Context, tenantID, templateID string) (common.TemplateInfo, error) {
+ return common.TemplateInfo{ID: templateID, Kind: templateID, Config: map[string]any{}}, nil
+}
+
+// installVariantTemplateResolver wires a template resolver so the synthetic
+// template id "tpl-<variant>" resolves to Kind <variant>. This lets a test
+// select a compiler variant through the production id -> kind -> variant path
+// (resolveTemplateSpecs -> KindToVariant) instead of misusing the variant name
+// as the compilation_template_id. Any other template id is delegated to the
+// package default; the default is restored on cleanup.
+func installVariantTemplateResolver(t *testing.T, variant string) {
+ t.Helper()
+ prev := testTemplateResolver
+ common.SetTemplateResolver(func(ctx context.Context, tenantID, templateID string) (common.TemplateInfo, error) {
+ if templateID == "tpl-"+variant {
+ return common.TemplateInfo{ID: templateID, Kind: variant, Config: map[string]any{}}, nil
+ }
+ return prev(ctx, tenantID, templateID)
+ })
+ t.Cleanup(func() { common.SetTemplateResolver(testTemplateResolver) })
+}
+
+var testGroupResolver common.GroupResolver = func(ctx context.Context, tenantID string, groupIDs []string) ([]string, error) {
+ return groupIDs, nil
+}
+
+// TestMain installs the stub resolvers for the variant unit tests.
+func TestMain(m *testing.M) {
+ common.SetTemplateResolver(testTemplateResolver)
+ common.SetGroupResolver(testGroupResolver)
+ os.Exit(m.Run())
+}
diff --git a/internal/ingestion/component/knowledge_compiler/golden/corpus.go b/internal/ingestion/component/knowledge_compiler/golden/corpus.go
index 66d18bfbb6..a24b5f64a5 100644
--- a/internal/ingestion/component/knowledge_compiler/golden/corpus.go
+++ b/internal/ingestion/component/knowledge_compiler/golden/corpus.go
@@ -17,7 +17,7 @@ import (
// FixedCorpus is the canonical sample document. It describes a small, coherent
// ecosystem of named entities (Alpha/Beta/Gamma/Delta/Epsilon) with dense
-// cross-references so structure extraction, wiki synthesis, and raptor
+// cross-references so structure extraction, wiki synthesis, and tree
// clustering all have meaningful signal.
func FixedCorpus() []common.Chunk {
texts := []string{
diff --git a/internal/ingestion/component/knowledge_compiler/golden/fixtures/raptor_baseline.json b/internal/ingestion/component/knowledge_compiler/golden/fixtures/tree_baseline.json
similarity index 100%
rename from internal/ingestion/component/knowledge_compiler/golden/fixtures/raptor_baseline.json
rename to internal/ingestion/component/knowledge_compiler/golden/fixtures/tree_baseline.json
diff --git a/internal/ingestion/component/knowledge_compiler/golden/golden_test.go b/internal/ingestion/component/knowledge_compiler/golden/golden_test.go
index 110f3e768b..8c55674021 100644
--- a/internal/ingestion/component/knowledge_compiler/golden/golden_test.go
+++ b/internal/ingestion/component/knowledge_compiler/golden/golden_test.go
@@ -44,27 +44,27 @@ func TestChunksToAny(t *testing.T) {
}
}
-func TestAnalyzeRaptorProducts_TreeShape(t *testing.T) {
+func TestAnalyzeTreeProducts_TreeShape(t *testing.T) {
vector := json.RawMessage(`[0.1,0.2,0.3]`)
chunks := []schema.ChunkDoc{
{Text: "root summary", Extra: mustExtras(t, map[string]any{
- "id": "r1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "raptor",
+ "id": "r1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
"kc_kind": "root", "kc_level": float64(-1), "q_3_vec": vector,
})},
{Text: "leaf A", Extra: mustExtras(t, map[string]any{
- "id": "a1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "raptor",
+ "id": "a1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
"kc_kind": "summary", "kc_level": float64(0), "parent_kwd": "r1", "q_3_vec": vector,
})},
{Text: "leaf B", Extra: mustExtras(t, map[string]any{
- "id": "b1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "raptor",
+ "id": "b1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
"kc_kind": "summary", "kc_level": float64(0), "parent_kwd": "r1", "q_3_vec": vector,
})},
{Text: "mid A", Extra: mustExtras(t, map[string]any{
- "id": "m1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "raptor",
+ "id": "m1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
"kc_kind": "summary", "kc_level": float64(1), "parent_kwd": "r1", "q_3_vec": vector,
})},
}
- m := AnalyzeRaptorProducts(chunks)
+ m := AnalyzeTreeProducts(chunks)
if m.RootCount != 1 {
t.Errorf("RootCount = %d, want 1", m.RootCount)
}
@@ -82,34 +82,73 @@ func TestAnalyzeRaptorProducts_TreeShape(t *testing.T) {
}
}
-func TestAnalyzeRaptorProducts_DetectsDanglingParent(t *testing.T) {
+func TestAnalyzeTreeProducts_DetectsDanglingParent(t *testing.T) {
chunks := []schema.ChunkDoc{
{Text: "root", Extra: mustExtras(t, map[string]any{
- "id": "r1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "raptor",
+ "id": "r1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
"kc_kind": "root",
})},
{Text: "orphan", Extra: mustExtras(t, map[string]any{
- "id": "o1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "raptor",
+ "id": "o1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
"kc_kind": "summary", "kc_level": float64(0), "parent_kwd": "missing-parent",
})},
}
- m := AnalyzeRaptorProducts(chunks)
+ m := AnalyzeTreeProducts(chunks)
if m.AllParented {
t.Error("AllParented = true, want false (o1's parent is missing)")
}
}
-func TestCoverageFraction_AllParentedOneRoot(t *testing.T) {
- m := TreeMetrics{RootCount: 1, LeafClusters: 3, AllParented: true}
+func TestCoverageFraction_AllSourcesCovered(t *testing.T) {
+ // Coverage is now measured from source_chunk_ids of level-0 leaf clusters,
+ // not structural well-formedness. When every input chunk is referenced, the
+ // fraction is 1.0 regardless of root/parent structure.
+ m := TreeMetrics{RootCount: 1, LeafClusters: 3, AllParented: true, CoveredSources: 12}
if cov := m.CoverageFraction(12); cov != 1.0 {
t.Errorf("CoverageFraction = %v, want 1.0", cov)
}
}
-func TestCoverageFraction_NoRootIsZero(t *testing.T) {
- m := TreeMetrics{LeafClusters: 3, AllParented: true}
+func TestCoverageFraction_NoCoveredSourcesIsZero(t *testing.T) {
+ // A structurally well-formed tree that references no source chunks covers 0.
+ m := TreeMetrics{RootCount: 1, LeafClusters: 3, AllParented: true, CoveredSources: 0}
if cov := m.CoverageFraction(12); cov != 0.0 {
- t.Errorf("CoverageFraction = %v, want 0.0 (no root)", cov)
+ t.Errorf("CoverageFraction = %v, want 0.0 (no covered sources)", cov)
+ }
+}
+
+func TestCoverageFraction_PartialCoverage(t *testing.T) {
+ // A tree that drops some source chunks scores below 1.0.
+ m := TreeMetrics{RootCount: 1, LeafClusters: 3, AllParented: true, CoveredSources: 9}
+ if cov := m.CoverageFraction(12); cov != 0.75 {
+ t.Errorf("CoverageFraction = %v, want 0.75", cov)
+ }
+}
+
+// TestAnalyzeTreeProducts_IgnoresUnknownSourceIDs verifies that an unknown
+// (leaked/garbage) ID inside source_chunk_ids does not inflate CoveredSources
+// past the corpus size, so CoverageFraction stays <= 1.0. A level-0 leaf that
+// references 2 valid + 1 unknown ID over an nChunks=2 corpus must still report
+// coverage 1.0, not 1.5.
+func TestAnalyzeTreeProducts_IgnoresUnknownSourceIDs(t *testing.T) {
+ vector := json.RawMessage(`[0.1,0.2,0.3]`)
+ chunks := []schema.ChunkDoc{
+ {Text: "root", Extra: mustExtras(t, map[string]any{
+ "id": "r1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
+ "kc_kind": "root", "kc_level": float64(-1), "q_3_vec": vector,
+ })},
+ {Text: "leaf", Extra: mustExtras(t, map[string]any{
+ "id": "a1", "doc_id": "d1", "tenant_id": "t1", "compile_kwd": "tree",
+ "kc_kind": "summary", "kc_level": float64(0), "parent_kwd": "r1", "q_3_vec": vector,
+ "source_chunk_ids": []string{"chunk-01", "chunk-02", "leaked-unknown-id"},
+ })},
+ }
+ m := AnalyzeTreeProducts(chunks, "chunk-01", "chunk-02")
+ if m.CoveredSources != 2 {
+ t.Errorf("CoveredSources = %d, want 2 (unknown id excluded)", m.CoveredSources)
+ }
+ if cov := m.CoverageFraction(2); cov != 1.0 {
+ t.Errorf("CoverageFraction = %v, want 1.0 (must not exceed 1.0)", cov)
}
}
diff --git a/internal/ingestion/component/knowledge_compiler/golden/metrics.go b/internal/ingestion/component/knowledge_compiler/golden/metrics.go
index 4ff03e917c..753cfa8ff2 100644
--- a/internal/ingestion/component/knowledge_compiler/golden/metrics.go
+++ b/internal/ingestion/component/knowledge_compiler/golden/metrics.go
@@ -7,7 +7,7 @@ import (
"ragflow/internal/ingestion/component/schema"
)
-// TreeMetrics summarizes the structural shape of a raptor product tree.
+// TreeMetrics summarizes the structural shape of a tree product tree.
type TreeMetrics struct {
ProductCount int
RootCount int
@@ -16,19 +16,38 @@ type TreeMetrics struct {
AllParented bool
VectorOK bool // every product carries a non-empty vector
SchemaOK bool // every product carries the schema fields
+ // CoveredSources is the number of distinct source chunk IDs referenced by
+ // level-0 leaf clusters via their source_chunk_ids meta. It measures how
+ // completely the input corpus is represented by the tree, independent of
+ // the tree's structural well-formedness.
+ CoveredSources int
+ // covered is the working set of distinct source chunk IDs seen so far.
+ covered map[string]bool
}
-// AnalyzeRaptorProducts validates tree integrity and computes structural
-// metrics from a flat chunk list (the compiled raptor output, expressed as
+// AnalyzeTreeProducts validates tree integrity and computes structural
+// metrics from a flat chunk list (the compiled tree output, expressed as
// schema.ChunkDoc values). Used by the 缺口 C golden gate.
-func AnalyzeRaptorProducts(chunks []schema.ChunkDoc) TreeMetrics {
+//
+// validSourceIDs, when provided, limits coverage counting to source chunk IDs
+// that actually belong to the input corpus. This prevents an untrusted
+// source_chunk_ids (e.g. a leaked/garbage ID) from inflating CoveredSources
+// past nChunks and pushing CoverageFraction above 1.0. When empty, all
+// source_chunk_ids are counted (backward compatible for unit tests that build
+// synthetic trees).
+func AnalyzeTreeProducts(chunks []schema.ChunkDoc, validSourceIDs ...string) TreeMetrics {
ids := make(map[string]bool, len(chunks))
for _, c := range chunks {
if id, ok := c.GetExtraString("id"); ok {
ids[id] = true
}
}
- m := TreeMetrics{ProductCount: len(chunks), AllParented: true, VectorOK: true, SchemaOK: true}
+ validSet := make(map[string]bool, len(validSourceIDs))
+ for _, id := range validSourceIDs {
+ validSet[id] = true
+ }
+ checkValid := len(validSet) > 0
+ m := TreeMetrics{ProductCount: len(chunks), AllParented: true, VectorOK: true, SchemaOK: true, covered: make(map[string]bool)}
maxLevel := -1
for _, c := range chunks {
kind, _ := c.GetExtraString("kc_kind")
@@ -42,6 +61,22 @@ func AnalyzeRaptorProducts(chunks []schema.ChunkDoc) TreeMetrics {
case "summary":
if level == 0 {
m.LeafClusters++
+ // Accumulate the distinct source chunk IDs this leaf cluster
+ // was built from. Every input chunk is assigned to exactly one
+ // level-0 cluster in buildTree, so the union of these sets is
+ // the set of covered source chunks. Only IDs that belong to the
+ // input corpus count, so an unknown ID cannot inflate coverage.
+ if src, ok := c.GetExtraStringSlice("source_chunk_ids"); ok {
+ for _, id := range src {
+ if checkValid && !validSet[id] {
+ continue
+ }
+ if !m.covered[id] {
+ m.covered[id] = true
+ m.CoveredSources++
+ }
+ }
+ }
}
if level > maxLevel {
maxLevel = level
@@ -70,20 +105,24 @@ func AnalyzeRaptorProducts(chunks []schema.ChunkDoc) TreeMetrics {
}
// CoverageFraction reports how completely the input chunks are represented by
-// the raptor tree. Every source chunk is assigned to exactly one level-0
-// cluster in buildTree, and each such cluster becomes a leaf summary node, so a
-// well-formed tree covers 100% of chunks. nChunks is the input chunk count.
+// the tree. It is the ratio of distinct source chunk IDs referenced by the
+// level-0 leaf clusters (CoveredSources) to the total input chunk count
+// (nChunks). This detects dropped source chunks: a structurally well-formed
+// tree that silently omits input chunks will score below 1.0.
func (m TreeMetrics) CoverageFraction(nChunks int) float64 {
if nChunks <= 0 {
return 0
}
- // Every chunk maps to a level-0 cluster; the number of covered chunks
- // equals the total chunk count when at least one leaf cluster exists and
- // the tree is well-formed (all parented, single root).
- if m.LeafClusters > 0 && m.RootCount == 1 && m.AllParented {
+ if m.CoveredSources <= 0 {
+ return 0
+ }
+ // Coverage can never exceed 1.0: a source chunk is covered at most once,
+ // and only corpus IDs are counted, so CoveredSources <= nChunks.
+ frac := float64(m.CoveredSources) / float64(nChunks)
+ if frac > 1.0 {
return 1.0
}
- return 0.0
+ return frac
}
// extraFloat reads a numeric Extra value by key.
diff --git a/internal/ingestion/component/knowledge_compiler/golden_test.go b/internal/ingestion/component/knowledge_compiler/golden_test.go
index 281d5089e7..09ed27dcc5 100644
--- a/internal/ingestion/component/knowledge_compiler/golden_test.go
+++ b/internal/ingestion/component/knowledge_compiler/golden_test.go
@@ -74,7 +74,8 @@ func runVariantChunks(t *testing.T, variant string, extra map[string]any) []sche
// inputs (e.g. parser_config for the structure variant's template shape).
func runVariantChunksWithInputs(t *testing.T, variant string, extra, inputsExtra map[string]any) []schema.ChunkDoc {
t.Helper()
- params := map[string]any{"variant": variant, "llm_id": "llm1", "embedding_model": "emb1"}
+ installVariantTemplateResolver(t, variant)
+ params := map[string]any{"compilation_template_id": "tpl-" + variant, "llm_id": "llm1", "embedding_model": "emb1"}
for k, v := range extra {
params[k] = v
}
@@ -224,15 +225,15 @@ func TestGolden_Wiki_ProductCount(t *testing.T) {
}
}
-// TestGolden_Raptor_Structure is the 缺口 C gate for the raptor variant: on the
-// fixed corpus, the watershed builder (raptor.go::watershed) must produce a
+// TestGolden_Tree_Structure is the 缺口 C gate for the tree variant: on the
+// fixed corpus, the watershed builder (tree/raptor.go::watershed) must produce a
// well-formed tree (single root, fully parented, vectors + schema intact, full
// chunk coverage, bounded depth/cluster count) across several tree_order
// settings. The structural contract is what a regression would violate; the
-// exact cluster counts are locked in raptor_baseline.json.
-func TestGolden_Raptor_Structure(t *testing.T) {
+// exact cluster counts are locked in tree_baseline.json.
+func TestGolden_Tree_Structure(t *testing.T) {
installSignedProseDeps(t)
- baseline := loadBaseline(t, "raptor_baseline.json")
+ baseline := loadBaseline(t, "tree_baseline.json")
nChunks := len(golden.FixedCorpus())
cases := []struct {
@@ -248,36 +249,36 @@ func TestGolden_Raptor_Structure(t *testing.T) {
for _, tc := range cases {
var prods []schema.ChunkDoc
if tc.order <= 0 {
- prods = runVariantChunks(t, "raptor", nil)
+ prods = runVariantChunks(t, "tree", nil)
} else {
- prods = runVariantChunks(t, "raptor", map[string]any{
+ prods = runVariantChunks(t, "tree", map[string]any{
"extra": map[string]any{"tree_order": tc.order},
})
}
- m := golden.AnalyzeRaptorProducts(prods)
- t.Logf("raptor(%s): products=%d root=%d leafClusters=%d maxDepth=%d coverage=%.2f",
+ m := golden.AnalyzeTreeProducts(prods, goldenCorpusIDs()...)
+ t.Logf("tree(%s): products=%d root=%d leafClusters=%d maxDepth=%d coverage=%.2f",
tc.name, m.ProductCount, m.RootCount, m.LeafClusters, m.MaxDepth, m.CoverageFraction(nChunks))
if m.RootCount != 1 {
- t.Errorf("raptor(%s): rootCount=%d, want 1", tc.name, m.RootCount)
+ t.Errorf("tree(%s): rootCount=%d, want 1", tc.name, m.RootCount)
}
if !m.AllParented {
- t.Errorf("raptor(%s): tree has dangling parent_id references", tc.name)
+ t.Errorf("tree(%s): tree has dangling parent_id references", tc.name)
}
if !m.VectorOK {
- t.Errorf("raptor(%s): some products missing vectors", tc.name)
+ t.Errorf("tree(%s): some products missing vectors", tc.name)
}
if !m.SchemaOK {
- t.Errorf("raptor(%s): some products missing schema fields", tc.name)
+ t.Errorf("tree(%s): some products missing schema fields", tc.name)
}
if cov := m.CoverageFraction(nChunks); cov < numFloat(baseline, "leaf_coverage_min") {
- t.Errorf("raptor(%s): leaf coverage %.2f < baseline %.2f", tc.name, cov, numFloat(baseline, "leaf_coverage_min"))
+ t.Errorf("tree(%s): leaf coverage %.2f < baseline %.2f", tc.name, cov, numFloat(baseline, "leaf_coverage_min"))
}
if m.LeafClusters < num(baseline, tc.minKey) || m.LeafClusters > num(baseline, tc.maxKey) {
- t.Errorf("raptor(%s): leafClusters=%d outside [%d,%d]", tc.name, m.LeafClusters, num(baseline, tc.minKey), num(baseline, tc.maxKey))
+ t.Errorf("tree(%s): leafClusters=%d outside [%d,%d]", tc.name, m.LeafClusters, num(baseline, tc.minKey), num(baseline, tc.maxKey))
}
if m.MaxDepth > num(baseline, "max_depth_max") {
- t.Errorf("raptor(%s): maxDepth=%d > baseline %d", tc.name, m.MaxDepth, num(baseline, "max_depth_max"))
+ t.Errorf("tree(%s): maxDepth=%d > baseline %d", tc.name, m.MaxDepth, num(baseline, "max_depth_max"))
}
}
}
@@ -291,3 +292,15 @@ func numFloat(m map[string]any, key string) float64 {
}
return 0
}
+
+// goldenCorpusIDs returns the IDs of every chunk in the fixed golden corpus.
+// Passed to AnalyzeTreeProducts so coverage only counts source chunk IDs that
+// belong to the input corpus (a leaked/unknown ID must not inflate coverage).
+func goldenCorpusIDs() []string {
+ corpus := golden.FixedCorpus()
+ ids := make([]string, 0, len(corpus))
+ for _, c := range corpus {
+ ids = append(ids, c.ID)
+ }
+ return ids
+}
diff --git a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go
index 5a78d0e5b2..52b8848d4b 100644
--- a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go
+++ b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go
@@ -12,6 +12,7 @@ package mindmap
import (
"context"
+ "encoding/json"
"fmt"
"strings"
@@ -33,14 +34,13 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
}
sections := chunkTexts(inputs.Chunks)
- if len(sections) == 0 {
- return common.Outputs{}, nil
- }
// One LLM task per token-budget batch (mirrors __call__'s task fan-out).
batches := packSections(sections, deps.Tokenizer)
results := make([]utility.OMap, len(batches))
- // One LLM task per token-budget batch (mirrors __call__'s task fan-out).
+ // Bounded concurrency. Python fans out with unbounded asyncio.gather plus a
+ // global chat_limiter semaphore; here we cap with a worker pool sized by
+ // MaxWorkers (defaulting to 1, i.e. serial).
n := param.MaxWorkers
if n <= 0 {
n = 1
@@ -119,6 +119,12 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
// becomes a "root" product whose content is the serialized {"id","children"}
// tree (Python's MindMapResult.output shape); each inner node becomes a
// "node" product linked via parent_id.
+//
+// NOTE: the per-node "node" products (content = the node title, plus a
+// meta["children"] list of immediate child titles) are a Go component-layer
+// contract for streaming the tree into the upstream chunk list. They go
+// beyond Python's nested dict output and exist so downstream code can index
+// each node independently with parent links.
func treeToProducts(tenantID, docID string, root *utility.Node) []common.Product {
var out []common.Product
rootID := common.StableRowID(tenantID, docID, string(common.VariantMindmap), "root")
@@ -179,42 +185,31 @@ func treeToProducts(tenantID, docID string, root *utility.Node) []common.Product
return out
}
-// serializeNode renders the tree in Python's MindMapResult.output shape:
-// {"id": ..., "children": [...]}.
-func serializeNode(node *utility.Node) string {
- var b strings.Builder
- b.WriteString(`{"id":`)
- b.WriteString(quoteJSON(node.ID))
- b.WriteString(`,"children":[`)
- for i, c := range node.Children {
- if i > 0 {
- b.WriteString(",")
- }
- b.WriteString(serializeNode(c))
+// jsonNode mirrors the {"id","children"} mind-map node shape (Python's
+// MindMapResult.output) for JSON serialization.
+type jsonNode struct {
+ ID string `json:"id"`
+ Children []*jsonNode `json:"children"`
+}
+
+func toJSONNode(n *utility.Node) *jsonNode {
+ jn := &jsonNode{ID: n.ID}
+ for _, c := range n.Children {
+ jn.Children = append(jn.Children, toJSONNode(c))
}
- b.WriteString("]}")
- return b.String()
+ return jn
}
-func quoteJSON(s string) string {
- var b strings.Builder
- b.WriteByte('"')
- for _, r := range s {
- switch r {
- case '"':
- b.WriteString("\\\"")
- case '\\':
- b.WriteString("\\\\")
- case '\n':
- b.WriteString("\\n")
- case '\t':
- b.WriteString("\\t")
- default:
- b.WriteRune(r)
- }
+// serializeNode renders the tree in Python's MindMapResult.output shape:
+// {"id": ..., "children": [...]}. Uses encoding/json so every control
+// character (including \r, \b, \f, etc.) is escaped correctly; the previous
+// hand-rolled serializer only escaped ", \, \n and \t.
+func serializeNode(node *utility.Node) string {
+ b, err := json.Marshal(toJSONNode(node))
+ if err != nil {
+ return `{"id":"` + node.ID + `","children":[]}`
}
- b.WriteByte('"')
- return b.String()
+ return string(b)
}
func chunkTexts(chunks []common.Chunk) []string {
diff --git a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.go b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.go
index b6eb5bc594..32e276ca99 100644
--- a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.go
+++ b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap_test.go
@@ -1,6 +1,7 @@
package mindmap
import (
+ "encoding/json"
"strings"
"testing"
@@ -78,3 +79,26 @@ func TestSerializeNode_Shape(t *testing.T) {
t.Errorf("serializeNode = %q", js)
}
}
+
+// TestSerializeNode_EscapesControlChars locks that serializeNode uses
+// encoding/json: control characters such as CR/LF/TAB/FF are escaped (not
+// embedded raw), so the output is always valid JSON.
+func TestSerializeNode_EscapesControlChars(t *testing.T) {
+ root := &utility.Node{ID: "a\rb\tc\fd", Children: []*utility.Node{{ID: "x\ny"}}}
+ js := serializeNode(root)
+ var dec struct {
+ ID string `json:"id"`
+ Children []struct {
+ ID string `json:"id"`
+ } `json:"children"`
+ }
+ if err := json.Unmarshal([]byte(js), &dec); err != nil {
+ t.Fatalf("serializeNode produced invalid JSON: %v (%q)", err, js)
+ }
+ if dec.ID != "a\rb\tc\fd" {
+ t.Errorf("root id round-trip = %q, want %q", dec.ID, "a\rb\tc\fd")
+ }
+ if len(dec.Children) != 1 || dec.Children[0].ID != "x\ny" {
+ t.Errorf("child id round-trip = %+v", dec.Children)
+ }
+}
diff --git a/internal/ingestion/component/knowledge_compiler/structure/structure.go b/internal/ingestion/component/knowledge_compiler/structure/structure.go
index 788abaed2c..9908e6bc1e 100644
--- a/internal/ingestion/component/knowledge_compiler/structure/structure.go
+++ b/internal/ingestion/component/knowledge_compiler/structure/structure.go
@@ -46,7 +46,7 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
Variant: common.VariantStructure,
Lang: param.Language,
ParserConfig: parserConfig,
- TemplateID: common.FirstNonEmpty(param.TemplateIDs...),
+ TemplateID: param.TemplateID,
}
nodePrompt, edgePromptTmpl := HypergraphPrompts(parserConfig, param.Language)
diff --git a/internal/ingestion/component/knowledge_compiler/raptor/raptor.go b/internal/ingestion/component/knowledge_compiler/tree/raptor.go
similarity index 92%
rename from internal/ingestion/component/knowledge_compiler/raptor/raptor.go
rename to internal/ingestion/component/knowledge_compiler/tree/raptor.go
index 3dde45ddf1..697b1a845e 100644
--- a/internal/ingestion/component/knowledge_compiler/raptor/raptor.go
+++ b/internal/ingestion/component/knowledge_compiler/tree/raptor.go
@@ -1,4 +1,4 @@
-// Package raptor implements the "raptor" variant of KnowledgeCompiler: a
+// Package tree implements the "tree" variant of KnowledgeCompiler: a
// recursive abstractive summarization tree (RAPTOR). It builds a clustering of
// chunk embeddings and summarizes each cluster with the LLM, recursing upward
// until a single root summary remains.
@@ -11,7 +11,7 @@
// defaults to DefaultTreeOrder.
//
// See PORT_PLAN.md §3.3.
-package raptor
+package tree
import (
"context"
@@ -25,13 +25,13 @@ import (
"ragflow/internal/tokenizer"
)
-// Run executes the raptor variant.
+// Run executes the tree variant.
func Run(ctx context.Context, deps common.Deps, param common.Param, inputs common.Inputs) (common.Outputs, error) {
if deps.Embed == nil {
- return common.Outputs{}, fmt.Errorf("raptor: embedder required")
+ return common.Outputs{}, fmt.Errorf("tree: embedder required")
}
if deps.Chat == nil {
- return common.Outputs{}, fmt.Errorf("raptor: chat model required")
+ return common.Outputs{}, fmt.Errorf("tree: chat model required")
}
docID := firstNonEmpty(inputs.DocID, deps.DatasetID)
if docID == "" {
@@ -132,7 +132,7 @@ func resolveMaxErrors(param common.Param) int {
// buildTree summarizes each cluster (level 0) and recurses upward: each parent
// node is the LLM summary of its child cluster's texts. The root is a single
-// "raptor" product with a stable id. Every node is appended to products (a
+// "tree" product with a stable id. Every node is appended to products (a
// caller-owned slice), and only the deepest-level summaries are retained for the
// root synthesis. buildTree reads the source texts, ids, and embeddings directly
// from the passed chunks; when the embeddings are missing or incomplete, it
@@ -171,7 +171,7 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str
groups[labels[i]] = append(groups[labels[i]], i)
}
- rootID := common.StableRowID(tenantID, docID, string(common.VariantRaptor), "root")
+ rootID := common.StableRowID(tenantID, docID, string(common.VariantTree), "root")
var (
maxLevel = -1
topLevelTexts []string
@@ -201,12 +201,12 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str
if err != nil {
errorCount++
if errorCount >= maxErrors {
- return fmt.Errorf("raptor: aborted after %d summarization errors: %w", errorCount, err)
+ return fmt.Errorf("tree: aborted after %d summarization errors: %w", errorCount, err)
}
- log.Printf("raptor: skipping cluster due to summarization error (continuing): %v", err)
+ log.Printf("tree: skipping cluster due to summarization error (continuing): %v", err)
continue
}
- nodeID := common.StableRowID(tenantID, docID, string(common.VariantRaptor),
+ nodeID := common.StableRowID(tenantID, docID, string(common.VariantTree),
fmt.Sprintf("L%d", task.level), summary)
embedding, err := deps.Embed.Encode(ctx, []string{summary})
if err != nil {
@@ -220,7 +220,7 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str
ID: nodeID,
DocID: docID,
TenantID: tenantID,
- Variant: common.VariantRaptor,
+ Variant: common.VariantTree,
Content: summary,
Vector: vec,
ParentID: task.parentID,
@@ -289,7 +289,7 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str
// No summaries survived (e.g. every deepest cluster failed while the
// error budget was not yet exhausted). Return the partial tree without a
// root node — Python drops the root in this case rather than crashing.
- log.Printf("raptor: no top-level summaries produced, skipping root node")
+ log.Printf("tree: no top-level summaries produced, skipping root node")
return nil
}
rootContent := buildClusterContent(topLevelTexts, allIndices(len(topLevelTexts)), deps.LLMMaxLength, maxToken)
@@ -299,23 +299,23 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str
if err != nil {
// A failed root must not abort the whole tree: Python drops the root
// node and returns the rest of the tree.
- log.Printf("raptor: root synthesis failed, skipping root node: %v", err)
+ log.Printf("tree: root synthesis failed, skipping root node: %v", err)
return nil
}
embedding, err := deps.Embed.Encode(ctx, []string{rootSummary})
if err != nil {
- log.Printf("raptor: root embedding failed, skipping root node: %v", err)
+ log.Printf("tree: root embedding failed, skipping root node: %v", err)
return nil
}
if len(embedding) == 0 {
- log.Printf("raptor: root embedding returned no vectors, skipping root node")
+ log.Printf("tree: root embedding returned no vectors, skipping root node")
return nil
}
*products = append(*products, common.Product{
ID: rootID,
DocID: docID,
TenantID: tenantID,
- Variant: common.VariantRaptor,
+ Variant: common.VariantTree,
Content: rootSummary,
Vector: embedding[0],
ParentID: "",
@@ -540,12 +540,14 @@ func toFloat64Matrix(vecs [][]float32) [][]float64 {
return out
}
-// defaultRaptorPrompt is the summary task template. It mirrors Python
-// rag/advanced_rag/knowlege_compile/raptor.py, whose _summarize_texts uses
-// "Please write a concise summary of the following texts:\n{cluster_content}".
-// The {cluster_content} placeholder is filled with the joined cluster text. A
-// caller may override it via extra["prompt"] (Python: raptor_cfg["prompt"]).
-const defaultRaptorPrompt = "Please write a concise summary of the following texts:\n{cluster_content}"
+// defaultRaptorPrompt is the summary task template. It mirrors the production
+// prompt from api/db/init_data/compilation_templates/tree.yaml (the tree
+// compilation template), NOT the Python fallback in compiler.py:128. The
+// {cluster_content} placeholder is filled with the joined cluster text. The
+// YAML literal block carries a base indent of 6 spaces before {cluster_content};
+// those 6 spaces are part of the prompt and MUST be preserved. A caller may
+// override it via extra["prompt"] (Python: raptor_cfg["prompt"]).
+const defaultRaptorPrompt = "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize."
// raptorSystemHelper mirrors the leading "You're a helpful assistant.\n\nHelp me
// with the following task.\n\n" wrapper Python prepends to the task prompt.
diff --git a/internal/ingestion/component/knowledge_compiler/raptor/raptor_test.go b/internal/ingestion/component/knowledge_compiler/tree/raptor_test.go
similarity index 82%
rename from internal/ingestion/component/knowledge_compiler/raptor/raptor_test.go
rename to internal/ingestion/component/knowledge_compiler/tree/raptor_test.go
index eb8d0c0f1a..67a09f3078 100644
--- a/internal/ingestion/component/knowledge_compiler/raptor/raptor_test.go
+++ b/internal/ingestion/component/knowledge_compiler/tree/raptor_test.go
@@ -1,4 +1,4 @@
-package raptor
+package tree
import (
"context"
@@ -146,3 +146,20 @@ func TestBuildTreeNoPanicWhenAllSummariesFail(t *testing.T) {
t.Fatalf("expected no products when every summary fails, got %d", len(products))
}
}
+
+// TestDefaultRaptorPromptMatchesTreeYAML locks the default summary prompt to the
+// production tree.yaml template. It must equal the Python tree compilation
+// template prompt (api/db/init_data/compilation_templates/tree.yaml), NOT the
+// compiler.py:128 fallback. Critically, the YAML literal block carries a base
+// indent of 6 spaces before {cluster_content}; those 6 spaces are part of the
+// prompt and must be preserved (Python does self._prompt.format(...), splicing
+// the cluster text after the 6-space indent).
+func TestDefaultRaptorPromptMatchesTreeYAML(t *testing.T) {
+ want := "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize."
+ if defaultRaptorPrompt != want {
+ t.Fatalf("defaultRaptorPrompt drifted from tree.yaml:\n got: %q\nwant: %q", defaultRaptorPrompt, want)
+ }
+ if !strings.Contains(defaultRaptorPrompt, "\n {cluster_content}") {
+ t.Errorf("defaultRaptorPrompt missing the 6-space indent before {cluster_content}: %q", defaultRaptorPrompt)
+ }
+}
diff --git a/internal/ingestion/component/knowledge_compiler/raptor/watershed.go b/internal/ingestion/component/knowledge_compiler/tree/watershed.go
similarity index 97%
rename from internal/ingestion/component/knowledge_compiler/raptor/watershed.go
rename to internal/ingestion/component/knowledge_compiler/tree/watershed.go
index c392ef27ac..9eb88f989a 100644
--- a/internal/ingestion/component/knowledge_compiler/raptor/watershed.go
+++ b/internal/ingestion/component/knowledge_compiler/tree/watershed.go
@@ -6,7 +6,7 @@
// agglomerative hierarchical clustering.
//
// See PORT_PLAN.md §3.3 and the M6 validation gate (缺口 C).
-package raptor
+package tree
import (
"errors"
@@ -46,7 +46,7 @@ func watershed(embeddings [][]float64, treeOrder int) ([]int, error) {
n := len(embeddings)
switch {
case n == 0:
- return nil, errors.New("raptor: watershed requires at least one embedding")
+ return nil, errors.New("tree: watershed requires at least one embedding")
case n == 1:
return []int{0}, nil
case n == 2:
diff --git a/internal/ingestion/component/knowledge_compiler/raptor/watershed_test.go b/internal/ingestion/component/knowledge_compiler/tree/watershed_test.go
similarity index 99%
rename from internal/ingestion/component/knowledge_compiler/raptor/watershed_test.go
rename to internal/ingestion/component/knowledge_compiler/tree/watershed_test.go
index f6b9735eba..42e917a15d 100644
--- a/internal/ingestion/component/knowledge_compiler/raptor/watershed_test.go
+++ b/internal/ingestion/component/knowledge_compiler/tree/watershed_test.go
@@ -1,4 +1,4 @@
-package raptor
+package tree
import (
"math"
diff --git a/internal/ingestion/pipeline/pipeline_knowledge_compiler_dsl_test.go b/internal/ingestion/pipeline/pipeline_knowledge_compiler_dsl_test.go
new file mode 100644
index 0000000000..8a73819b54
--- /dev/null
+++ b/internal/ingestion/pipeline/pipeline_knowledge_compiler_dsl_test.go
@@ -0,0 +1,316 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pipeline
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+
+ // Named import so this file can drive the runtime factory directly.
+ "ragflow/internal/agent/runtime"
+ // Blank-imports fire the runtime.MustRegister init hooks that populate
+ // DefaultRegistry with File/Parser/TokenChunker/KnowledgeCompiler.
+ _ "ragflow/internal/ingestion/component"
+ _ "ragflow/internal/ingestion/component/chunker"
+ _ "ragflow/internal/ingestion/component/knowledge_compiler"
+
+ kc "ragflow/internal/ingestion/component/knowledge_compiler/common"
+)
+
+// installStubResolvers wires in-memory GroupResolver / TemplateResolver stubs so
+// the DSL tests can exercise the template-resolution path without MySQL.
+func installStubResolvers(t *testing.T, groupToTemplates map[string][]string, templates map[string]kc.TemplateInfo) {
+ t.Helper()
+ kc.SetGroupResolver(func(ctx context.Context, tenantID string, groupIDs []string) ([]string, error) {
+ var out []string
+ for _, g := range groupIDs {
+ out = append(out, groupToTemplates[g]...)
+ }
+ return out, nil
+ })
+ kc.SetTemplateResolver(func(ctx context.Context, tenantID, templateID string) (kc.TemplateInfo, error) {
+ info, ok := templates[templateID]
+ if !ok {
+ return kc.TemplateInfo{}, fmt.Errorf("no stub template %q", templateID)
+ }
+ return info, nil
+ })
+ t.Cleanup(func() { kc.SetGroupResolver(nil); kc.SetTemplateResolver(nil) })
+}
+
+// TestKnowledgeCompilerDSL_FixtureDecodesAndBindsParams loads the
+// frontend-authored pipeline DSL (agent/templates/compiler.json) and verifies
+// it decodes through NewPipelineFromDSL and that the Compiler node's authored
+// parameters survive intact. The frontend emits the operator as "Compiler" (its
+// canvas label) and does NOT set a "variant" — the variant is derived at runtime
+// from the compilation_template's kind. The single fixed operator carries a
+// singular compilation_template_group_id (see the frontend
+// accordion-operators.tsx restriction and the pipeline.tsx default).
+func TestKnowledgeCompilerDSL_FixtureDecodesAndBindsParams(t *testing.T) {
+ raw, err := os.ReadFile(filepath.Join(repoRootFromPipelineTest(t), "agent", "templates", "compiler.json"))
+ if err != nil {
+ t.Fatalf("read fixture: %v", err)
+ }
+ if _, err := NewPipelineFromDSL(raw, "kc-fixture"); err != nil {
+ t.Fatalf("NewPipelineFromDSL: %v", err)
+ }
+
+ schemas, err := ExtractAllComponentParams(raw)
+ if err != nil {
+ t.Fatalf("ExtractAllComponentParams: %v", err)
+ }
+ var compiler *ComponentParamsSchema
+ for i := range schemas {
+ if schemas[i].ComponentName == "Compiler" {
+ compiler = &schemas[i]
+ break
+ }
+ }
+ if compiler == nil {
+ t.Fatal("fixture has no component with component_name \"Compiler\"")
+ }
+ params := compiler.ParamsDefaults
+
+ // The frontend Compiler DSL omits "variant"; the Go component derives it
+ // from the template kind, so absence is expected.
+ if _, ok := params["variant"]; ok {
+ t.Errorf("fixture unexpectedly sets variant; frontend Compiler DSL omits it")
+ }
+
+ // Authored as a single string group id in the frontend form.
+ gid, ok := params["compilation_template_group_id"].(string)
+ if !ok || gid != "c3aa748c8b2111f191f3047c16ec874f" {
+ t.Fatalf("compilation_template_group_id = %v, want single string id", params["compilation_template_group_id"])
+ }
+}
+
+// TestKnowledgeCompilerDSL_FrontendDSLDecodesAndConstructs documents the
+// contract between the frontend-authored Compiler DSL and the Go component: as
+// emitted, the DSL has no "variant" but carries compilation_template_group_id,
+// so ParseParam (and thus the runtime factory) succeeds without any server-side
+// variant injection. What the component does need at Invoke time is the
+// TemplateResolver seam — without it, resolving the template fails loudly
+// rather than compiling with no template config.
+func TestKnowledgeCompilerDSL_FrontendDSLDecodesAndConstructs(t *testing.T) {
+ raw, err := os.ReadFile(filepath.Join(repoRootFromPipelineTest(t), "agent", "templates", "compiler.json"))
+ if err != nil {
+ t.Fatalf("read fixture: %v", err)
+ }
+ schemas, err := ExtractAllComponentParams(raw)
+ if err != nil {
+ t.Fatalf("ExtractAllComponentParams: %v", err)
+ }
+ var compiler *ComponentParamsSchema
+ for i := range schemas {
+ if schemas[i].ComponentName == "Compiler" {
+ compiler = &schemas[i]
+ break
+ }
+ }
+ if compiler == nil {
+ t.Fatal("fixture has no component with component_name \"Compiler\"")
+ }
+ params := compiler.ParamsDefaults
+
+ runtime.InstallDefaultRegistryFactory()
+ f := runtime.DefaultFactory()
+ if f == nil {
+ t.Fatal("default runtime factory not installed")
+ }
+
+ // The fixture params (single group id, no variant) construct fine.
+ comp, err := f("KnowledgeCompiler", params)
+ if err != nil {
+ t.Fatalf("construct from fixture params: %v", err)
+ }
+ if comp == nil {
+ t.Fatal("construct from fixture params: nil component")
+ }
+
+ // Without an installed TemplateResolver, the resolution seam fails loud.
+ kc.SetTemplateResolver(nil)
+ if _, err := kc.ResolveTemplate(context.Background(), "tenant", "t1"); err == nil {
+ t.Fatal("ResolveTemplate without resolver: expected error")
+ }
+}
+
+// TestKnowledgeCompilerDSL_RegisteredAndConstructible confirms the Go runtime
+// registers the component under the canonical name "KnowledgeCompiler" and that
+// the runtime factory can build a component instance from a DSL params map that
+// carries either compilation_template_id or compilation_template_group_id (the
+// variant is no longer part of the DSL surface). The frontend label "Compiler"
+// (see the fixture tests) maps to this runtime name at the API/canvas layer,
+// not inside the pipeline DSL decoder.
+func TestKnowledgeCompilerDSL_RegisteredAndConstructible(t *testing.T) {
+ runtime.InstallDefaultRegistryFactory()
+ if _, _, _, ok := runtime.DefaultRegistry.Lookup("KnowledgeCompiler"); !ok {
+ t.Fatal("KnowledgeCompiler not registered in the runtime factory")
+ }
+ f := runtime.DefaultFactory()
+ if f == nil {
+ t.Fatal("default runtime factory not installed")
+ }
+
+ cases := []map[string]any{
+ {"compilation_template_id": "t1"},
+ {"compilation_template_group_id": "g1"},
+ // Both present: compilation_template_id wins (priority id > group_id).
+ {"compilation_template_id": "t1", "compilation_template_group_id": "g1"},
+ }
+ for i, params := range cases {
+ comp, err := f("KnowledgeCompiler", params)
+ if err != nil {
+ t.Fatalf("case %d construct: %v", i, err)
+ }
+ if comp == nil {
+ t.Fatalf("case %d: nil component", i)
+ }
+ }
+
+ // Param map with neither id resolves to a parse error.
+ if _, err := f("KnowledgeCompiler", map[string]any{}); err == nil {
+ t.Fatal("construct with no template spec: expected error")
+ }
+}
+
+// TestKnowledgeCompilerDSL_ParamBinding exercises the DSL params map ->
+// common.Param translation directly, including the id > group_id priority, the
+// scalar/extra/default parsing that the pipeline wires from the canvas node
+// params, and that the variant is intentionally NOT part of the DSL surface.
+func TestKnowledgeCompilerDSL_ParamBinding(t *testing.T) {
+ // compilation_template_id wins over compilation_template_group_id.
+ p, err := kc.ParseParam(map[string]any{
+ "compilation_template_id": "t1",
+ "compilation_template_group_id": "g1",
+ "llm_id": "llm-1",
+ "embedding_model": "emb-1",
+ "language": "Chinese",
+ "extra": map[string]any{"prompt": "summarize"},
+ })
+ if err != nil {
+ t.Fatalf("ParseParam: %v", err)
+ }
+ if p.CompilationTemplateID != "t1" {
+ t.Errorf("CompilationTemplateID = %q, want t1", p.CompilationTemplateID)
+ }
+ if p.CompilationTemplateGroupID != "g1" {
+ t.Errorf("CompilationTemplateGroupID = %q, want g1", p.CompilationTemplateGroupID)
+ }
+ if p.Variant != "" {
+ t.Errorf("Variant should be empty after ParseParam (derived from kind later), got %q", p.Variant)
+ }
+ if p.LLMID != "llm-1" || p.EmbeddingModel != "emb-1" || p.Language != "Chinese" {
+ t.Errorf("scalar fields = %+v", p)
+ }
+ if p.Extra["prompt"] != "summarize" {
+ t.Errorf("Extra = %v, want prompt=summarize", p.Extra)
+ }
+ // Defaults must be applied for fields the DSL omits.
+ if p.MaxWorkers != 4 {
+ t.Errorf("MaxWorkers default = %d, want 4", p.MaxWorkers)
+ }
+ if p.SimilarityThreshold != 0.99 {
+ t.Errorf("SimilarityThreshold default = %v, want 0.99", p.SimilarityThreshold)
+ }
+ if p.EnableHistoricalDedup {
+ t.Errorf("EnableHistoricalDedup default = true, want false")
+ }
+}
+
+// TestKnowledgeCompilerDSL_KindToVariant locks the compilation_template.kind ->
+// Go Variant mapping. tree->tree, mind_map->mindmap, wiki->wiki; the
+// graph-family kinds (page_index / session_essence / session_graph / timeline /
+// knowledge_graph) all collapse onto the structure (graph) variant; unknown
+// kinds return ErrUnknownVariant.
+func TestKnowledgeCompilerDSL_KindToVariant(t *testing.T) {
+ cases := []struct {
+ kind string
+ want kc.Variant
+ }{
+ {"tree", kc.VariantTree},
+ {"mind_map", kc.VariantMindmap},
+ {"wiki", kc.VariantWiki},
+ {"page_index", kc.VariantStructure},
+ {"session_essence", kc.VariantStructure},
+ {"session_graph", kc.VariantStructure},
+ {"timeline", kc.VariantStructure},
+ {"knowledge_graph", kc.VariantStructure},
+ }
+ for _, c := range cases {
+ got, err := kc.KindToVariant(c.kind)
+ if err != nil {
+ t.Fatalf("KindToVariant(%q): %v", c.kind, err)
+ }
+ if got != c.want {
+ t.Errorf("KindToVariant(%q) = %q, want %q", c.kind, got, c.want)
+ }
+ }
+
+ if _, err := kc.KindToVariant("not-a-real-kind"); err == nil {
+ t.Fatal("KindToVariant(unknown): expected ErrUnknownVariant")
+ }
+}
+
+// TestKnowledgeCompilerDSL_VariantValidation documents the parse-time validation
+// boundary reachable without a running LLM/embedder: at least one of
+// compilation_template_id / compilation_template_group_id is required. The
+// variant is not parsed from the DSL at all (it is derived from the resolved
+// template's kind at Invoke time).
+func TestKnowledgeCompilerDSL_VariantValidation(t *testing.T) {
+ if _, err := kc.ParseParam(map[string]any{}); err == nil {
+ t.Fatal("ParseParam with no template spec: expected error")
+ }
+ if _, err := kc.ParseParam(map[string]any{"compilation_template_group_id": ""}); err == nil {
+ t.Fatal("ParseParam with empty group id: expected error")
+ }
+ if _, err := kc.ParseParam(map[string]any{"compilation_template_id": "t1"}); err != nil {
+ t.Fatalf("ParseParam with id only: unexpected error %v", err)
+ }
+}
+
+// TestKnowledgeCompilerDSL_ResolveTemplateSeam verifies the group->template
+// resolution path uses the installed seams: compilation_template_group_id
+// resolves to child template ids (GroupResolver) and each is loaded by
+// TemplateResolver, which yields the kind that selects the variant.
+func TestKnowledgeCompilerDSL_ResolveTemplateSeam(t *testing.T) {
+ installStubResolvers(t, map[string][]string{
+ "g1": {"t1", "t2"},
+ }, map[string]kc.TemplateInfo{
+ "t1": {ID: "t1", Kind: "mind_map", Config: map[string]any{"language": "English"}},
+ "t2": {ID: "t2", Kind: "tree", Config: map[string]any{}},
+ })
+
+ ids, err := kc.ResolveGroupTemplateIDs(context.Background(), "tenant", []string{"g1"})
+ if err != nil {
+ t.Fatalf("ResolveGroupTemplateIDs: %v", err)
+ }
+ if len(ids) != 2 || ids[0] != "t1" || ids[1] != "t2" {
+ t.Fatalf("group resolved to %v, want [t1 t2]", ids)
+ }
+
+ info, err := kc.ResolveTemplate(context.Background(), "tenant", "t1")
+ if err != nil {
+ t.Fatalf("ResolveTemplate t1: %v", err)
+ }
+ if info.Kind != "mind_map" {
+ t.Errorf("template kind = %q, want mind_map", info.Kind)
+ }
+ if v, _ := kc.KindToVariant(info.Kind); v != kc.VariantMindmap {
+ t.Errorf("kind %q -> variant %q, want mindmap", info.Kind, v)
+ }
+}
diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.go
index 06a651986b..44ba0ef41a 100644
--- a/internal/ingestion/task/knowledge_compiler_wiring.go
+++ b/internal/ingestion/task/knowledge_compiler_wiring.go
@@ -47,11 +47,12 @@ import (
func init() {
kc.SetDepsResolver(newKnowledgeCompilerDepsResolver())
kc.SetGroupResolver(newKnowledgeCompilerGroupResolver())
+ kc.SetTemplateResolver(newKnowledgeCompilerTemplateResolver())
}
// newKnowledgeCompilerGroupResolver builds the production GroupResolver backed by
// the compilation_template DAO. Without it, any config carrying
-// compilation_template_group_ids would fail loud at runtime (the component
+// compilation_template_group_id would fail loud at runtime (the component
// refuses to silently drop the compilation_template_ids stamp). It resolves each
// group id to its child template ids so group-based configs stamp the full set
// on every compiled unit.
@@ -62,6 +63,26 @@ func newKnowledgeCompilerGroupResolver() kc.GroupResolver {
}
}
+// newKnowledgeCompilerTemplateResolver builds the production TemplateResolver
+// backed by the compilation_template DAO. It loads a single template by id and
+// returns its id, kind (which selects the Go variant via common.KindToVariant),
+// and config (the template "content"). Without it, any config carrying
+// compilation_template_id would fail loudly at runtime.
+func newKnowledgeCompilerTemplateResolver() kc.TemplateResolver {
+ tmplDAO := dao.NewCompilationTemplateDAO()
+ return func(ctx context.Context, tenantID, templateID string) (kc.TemplateInfo, error) {
+ t, err := tmplDAO.GetTemplate(ctx, tenantID, templateID)
+ if err != nil {
+ return kc.TemplateInfo{}, err
+ }
+ return kc.TemplateInfo{
+ ID: t.ID,
+ Kind: t.Kind,
+ Config: map[string]any(t.Config),
+ }, nil
+ }
+}
+
// newKnowledgeCompilerDepsResolver builds the production DepsResolver. Each call
// yields a fresh Deps whose ChatInvoker / Embedder are bound to the resolved
// tenant + model ids (captured in the closure), mirroring how the Tokenizer
diff --git a/rag/flow/compiler/compiler.py b/rag/flow/compiler/compiler.py
index eb1bbfb078..ebc544e601 100644
--- a/rag/flow/compiler/compiler.py
+++ b/rag/flow/compiler/compiler.py
@@ -43,20 +43,29 @@ class CompilerParam(ProcessParamBase, LLMParam):
"""Parameters for the knowledge-Compiler flow component.
Same LLM-backed shape as the Extractor, but instead of a single inline
- ``knowledge_compilation`` config it drives compilation from one or more
- saved **compilation-template groups** (``compilation_template_group_ids``).
- Each group resolves to a set of templates, each of which carries its own
- structure-compilation config (kind, fields, synthesis, ...).
+ ``knowledge_compilation`` config it drives compilation from a saved
+ **compilation-template group** (``compilation_template_group_id``). The group
+ resolves to a set of templates, each of which carries its own
+ structure-compilation config (kind, fields, synthesis, ...). The legacy
+ plural ``compilation_template_group_ids`` key is still accepted as a
+ fallback so existing pipelines keep working.
"""
def __init__(self):
super().__init__()
+ self.compilation_template_group_id = ""
self.compilation_template_group_ids = []
def check(self):
- self.check_empty(self.compilation_template_group_ids, "Compilation Template Groups")
- if isinstance(self.compilation_template_group_ids, str):
- self.compilation_template_group_ids = [self.compilation_template_group_ids]
+ # Prefer the singular group id emitted by the frontend; fall back to the
+ # legacy plural key for backward compatibility. check() normalises the
+ # chosen value into self.compilation_template_group_ids (a list) so the
+ # rest of the Compiler only ever reads that.
+ group = self.compilation_template_group_id or self.compilation_template_group_ids
+ self.check_empty(group, "Compilation Template Group")
+ if isinstance(group, str):
+ group = [group]
+ self.compilation_template_group_ids = group
class Compiler(ProcessBase, LLM):
diff --git a/web/src/pages/agent/canvas/node/compilation-node.tsx b/web/src/pages/agent/canvas/node/compilation-node.tsx
index 409926d0ef..b86659d51f 100644
--- a/web/src/pages/agent/canvas/node/compilation-node.tsx
+++ b/web/src/pages/agent/canvas/node/compilation-node.tsx
@@ -10,7 +10,7 @@ export function CompilationNode({ ...props }: NodeProps<IRagNode>) {
const { data } = props;
const { t } = useTranslation();
const options = useCompilationTemplateGroupOptions();
- const groupId = get(data, 'form.compilation_template_group_ids');
+ const groupId = get(data, 'form.compilation_template_group_id');
const groupName =
options.find((option) => option.value === groupId)?.label ?? groupId;
diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsx
index 90348f3cfc..dbfc0e3372 100644
--- a/web/src/pages/agent/constant/pipeline.tsx
+++ b/web/src/pages/agent/constant/pipeline.tsx
@@ -362,7 +362,7 @@ export const initialExtractorValues = {
};
export const initialCompilationValues = {
- compilation_template_group_ids: [],
+ compilation_template_group_id: '',
outputs: {
chunks: { type: 'Array<Object>', value: [] },
},
diff --git a/web/src/pages/agent/form/compilation-form/index.tsx b/web/src/pages/agent/form/compilation-form/index.tsx
index c3cf9cccf7..3e2a7d3fdf 100644
--- a/web/src/pages/agent/form/compilation-form/index.tsx
+++ b/web/src/pages/agent/form/compilation-form/index.tsx
@@ -13,7 +13,7 @@ import { FormWrapper } from '../components/form-wrapper';
import { Output } from '../components/output';
export const FormSchema = z.object({
- compilation_template_group_ids: z.string().optional(),
+ compilation_template_group_id: z.string().optional(),
});
export type CompilationFormSchemaType = z.infer<typeof FormSchema>;
@@ -33,7 +33,7 @@ const CompilationForm = ({ node }: INextOperatorForm) => {
return (
<Form {...form}>
<FormWrapper>
- <CompilationTemplateFormField name="compilation_template_group_ids"></CompilationTemplateFormField>
+ <CompilationTemplateFormField name="compilation_template_group_id"></CompilationTemplateFormField>
<Output list={outputList}></Output>
</FormWrapper>
</Form>