RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Go knowledge compiler with scheduler-driven dataset compilation (#17913)
Zhichang Yu2e37997ab9c9
diff --git a/AGENTS.md b/AGENTS.mdindex 733bd0ec42..37f9f28a51 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -77,6 +77,7 @@ Rules:## Working Rules- Before editing, inspect the nearest code path that actually owns the behavior.+- When handling review comments, independently verify each substantive claim against the current code or tests before accepting, rejecting, or acting on it.- Keep changes small and local unless the task is explicitly a broader refactor.- Prefer one implementation path instead of preserving old and new versions side by side.- Preserve behavior with focused tests when the behavior is still valid; do not keep tests that protect obsolete behavior.diff --git a/agent/templates/compiler.json b/agent/templates/compiler.jsonindex f38526883c..437524b766 100644--- a/agent/templates/compiler.json+++ b/agent/templates/compiler.json@@ -5,7 +5,8 @@"obj": {"component_name": "Compiler","params": {- "compilation_template_group_id": "c3aa748c8b2111f191f3047c16ec874f",+ "compilation_template_group_id": "",+ "llm_id": "","outputs": {"chunks": {"type": "Array<Object>",@@ -408,7 +409,8 @@{"data": {"form": {- "compilation_template_group_id": "c3aa748c8b2111f191f3047c16ec874f",+ "compilation_template_group_id": "",+ "llm_id": "","outputs": {"chunks": {"type": "Array<Object>",diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.goindex ea77ed5fdb..61a55280cf 100644--- a/cmd/ragflow_server.go+++ b/cmd/ragflow_server.go@@ -30,6 +30,7 @@ import (agenttool "ragflow/internal/agent/tool""ragflow/internal/channels""ragflow/internal/handler"+ "ragflow/internal/ingestion/knowledge_compile"ingestion "ragflow/internal/ingestion/service""ragflow/internal/mcp""ragflow/internal/router"@@ -41,6 +42,7 @@ import ("ragflow/internal/service/file""ragflow/internal/service/nav""ragflow/internal/service/nlp"+ "ragflow/internal/service/wikisearch""ragflow/internal/storage""ragflow/internal/syncer""ragflow/internal/tokenizer"@@ -554,7 +556,15 @@ func runIngestor(ctx context.Context, cancel context.CancelFunc, args *serverArg// writes available_int=0 compiled chunks; they just won't be merged until// the consumer is available.globalConfig := server.GetConfig()- ingestor := ingestion.NewIngestor(*args.name, 2, []string{"pdf", "docx", "txt"})+ ingestorCfg := globalConfig.GetIngestorConfig()+ const maxIngestorConcurrency = int32(1<<30 - 1)+ if ingestorCfg.MaxConcurrentWorkers > int(maxIngestorConcurrency) {+ return fmt.Errorf("ingestor max_concurrent_workers %d exceeds maximum %d", ingestorCfg.MaxConcurrentWorkers, maxIngestorConcurrency)+ }+ // Apply the configured compiler pool size (no-op when 0; the pool keeps its+ // vCPU default, overridable via KC_COMPILE_CONCURRENCY).+ knowledge_compile.SetCompilerConcurrency(ingestorCfg.CompilerPoolSize)+ ingestor := ingestion.NewIngestor(*args.name, int32(ingestorCfg.MaxConcurrentWorkers), []string{"pdf", "docx", "txt"})ingestor.SetKnowledgeCompileModelConfig(globalConfig.GetDefaultChatModel().Name,globalConfig.GetDefaultEmbeddingModel().Name,@@ -850,6 +860,14 @@ func startServer(ctx context.Context) {// on demand so Search/UpsertDoc can embed queries/summaries automatically.nav.SetNavService(nlp.NewNavService(service.NewNavEmbedder(modelProviderService, "")))+ // Install the compiled-wiki search service. It is backed directly by the+ // document engine: QueryPages filters the tenant-scoped index to+ // compile_kwd="wiki_page" (+ supported kinds) so ordinary source chunks are+ // never relabeled as wiki pages, and BackfillChunks fetches original chunks+ // by id. When the engine is unavailable the service degrades to empty so the+ // agent falls back to hybrid search (no failing call).+ wikisearch.SetService(wikisearch.NewEngineService(engine.Get()))+// Initialize routerr := router.NewRouter(authHandler,userHandler,diff --git a/conf/service_conf.yaml b/conf/service_conf.yamlindex 25073a0fa3..59f7de50e2 100644--- a/conf/service_conf.yaml+++ b/conf/service_conf.yaml@@ -69,6 +69,7 @@ otel:ingestor:mq_type: 'nats'max_concurrent_workers: 1+ compiler_pool_size: 0file_syncer:max_concurrent_syncs: 1sync_interval: 3diff --git a/docker/service_conf.yaml.template b/docker/service_conf.yaml.templateindex 5b77399425..531f29a725 100644--- a/docker/service_conf.yaml.template+++ b/docker/service_conf.yaml.template@@ -87,6 +87,7 @@ otel:ingestor:mq_type: 'nats'max_concurrent_workers: 1+ compiler_pool_size: 0file_syncer:max_concurrent_syncs: 1sync_interval: 3diff --git a/internal/agent/harness/agentic_rag.go b/internal/agent/harness/agentic_rag.goindex 24b419199d..d843ff96ac 100644--- a/internal/agent/harness/agentic_rag.go+++ b/internal/agent/harness/agentic_rag.go@@ -46,7 +46,18 @@ type AgenticState struct {// - medium+ (decompose_and_search / agentic_research / deep_research):// pre_search grounds the planner, then decompose-and-search runs until a// sufficiency verdict stops it.+//+// RunAgenticRAG drives the agentic-search graph. It computes the route itself+// and delegates to RunAgenticRAGWithRoute so production runners that need a+// route-aware search strategy (e.g. prefer wiki_query on a wiki suggestion) can+// reuse the same flow with a pre-computed route.func RunAgenticRAG(ctx context.Context, db *gorm.DB, question, keywords, modeLabel string, search SearchFn) AnswerResult {+ return RunAgenticRAGWithRoute(ctx, db, question, keywords, modeLabel, RouteNode(ctx, db, question, modeLabel), search)+}++// RunAgenticRAGWithRoute is the route-aware core of RunAgenticRAG. It performs+// pre_search → planner → orchestrator → formalize_answer with the given route.+func RunAgenticRAGWithRoute(ctx context.Context, db *gorm.DB, question, keywords, modeLabel string, route RouteDecision, search SearchFn) AnswerResult {state := &AgenticState{Question: strings.TrimSpace(question),Keywords: keywords,@@ -56,8 +67,8 @@ func RunAgenticRAG(ctx context.Context, db *gorm.DB, question, keywords, modeLabreturn AnswerResult{FinalAnswer: emptyResultMessage, Empty: true}}- // ── route ──- state.Route = RouteNode(ctx, db, state.Question, modeLabel)+ // ── route (pre-computed by the caller) ──+ state.Route = route// ── pre_search (decomposition modes only) ──if state.Route.RequiresDecomposition {diff --git a/internal/agent/harness/production.go b/internal/agent/harness/production.goindex 2a6f3a9139..0b1f739585 100644--- a/internal/agent/harness/production.go+++ b/internal/agent/harness/production.go@@ -27,24 +27,34 @@ import ("gorm.io/gorm""ragflow/internal/agent/tool"+ "ragflow/internal/common""ragflow/internal/service/nav"+ "ragflow/internal/service/wikisearch")// ProductionRunner wires the real agentic-search tools (hybrid_search,-// dataset_navigation_by_tree) into the RunAgenticRAG flow, so the tools are-// actually invoked rather than merely registered. This is the production-// counterpart to the unit-testable SearchFn seam.+// dataset_navigation_by_tree, wiki_query) into the RunAgenticRAG flow, so the+// tools are actually invoked rather than merely registered. This is the+// production counterpart to the unit-testable SearchFn seam.type ProductionRunner struct {db *gorm.DBtenantID stringdatasetIDs []stringsearchTool einotool.InvokableToolnavSvc nav.NavService // defaults to nav.GetNavService() when nil+ wikiSvc wikisearch.Service+ // webTool is an optional, already-configured web search tool. When nil the+ // runner never exposes web fallback (P8: no web provider configured => the+ // agent does not attempt web search and no failing tool call is made).+ webTool einotool.InvokableTool}// NewProductionRunner builds a ProductionRunner backed by the real tools. The// dataset-nav router (harness.NavigateDatasetByTree) resolves its NavService-// lazily via nav.GetNavService().+// lazily via nav.GetNavService(). When a web provider is configured (a Tavily+// API key is present), the runner also wires the web fallback tool so+// high/ultra modes can fill an empty KB result from the web; otherwise no web+// tool is attached and no web call is ever attempted (P8/R2).func NewProductionRunner(db *gorm.DB, tenantID string, datasetIDs []string) (*ProductionRunner, error) {searchBase, err := tool.BuildByName("hybrid_search", nil)if err != nil {@@ -54,7 +64,11 @@ func NewProductionRunner(db *gorm.DB, tenantID string, datasetIDs []string) (*Prif !ok {return nil, fmt.Errorf("hybrid_search is not invokable")}- return &ProductionRunner{db: db, tenantID: tenantID, datasetIDs: datasetIDs, searchTool: search}, nil+ r := &ProductionRunner{db: db, tenantID: tenantID, datasetIDs: datasetIDs, searchTool: search}+ if common.GetEnv(common.EnvTavilyApiKey) != "" {+ r.webTool = tool.NewTavilyTool()+ }+ return r, nil}// newProductionRunnerWithTools builds a ProductionRunner with an injected@@ -64,19 +78,61 @@ func newProductionRunnerWithTools(db *gorm.DB, tenantID string, datasetIDs []strreturn &ProductionRunner{db: db, tenantID: tenantID, datasetIDs: datasetIDs, searchTool: searchTool, navSvc: navSvc}}-// Run executes the agentic-search graph with the real tools. It returns the-// final answer.+// Run executes the agentic-search graph with the real tools. It computes the+// route once and uses it to pick a search strategy: when the route suggests a+// wiki compilation and the bound KBs actually carry wiki artifacts, the runner+// tries wiki_query first and falls back to general hybrid search on an empty+// result; otherwise it uses hybrid search. Web fallback is only reachable when a+// web provider is configured (P8). Returns the final answer.func (r *ProductionRunner) Run(ctx context.Context, question, keywords, modeLabel string) AnswerResult {if r.searchTool == nil {log.Printf("agentic_rag: production runner not fully wired (search tool missing)")return AnswerResult{FinalAnswer: emptyResultMessage, Empty: true}}- // The router tool returns a doc list; feed it as the search DocScope.+ route := RouteNode(ctx, r.db, question, modeLabel)++ // Base hybrid search, optionally scoped by the nav router for decomposition+ // modes.+ searchFn := r.hybridSearchFn(ctx, question, keywords, modeLabel)++ // P8/R4: web fallback is phase-gated — only wired for modes whose+ // AvailableTools actually include web_search (high/ultra), AND only when a+ // web provider is configured. Low/medium never trigger external web requests+ // from an empty KB result. Unconfigured => no web tool call is ever attempted.+ if modeAllowsWeb(modeLabel) {+ searchFn = r.webFallbackFn(searchFn)+ }++ // P5: prefer wiki when the route suggests it AND the bound KBs carry the+ // artifact; fall back to hybrid on empty/absent wiki results.+ if route.SuggestsCompilation == "wiki" && r.wikiAvailable(ctx) {+ searchFn = r.wikiPreferredSearchFn(searchFn)+ }+ return RunAgenticRAGWithRoute(ctx, r.db, question, keywords, modeLabel, route, searchFn)+}++// modeAllowsWeb reports whether the mode's AvailableTools include web_search, so+// web fallback is only reachable in the modes that are supposed to have it+// (high/ultra). Unknown modes are treated as not allowing web.+func modeAllowsWeb(modeLabel string) bool {+ mode, ok := GetMode(modeLabel)+ if !ok {+ return false+ }+ for _, name := range mode.AvailableTools {+ if name == "web_search" {+ return true+ }+ }+ return false+}++// hybridSearchFn builds the base hybrid search closure (optionally doc-scoped+// for decomposition modes).+func (r *ProductionRunner) hybridSearchFn(ctx context.Context, question, keywords, modeLabel string) SearchFn {searchFn := func(ctx context.Context, query, kws string) ([]map[string]interface{}, []map[string]interface{}) {return r.search(ctx, query, kws, nil)}-- // For decomposition modes, route the doc scope first via the nav tool.if mode, _ := GetMode(modeLabel); mode.RequiresDecomposition {docs := r.routeDocs(ctx, question, keywords)if len(docs) > 0 {@@ -85,7 +141,55 @@ func (r *ProductionRunner) Run(ctx context.Context, question, keywords, modeLabe}}}- return RunAgenticRAG(ctx, r.db, question, keywords, modeLabel, searchFn)+ return searchFn+}++// wikiPreferredSearchFn wraps the hybrid searchFn so that each search first asks+// the compiled wiki for the query and only falls back to hybrid when the wiki+// returns nothing (or the wiki backend is unavailable). This is the P5 route+// consumption: a wiki suggestion selects the wiki path without discarding the+// hybrid fallback.+func (r *ProductionRunner) wikiPreferredSearchFn(hybrid SearchFn) SearchFn {+ return func(ctx context.Context, query, kws string) ([]map[string]interface{}, []map[string]interface{}) {+ chunks, aggs := r.wikiSearch(ctx, query, kws)+ if len(chunks) > 0 {+ return chunks, aggs+ }+ return hybrid(ctx, query, kws)+ }+}++// wikiAvailable reports whether the bound datasets carry searchable wiki+// artifacts, so the runner only selects the wiki path when it can actually serve.+func (r *ProductionRunner) wikiAvailable(ctx context.Context) bool {+ ws := r.wikiSvc+ if ws == nil {+ ws = wikisearch.GetService()+ }+ if ws == nil {+ return false+ }+ return ws.AvailableFor(ctx, r.tenantID, r.datasetIDs)+}++// wikiSearch invokes the wiki_query tool against the compiled wiki. It returns+// empty chunks (never a hard error) when the service is unavailable or yields+// nothing, so the caller falls back to hybrid search.+func (r *ProductionRunner) wikiSearch(ctx context.Context, query, keywords string) ([]map[string]interface{}, []map[string]interface{}) {+ ws := r.wikiSvc+ if ws == nil {+ ws = wikisearch.GetService()+ }+ if ws == nil || !ws.AvailableFor(ctx, r.tenantID, r.datasetIDs) {+ return nil, nil+ }+ res, err := ws.QueryPages(ctx, r.tenantID, r.datasetIDs, query, keywords, 12)+ if err != nil || len(res.Chunks) == 0 {+ return nil, nil+ }+ // P7: backfill the original source chunks referenced by the compiled page+ // hits, deduped and bounded, so the answer can cite raw evidence.+ return r.expandCompiledEvidence(ctx, res.Chunks, res.DocAggs)}// search invokes the hybrid_search tool and normalizes its chunk output.@@ -108,6 +212,204 @@ func (r *ProductionRunner) search(ctx context.Context, query, keywords string, dreturn res.Chunks, nil}+// expandCompiledEvidence backfills the ORIGINAL source chunks a compiled-page+// hit was built from (P7/R3). It collects the page hits' source_chunk_ids+// (bounded per page and in total), then asks the concrete wiki service to fetch+// them BY ID — scoped to the tenant + datasets — so the answer can cite raw+// evidence. When the page hits carry no source ids, the service is unavailable,+// or none of the ids resolve, the page results are kept as-is (safe degradation;+// nothing is fabricated).+func (r *ProductionRunner) expandCompiledEvidence(ctx context.Context, chunks, aggs []map[string]interface{}) ([]map[string]interface{}, []map[string]interface{}) {+ if len(chunks) == 0 {+ return chunks, aggs+ }+ ws := r.wikiSvc+ if ws == nil {+ ws = wikisearch.GetService()+ }+ if ws == nil {+ return chunks, aggs+ }+ const maxEvidencePerPage = 4+ const maxEvidenceTotal = 12++ // Collect bounded source-chunk ids from the page hits (deduped, in page+ // order), grouped by dataset so the backfill stays within each KB's scope.+ var sourceIDs []string+ seen := map[string]bool{}+ datasets := map[string]bool{}+ for _, c := range chunks {+ if len(sourceIDs) >= maxEvidenceTotal {+ break+ }+ ids := stringSlice(c["source_chunk_ids"])+ count := 0+ for _, id := range ids {+ if count >= maxEvidencePerPage {+ break+ }+ if id == "" || seen[id] {+ continue+ }+ seen[id] = true+ count+++ sourceIDs = append(sourceIDs, id)+ if ds := stringValue(c["dataset_id"]); ds != "" {+ datasets[ds] = true+ }+ if len(sourceIDs) >= maxEvidenceTotal {+ break+ }+ }+ }+ if len(sourceIDs) == 0 {+ return chunks, aggs+ }+ // Scope the backfill to the page hits' datasets (fall back to all bound+ // datasets when the page hits carry none). Build a fresh slice: never mutate+ // r.datasetIDs.+ scope := make([]string, 0, len(r.datasetIDs))+ if len(datasets) == 0 {+ scope = append(scope, r.datasetIDs...)+ } else {+ for ds := range datasets {+ scope = append(scope, ds)+ }+ }+ evidence, err := ws.BackfillChunks(ctx, r.tenantID, scope, sourceIDs)+ if err != nil || len(evidence) == 0 {+ return chunks, aggs+ }++ // Stable merge: page results first (in retrieval order), then the backfilled+ // evidence, deduped by chunk key.+ merged := append([]map[string]interface{}(nil), chunks...)+ keys := map[string]bool{}+ for _, c := range chunks {+ if k := chunkKey(c); k != "" {+ keys[k] = true+ }+ }+ for _, e := range evidence {+ k := chunkKey(e)+ if k != "" && !keys[k] {+ keys[k] = true+ merged = append(merged, e)+ }+ }+ // Doc aggs: union the page doc aggs with the evidence docs.+ dseen := map[string]bool{}+ for _, d := range aggs {+ if id, _ := d["doc_id"].(string); id != "" {+ dseen[id] = true+ }+ }+ for _, e := range evidence {+ id := stringValue(e["doc_id"])+ if id == "" {+ continue+ }+ if !dseen[id] {+ dseen[id] = true+ aggs = append(aggs, map[string]interface{}{"doc_id": id, "doc_name": stringValue(e["docnm_kwd"])})+ }+ }+ return merged, aggs+}++// webFallbackFn wraps a SearchFn so that, when the KB search returns nothing, a+// configured web provider is invoked to fill the gap (P8). It is only used when+// webTool is non-nil; otherwise it returns the hybrid path unchanged and no web+// tool call is ever attempted (no failing call when unconfigured).+func (r *ProductionRunner) webFallbackFn(hybrid SearchFn) SearchFn {+ if r.webTool == nil {+ return hybrid+ }+ return func(ctx context.Context, query, kws string) ([]map[string]interface{}, []map[string]interface{}) {+ chunks, aggs := hybrid(ctx, query, kws)+ if len(chunks) > 0 {+ return chunks, aggs+ }+ raw, err := r.webTool.InvokableRun(ctx, mustJSON(map[string]interface{}{"query": query, "keywords": kws}))+ if err != nil {+ return nil, nil+ }+ var res struct {+ Chunks []map[string]interface{} `json:"chunks"`+ Results []map[string]interface{} `json:"results"`+ }+ if err := json.Unmarshal([]byte(raw), &res); err != nil {+ return nil, nil+ }+ // Normalize web evidence into the same agentic evidence shape as KB+ // chunks. Accept both the agent "chunks" envelope and the Tavily+ // "results" envelope (tavily.go returns {"results":[...]}); each result+ // contributes content + a doc_id reference so the answer can retain the+ // source URL.+ src := res.Chunks+ if len(src) == 0 {+ src = res.Results+ }+ out := make([]map[string]interface{}, 0, len(src))+ for _, c := range src {+ url := firstNonEmpty(stringValue(c["url"]), stringValue(c["link"]), stringValue(c["source"]))+ if url == "" {+ continue+ }+ content := firstNonEmpty(stringValue(c["content"]), stringValue(c["raw_content"]), stringValue(c["text"]))+ if content == "" {+ continue+ }+ docID := stringValue(c["doc_id"])+ if docID == "" {+ docID = url + "|" + stringValue(c["source"])+ }+ out = append(out, map[string]interface{}{+ "chunk_id": docID, "content_with_weight": content,+ "doc_id": docID, "docnm_kwd": firstNonEmpty(stringValue(c["title"]), stringValue(c["source"])),+ "dataset_id": stringValue(c["dataset_id"]), "url": url, "source": "web",+ })+ }+ if len(out) == 0 {+ return nil, nil+ }+ return out, nil+ }+}++func stringValue(v interface{}) string {+ if s, ok := v.(string); ok {+ return s+ }+ return ""+}++func firstNonEmpty(ss ...string) string {+ for _, s := range ss {+ if strings.TrimSpace(s) != "" {+ return s+ }+ }+ return ""+}++func stringSlice(v interface{}) []string {+ if raw, ok := v.([]string); ok {+ return raw+ }+ arr, ok := v.([]interface{})+ if !ok {+ return nil+ }+ out := make([]string, 0, len(arr))+ for _, item := range arr {+ if s, ok := item.(string); ok {+ out = append(out, s)+ }+ }+ return out+}+// routeDocs derives the doc scope via the canonical dataset-nav router// (harness.NavigateDatasetByTree — the full LLM two-round selection). It routes// across ALL bound datasets and merges the doc ids, so every KB contributes itsdiff --git a/internal/agent/harness/production_wiki_test.go b/internal/agent/harness/production_wiki_test.gonew file mode 100644index 0000000000..31c9344814--- /dev/null+++ b/internal/agent/harness/production_wiki_test.go@@ -0,0 +1,276 @@+package harness++import (+ "context"+ "strings"+ "sync"+ "testing"++ "gorm.io/gorm"++ "ragflow/internal/agent/component"+ "ragflow/internal/service/wikisearch"+)++// fakeWikiSvcHarness is a deterministic wikisearch.Service double for the+// production runner.+type fakeWikiSvcHarness struct {+ available bool+ // pages are pre-shaped page chunks (map form) returned by QueryPages.+ pages []map[string]interface{}+ // backfill maps chunk id -> evidence content for BackfillChunks.+ backfill map[string]string+ calls []string+ queryCount int+ mu sync.Mutex+}++func (f *fakeWikiSvcHarness) AvailableFor(_ context.Context, _ string, _ []string) bool {+ return f.available+}++func (f *fakeWikiSvcHarness) QueryPages(_ context.Context, _ string, _ []string, query, _ string, _ int) (wikisearch.SearchResult, error) {+ f.mu.Lock()+ f.queryCount+++ f.calls = append(f.calls, query)+ f.mu.Unlock()+ res := wikisearch.SearchResult{Chunks: append([]map[string]interface{}(nil), f.pages...), DocAggs: []map[string]interface{}{}}+ for _, c := range res.Chunks {+ if docID, _ := c["doc_id"].(string); docID != "" {+ res.DocAggs = append(res.DocAggs, map[string]interface{}{"doc_id": docID, "doc_name": c["docnm_kwd"]})+ }+ }+ return res, nil+}++func (f *fakeWikiSvcHarness) seenQueries() []string {+ f.mu.Lock()+ defer f.mu.Unlock()+ return append([]string(nil), f.calls...)+}++// backfill is the fake's per-id evidence map (chunk id -> content).+func (f *fakeWikiSvcHarness) BackfillChunks(_ context.Context, _ string, _ []string, chunkIDs []string) ([]map[string]interface{}, error) {+ out := make([]map[string]interface{}, 0, len(chunkIDs))+ for _, id := range chunkIDs {+ content, ok := f.backfill[id]+ if !ok {+ continue+ }+ out = append(out, map[string]interface{}{+ "chunk_id": id, "content_with_weight": content, "doc_id": "d1", "docnm_kwd": "Doc", "dataset_id": "kb1",+ })+ }+ return out, nil+}++// wikiRouteChat returns a route JSON suggesting wiki for the route stage and a+// plain final answer otherwise.+type wikiRouteChat struct{}++func (wikiRouteChat) Invoke(_ context.Context, _ *gorm.DB, req component.ChatInvokeRequest) (*component.ChatInvokeResponse, error) {+ // The route stage carries the route prompt (with "suggests_compilation") as+ // a system message; detect it across any message (system or user).+ isRoute := false+ for _, m := range req.Messages {+ if strings.Contains(m.Content, "suggests_compilation") {+ isRoute = true+ break+ }+ }+ if isRoute {+ return &component.ChatInvokeResponse{Content: `{"question_type":"analytical","requires_decomposition":false,"suggests_compilation":"wiki"}`}, nil+ }+ return &component.ChatInvokeResponse{Content: "final wiki answer"}, nil+}++func installWikiRouteChat(t *testing.T) {+ t.Helper()+ component.SetDefaultChatInvoker(wikiRouteChat{})+ t.Cleanup(func() { component.SetDefaultChatInvoker(nil) })+}++// TestProductionRunner_WikiPreferred_WhenSuggested drives a low-mode run with a+// wiki suggestion and an available wiki service, and asserts the wiki service is+// queried (P5: route suggestion selects the wiki path).+func TestProductionRunner_WikiPreferred_WhenSuggested(t *testing.T) {+ installWikiRouteChat(t)+ hybrid := &fakeInvokableTool{name: "hybrid_search", fn: func(_ context.Context, _ string) string {+ return `{"chunks":[]}`+ }}+ wikiSvc := &fakeWikiSvcHarness{available: true, pages: []map[string]interface{}{+ {"chunk_id": "wiki/entity/alpha", "content_with_weight": "# Alpha", "doc_id": "kb1", "docnm_kwd": "Alpha", "wiki_slug_kwd": "entity/alpha", "dataset_id": "kb1"},+ }}+ runner := newProductionRunnerWithTools(nil, "t1", []string{"kb1"}, hybrid, nil)+ runner.wikiSvc = wikiSvc+ res := runner.Run(context.Background(), "What is Alpha?", "", "low")+ if res.FinalAnswer != "final wiki answer" {+ t.Errorf("final answer = %q, want wiki chat output", res.FinalAnswer)+ }+ if len(wikiSvc.seenQueries()) == 0 {+ t.Errorf("wiki service was not queried despite a wiki suggestion")+ }+}++// TestProductionRunner_WikiEmpty_FallsBackToHybrid asserts that when the wiki+// service yields nothing, the runner falls back to hybrid search (P5 must not+// discard the general retrieval fallback).+func TestProductionRunner_WikiEmpty_FallsBackToHybrid(t *testing.T) {+ installWikiRouteChat(t)+ hybrid := &fakeInvokableTool{name: "hybrid_search", fn: func(_ context.Context, _ string) string {+ return `{"chunks":[{"chunk_id":"c1","content_with_weight":"hybrid evidence"}]}`+ }}+ wikiSvc := &fakeWikiSvcHarness{available: true} // no pages+ runner := newProductionRunnerWithTools(nil, "t1", []string{"kb1"}, hybrid, nil)+ runner.wikiSvc = wikiSvc+ res := runner.Run(context.Background(), "What is Alpha?", "", "low")+ if len(wikiSvc.seenQueries()) == 0 {+ t.Errorf("wiki service should have been attempted")+ }+ if !strings.Contains(hybrid.args(), `"query":"What is Alpha?"`) {+ t.Errorf("hybrid fallback was not invoked after empty wiki; args=%s", hybrid.args())+ }+ if res.FinalAnswer == "" {+ t.Errorf("expected a final answer from the hybrid fallback")+ }+}++// TestProductionRunner_NoWikiWithoutSuggestion asserts the wiki service is NOT+// queried when the route does not suggest wiki.+func TestProductionRunner_NoWikiWithoutSuggestion(t *testing.T) {+ // route chat returns no suggestion for a generic route call+ installRouteChat(t) // routeChat returns plain text -> route falls back (no suggestion)+ hybrid := &fakeInvokableTool{name: "hybrid_search", fn: func(_ context.Context, _ string) string {+ return `{"chunks":[{"chunk_id":"c1","content_with_weight":"evidence"}]}`+ }}+ wikiSvc := &fakeWikiSvcHarness{available: true, pages: []map[string]interface{}{+ {"chunk_id": "wiki/s", "content_with_weight": "c", "doc_id": "kb1", "docnm_kwd": "T", "wiki_slug_kwd": "s", "dataset_id": "kb1"},+ }}+ runner := newProductionRunnerWithTools(nil, "t1", []string{"kb1"}, hybrid, nil)+ runner.wikiSvc = wikiSvc+ runner.Run(context.Background(), "What is Alpha?", "", "low")+ if len(wikiSvc.seenQueries()) != 0 {+ t.Errorf("wiki service must not be queried without a wiki suggestion; calls=%v", wikiSvc.seenQueries())+ }+}++// TestProductionRunner_WebFallback_UnconfiguredIsNoop asserts that with no web+// tool configured the runner never attempts a web call (P8 gate).+func TestProductionRunner_WebFallback_UnconfiguredIsNoop(t *testing.T) {+ webTool := &fakeInvokableTool{name: "web_search", fn: func(_ context.Context, _ string) string {+ return `{"chunks":[{"chunk_id":"w1","content_with_weight":"web evidence"}]}`+ }}+ hybrid := &fakeInvokableTool{name: "hybrid_search", fn: func(_ context.Context, _ string) string {+ return `{"chunks":[]}`+ }}+ // webTool is NOT set on the runner -> the runner must not invoke it.+ runner := newProductionRunnerWithTools(nil, "t1", []string{"kb1"}, hybrid, nil)+ // With webTool nil, webFallbackFn returns the hybrid path unchanged, so the+ // web tool is never called.+ fn := runner.webFallbackFn(func(ctx context.Context, q, k string) ([]map[string]interface{}, []map[string]interface{}) {+ return nil, nil+ })+ chunks, _ := fn(context.Background(), "q", "k")+ if len(chunks) != 0 {+ t.Errorf("expected no chunks from the no-web fallback path")+ }+ if webTool.args() != "" {+ t.Errorf("web tool must not be called when unconfigured; args=%s", webTool.args())+ }+}++// TestProductionRunner_WebFallback_TavilyResultsNormalized asserts R2: the web+// fallback consumes the Tavily `{"results":[...]}` envelope (tavily.go contract)+// and normalizes each result into an agent evidence chunk with a doc_id+// reference, not the agent `chunks` shape.+func TestProductionRunner_WebFallback_TavilyResultsNormalized(t *testing.T) {+ webTool := &fakeInvokableTool{name: "web_search", fn: func(_ context.Context, _ string) string {+ return `{"results":[{"title":"Alpha docs","url":"https://example.com/x","content":"web evidence body","source":"example.com"}]}`+ }}+ runner := newProductionRunnerWithTools(nil, "t1", []string{"kb1"}, nil, nil)+ runner.webTool = webTool+ fn := runner.webFallbackFn(func(ctx context.Context, q, k string) ([]map[string]interface{}, []map[string]interface{}) {+ return nil, nil+ })+ chunks, _ := fn(context.Background(), "q", "k")+ if len(chunks) != 1 {+ t.Fatalf("chunks = %d, want 1 normalized Tavily result", len(chunks))+ }+ if chunks[0]["content_with_weight"] != "web evidence body" {+ t.Errorf("content = %v, want web evidence body", chunks[0]["content_with_weight"])+ }+ if chunks[0]["doc_id"] == "" || !strings.Contains(chunks[0]["doc_id"].(string), "https://example.com/x") {+ t.Errorf("doc_id must reference the source url: %v", chunks[0]["doc_id"])+ }+ if chunks[0]["source"] != "web" {+ t.Errorf("source = %v, want web", chunks[0]["source"])+ }+ if webTool.args() == "" {+ t.Errorf("web tool was not invoked when hybrid was empty")+ }+}++// TestModeAllowsWeb asserts R4 gating: only modes whose AvailableTools include+// web_search (high/ultra) allow web fallback; low/medium/unknown do not.+func TestModeAllowsWeb(t *testing.T) {+ if !modeAllowsWeb("high") || !modeAllowsWeb("ultra") {+ t.Errorf("high/ultra must allow web search")+ }+ for _, m := range []string{"low", "medium", "fast", "unknown"} {+ if modeAllowsWeb(m) {+ t.Errorf("mode %q must NOT allow web search", m)+ }+ }+}++// TestProductionRunner_CompiledEvidenceExpansion asserts P7/R3: page hits+// carrying source_chunk_ids get the ORIGINAL chunks fetched by id (via the wiki+// service BackfillChunks) appended after the page results, deduped.+func TestProductionRunner_CompiledEvidenceExpansion(t *testing.T) {+ wikiSvc := &fakeWikiSvcHarness{available: true, backfill: map[string]string{"c1": "raw chunk 1", "c2": "raw chunk 2"}}+ runner := newProductionRunnerWithTools(nil, "t1", []string{"kb1"}, nil, nil)+ runner.wikiSvc = wikiSvc+ chunks := []map[string]interface{}{+ {+ "chunk_id": "wiki/s", "content_with_weight": "# Page", "doc_id": "d1", "docnm_kwd": "Page", "dataset_id": "kb1",+ "source_chunk_ids": []interface{}{"c1", "c2"},+ },+ }+ merged, aggs := runner.expandCompiledEvidence(context.Background(), chunks, []map[string]interface{}{})+ if len(merged) != 3 {+ t.Fatalf("merged = %d, want 3 (page + 2 backfilled evidence chunks)", len(merged))+ }+ if merged[0]["chunk_id"] != "wiki/s" || merged[1]["chunk_id"] != "c1" || merged[2]["chunk_id"] != "c2" {+ t.Errorf("merge order wrong: %v", merged)+ }+ if merged[1]["content_with_weight"] != "raw chunk 1" {+ t.Errorf("evidence chunk content = %v, want raw chunk 1 (by-id backfill)", merged[1]["content_with_weight"])+ }+ // Doc aggs must include the evidence doc.+ found := false+ for _, d := range aggs {+ if d["doc_id"] == "d1" {+ found = true+ }+ }+ if !found {+ t.Errorf("evidence doc not unioned into doc aggs: %v", aggs)+ }+}++// TestProductionRunner_CompiledEvidence_NoServiceIsNoop asserts P7/R3 degrades+// safely: when the wiki service is unavailable, page hits (even with+// source_chunk_ids) keep the page results unchanged and fabricate nothing.+func TestProductionRunner_CompiledEvidence_NoServiceIsNoop(t *testing.T) {+ runner := newProductionRunnerWithTools(nil, "t1", []string{"kb1"}, nil, nil) // wikiSvc nil+ chunks := []map[string]interface{}{+ {"chunk_id": "wiki/s", "content_with_weight": "# Page", "dataset_id": "kb1", "source_chunk_ids": []interface{}{"c1", "c2"}},+ }+ merged, _ := runner.expandCompiledEvidence(context.Background(), chunks, []map[string]interface{}{})+ if len(merged) != 1 {+ t.Fatalf("merged = %d, want 1 (no service => no fabricated evidence)", len(merged))+ }+ if merged[0]["chunk_id"] != "wiki/s" {+ t.Errorf("page result changed: %v", merged[0])+ }+}diff --git a/internal/agent/harness/route.go b/internal/agent/harness/route.goindex 5459cfd0d5..9e5ca228c1 100644--- a/internal/agent/harness/route.go+++ b/internal/agent/harness/route.go@@ -50,9 +50,10 @@ Output format (JSON):`type routeResult struct {- QuestionType string `json:"question_type"`- RequiresDecomp *bool `json:"requires_decomposition"`- Reasoning string `json:"reasoning"`+ QuestionType string `json:"question_type"`+ RequiresDecomp *bool `json:"requires_decomposition"`+ SuggestsCompilation string `json:"suggests_compilation"`+ Reasoning string `json:"reasoning"`}// RouteNode mirrors Python route_node. It classifies the question into a@@ -104,10 +105,28 @@ func decide(question, modeLabel string, res routeResult) RouteDecision {QuestionType: qType,RequiresDecomposition: mode.RequiresDecomposition && needDecomp,ExecutionStrategy: mode.Strategy,+ SuggestsCompilation: normalizeCompilationSuggestion(res.SuggestsCompilation),Reasoning: res.Reasoning,}}+// normalizeCompilationSuggestion maps the LLM's free-text suggestion to a+// canonical compiled-artifact key: "", "toc", "graph", or "wiki". Anything else+// (including "null") collapses to "" so the production runner never routes on an+// unknown artifact name.+func normalizeCompilationSuggestion(s string) string {+ switch strings.ToLower(strings.TrimSpace(s)) {+ case "toc":+ return "toc"+ case "graph", "knowledge_graph", "kg":+ return "graph"+ case "wiki", "compiled":+ return "wiki"+ default:+ return ""+ }+}+func fallbackRoute(question, modeLabel, reason string) RouteDecision {return RouteDecision{Question: question, ThinkingMode: modeLabel, QuestionType: "factual",diff --git a/internal/agent/harness/route_test.go b/internal/agent/harness/route_test.goindex 03d9170079..da9eec04a7 100644--- a/internal/agent/harness/route_test.go+++ b/internal/agent/harness/route_test.go@@ -74,6 +74,48 @@ func TestRouteNode_EmptyQuestionFallsBack(t *testing.T) {}}+// TestRouteNode_SuggestsCompilation asserts the route preserves a normalized+// compiled-artifact suggestion (P5) so the production runner can prefer the wiki+// tool.+func TestRouteNode_SuggestsCompilation(t *testing.T) {+ installChat(t, `{"question_type":"analytical","requires_decomposition":true,"suggests_compilation":"wiki"}`)+ r := RouteNode(context.Background(), nil, "What does the domain say about X?", "medium")+ if r.SuggestsCompilation != "wiki" {+ t.Errorf("suggests_compilation = %q, want wiki", r.SuggestsCompilation)+ }+}++// TestNormalizeCompilationSuggestion asserts free-text suggestions map to the+// canonical keys and unknown/null collapse to "".+func TestNormalizeCompilationSuggestion(t *testing.T) {+ cases := map[string]string{+ "wiki": "wiki",+ "WIKI": "wiki",+ "compiled": "wiki",+ "graph": "graph",+ "kg": "graph",+ "toc": "toc",+ "null": "",+ "": "",+ "spaghetti": "",+ }+ for in, want := range cases {+ if got := normalizeCompilationSuggestion(in); got != want {+ t.Errorf("normalizeCompilationSuggestion(%q) = %q, want %q", in, got, want)+ }+ }+}++// TestRouteNode_WikiSuggestionSurvivesFence asserts a fenced JSON route still+// carries the wiki suggestion through decide().+func TestRouteNode_WikiSuggestionSurvivesFence(t *testing.T) {+ installChat(t, "```json\n{\"question_type\":\"procedural\",\"requires_decomposition\":false,\"suggests_compilation\":\"graph\"}\n```")+ r := RouteNode(context.Background(), nil, "How is this structured?", "medium")+ if r.SuggestsCompilation != "graph" {+ t.Errorf("suggests_compilation = %q, want graph", r.SuggestsCompilation)+ }+}+// TestPlannerNode_DirectMode asserts a non-decomposed route yields one coarse// claim without calling the LLM.func TestPlannerNode_DirectMode(t *testing.T) {diff --git a/internal/agent/harness/types.go b/internal/agent/harness/types.goindex 3b9c56d357..2f3393d1c8 100644--- a/internal/agent/harness/types.go+++ b/internal/agent/harness/types.go@@ -26,7 +26,13 @@ type RouteDecision struct {QuestionType string // factual | comparative | analytical | procedural | exploratory | verification | summarizationRequiresDecomposition boolExecutionStrategy string // direct_search | decompose_and_search | agentic_research | deep_research- Reasoning string+ // SuggestsCompilation is the compiled-artifact type the route suggests using+ // for retrieval: "" (none) | "toc" | "graph" | "wiki". Mirrors Python's+ // route `suggests_compilation`. It is preserved so the production runner can+ // prefer a compiled wiki/graph tool when the bound KBs actually carry the+ // artifact, and fall back to general hybrid search otherwise.+ SuggestsCompilation string+ Reasoning string}// ClaimTarget mirrors Python ClaimTarget.diff --git a/internal/agent/tool/registry.go b/internal/agent/tool/registry.goindex 89fafeefd7..f5143b9ec2 100644--- a/internal/agent/tool/registry.go+++ b/internal/agent/tool/registry.go@@ -40,6 +40,7 @@ var registry = map[string]Factory{"hybrid_search": noConfig("hybrid_search", func() einotool.BaseTool { return NewAgenticSearchTool(toolHybridSearch) }),"vector_search": noConfig("vector_search", func() einotool.BaseTool { return NewAgenticSearchTool(toolVectorSearch) }),"bm25_search": noConfig("bm25_search", func() einotool.BaseTool { return NewAgenticSearchTool(toolBM25Search) }),+ "wiki_query": noConfig("wiki_query", func() einotool.BaseTool { return NewWikiQueryTool() }),"deepl": noConfig("deepl", func() einotool.BaseTool { return NewDeepLTool() }),"duckduckgo": buildDuckDuckGoTool,"email": buildEmailTool,diff --git a/internal/agent/tool/wiki_query.go b/internal/agent/tool/wiki_query.gonew file mode 100644index 0000000000..bd02c9a165--- /dev/null+++ b/internal/agent/tool/wiki_query.go@@ -0,0 +1,115 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package tool++import (+ "context"+ "encoding/json"+ "fmt"+ "strings"++ einotool "github.com/cloudwego/eino/components/tool"+ "github.com/cloudwego/eino/schema"++ "ragflow/internal/service/wikisearch"+)++// WikiQueryTool is the wiki_query agent tool (Python harness/tools/exploration.py+// wiki_query). It hybrid-searches the compiled wiki/artifact pages of the bound+// datasets and returns each page's rendered markdown as a chunk, narrowed by+// keywords. Input keeps query + keywords so the LLM's tool schema matches the+// other search tools.+//+// The tool is scoped to the calling tenant and bound datasets, so results never+// leak across tenants/KBs. When the wiki-search service is not configured, or the+// bound datasets have no wiki artifacts, it returns an empty result so the agent+// falls back to general hybrid search.+type WikiQueryTool struct {+ service wikisearch.Service // nil => resolve lazily via wikisearch.GetService()+ topN int+}++// NewWikiQueryTool returns the wiki_query tool. The service is resolved lazily+// from the wikisearch singleton unless overridden for tests.+func NewWikiQueryTool() *WikiQueryTool {+ return &WikiQueryTool{topN: 12}+}++// newWikiQueryToolWithService builds a tool with an injected service, for tests.+func newWikiQueryToolWithService(service wikisearch.Service) *WikiQueryTool {+ return &WikiQueryTool{service: service, topN: 12}+}++type wikiQueryArgs struct {+ Query string `json:"query"`+ Keywords string `json:"keywords,omitempty"`+}++func (w *WikiQueryTool) Info(_ context.Context) (*schema.ToolInfo, error) {+ return &schema.ToolInfo{+ Name: "wiki_query",+ Desc: "Search the compiled wiki of the bound knowledge base(s). Returns rendered wiki page content as passages.",+ ParamsOneOf: schema.NewParamsOneOfByParams(map[string]*schema.ParameterInfo{+ "query": {Type: schema.String, Required: true, Desc: "The search query."},+ "keywords": {Type: schema.String, Desc: "Comma-separated keywords to narrow results."},+ }),+ }, nil+}++// InvokableRun executes a wiki lookup. It never returns a hard error for an+// empty/unconfigured backend so the agent can fall back to hybrid search.+func (w *WikiQueryTool) InvokableRun(ctx context.Context, argumentsInJSON string, _ ...einotool.Option) (string, error) {+ var args wikiQueryArgs+ if err := json.Unmarshal([]byte(argumentsInJSON), &args); err != nil {+ return "", fmt.Errorf("wiki_query: parse arguments: %w", err)+ }+ svc := w.service+ if svc == nil {+ svc = wikisearch.GetService()+ }+ tenantID := canvasTenantID(ctx)+ datasetIDs := canvasDatasetIDs(ctx, nil)+ if svc == nil || tenantID == "" || len(datasetIDs) == 0 {+ return emptyWikiResult(), nil+ }+ if !svc.AvailableFor(ctx, tenantID, datasetIDs) {+ return emptyWikiResult(), nil+ }+ topN := w.topN+ if topN <= 0 {+ topN = 12+ }+ res, err := svc.QueryPages(ctx, tenantID, datasetIDs, strings.TrimSpace(args.Query), args.Keywords, topN)+ if err != nil {+ return emptyWikiResult(), nil+ }+ if res.Chunks == nil {+ res.Chunks = []map[string]interface{}{}+ }+ if res.DocAggs == nil {+ res.DocAggs = []map[string]interface{}{}+ }+ out, err := json.Marshal(map[string]interface{}{"answer": "", "chunks": res.Chunks, "doc_aggs": res.DocAggs})+ if err != nil {+ return emptyWikiResult(), nil+ }+ return string(out), nil+}++func emptyWikiResult() string {+ return `{"answer":"","chunks":[],"doc_aggs":[]}`+}diff --git a/internal/agent/tool/wiki_query_test.go b/internal/agent/tool/wiki_query_test.gonew file mode 100644index 0000000000..11e6911187--- /dev/null+++ b/internal/agent/tool/wiki_query_test.go@@ -0,0 +1,125 @@+package tool++import (+ "context"+ "encoding/json"+ "testing"++ "ragflow/internal/agent/runtime"+ "ragflow/internal/service/wikisearch"+)++// fakeWikiService is a deterministic wikisearch.Service double.+type fakeWikiService struct {+ available map[string]bool // datasetID -> has artifact+ pages func(query string) []map[string]interface{}+ backfill map[string]string // chunk id -> content+ callCount int+}++func (f *fakeWikiService) AvailableFor(_ context.Context, _ string, datasetIDs []string) bool {+ for _, ds := range datasetIDs {+ if f.available[ds] {+ return true+ }+ }+ return false+}++func (f *fakeWikiService) QueryPages(_ context.Context, _ string, _ []string, query, _ string, topN int) (wikisearch.SearchResult, error) {+ f.callCount+++ if f.pages == nil || len(f.pages(query)) == 0 {+ return wikisearch.SearchResult{Chunks: []map[string]interface{}{}, DocAggs: []map[string]interface{}{}}, nil+ }+ res := wikisearch.SearchResult{Chunks: append([]map[string]interface{}(nil), f.pages(query)...), DocAggs: []map[string]interface{}{}}+ seen := map[string]bool{}+ for _, c := range res.Chunks {+ if docID, _ := c["doc_id"].(string); docID != "" && !seen[docID] {+ seen[docID] = true+ res.DocAggs = append(res.DocAggs, map[string]interface{}{"doc_id": docID, "doc_name": c["docnm_kwd"]})+ }+ }+ return res, nil+}++func (f *fakeWikiService) BackfillChunks(_ context.Context, _ string, _ []string, chunkIDs []string) ([]map[string]interface{}, error) {+ out := make([]map[string]interface{}, 0, len(chunkIDs))+ for _, id := range chunkIDs {+ content, ok := f.backfill[id]+ if !ok {+ continue+ }+ out = append(out, map[string]interface{}{"chunk_id": id, "content_with_weight": content, "doc_id": "d1", "docnm_kwd": "Doc"})+ }+ return out, nil+}++// wikiToolRun runs the wiki_query tool with a single-dataset canvas context+// (the tool derives tenant + dataset scope from canvas state).+func wikiToolRun(t *testing.T, svc wikisearch.Service, tenant string, kb string, query string) map[string]interface{} {+ t.Helper()+ state := runtime.NewCanvasState("run-1", "task-1")+ state.Sys["tenant_id"] = tenant+ state.Sys["dataset_id"] = kb+ ctx := runtime.WithState(context.Background(), state)+ tool := newWikiQueryToolWithService(svc)+ args, _ := json.Marshal(map[string]interface{}{"query": query})+ raw, err := tool.InvokableRun(ctx, string(args))+ if err != nil {+ t.Fatalf("wiki_query InvokableRun err = %v", err)+ }+ var out map[string]interface{}+ if err := json.Unmarshal([]byte(raw), &out); err != nil {+ t.Fatalf("bad wiki_query output: %v", err)+ }+ return out+}++func TestWikiQueryTool_ReturnsPages(t *testing.T) {+ svc := &fakeWikiService{+ available: map[string]bool{"kb1": true},+ pages: func(q string) []map[string]interface{} {+ return []map[string]interface{}{{"chunk_id": "wiki/entity/alpha", "content_with_weight": "# Alpha", "doc_id": "kb1", "docnm_kwd": "Alpha", "wiki_slug_kwd": "entity/alpha", "dataset_id": "kb1"}}+ },+ }+ out := wikiToolRun(t, svc, "t1", "kb1", "alpha")+ chunks, _ := out["chunks"].([]interface{})+ if len(chunks) != 1 {+ t.Fatalf("chunks = %d, want 1", len(chunks))+ }+ first := chunks[0].(map[string]interface{})+ if first["content_with_weight"] != "# Alpha" {+ t.Errorf("page content = %v, want # Alpha", first["content_with_weight"])+ }+ if first["wiki_slug_kwd"] != "entity/alpha" {+ t.Errorf("slug = %v, want entity/alpha", first["wiki_slug_kwd"])+ }+}++func TestWikiQueryTool_EmptyWhenNoArtifact(t *testing.T) {+ svc := &fakeWikiService{available: map[string]bool{"kb2": true}}+ out := wikiToolRun(t, svc, "t1", "kb1", "alpha") // kb1 has no artifact+ if chunks, _ := out["chunks"].([]interface{}); len(chunks) != 0 {+ t.Fatalf("chunks = %d, want 0 (kb1 has no wiki artifact)", len(chunks))+ }+}++func TestWikiQueryTool_EmptyWhenNoService(t *testing.T) {+ out := wikiToolRun(t, nil, "t1", "kb1", "alpha")+ if chunks, _ := out["chunks"].([]interface{}); len(chunks) != 0 {+ t.Fatalf("chunks = %d, want 0 (no service configured)", len(chunks))+ }+}++func TestWikiQueryTool_ScopeRespected(t *testing.T) {+ svc := &fakeWikiService{+ available: map[string]bool{"kb1": true},+ pages: func(q string) []map[string]interface{} {+ return []map[string]interface{}{{"chunk_id": "wiki/s", "content_with_weight": "c", "doc_id": "kb1", "docnm_kwd": "T", "wiki_slug_kwd": "s", "dataset_id": "kb1"}}+ },+ }+ out := wikiToolRun(t, svc, "t1", "kb1", "alpha")+ if _, ok := out["chunks"]; !ok {+ t.Fatalf("missing chunks key")+ }+}diff --git a/internal/common/parser_config.go b/internal/common/parser_config.goindex adc1793b96..df3c53588f 100644--- a/internal/common/parser_config.go+++ b/internal/common/parser_config.go@@ -1,129 +1,5 @@package common-import "strings"--// InjectExtractorLLMID finds all Extractor component entries (keys prefixed-// with "extractor:" or "extractor_") in parserConfig and sets their llm_id-// to the given value. Returns whether any entry was updated.-func InjectExtractorLLMID(parserConfig map[string]interface{}, llmID string) bool {- if parserConfig == nil || llmID == "" {- return false- }- updated := false- for cid, raw := range parserConfig {- compMap, ok := raw.(map[string]interface{})- if !ok {- continue- }- cidLower := strings.ToLower(cid)- if strings.HasPrefix(cidLower, "extractor:") || strings.HasPrefix(cidLower, "extractor_") {- if current, ok := compMap["llm_id"].(string); !ok || current == "" {- compMap["llm_id"] = llmID- updated = true- }- }- }- return updated-}--// InjectExtractorEnableMetadata enables auto-metadata (enable_metadata) extraction-// on every Extractor node when the dataset has enable_metadata on and a-// non-empty field set (metadata and/or built_in_metadata). The dataset-level-// enable_metadata flag is authoritative (mirrors Python task_executor.py:519,-// which reads parser_config directly and never consults a per-node flag): a-// shipped DSL that defaults enable_metadata to 0 is still turned on. Only a-// node the user already turned ON (enable_metadata truthy) is left untouched,-// so an explicit per-node enablement keeps its own config. The field schema is-// taken from parserConfig["metadata"] and parserConfig["built_in_metadata"]-// (combined). Returns whether any entry was updated.-func InjectExtractorEnableMetadata(parserConfig map[string]interface{}) bool {- if parserConfig == nil {- return false- }- if !isTruthy(parserConfig["enable_metadata"]) {- return false- }- fields := metadataFieldDefs(parserConfig)- if len(fields) == 0 {- return false- }- updated := false- for cid, raw := range parserConfig {- compMap, ok := raw.(map[string]interface{})- if !ok {- continue- }- cidLower := strings.ToLower(cid)- if !strings.HasPrefix(cidLower, "extractor:") && !strings.HasPrefix(cidLower, "extractor_") {- continue- }- // The dataset-level enable_metadata flag is authoritative (mirrors- // Python task_executor.py:519, which reads parser_config directly and- // never consults a per-node flag). Only a node the user already turned- // ON (truthy) is left alone; a shipped DSL that defaults the field to- // 0 must still be enabled by the dataset flag, otherwise auto-metadata- // could never turn on for any of the built-in pipelines.- if isTruthy(compMap["enable_metadata"]) {- continue- }- compMap["enable_metadata"] = 1- compMap["metadata"] = fields- updated = true- }- return updated-}--// metadataFieldDefs combines parserConfig["metadata"] and-// parserConfig["built_in_metadata"] into the field list injected as-// metadata (each entry keeps key/type/description/enum, mirroring the-// stored shape from dataset/helpers.go normalizeMetadataConfigFields).-//-// It returns []any (i.e. []interface{}) rather than []map[string]interface{}-// because the injected value is handed to NewExtractorComponent, which reads-// params["metadata"].([]any). A []map[string]interface{} value would fail that-// type assertion (Go slice types are not covariant) and the field schema would-// be silently dropped, so auto-metadata never reached ExtractorParam.Metadata.-func metadataFieldDefs(parserConfig map[string]interface{}) []any {- var out []any- for _, key := range []string{"metadata", "built_in_metadata"} {- raw, ok := parserConfig[key].([]interface{})- if !ok {- continue- }- for _, item := range raw {- m, ok := item.(map[string]interface{})- if !ok {- continue- }- k, _ := m["key"].(string)- if strings.TrimSpace(k) == "" {- continue- }- out = append(out, m)- }- }- return out-}--// isTruthy reports whether a parserConfig flag (e.g. enable_metadata) is on.-// It tolerates bool, numeric >0 and the strings "true"/"1" so storage-// representation differences don't silently disable the feature.-func isTruthy(v interface{}) bool {- switch t := v.(type) {- case bool:- return t- case string:- return t == "true" || t == "1" || t == "True" || t == "TRUE"- case float64:- return t > 0- case int:- return t > 0- case int64:- return t > 0- }- return false-}-// deepCopyMap duplicates a JSON-like map so later merges do not mutate shared defaults.func deepCopyMap(source map[string]interface{}) map[string]interface{} {if source == nil {diff --git a/internal/common/parser_config_test.go b/internal/common/parser_config_test.godeleted file mode 100644index e6d1195eba..0000000000--- a/internal/common/parser_config_test.go+++ /dev/null@@ -1,135 +0,0 @@-package common--import "testing"--func TestInjectExtractorLLMID_SkipWhenUUID(t *testing.T) {- uuid := "9e819c2442b14f9dab46062916e29195"- pc := map[string]interface{}{- "Extractor:A": map[string]interface{}{- "llm_id": uuid,- },- }- InjectExtractorLLMID(pc, "Qwen/Qwen3-8B@siliconflow")- id := pc["Extractor:A"].(map[string]interface{})["llm_id"].(string)- if id != uuid {- t.Fatalf("expected UUID preserved, got %q", id)- }-}--func TestInjectExtractorLLMID_SkipWhenComposite(t *testing.T) {- composite := "Qwen/Qwen3-8B@siliconflow"- pc := map[string]interface{}{- "Extractor:B": map[string]interface{}{- "llm_id": composite,- },- }- InjectExtractorLLMID(pc, "DeepSeek@siliconflow")- id := pc["Extractor:B"].(map[string]interface{})["llm_id"].(string)- if id != composite {- t.Fatalf("expected composite preserved, got %q", id)- }-}--func TestInjectExtractorLLMID_InjectWhenEmpty(t *testing.T) {- defaultLLM := "Qwen/Qwen3-8B@siliconflow"- pc := map[string]interface{}{- "Extractor:C": map[string]interface{}{},- }- InjectExtractorLLMID(pc, defaultLLM)- id := pc["Extractor:C"].(map[string]interface{})["llm_id"].(string)- if id != defaultLLM {- t.Fatalf("expected %q injected, got %q", defaultLLM, id)- }-}--func TestInjectExtractorLLMID_NoExtractor(t *testing.T) {- pc := map[string]interface{}{- "Parser:X": map[string]interface{}{"llm_id": ""},- }- InjectExtractorLLMID(pc, "default@provider")- if _, ok := pc["Parser:X"]; !ok {- t.Fatal("expected Parser:X still present")- }-}--func TestInjectExtractorEnableMetadata_Disabled(t *testing.T) {- pc := map[string]interface{}{- "Extractor:A": map[string]interface{}{},- }- if InjectExtractorEnableMetadata(pc) {- t.Fatal("expected no update when enable_metadata is off")- }- if _, ok := pc["Extractor:A"].(map[string]interface{})["enable_metadata"]; ok {- t.Fatal("enable_metadata should not be set")- }-}--func TestInjectExtractorEnableMetadata_InjectsAndMerges(t *testing.T) {- pc := map[string]interface{}{- "enable_metadata": true,- "metadata": []interface{}{- map[string]interface{}{"key": "author", "type": "string", "description": "doc author", "enum": []interface{}{"Alice", "Bob"}},- },- "built_in_metadata": []interface{}{- map[string]interface{}{"key": "year", "type": "number"},- },- "Extractor:A": map[string]interface{}{},- "Parser:X": map[string]interface{}{},- }- if !InjectExtractorEnableMetadata(pc) {- t.Fatal("expected update")- }- ext := pc["Extractor:A"].(map[string]interface{})- if got, _ := ext["enable_metadata"].(int); got != 1 {- t.Fatalf("expected enable_metadata=1, got %v", ext["enable_metadata"])- }- fields, ok := ext["metadata"].([]any)- if !ok || len(fields) != 2 {- t.Fatalf("expected 2 merged metadata, got %#v", ext["metadata"])- }- if _, ok := pc["Parser:X"].(map[string]interface{})["enable_metadata"]; ok {- t.Fatal("Parser node must not be touched")- }-}--func TestInjectExtractorEnableMetadata_RespectsExplicitEnabled(t *testing.T) {- // A node the user already turned ON keeps its own config (no clobber).- pc := map[string]interface{}{- "enable_metadata": true,- "metadata": []interface{}{map[string]interface{}{"key": "author"}},- "Extractor:A": map[string]interface{}{"enable_metadata": 1},- }- if InjectExtractorEnableMetadata(pc) {- t.Fatal("expected no update when user explicitly enabled enable_metadata")- }-}--func TestInjectExtractorEnableMetadata_OverridesDefaultZero(t *testing.T) {- // Shipped DSLs default enable_metadata to 0; the dataset flag must still- // turn them on (otherwise auto-metadata could never activate).- pc := map[string]interface{}{- "enable_metadata": true,- "metadata": []interface{}{map[string]interface{}{"key": "author"}},- "Extractor:A": map[string]interface{}{"enable_metadata": 0},- }- if !InjectExtractorEnableMetadata(pc) {- t.Fatal("expected update: dataset flag must override default enable_metadata=0")- }- ext := pc["Extractor:A"].(map[string]interface{})- if got, _ := ext["enable_metadata"].(int); got != 1 {- t.Fatalf("expected enable_metadata=1 after override, got %v", ext["enable_metadata"])- }- if _, ok := ext["metadata"].([]any); !ok {- t.Fatalf("expected metadata field schema injected, got %#v", ext["metadata"])- }-}--func TestInjectExtractorEnableMetadata_NoFields(t *testing.T) {- pc := map[string]interface{}{- "enable_metadata": true,- "Extractor:A": map[string]interface{}{},- }- if InjectExtractorEnableMetadata(pc) {- t.Fatal("expected no update when no fields configured")- }-}diff --git a/internal/entity/knowledge_compile_doc.go b/internal/entity/knowledge_compile_doc.goindex 84f144b0c6..1c38685919 100644--- a/internal/entity/knowledge_compile_doc.go+++ b/internal/entity/knowledge_compile_doc.go@@ -17,6 +17,16 @@ package entityimport "time"+// Dataset-level compile lifecycle states. Shared source of truth for the+// scheduler (which writes State on the knowledge_compile_docs row) and the+// dataset compilation-status API (which reads it back).+const (+ DatasetStateIdle = "idle" // no scheduling row / nothing to do+ DatasetStatePending = "pending" // backlog non-empty, awaiting claim+ DatasetStateRunning = "running" // a worker holds the lease and is merging+ DatasetStateCompleted = "completed" // backlog drained to empty+)+// KnowledgeCompileDataset is the MySQL scheduling row for the dataset-level// post-processing consumer (knowledge_compile_design.md §11.4, Option E). It is// the scheduling system of record: backlog_doc_ids holds the not-yet-processed@@ -42,8 +52,20 @@ type KnowledgeCompileDataset struct {ClaimToken string `gorm:"column:claim_token;size:64;not null;default:''" json:"claim_token"`ClaimExpiresAt *time.Time `gorm:"column:claim_expires_at;default:null" json:"claim_expires_at"`Priority int `gorm:"column:priority;not null;default:0" json:"priority"`- CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"created_at"`- UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime" json:"updated_at"`+ // State is the dataset-level compile lifecycle state surfaced to the API:+ // idle | pending | running | completed. It is written by the scheduler and+ // consumer; the API never derives it from the backlog alone. Default is a+ // scalar string literal on a varchar column, which MySQL allows (Error 1101+ // only affects TEXT/BLOB, not varchar).+ State string `gorm:"column:state;size:16;not null;default:'idle'" json:"state"`+ // ErrorMsg is the most recent failure/retry diagnostic. It is TEXT with no+ // DDL default (MySQL 8.0.13+ rejects a literal default on TEXT, Error 1101);+ // the application always writes it explicitly when set.+ ErrorMsg string `gorm:"column:error_msg;type:text;not null" json:"error_msg"`+ // LastCompletedAt records the last time the backlog drained to empty.+ LastCompletedAt *time.Time `gorm:"column:last_completed_at;default:null" json:"last_completed_at"`+ CreatedAt time.Time `gorm:"column:created_at;autoCreateTime" json:"created_at"`+ UpdatedAt time.Time `gorm:"column:updated_at;autoUpdateTime" json:"updated_at"`}// TableName pins the scheduling table name.diff --git a/internal/entity/models/bedrock.go b/internal/entity/models/bedrock.goindex 5d461cfb3e..604e7c60eb 100644--- a/internal/entity/models/bedrock.go+++ b/internal/entity/models/bedrock.go@@ -499,10 +499,6 @@ func mapChatConfigToInference(cfg *ChatConfig) *bedrockInferenceConfig {}inf := &bedrockInferenceConfig{}hasField := false- if cfg.MaxTokens != nil {- inf.MaxTokens = cfg.MaxTokens- hasField = true- }if cfg.Temperature != nil {inf.Temperature = cfg.TemperaturehasField = truediff --git a/internal/entity/models/bedrock_test.go b/internal/entity/models/bedrock_test.goindex c3a51cc860..968636d200 100644--- a/internal/entity/models/bedrock_test.go+++ b/internal/entity/models/bedrock_test.go@@ -207,8 +207,8 @@ func TestMapChatConfigToInferenceForwardsAllFields(t *testing.T) {if inf == nil {t.Fatal("expected non-nil inferenceConfig")}- if inf.MaxTokens == nil || *inf.MaxTokens != 4096 {- t.Errorf("maxTokens=%v", inf.MaxTokens)+ if inf.MaxTokens != nil {+ t.Errorf("maxTokens should be omitted, got %v", inf.MaxTokens)}if inf.Temperature == nil || *inf.Temperature != 0.5 {t.Errorf("temperature=%v", inf.Temperature)diff --git a/internal/entity/models/model.go b/internal/entity/models/model.goindex 0d7b4e186f..17069d9095 100644--- a/internal/entity/models/model.go+++ b/internal/entity/models/model.go@@ -23,6 +23,7 @@ import ("os""path/filepath""sort"+ "strconv""strings")@@ -167,6 +168,7 @@ type Model struct {Class *string `json:"class"`MaxDimension *int `json:"max_dimension"` // used by embedding modelsDimensions []int `json:"dimensions"`+ BatchSize *int `json:"batch_size"` // max texts per Embed request; used by embedding modelsAlias []string `json:"alias"`Rank *int `json:"rank"`ModelTypeMap map[string]bool@@ -418,6 +420,9 @@ func (pm *ProviderManager) ListAllModels() ([]map[string]interface{}, error) {if len(model.Dimensions) > 0 {modelData["dimensions"] = model.Dimensions}+ if model.BatchSize != nil {+ modelData["batch_size"] = *model.BatchSize+ }if model.Thinking != nil {modelData["thinking"] = "supported"}@@ -444,6 +449,34 @@ func (pm *ProviderManager) GetModelByNameOrAlias(modelName string) *Model {return nil}+// DefaultEmbeddingBatchSize is the fallback per-request embedding input count+// when a model reports no batch_size capability (mirrors Python's+// settings.EMBEDDING_BATCH_SIZE).+const DefaultEmbeddingBatchSize = 16++// GetEmbeddingBatchSize returns the max texts per Embed request for the named+// model. It consults the model provider capability (batch_size, added to+// all_models.json by #17877/#17878) and falls back to+// DefaultEmbeddingBatchSize when unset or unknown. The+// TOKENIZER_EMBEDDING_BATCH_SIZE env var, when valid, overrides everything for+// local tuning.+func GetEmbeddingBatchSize(modelName string) int {+ if v := os.Getenv("TOKENIZER_EMBEDDING_BATCH_SIZE"); v != "" {+ if n, err := strconv.Atoi(v); err == nil && n > 0 {+ return n+ }+ }+ if modelName != "" {+ if pm := GetProviderManager(); pm != nil {+ if m := pm.GetModelByNameOrAlias(modelName); m != nil && m.BatchSize != nil && *m.BatchSize > 0 {+ return *m.BatchSize+ }+ }+ }+ return DefaultEmbeddingBatchSize+}++// 2. Show specific provider information (including base_url)func (pm *ProviderManager) GetProviderByName(providerName string) (map[string]interface{}, error) {provider := pm.FindProvider(providerName)@@ -489,6 +522,9 @@ func (pm *ProviderManager) ListModels(providerName string) ([]map[string]interfa"max_dimension": model.MaxDimension,"dimensions": model.Dimensions,}+ if model.BatchSize != nil {+ modelData["batch_size"] = *model.BatchSize+ }if model.Thinking != nil {modelData["thinking"] = "supported"}diff --git a/internal/entity/models/replicate.go b/internal/entity/models/replicate.goindex 3c6a658721..93b653d14d 100644--- a/internal/entity/models/replicate.go+++ b/internal/entity/models/replicate.go@@ -164,9 +164,6 @@ func replicateInputFromMessages(messages []Message, chatModelConfig *ChatConfig)input["system_prompt"] = systemPrompt}if chatModelConfig != nil {- if chatModelConfig.MaxTokens != nil {- input["max_new_tokens"] = *chatModelConfig.MaxTokens- }if chatModelConfig.Temperature != nil {input["temperature"] = *chatModelConfig.Temperature}diff --git a/internal/entity/models/replicate_test.go b/internal/entity/models/replicate_test.goindex d4e80bbdbc..8ea68352b1 100644--- a/internal/entity/models/replicate_test.go+++ b/internal/entity/models/replicate_test.go@@ -105,8 +105,8 @@ func TestReplicateOfficialChatHappyPath(t *testing.T) {if input["system_prompt"] != "be helpful" {t.Errorf("system_prompt=%v", input["system_prompt"])}- if input["max_new_tokens"] != float64(128) {- t.Errorf("max_new_tokens=%v", input["max_new_tokens"])+ if _, ok := input["max_new_tokens"]; ok {+ t.Errorf("max_new_tokens should be omitted, got %v", input["max_new_tokens"])}// Stop is deliberately filtered out because Replicate model// inputs are model-specific and upstream support is undefined.diff --git a/internal/entity/models/siliconflow.go b/internal/entity/models/siliconflow.goindex 5dac0642a6..4b9b1cae91 100644--- a/internal/entity/models/siliconflow.go+++ b/internal/entity/models/siliconflow.go@@ -139,9 +139,6 @@ type siliconflowEmbeddingResponse struct {} `json:"usage"`}-// siliconflowMaxBatchSize is the per-request input limit documented at-const siliconflowMaxBatchSize = 32-// Embed embeds a list of texts into embeddingsfunc (s *SiliconflowModel) Embed(ctx context.Context, modelName *string, texts []string, apiConfig *APIConfig, embeddingConfig *EmbeddingConfig, modelUsage *common.ModelUsage) ([]EmbeddingData, error) {if err := s.baseModel.APIConfigCheck(apiConfig); err != nil {@@ -151,8 +148,16 @@ func (s *SiliconflowModel) Embed(ctx context.Context, modelName *string, texts [if len(texts) == 0 {return []EmbeddingData{}, nil}- if len(texts) > siliconflowMaxBatchSize {- return nil, fmt.Errorf("siliconflow supports a maximum of %d inputs per request", siliconflowMaxBatchSize)+ // Per-request input cap: resolved from the provider capability (batch_size+ // added to all_models.json by #17877/#17878) and falling back to a safe+ // default. This defends against callers that bypass batch splitting upstream.+ var modelNameStr string+ if modelName != nil {+ modelNameStr = *modelName+ }+ maxBatch := GetEmbeddingBatchSize(modelNameStr)+ if len(texts) > maxBatch {+ return nil, fmt.Errorf("siliconflow supports a maximum of %d inputs per request", maxBatch)}if modelName == nil || *modelName == "" {diff --git a/internal/entity/models/types.go b/internal/entity/models/types.goindex bfcc887df9..1305c8c72f 100644--- a/internal/entity/models/types.go+++ b/internal/entity/models/types.go@@ -219,10 +219,11 @@ type ParseFileConfig struct {// EmbeddingModel wraps a ModelDriver with embedding-specific configurationtype EmbeddingModel struct {- ModelDriver ModelDriver- ModelName *string- APIConfig *APIConfig- MaxTokens int // Max input tokens for the embedding model, used for text truncation+ ModelDriver ModelDriver+ ModelName *string+ APIConfig *APIConfig+ MaxTokens int // Max input tokens for the embedding model, used for text truncation+ MaxBatchSize *int // Max texts per Embed request; nil means "resolve from provider capability at use site"}// NewEmbeddingModel creates a new EmbeddingModel@@ -235,6 +236,22 @@ func NewEmbeddingModel(driver ModelDriver, modelName *string, apiConfig *APIConf}}+// ResolveBatchSize returns the max texts per Embed request for this embedding+// model. It prefers an explicit MaxBatchSize set at construction time and falls+// back to the provider capability (all_models.json batch_size, added by+// #17877/#17878) via GetEmbeddingBatchSize, which itself defaults to+// DefaultEmbeddingBatchSize.+func (m *EmbeddingModel) ResolveBatchSize() int {+ if m != nil && m.MaxBatchSize != nil && *m.MaxBatchSize > 0 {+ return *m.MaxBatchSize+ }+ var name string+ if m != nil && m.ModelName != nil {+ name = *m.ModelName+ }+ return GetEmbeddingBatchSize(name)+}+// RerankModel wraps a ModelDriver with rerank-specific configurationtype RerankModel struct {ModelDriver ModelDriverdiff --git a/internal/entity/models/zhipu-ai.go b/internal/entity/models/zhipu-ai.goindex 98920be0fe..75bebb9ad7 100644--- a/internal/entity/models/zhipu-ai.go+++ b/internal/entity/models/zhipu-ai.go@@ -55,7 +55,7 @@ func (z *ZhipuAIModel) Name() string {return "zhipu"}-// ChatWithMessages sends multiple messages with roles and returns response+// ChatWithMessages sends multiple messages with roles and returns response.func (z *ZhipuAIModel) ChatWithMessages(ctx context.Context, modelName string, messages []Message, apiConfig *APIConfig, chatModelConfig *ChatConfig, modelUsage *common.ModelUsage) (*ChatResponse, error) {if err := z.baseModel.APIConfigCheck(apiConfig); err != nil {return nil, errdiff --git a/internal/handler/compilation_status_test.go b/internal/handler/compilation_status_test.gonew file mode 100644index 0000000000..b76b490685--- /dev/null+++ b/internal/handler/compilation_status_test.go@@ -0,0 +1,195 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package handler++import (+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "net/url"+ "testing"++ "github.com/gin-gonic/gin"+ "github.com/glebarez/sqlite"+ "gorm.io/gorm"++ "ragflow/internal/common"+ "ragflow/internal/dao"+ "ragflow/internal/entity"+ dataset "ragflow/internal/service/dataset"+)++// setupCompilationStatusHandlerDB migrates the minimal schema for the+// GET /datasets/:id/compilation/status handler and pushes it onto dao.DB.+func setupCompilationStatusHandlerDB(t *testing.T) *gorm.DB {+ t.Helper()+ db, err := gorm.Open(sqlite.Open("file:"+url.QueryEscape(t.Name())+"?mode=memory&cache=shared"), &gorm.Config{+ TranslateError: true,+ })+ if err != nil {+ t.Fatalf("failed to open sqlite: %v", err)+ }+ if err := db.AutoMigrate(+ &entity.Knowledgebase{},+ &entity.KnowledgeCompileDataset{},+ ); err != nil {+ t.Fatalf("failed to migrate test schema: %v", err)+ }+ origDB := dao.DB+ dao.DB = db+ t.Cleanup(func() { dao.DB = origDB })+ return db+}++func insertCompilationStatusHandlerKB(t *testing.T, kbID, ownerID string) {+ t.Helper()+ status := string(entity.StatusValid)+ kb := &entity.Knowledgebase{+ ID: kbID,+ TenantID: ownerID,+ Name: "compile-status-handler-kb",+ EmbdID: "BAAI/bge-large-zh-v1.5@Builtin",+ CreatedBy: ownerID,+ Permission: string(entity.TenantPermissionMe),+ Status: &status,+ }+ if err := dao.DB.Create(kb).Error; err != nil {+ t.Fatalf("insert kb: %v", err)+ }+}++func newCompilationStatusHandlerRouter() *gin.Engine {+ gin.SetMode(gin.TestMode)+ h := NewDatasetsHandler(dataset.NewDatasetService(), nil)+ r := gin.New()+ r.GET("/api/v1/datasets/:dataset_id/compilation/status", func(c *gin.Context) {+ c.Set("user", &entity.User{ID: "user-1"})+ h.GetCompilationStatus(c)+ })+ return r+}++type compilationStatusResponse struct {+ Code int `json:"code"`+ Message string `json:"message"`+ Data map[string]interface{} `json:"data"`+}++func getCompilationStatus(t *testing.T, r *gin.Engine, datasetID string) (int, compilationStatusResponse) {+ t.Helper()+ resp := httptest.NewRecorder()+ req := httptest.NewRequest(http.MethodGet,+ "/api/v1/datasets/"+datasetID+"/compilation/status", nil)+ r.ServeHTTP(resp, req)+ var body compilationStatusResponse+ if err := json.Unmarshal(resp.Body.Bytes(), &body); err != nil {+ t.Fatalf("unmarshal response: %v body=%s", err, resp.Body.String())+ }+ return resp.Code, body+}++// TestCompilationStatusHandler_NoRowIdle verifies a dataset with no scheduling+// row returns idle with zero counts.+func TestCompilationStatusHandler_NoRowIdle(t *testing.T) {+ db := setupCompilationStatusHandlerDB(t)+ insertCompilationStatusHandlerKB(t, "kb-status-idle", "user-1")+ _ = db++ status, body := getCompilationStatus(t, newCompilationStatusHandlerRouter(), "kb-status-idle")+ if status != http.StatusOK {+ t.Fatalf("status=%d want 200", status)+ }+ if body.Code != int(common.CodeSuccess) {+ t.Fatalf("code=%d message=%q", body.Code, body.Message)+ }+ if body.Data["state"] != entity.DatasetStateIdle {+ t.Fatalf("state=%v want idle", body.Data["state"])+ }+ if n, _ := body.Data["inflight"].(float64); n != 0 {+ t.Fatalf("inflight=%v want 0", body.Data["inflight"])+ }+ if n, _ := body.Data["backlog"].(float64); n != 0 {+ t.Fatalf("backlog=%v want 0", body.Data["backlog"])+ }+}++// TestCompilationStatusHandler_FullOutput locks the JSON contract for a row+// with state, inflight/backlog counts, and error diagnostic.+func TestCompilationStatusHandler_FullOutput(t *testing.T) {+ db := setupCompilationStatusHandlerDB(t)+ insertCompilationStatusHandlerKB(t, "kb-status-full", "user-1")++ row := entity.KnowledgeCompileDataset{+ DatasetID: "kb-status-full",+ TenantID: "user-1",+ BacklogDocIDs: `[{"doc_id":"d2","event_type":"completed","seq":2}]`,+ InflightDocIDs: `[{"doc_id":"d1","event_type":"completed","seq":1}]`,+ State: entity.DatasetStatePending,+ ErrorMsg: "merge failed: boom",+ }+ if err := db.Create(&row).Error; err != nil {+ t.Fatalf("insert scheduling row: %v", err)+ }++ status, body := getCompilationStatus(t, newCompilationStatusHandlerRouter(), "kb-status-full")+ if status != http.StatusOK {+ t.Fatalf("status=%d want 200", status)+ }+ if body.Code != int(common.CodeSuccess) {+ t.Fatalf("code=%d message=%q", body.Code, body.Message)+ }+ if body.Data["state"] != entity.DatasetStatePending {+ t.Fatalf("state=%v want pending", body.Data["state"])+ }+ if n, _ := body.Data["inflight"].(float64); n != 1 {+ t.Fatalf("inflight=%v want 1", body.Data["inflight"])+ }+ if n, _ := body.Data["backlog"].(float64); n != 1 {+ t.Fatalf("backlog=%v want 1", body.Data["backlog"])+ }+ if body.Data["error"] != "merge failed: boom" {+ t.Fatalf("error=%v want %q", body.Data["error"], "merge failed: boom")+ }+}++// TestCompilationStatusHandler_Unauthorized verifies a user who does not own the+// dataset is rejected with a data error (HTTP 200 + non-zero code, matching the+// handler's ErrorWithCode contract).+func TestCompilationStatusHandler_Unauthorized(t *testing.T) {+ db := setupCompilationStatusHandlerDB(t)+ // KB is owned by user-1; the router sets user to user-1, so this test must+ // exercise the case where the KB belongs to a different owner. We insert the+ // KB under a different owner tenant than the request user by re-pointing the+ // KB owner to "other-owner".+ insertCompilationStatusHandlerKB(t, "kb-status-forbidden", "other-owner")+ if err := db.Create(&entity.KnowledgeCompileDataset{+ DatasetID: "kb-status-forbidden",+ TenantID: "other-owner",+ BacklogDocIDs: "[]",+ InflightDocIDs: "[]",+ State: entity.DatasetStateRunning,+ }).Error; err != nil {+ t.Fatalf("insert scheduling row: %v", err)+ }++ _, body := getCompilationStatus(t, newCompilationStatusHandlerRouter(), "kb-status-forbidden")+ if body.Code != int(common.CodeDataError) {+ t.Fatalf("code=%d want %d", body.Code, common.CodeDataError)+ }+ if body.Message != "no authorization" {+ t.Fatalf("message=%q want %q", body.Message, "no authorization")+ }+}diff --git a/internal/handler/components_testpkg/components_handler_test.go b/internal/handler/components_testpkg/components_handler_test.goindex 59c9f2df47..1fefbb7650 100644--- a/internal/handler/components_testpkg/components_handler_test.go+++ b/internal/handler/components_testpkg/components_handler_test.go@@ -134,8 +134,8 @@ func TestComponentsHandler_NoFilter(t *testing.T) {// TestComponentsHandler_FilterIngestion verifies the// ?category=ingestion filter returns the ingestion components-// (Extractor, File, Parser, Tokenizer + 9 chunker variants). Names-// must be sorted ascending (plan §4 task 1 stable output).+// (Compiler, Extractor, File, Parser, Tokenizer + 9 chunker variants).+// Names must be sorted ascending (plan §4 task 1 stable output).func TestComponentsHandler_FilterIngestion(t *testing.T) {eng := newComponentsTestRig(t)w := doRequest(t, eng, "/api/v1/components?category=ingestion")@@ -146,7 +146,7 @@ func TestComponentsHandler_FilterIngestion(t *testing.T) {_, _, data := decodeEnvelope(t, w.Body.Bytes())wantNames := []string{- "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",+ "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker","onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker","titlechunker", "tokenchunker", "tokenizer",}@@ -172,7 +172,7 @@ func TestComponentsHandler_FilterMultiple(t *testing.T) {_, _, data := decodeEnvelope(t, w.Body.Bytes())wantNames := []string{- "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",+ "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker","onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker","titlechunker", "tokenchunker", "tokenizer",}@@ -273,7 +273,7 @@ func TestComponentsHandler_CaseInsensitive(t *testing.T) {}_, _, data := decodeEnvelope(t, w.Body.Bytes())wantNames := []string{- "extractor", "file", "grouptitlechunker", "hierarchytitlechunker",+ "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker","onechunker", "parser", "presentationchunker", "qachunker", "tablechunker", "tagchunker","titlechunker", "tokenchunker", "tokenizer",}diff --git a/internal/handler/dataset.go b/internal/handler/dataset.goindex 37feaec5e4..1fbaf8509e 100644--- a/internal/handler/dataset.go+++ b/internal/handler/dataset.go@@ -1008,114 +1008,28 @@ func (h *DatasetsHandler) AggregateTags(c *gin.Context) {common.SuccessWithData(c, result, "success")}-// RunIndex Run an indexing task (graph/raptor/mindmap) for a dataset.-func (h *DatasetsHandler) RunIndex(c *gin.Context) {+// GetCompilationStatus returns the dataset-level knowledge-compile lifecycle+// state (scheduler contract for API_PROXY_SCHEME=go/hybrid). It replaces the+// Python-era TraceIndex task-progress endpoint for the Go backend.+func (h *DatasetsHandler) GetCompilationStatus(c *gin.Context) {user, errorCode, errorMessage := GetUser(c)if errorCode != common.CodeSuccess {common.ErrorWithCode(c, errorCode, errorMessage)return}-- datasetID := strings.TrimSpace(c.Param("dataset_id"))- if datasetID == "" {- common.ResponseWithCodeData(c, common.CodeDataError, nil, "dataset_id is required")- return- }-- userID := strings.TrimSpace(user.ID)- if userID == "" {- common.ResponseWithCodeData(c, common.CodeDataError, nil, "user_id is required")- return- }-- ctx := c.Request.Context()- indexType := strings.ToLower(strings.TrimSpace(c.Query("type")))- data, code, err := h.datasetsService.RunIndex(ctx, userID, datasetID, indexType)- if err != nil {- common.ErrorWithCode(c, code, err.Error())- return- }-- common.SuccessWithData(c, data, "success")-}--// TraceIndex Trace an indexing task (graph/raptor/mindmap) for a dataset.-func (h *DatasetsHandler) TraceIndex(c *gin.Context) {- user, errorCode, errorMessage := GetUser(c)- if errorCode != common.CodeSuccess {- common.ErrorWithCode(c, errorCode, errorMessage)- return- }-- datasetID := strings.TrimSpace(c.Param("dataset_id"))- if datasetID == "" {- common.ResponseWithCodeData(c, common.CodeDataError, nil, "dataset_id is required")- return- }-- userID := strings.TrimSpace(user.ID)- if userID == "" {- common.ResponseWithCodeData(c, common.CodeDataError, nil, "user_id is required")- return- }-- ctx := c.Request.Context()-- indexType := strings.ToLower(strings.TrimSpace(c.Query("type")))- result, code, err := h.datasetsService.TraceIndex(ctx, datasetID, userID, indexType)- if err != nil {- common.ErrorWithCode(c, code, err.Error())- return- }- if result == nil {- common.SuccessWithData(c, map[string]interface{}{}, "success")- return- }-- common.SuccessWithData(c, result, "success")-}--// DeleteIndex Delete an indexing task (graph/raptor/mindmap) for a dataset.-func (h *DatasetsHandler) DeleteIndex(c *gin.Context) {- user, errorCode, errorMessage := GetUser(c)- if errorCode != common.CodeSuccess {- common.ErrorWithCode(c, errorCode, errorMessage)- return- }-datasetID := strings.TrimSpace(c.Param("dataset_id"))if datasetID == "" {common.ResponseWithCodeData(c, common.CodeDataError, nil, "dataset_id is required")return}-userID := strings.TrimSpace(user.ID)- if userID == "" {- common.ResponseWithCodeData(c, common.CodeDataError, nil, "user_id is required")- return- }-- indexType := strings.ToLower(strings.TrimSpace(c.Param("index_type")))- if indexType == "" {- indexType = strings.ToLower(strings.TrimSpace(c.Query("type")))- }-- wipeArg := strings.ToLower(strings.TrimSpace(c.DefaultQuery("wipe", "true")))- wipe := true- switch wipeArg {- case "false", "0", "no", "off":- wipe = false- }-ctx := c.Request.Context()-- code, err := h.datasetsService.DeleteIndex(ctx, userID, datasetID, indexType, wipe)+ st, code, err := h.datasetsService.GetDatasetCompilationStatus(ctx, userID, datasetID)if err != nil {common.ErrorWithCode(c, code, err.Error())return}-- common.SuccessWithData(c, map[string]interface{}{}, "success")+ common.SuccessWithData(c, st, "success")}// ListMetadataFlattened handles GET /api/v1/datasets/metadata/flattened.diff --git a/internal/handler/dataset_artifact.go b/internal/handler/dataset_artifact.goindex 1c898f6469..dcddfddda7 100644--- a/internal/handler/dataset_artifact.go+++ b/internal/handler/dataset_artifact.go@@ -119,7 +119,9 @@ func (h *DatasetArtifactHandler) ListArtifacts(c *gin.Context) {common.ErrorWithCode(c, common.CodeDataError, err.Error())return}- common.SuccessWithData(c, gin.H{"total": total, "pages": items}, "success")+ // Python's list_wiki_pages returns {total, items}; align the Go port so the+ // shared frontend (which reads data.items) stays compatible.+ common.SuccessWithData(c, gin.H{"total": total, "items": items}, "success")}// UpdateArtifact handles PUT /artifacts/<page_type>/<slug> — edit a wiki page.@@ -239,7 +241,9 @@ func (h *DatasetArtifactHandler) ListArtifactTopics(c *gin.Context) {common.ErrorWithCode(c, common.CodeDataError, err.Error())return}- common.SuccessWithData(c, gin.H{"total": total, "topics": items}, "success")+ // Python's list_wiki_topics returns {total, items}; align the Go port so the+ // shared frontend (which reads data.items) stays compatible.+ common.SuccessWithData(c, gin.H{"total": total, "items": items}, "success")}// GetArtifactAlteration handles GET /artifacts/alteration — wiki alteration summary.diff --git a/internal/ingestion/component/knowledge_compiler/common/deps.go b/internal/ingestion/component/knowledge_compiler/common/deps.goindex 0f37d2c4be..2c4d00fefa 100644--- a/internal/ingestion/component/knowledge_compiler/common/deps.go+++ b/internal/ingestion/component/knowledge_compiler/common/deps.go@@ -106,10 +106,11 @@ type Deps struct {Redis RedisClient // optional (datasetnav)TenantID stringDatasetID string- // LLMMaxLength is the chat model's context window in tokens. RAPTOR uses it- // to truncate each cluster's texts so the summary prompt fits the window- // (mirrors Python self._llm_model.max_length).- LLMMaxLength int+ // ModelContextLen is the chat model's context window in tokens+ // (content_length). The prompt-budget helpers (wikiMapMaxTokens,+ // deriveWikiPlanBudget, buildClusterContent) use it to size the input/output+ // quotas (mirrors Python self._llm_model.max_length).+ ModelContextLen int}// DepsResolver resolves the per-run Deps from a tenant/llm/embedding triple.diff --git a/internal/ingestion/component/knowledge_compiler/common/jsonchat.go b/internal/ingestion/component/knowledge_compiler/common/jsonchat.goindex e6f868b91e..d5937d0a41 100644--- a/internal/ingestion/component/knowledge_compiler/common/jsonchat.go+++ b/internal/ingestion/component/knowledge_compiler/common/jsonchat.go@@ -4,6 +4,7 @@ import ("context""encoding/json""fmt"+ "log""regexp""strings")@@ -34,6 +35,13 @@ func GenJSON(ctx context.Context, chat ChatInvoker, req ChatRequest) (map[stringreturn m, nil}}+ // Diagnostic: surface how far parsing got so a formatting failure is not+ // opaque. Each candidate is reported with its own unmarshal error so we can+ // tell an unfenced/truncated payload from a genuine syntax error.+ for i, candidate := range jsonCandidates(resp.Content) {+ _, err := tryUnmarshalJSONErr(candidate)+ log.Printf("knowledge_compiler: GenJSON candidate[%d] len=%d parse_err=%v body=%q", i, len(candidate), err, truncate(candidate, 300))+ }return nil, fmt.Errorf("knowledge_compiler: LLM response is not parseable JSON: %q", truncate(resp.Content, 200))}@@ -54,15 +62,20 @@ func jsonCandidates(s string) []string {}func tryUnmarshalJSON(s string) (map[string]any, bool) {+ m, err := tryUnmarshalJSONErr(s)+ return m, err == nil+}++func tryUnmarshalJSONErr(s string) (map[string]any, error) {s = strings.TrimSpace(s)if s == "" {- return nil, false+ return nil, fmt.Errorf("empty candidate")}var m map[string]anyif err := json.Unmarshal([]byte(s), &m); err != nil {- return nil, false+ return nil, err}- return m, true+ return m, nil}func truncate(s string, n int) string {diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.goindex b88f762b93..e1e1f25ac9 100644--- a/internal/ingestion/component/knowledge_compiler/component.go+++ b/internal/ingestion/component/knowledge_compiler/component.go@@ -9,9 +9,11 @@ import ("encoding/json""fmt""log"+ "sort""strings""ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/component/globals""ragflow/internal/ingestion/component/knowledge_compiler/common""ragflow/internal/ingestion/component/knowledge_compiler/mindmap""ragflow/internal/ingestion/component/knowledge_compiler/structure"@@ -37,7 +39,12 @@ var chunkerOutputs = map[string]string{"_ERROR": "Set only on validation failure.",}-const componentNameKnowledgeCompiler = "KnowledgeCompiler"+// componentNameCompiler is the canonical, unified component name for the+// knowledge-compilation flow. It matches the Python side+// (rag/flow/compiler/compiler.py registers component_name = "Compiler"), so a+// canvas saved by the Python frontend and Go's built-in ingestion templates+// both reference the node as "Compiler" and resolve to the same component.+const componentNameCompiler = "Compiler"// KnowledgeCompilerComponent is the runtime.Component surface. Param is set at// construction from the DSL; per-call overrides flow through the inputs map.@@ -60,10 +67,6 @@ func NewKnowledgeCompilerComponent(name string, params map[string]any) (runtime.func (c *KnowledgeCompilerComponent) Inputs() map[string]string {return map[string]string{"chunks": "List of map[string]any from upstream chunker/parser; each must carry id + text/content_with_weight.",- "llm_id": "Optional per-call LLM id override.",- "embedding_model": "Optional per-call embedding model override.",- "tenant_id": "Optional tenant scope (defaults to resolver context).",- "dataset_id": "Optional dataset scope (wiki historical dedup).","historical_candidates": "Optional []common.Candidate override for historical dedup (test/offline).",}}@@ -85,14 +88,13 @@ func (c *KnowledgeCompilerComponent) Outputs() map[string]string {func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {_ = dbparam := c.Param- if v, ok := inputs["llm_id"].(string); ok && v != "" {- param.LLMID = v- }- if v, ok := inputs["embedding_model"].(string); ok && v != "" {- param.EmbeddingModel = v- }- tenantID, _ := inputs["tenant_id"].(string)- datasetID, _ := inputs["dataset_id"].(string)+ // Resolve the run-level tenant scope from the shared CanvasState.Globals+ // bag first (seeded by the pipeline at run start), falling back to the+ // component's own input map. Mirrors parser.go: it keeps the tenant id from+ // being lost when the upstream output map narrows it, which would otherwise+ // leave the template-group lookup with an empty tenant and fail loudly.+ tenantID := globals.GlobalOrInput(ctx, inputs, "tenant_id", "")+ datasetID := globals.GlobalOrInput(ctx, inputs, "dataset_id", "")// Resolve the compilation template spec(s). Priority:// compilation_template_id > compilation_template_group_id. The variant is@@ -415,10 +417,13 @@ func kindOrVariant(p common.Product) string {// variantCompileKWD maps each Go variant to the compile_kwd discriminator value// Python writes into ES (rag/advanced_rag/knowlege_compile). It is the primary// key that distinguishes compiled knowledge units from ordinary chunks and-// routes retrieval-side filters (e.g. "compile_kwd": ["artifact_page"]).+// routes retrieval-side filters. The wiki value MUST be "wiki_page" (Python's+// canonical WIKI_PAGE_COMPILE_KWD in wiki.py:1661 / wiki_incremental.py:44 /+// dataset_wiki_generator.py:108) so Go-produced wiki pages are visible to the+// artifact API (dataset_artifact_service.go reads compile_kwd="wiki_page").var variantCompileKWD = map[common.Variant]string{common.VariantStructure: "structure",- common.VariantWiki: "artifact_page",+ common.VariantWiki: "wiki_page",common.VariantTree: "tree",common.VariantMindmap: "mindmap",}@@ -470,6 +475,14 @@ func productsToChunkDocs(products []common.Product) ([]schema.ChunkDoc, error) {if v := metaString(p.Meta, "compile_kwd"); v != "" {compileKWD = v}+ // Wiki sub-parts: sections get their own compile_kwd so that a page+ // search on compile_kwd="wiki_page" returns pages only (page.go emits+ // both kind:"page" and kind:"section" rows under VariantWiki). This is+ // the schema-backed page/section discriminator: "wiki_page" == page,+ // "wiki_section" == a page sub-section.+ if p.Variant == common.VariantWiki && metaString(p.Meta, "kind") == "section" && compileKWD == "wiki_page" {+ compileKWD = "wiki_section"+ }if compileKWD == "" {compileKWD = string(p.Variant)}@@ -574,12 +587,25 @@ func applyVariantColumns(doc *schema.ChunkDoc, p common.Product) error {case common.VariantWiki:// One artifact_page row per wiki page; section rows reuse the same// page-level columns so retrieval-side filters work uniformly.- if v := metaString(p.Meta, "slug"); v != "" {- if err := doc.SetExtraValue("slug_kwd", v); err != nil {- return err- }- if err := doc.SetExtraValue("artifact_slug_kwd", v); err != nil {- return err+ // Match the Python writer contract (api/db/db_models.py slug_kwd):+ // slug_kwd stores the full "<page_type>/<slug>" form, so retrieval+ // filters (GetWikiPage) can reconstruct it directly. page_type is also+ // stored separately for topic grouping.+ if pageType := metaString(p.Meta, "page_type"); pageType != "" {+ if slug := metaString(p.Meta, "slug"); slug != "" {+ // Normalize to the full "<page_type>/<slug>" form (Python writer+ // contract). Idempotent: a slug that already carries the prefix+ // (some producers emit pageType/slug directly) is left as-is.+ fullSlug := slug+ if !strings.Contains(slug, "/") {+ fullSlug = pageType + "/" + slug+ }+ if err := doc.SetExtraValue("slug_kwd", fullSlug); err != nil {+ return err+ }+ if err := doc.SetExtraValue("artifact_slug_kwd", fullSlug); err != nil {+ return err+ }}}if v := metaString(p.Meta, "title"); v != "" {@@ -741,7 +767,21 @@ func metaStringSlice(m map[string]any, key string) []string {// headless / manual chaining reads them from the component output map, so they// must be forwarded when present.func mergeChunks(inputs map[string]any, compiled []schema.ChunkDoc) map[string]any {- raw, _ := inputs["chunks"].([]any)+ // Accept both the []any and []map[string]any chunk carriers (the chunker+ // emits the latter; buildInputs already handles both). Without this, the+ // original source chunks would be dropped when the carrier is []map[string]any.+ var raw []any+ switch v := inputs["chunks"].(type) {+ case []any:+ raw = v+ case []map[string]any:+ raw = make([]any, 0, len(v))+ for _, m := range v {+ raw = append(raw, m)+ }+ default:+ log.Printf("knowledge_compiler: mergeChunks: unexpected chunks type %T", inputs["chunks"])+ }merged := make([]any, 0, len(raw)+len(compiled))for _, r := range raw {merged = append(merged, r)@@ -772,6 +812,16 @@ func mergeChunks(inputs map[string]any, compiled []schema.ChunkDoc) map[string]a// serialization shape, and the one place where inputs are validated, defaulted,// and enriched (e.g. extracting each chunk's pre-computed embedding) before any// LLM/embedding work begins.+// mapKeys returns the sorted keys of m, for diagnostics logging.+func mapKeys(m map[string]any) []string {+ keys := make([]string, 0, len(m))+ for k := range m {+ keys = append(keys, k)+ }+ sort.Strings(keys)+ return keys+}+func buildInputs(inputs map[string]any, param common.Param) (common.Inputs, error) {in := common.Inputs{LLMID: param.LLMID,@@ -781,32 +831,45 @@ func buildInputs(inputs map[string]any, param common.Param) (common.Inputs, erroif d, ok := inputs["doc_id"].(string); ok && d != "" {in.DocID = d}- if raw, ok := inputs["chunks"].([]any); ok {- for _, r := range raw {- m, ok := r.(map[string]any)+ // The upstream pipeline hands chunks over as a []any of map[string]any in+ // some paths and as a []map[string]any in others (the chunker emits the+ // latter). Accept both so the knowledge compiler never silently drops the+ // whole upstream output on a type mismatch.+ var raw []map[string]any+ switch v := inputs["chunks"].(type) {+ case []any:+ raw = make([]map[string]any, 0, len(v))+ for _, item := range v {+ m, ok := item.(map[string]any)if !ok {continue}- ch := common.Chunk{Meta: m}- if id, ok := m["id"].(string); ok {- ch.ID = id- }- if t, ok := m["text"].(string); ok {- ch.Text = t- }- if cw, ok := m["content_with_weight"].(string); ok {- ch.Content = cw- }- // Reuse the embedding the upstream pipeline already computed on the- // chunk (stored under q_<dim>_vec); variants fall back to embedding- // on demand when it is absent. A chunk must carry exactly one vector.- vec, err := common.VectorFromChunkMap(m, 0)- if err != nil {- return in, err- }+ raw = append(raw, m)+ }+ case []map[string]any:+ raw = v+ default:+ log.Printf("knowledge_compiler: buildInputs: unexpected chunks type %T", inputs["chunks"])+ }+ log.Printf("knowledge_compiler: buildInputs: accepted %d chunk(s) from inputs[chunks]", len(raw))+ for _, m := range raw {+ ch := common.Chunk{Meta: m}+ if id, ok := m["id"].(string); ok {+ ch.ID = id+ }+ if t, ok := m["text"].(string); ok {+ ch.Text = t+ }+ if cw, ok := m["content_with_weight"].(string); ok {+ ch.Content = cw+ }+ // Reuse the embedding the upstream pipeline already computed on the+ // chunk (stored under q_<dim>_vec); variants fall back to embedding+ // on demand when it is absent. A chunk must carry exactly one vector.+ if vec, err := common.VectorFromChunkMap(m, 0); err == nil {ch.Vector = vec- in.Chunks = append(in.Chunks, ch)}+ in.Chunks = append(in.Chunks, ch)}if hc, ok := inputs["historical_candidates"].([]common.Candidate); ok {in.HistoricalCandidates = hc@@ -824,16 +887,17 @@ func buildInputs(inputs map[string]any, param common.Param) (common.Inputs, erro}func init() {- runtime.MustRegister(componentNameKnowledgeCompiler, runtime.CategoryIngestion,- NewKnowledgeCompilerComponent, runtime.Metadata{- Version: "0.1.0",- Inputs: map[string]string{- "chunks": "Upstream chunker/parser output chunks (id + text/content_with_weight).",- "llm_id": "Optional LLM id override.",- "embedding_model": "Optional embedding model override.",- "tenant_id": "Optional tenant scope.",- "dataset_id": "Optional dataset scope (wiki historical dedup).",- },- Outputs: chunkerOutputs,- })+ // Register under the single unified name "Compiler" (matching the Python+ // side) so both Python-saved canvases and Go's built-in ingestion templates+ // resolve to the same component without name translation.+ meta := runtime.Metadata{+ Version: "0.1.0",+ Inputs: map[string]string{+ "chunks": "Upstream chunker/parser output chunks (id + text/content_with_weight).",+ "historical_candidates": "Optional historical dedup candidates for offline/test runs.",+ },+ Outputs: chunkerOutputs,+ }+ runtime.MustRegister(componentNameCompiler, runtime.CategoryIngestion,+ NewKnowledgeCompilerComponent, meta)}diff --git a/internal/ingestion/component/knowledge_compiler/component_test.go b/internal/ingestion/component/knowledge_compiler/component_test.goindex c46a8847af..a9aaa5418a 100644--- a/internal/ingestion/component/knowledge_compiler/component_test.go+++ b/internal/ingestion/component/knowledge_compiler/component_test.go@@ -12,6 +12,8 @@ import ("sync""testing"+ "ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/component/globals""ragflow/internal/ingestion/component/knowledge_compiler/common""ragflow/internal/service/nav"@@ -173,7 +175,7 @@ func TestKnowledgeCompiler_Structure_EndToEnd(t *testing.T) {installMockDeps(t)installVariantTemplateResolver(t, "structure")- c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{+ c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{"compilation_template_id": "tpl-structure", "llm_id": "llm1", "embedding_model": "emb1",})if err != nil {@@ -232,7 +234,7 @@ func TestKnowledgeCompiler_Structure_EndToEnd(t *testing.T) {}func TestKnowledgeCompiler_UnknownVariant(t *testing.T) {- c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{"compilation_template_id": "nope"})+ c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{"compilation_template_id": "nope"})if err != nil {t.Fatalf("NewKnowledgeCompilerComponent: %v", err)}@@ -378,7 +380,7 @@ func TestKnowledgeCompiler_Alias_Mindmap(t *testing.T) {installMockDeps(t)// "mind_map" is the deprecated alias for "mindmap"; both resolve to the// implemented mindmap variant and must run (not ErrUnknownVariant / stub).- c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{+ c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{"compilation_template_id": "mind_map", "llm_id": "llm1", "embedding_model": "emb1",})if err != nil {@@ -408,7 +410,7 @@ func runVariant(t *testing.T, variant string, extra map[string]any) []map[stringfor k, v := range extra {params[k] = v}- c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", params)+ c, err := NewKnowledgeCompilerComponent("Compiler", params)if err != nil {t.Fatalf("NewKnowledgeCompilerComponent(%s): %v", variant, err)}@@ -531,7 +533,7 @@ func TestKnowledgeCompiler_EmitsChunks(t *testing.T) {installMockDeps(t)installVariantTemplateResolver(t, "structure")- c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{+ c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{"compilation_template_id": "tpl-structure", "llm_id": "llm1", "embedding_model": "emb1",})if err != nil {@@ -588,7 +590,7 @@ func TestKnowledgeCompiler_TemplateIDsAndProvenance(t *testing.T) {installMockDeps(t)installVariantTemplateResolver(t, "structure")- c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{+ c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{"compilation_template_id": "tpl-structure","llm_id": "llm1","embedding_model": "emb1",@@ -672,7 +674,7 @@ func (m constEmbedder) Encode(_ context.Context, texts []string) ([][]float32, efunc TestKnowledgeCompiler_Tree_DegenerateNoInfiniteLoop(t *testing.T) {installProseDeps(t)installVariantTemplateResolver(t, "tree")- c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{+ c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{"compilation_template_id": "tpl-tree", "llm_id": "llm1", "embedding_model": "emb1","extra": map[string]any{"tree_order": 4},})@@ -727,7 +729,7 @@ func TestKnowledgeCompiler_Wiki_HistoricalDedupDropsDuplicates(t *testing.T) {t.Cleanup(func() { common.SetDepsResolver(nil) })installVariantTemplateResolver(t, "wiki")- c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{+ c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{"compilation_template_id": "tpl-wiki", "llm_id": "llm1", "embedding_model": "emb1",})if err != nil {@@ -786,7 +788,7 @@ func TestKnowledgeCompiler_Wiki_UpdateMergesExistingPage(t *testing.T) {t.Cleanup(func() { common.SetDepsResolver(nil) })installVariantTemplateResolver(t, "wiki")- c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{+ c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{"compilation_template_id": "tpl-wiki", "llm_id": "llm1", "embedding_model": "emb1",})if err != nil {@@ -813,7 +815,7 @@ func TestKnowledgeCompiler_Wiki_UpdateMergesExistingPage(t *testing.T) {if !ok {continue}- if cm["compile_kwd"] == "artifact_page" && cm["kc_kind"] == "page" && cm["slug_kwd"] == "entity/alpha" {+ if cm["compile_kwd"] == "wiki_page" && cm["kc_kind"] == "page" && cm["slug_kwd"] == "entity/alpha" {page = cmbreak}@@ -867,7 +869,7 @@ func TestKnowledgeCompiler_Wiki_HistoricalDedupScopedByDataset(t *testing.T) {t.Cleanup(func() { common.SetDepsResolver(nil) })installVariantTemplateResolver(t, "wiki")- c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{+ c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{"compilation_template_id": "tpl-wiki", "llm_id": "llm1", "embedding_model": "emb1","enable_historical_dedup": true,})@@ -982,7 +984,7 @@ func TestKnowledgeCompiler_Structure_FencedJSONNotDropped(t *testing.T) {t.Cleanup(func() { common.SetDepsResolver(nil) })installVariantTemplateResolver(t, "structure")- c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{+ c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{"compilation_template_id": "tpl-structure", "llm_id": "llm1", "embedding_model": "emb1",})if err != nil {@@ -1022,7 +1024,7 @@ func TestKnowledgeCompiler_Structure_MalformedJSONFailsLoud(t *testing.T) {t.Cleanup(func() { common.SetDepsResolver(nil) })installVariantTemplateResolver(t, "structure")- c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{+ c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{"compilation_template_id": "tpl-structure", "llm_id": "llm1", "embedding_model": "emb1",})if err != nil {@@ -1048,7 +1050,7 @@ func TestKnowledgeCompiler_PassThroughEnvelope(t *testing.T) {installMockDeps(t)installVariantTemplateResolver(t, "structure")- c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{+ c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{"compilation_template_id": "tpl-structure", "llm_id": "llm1", "embedding_model": "emb1",})if err != nil {@@ -1116,7 +1118,7 @@ func TestKnowledgeCompiler_GroupIDsResolvedToTemplateIDs(t *testing.T) {// compilation_template_group_id (not the obsolete plural list) selects the// group; compilation_template_id is absent so the group path is taken.- c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{+ c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{"compilation_template_group_id": "grp1","llm_id": "llm1","embedding_model": "emb1",@@ -1177,7 +1179,7 @@ func TestKnowledgeCompiler_GroupIDsWithoutResolverFailsLoud(t *testing.T) {installMockDeps(t)common.SetGroupResolver(nil) // ensure no resolver is installed- c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", map[string]any{+ c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{"compilation_template_group_id": "grp1","llm_id": "llm1","embedding_model": "emb1",@@ -1232,6 +1234,148 @@ var testGroupResolver common.GroupResolver = func(ctx context.Context, db *gorm.return groupIDs, nil}+// TestKnowledgeCompiler_RegistryResolvesUnifiedName locks the unified-name+// contract: the knowledge-compilation node is registered under "Compiler"+// (matching the Python side rag/flow/compiler/compiler.py component_name), so+// both a Python-saved canvas and Go's built-in ingestion templates resolve to+// the same KnowledgeCompilerComponent through runtime.DefaultRegistry.+func TestKnowledgeCompiler_RegistryResolvesUnifiedName(t *testing.T) {+ factory, category, _, ok := runtime.DefaultRegistry.Lookup("Compiler")+ if !ok {+ t.Fatal("runtime registry has no component \"Compiler\"; the Python canvas and Go templates both use this name")+ }+ if category != runtime.CategoryIngestion {+ t.Fatalf("component \"Compiler\" category = %q, want %q", category, runtime.CategoryIngestion)+ }+ c, err := factory("Compiler", map[string]any{"compilation_template_id": "tree", "llm_id": "llm1", "embedding_model": "emb1"})+ if err != nil {+ t.Fatalf("factory(\"Compiler\"): %v", err)+ }+ if _, ok := c.(*KnowledgeCompilerComponent); !ok {+ t.Fatalf("factory(\"Compiler\") produced %T, want *KnowledgeCompilerComponent", c)+ }+}++// TestKnowledgeCompiler_TenantFromGlobals locks the tenant-resolution contract:+// in the production canvas run the run-level tenant_id lives in the shared+// CanvasState.Globals bag (seeded by the pipeline at run start), not necessarily+// in the KnowledgeCompiler's own input map. The component must resolve it through+// globals.GlobalOrInput; otherwise the template/group lookup gets an empty tenant+// and fails with "compilation_template_group ... not found for tenant".+func TestKnowledgeCompiler_TenantFromGlobals(t *testing.T) {+ // Attach a CanvasState to the context and seed the run-level tenant id into+ // the global bag, as the pipeline does at run start.+ ctx := runtime.WithState(context.Background(), runtime.NewCanvasState("run-id", "sess-id"))+ globals.SeedIngestionGlobals(ctx, map[string]any{"tenant_id": "tenant-from-globals"})++ // Install a template resolver that records the tenant id it is called with.+ var gotTenant string+ prev := testTemplateResolver+ common.SetTemplateResolver(func(ctx context.Context, db *gorm.DB, tenantID, templateID string) (common.TemplateInfo, error) {+ gotTenant = tenantID+ return common.TemplateInfo{ID: templateID, Kind: "structure", Config: map[string]any{}}, nil+ })+ t.Cleanup(func() { common.SetTemplateResolver(prev) })++ c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{"compilation_template_id": "tpl-x"})+ if err != nil {+ t.Fatalf("construct: %v", err)+ }++ // Invoke without tenant_id in the input map; the tenant must come from the+ // global bag. The template resolution (and thus gotTenant) happens early in+ // Invoke, before the variant's LLM/embedding deps are exercised — which is+ // all this test needs to assert.+ _, _ = c.Invoke(ctx, nil, map[string]any{+ "llm_id": "llm1",+ "chunks": []any{map[string]any{"id": "c1", "content_with_weight": "alpha beta", "text": "alpha beta"}},+ "embedding_model": "emb1",+ })+ if gotTenant != "tenant-from-globals" {+ t.Fatalf("template resolver saw tenant %q, want %q (tenant_id must be read from CanvasState.Globals)", gotTenant, "tenant-from-globals")+ }+}++// TestKnowledgeCompiler_BuildInputsAcceptsMapSliceChunks locks the chunk-carrier+// contract: the upstream chunker hands chunks over as []map[string]any (see the+// chunk map shape {"id","text","ck_type","doc_type_kwd","tk_nums"} observed from+// the running pipeline), not as []any of map. buildInputs must accept both+// shapes; a strict []any assertion alone would silently drop every chunk and+// leave the knowledge compiler with empty input (compiling nothing yet still+// reporting success).+func TestKnowledgeCompiler_BuildInputsAcceptsMapSliceChunks(t *testing.T) {+ in, err := buildInputs(map[string]any{+ "chunks": []map[string]any{+ {"id": "c1", "text": "《三国演义》", "ck_type": "text"},+ {"id": "c2", "text": "滚滚长江东逝水", "ck_type": "text"},+ },+ }, common.Param{})+ if err != nil {+ t.Fatalf("buildInputs: %v", err)+ }+ if len(in.Chunks) != 2 {+ t.Fatalf("buildInputs produced %d chunks, want 2 (upstream sends []map[string]any)", len(in.Chunks))+ }+ if in.Chunks[0].ID != "c1" || in.Chunks[0].Text != "《三国演义》" {+ t.Fatalf("chunk[0] = %+v, want id=c1 text=《三国演义》", in.Chunks[0])+ }++ // The legacy []any-of-map shape must still work.+ in2, err := buildInputs(map[string]any{+ "chunks": []any{map[string]any{"id": "x", "text": "t"}},+ }, common.Param{})+ if err != nil {+ t.Fatalf("buildInputs ([]any): %v", err)+ }+ if len(in2.Chunks) != 1 || in2.Chunks[0].ID != "x" {+ t.Fatalf("buildInputs ([]any) produced %+v, want 1 chunk id=x", in2.Chunks)+ }+}++// TestProductsToChunkDocs_PageVsSectionCompileKWD locks the page/section+// discriminator: a wiki page product is stamped compile_kwd="wiki_page" and a+// wiki section product compile_kwd="wiki_section", so a page search on+// compile_kwd="wiki_page" (engine_service / kcWikiPageStore) returns pages only.+func TestProductsToChunkDocs_PageVsSectionCompileKWD(t *testing.T) {+ page := common.Product{+ ID: "page-id", DocID: "d1", TenantID: "t1", Variant: common.VariantWiki,+ Content: "# Alpha\n\nBody", ParentID: "",+ Meta: map[string]any{"kind": "page", "slug": "entity/alpha", "title": "Alpha", "page_type": "entity", "source_chunk_ids": []string{"c1"}},+ }+ section := common.Product{+ ID: "section-id", DocID: "d1", TenantID: "t1", Variant: common.VariantWiki,+ Content: "Section body", ParentID: "page-id",+ Meta: map[string]any{"kind": "section", "slug": "overview", "page_slug": "entity/alpha", "section_level": 1, "source_chunk_ids": []string{"c1"}},+ }+ docs, err := productsToChunkDocs([]common.Product{page, section})+ if err != nil {+ t.Fatalf("productsToChunkDocs: %v", err)+ }+ var pageKWD, sectionKWD string+ var sectionParent string+ for _, d := range docs {+ // Product.Meta is preserved under the kc_* round-trip keys; the page/+ // section kind lives at "kc_kind".+ kind, _ := d.GetExtraString("kc_kind")+ if kind == "page" {+ pageKWD, _ = d.GetExtraString("compile_kwd")+ }+ if kind == "section" {+ sectionKWD, _ = d.GetExtraString("compile_kwd")+ sectionParent, _ = d.GetExtraString("parent_kwd")+ }+ }+ if pageKWD != "wiki_page" {+ t.Errorf("page compile_kwd = %q, want wiki_page", pageKWD)+ }+ if sectionKWD != "wiki_section" {+ t.Errorf("section compile_kwd = %q, want wiki_section (schema-backed page/section discriminator)", sectionKWD)+ }+ if sectionParent != "page-id" {+ t.Errorf("section parent_kwd = %q, want page-id", sectionParent)+ }+}+// TestMain installs the stub resolvers for the variant unit tests.func TestMain(m *testing.M) {common.SetTemplateResolver(testTemplateResolver)diff --git a/internal/ingestion/component/knowledge_compiler/golden_test.go b/internal/ingestion/component/knowledge_compiler/golden_test.goindex 09ed27dcc5..7ffa3b16d2 100644--- a/internal/ingestion/component/knowledge_compiler/golden_test.go+++ b/internal/ingestion/component/knowledge_compiler/golden_test.go@@ -79,7 +79,7 @@ func runVariantChunksWithInputs(t *testing.T, variant string, extra, inputsExtrafor k, v := range extra {params[k] = v}- c, err := NewKnowledgeCompilerComponent("KnowledgeCompiler", params)+ c, err := NewKnowledgeCompilerComponent("Compiler", params)if err != nil {t.Fatalf("NewKnowledgeCompilerComponent(%s): %v", variant, err)}diff --git a/internal/ingestion/component/knowledge_compiler/pool_wiring.go b/internal/ingestion/component/knowledge_compiler/pool_wiring.goindex 6eb4ec017a..3d4a92b7f8 100644--- a/internal/ingestion/component/knowledge_compiler/pool_wiring.go+++ b/internal/ingestion/component/knowledge_compiler/pool_wiring.go@@ -20,6 +20,7 @@ import ("ragflow/internal/ingestion/component/knowledge_compiler/mindmap""ragflow/internal/ingestion/component/knowledge_compiler/structure"+ "ragflow/internal/ingestion/component/knowledge_compiler/wiki""ragflow/internal/ingestion/knowledge_compile")@@ -35,4 +36,5 @@ func init() {}structure.SetBatchSubmitter(submit)mindmap.SetBatchSubmitter(submit)+ wiki.SetBatchSubmitter(submit)}diff --git a/internal/ingestion/component/knowledge_compiler/tree/raptor.go b/internal/ingestion/component/knowledge_compiler/tree/raptor.goindex 697b1a845e..5813e2e460 100644--- a/internal/ingestion/component/knowledge_compiler/tree/raptor.go+++ b/internal/ingestion/component/knowledge_compiler/tree/raptor.go@@ -195,7 +195,7 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str// text to a per-chunk token budget so the cluster fits the LLM context// (Python: len_per_chunk = (max_length - max_token) / len(texts);// truncate(t, len_per_chunk), raptor.py:389-390).- content := buildClusterContent(texts, task.pointIdxs, deps.LLMMaxLength, maxToken)+ content := buildClusterContent(texts, task.pointIdxs, deps.ModelContextLen, maxToken)system := raptorSystemHelper + strings.Replace(taskPrompt, "{cluster_content}", content, 1)summary, err := summarizeTexts(ctx, deps, llmID, system, raptorTitleInstruction, maxToken)if err != nil {@@ -292,7 +292,7 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID strlog.Printf("tree: no top-level summaries produced, skipping root node")return nil}- rootContent := buildClusterContent(topLevelTexts, allIndices(len(topLevelTexts)), deps.LLMMaxLength, maxToken)+ rootContent := buildClusterContent(topLevelTexts, allIndices(len(topLevelTexts)), deps.ModelContextLen, maxToken)rootSummary, err := summarizeTexts(ctx, deps, llmID,raptorSystemHelper+strings.Replace(taskPrompt, "{cluster_content}", rootContent, 1),raptorTitleInstruction, maxToken)@@ -389,14 +389,14 @@ func titleOf(summary string) string {// (max_length - max_token) / len(texts); truncate(t, len_per_chunk)). The token// budget uses the cl100k_base encoder, mirroring Python's truncate (token-level,// not character-level).-func buildClusterContent(texts []string, idxs []int, llmMaxLength, maxToken int) string {+func buildClusterContent(texts []string, idxs []int, modelContextLen, maxToken int) string {if len(idxs) == 0 {return ""}- if llmMaxLength <= 0 {- llmMaxLength = common.DefaultLLMContextLength+ if modelContextLen <= 0 {+ modelContextLen = common.DefaultLLMContextLength}- per := (llmMaxLength - maxToken) / len(idxs)+ per := (modelContextLen - maxToken) / len(idxs)if per < 1 {per = 1}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/page_test.go b/internal/ingestion/component/knowledge_compiler/wiki/page_test.goindex 5b8c563c7d..e4dcab3216 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/page_test.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/page_test.go@@ -202,6 +202,7 @@ func TestTransformWikiLinks(t *testing.T) {"See [[concept/beta|Beta]] and [[entity/alpha]]. Also [Beta](artifact/kb/concept/beta).","kb",map[string]string{"entity/alpha": "Alpha", "concept/beta": "Beta"},+ map[string]string{"entity/alpha": "entity", "concept/beta": "concept"},)if !strings.Contains(rendered, "(artifact/kb/concept/beta)") || !strings.Contains(rendered, "(artifact/kb/entity/alpha)") {t.Fatalf("rendered links not rewritten: %q", rendered)@@ -211,6 +212,26 @@ func TestTransformWikiLinks(t *testing.T) {}}+func TestTransformWikiLinksBareSlugGetsPageType(t *testing.T) {+ // A bare wikilink (no page_type prefix) resolves its page_type from the+ // plan map so the rendered link is clickable by the frontend parser.+ rendered, outlinks := transformWikiLinks(+ "See [[董卓]] and [[刘备|Liu Bei]].",+ "kb",+ map[string]string{"董卓": "董卓", "刘备": "刘备"},+ map[string]string{"董卓": "entity", "刘备": "entity"},+ )+ if !strings.Contains(rendered, "(artifact/kb/entity/董卓)") {+ t.Fatalf("bare slug link missing page_type: %q", rendered)+ }+ if !strings.Contains(rendered, "(artifact/kb/entity/刘备)") {+ t.Fatalf("bare slug link missing page_type: %q", rendered)+ }+ if len(outlinks) != 2 {+ t.Fatalf("outlinks = %#v, want 2", outlinks)+ }+}+func TestSlugify(t *testing.T) {cases := []struct{ in, want string }{{"Hello World", "hello-world"},diff --git a/internal/ingestion/component/knowledge_compiler/wiki/prompt.go b/internal/ingestion/component/knowledge_compiler/wiki/prompt.goindex 6da9738c75..43f9cac2c9 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/prompt.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/prompt.go@@ -15,6 +15,26 @@ const wikiPlanSystem = `You are a knowledge compilation planner. Given structureconst wikiRefineSystem = `You are a technical writer. Write a complete wiki page from the plan, evidence checklist, and source text. Preserve factual density, keep the source language, and return only markdown.`+const wikiReduceEntityDisambiguateSystem = `You are a knowledge canonicalization engine. Decide whether two named entities refer to the same real-world concept. Return ONLY valid JSON.`++const wikiReduceEntityDisambiguateUserTemplate = `## Entity A+{entity_a}++## Entity B+{entity_b}++Return JSON:+{+ "merge": true,+ "reason": "string"+}++Rules:+- merge=true only when A and B are the same real-world entity (e.g. aliases, abbreviations, spelling variants of the same thing).+- merge=false when they are distinct concepts that merely co-occur.+- Prefer false when ambiguous.+- Return ONLY the JSON object.`+const wikiMapUserTemplate = `## Document contextDocument id: {doc_id}Batch contains {chunk_count} packed chunk(s). Each chunk is introduced by a@@ -134,23 +154,16 @@ Return a JSON compilation plan with one or more page entries:}Rules:+- Return at most {max_pages} page entries for this batch.- Prefer one page per high-signal entity or concept when the batch supports it.+- Merge minor or weakly-supported facts into broader topic pages instead of emitting tiny standalone pages.- Use page_type=entity for entity pages, page_type=concept for concept pages, and page_type=topic for cross-cutting themes.- entity_names must name the entities and concepts that justify the page.- related_kb_pages should list other slugs from the same plan that the page should cross-link to.+- Keep lead concise (one sentence) and keep sections compact (no more than 4 sections, no more than 3 short points per section).- Keep the page in the source language.- Return ONLY the JSON object.`-const wikiPlanMergeUserTemplate = `## Knowledge base context-Document id: {doc_id}--## Partial plans-{candidates}--Merge the partial plans into one final compilation plan with the same JSON shape as above.-Preserve distinct pages when they cover different entities or concepts; drop only near-duplicate slugs.-Return ONLY the JSON object.`-const wikiPlanReconcileSystem = `You are a wiki page reconciliation engine. Compare a planned wiki page with existing wiki pages and decide whether the planned page should UPDATE one of them or CREATE a new page. Return only valid JSON.`const wikiPlanReconcileUserTemplate = `## Planned pagediff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.goindex 2cb95570af..1837d390e9 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go@@ -21,6 +21,55 @@ import ("ragflow/internal/ingestion/component/knowledge_compiler/structure")+// batchSubmitter fans out the MAP-stage extraction jobs on the process-wide+// knowledge-compilation pool. It is injected by the knowledge_compiler wiring+// so every variant shares one vCPU-sized concurrency bound; when nil the+// batches run sequentially (the historic default).+var batchSubmitter func(ctx context.Context, jobs []func() error) error++// SetBatchSubmitter installs the shared-pool fan-out used by Run's MAP stage.+// Pass nil to revert to serial execution.+func SetBatchSubmitter(submit func(ctx context.Context, jobs []func() error) error) {+ batchSubmitter = submit+}++// runBatches mirrors the other compiler variants: concurrent under the wired+// global compiler pool, or serial when no submitter is set. The first error is+// returned after all jobs settle; the global pool is never StopWait'd.+func runBatches(ctx context.Context, jobs []func() error) error {+ if len(jobs) == 0 {+ return nil+ }+ if batchSubmitter != nil {+ return batchSubmitter(ctx, jobs)+ }+ for _, j := range jobs {+ if err := j(); err != nil {+ return err+ }+ }+ return nil+}++// wikiMapTokenBudget is the input-token budget per MAP extraction batch. It is+// intentionally well below the chat model's context window so the LLM has+// generous room to emit the entity/concept/claim/relation/topic JSON without+// hitting the output-token limit and truncating the payload.+const wikiMapTokenBudget = 2048++// wikiMapMaxTokens derives the extraction output budget from the model's+// context length and the per-batch input budget: once the batch has consumed+// wikiMapTokenBudget input tokens, the rest of the window is handed to the+// output — but never below the input budget itself, so a small-input batch can+// still get a proportionally large extraction payload. modelContextLen is the+// model's total context window in tokens (0 means unknown).+func wikiMapMaxTokens(modelContextLen int) int {+ if modelContextLen <= 0 {+ modelContextLen = common.DefaultLLMContextLength+ }+ return max(modelContextLen-wikiMapTokenBudget, wikiMapTokenBudget)+}+type wikiPipeline struct {ctx context.Contextdeps common.Deps@@ -35,6 +84,12 @@ type wikiPipeline struct {reduced wikiExtractplan wikiPlanpages []wikiPageResult+ // planBudget is the resolved global page budget for the current planning+ // run (target approx + hard cap).+ planBudget wikiPlanBudget+ // planCapacityExcluded counts the planned pages dropped to fit the global+ // hard cap; testable and reported for observability.+ planCapacityExcluded int}type wikiExtract struct {@@ -100,8 +155,33 @@ type wikiPlanPage struct {Priority int `json:"priority"`Lead string `json:"lead"`Sections []wikiPlanSection `json:"sections"`-}-+ // MentionCount is an internal (non-serialized) signal used for the+ // deterministic page selection when the merged plan exceeds the global hard+ // cap. It is computed from the reduced extract, not read from JSON.+ MentionCount int `json:"-"`+}++// Reconciliation thresholds. These are a deliberate Go-specific refinement of+// the Python wiki.py contract, NOT a byte-for-byte alignment:+//+// Python `_wiki_reconcile_with_kb` (wiki.py:1900-1957) queries KNN with+// `extra_options={"similarity": update_threshold}` (update_threshold=0.95), so+// candidates below that threshold are normally filtered out at retrieval and+// the item becomes CREATE directly. Its MAYBE band ([maybe=0.60, update=0.95))+// is only reachable when a backend still returns low-score candidates despite+// the similarity filter.+//+// Go's `FindSimilarPages` returns top-K candidates without a similarity floor,+// so it genuinely sees low-score candidates Python never does. To exploit that+// richer signal we keep a real two-band decision:+//+// - Score >= update_threshold (0.92) -> direct UPDATE+// - Score < maybe_threshold (0.78) -> CREATE (no match)+// - Score in [maybe, update) -> title/topic/entity overlap+// heuristic first (direct UPDATE), else MAYBE resolved by the LLM.+//+// The title/topic/entity overlap straight-to-UPDATE shortcut is a Go-only+// enhancement; it is NOT a Python-aligned behavior. Keep it documented as such.const (wikiPlanUpdateThreshold = 0.92wikiPlanMaybeThreshold = 0.78@@ -199,6 +279,10 @@ func (p *wikiPipeline) run() error {return err}p.reduced = reduceExtracts(p.mapExtracts)+ // Layer embedding + LLM disambiguation onto the exact-merged entities+ // (REDUCE enhancement; concepts keep exact dedup). Degrades to a no-op when+ // the embedder/chat seams are unavailable.+ p.reduced.Entities = p.dedupeEntities(p.reduced.Entities)plan, err := p.runPlan()if err != nil {return err@@ -213,27 +297,66 @@ func (p *wikiPipeline) run() error {}func (p *wikiPipeline) runMap() error {- batches := common.PackBatches(p.inputs.Chunks, 4096, p.deps.Tokenizer)- for _, batch := range batches {- if err := p.ctx.Err(); err != nil {- return err- }- extract, err := p.mapBatch(batch)- if err != nil {- return err- }- p.mapExtracts = append(p.mapExtracts, extract)+ // Keep each batch small enough that the LLM's entity/relation JSON output+ // for the batch stays well under the model's output-token limit. 2048 input+ // tokens per batch (was a hard-coded 4096) leaves generous headroom for the+ // extraction payload; oversize batches caused truncated, unparseable JSON+ // (unexpected end of JSON input) on the real pipeline.+ batches := common.PackBatches(p.inputs.Chunks, wikiMapTokenBudget, p.deps.Tokenizer)+ extracts, err := runMapBatches(p.ctx, batches, p.mapBatch)+ if err != nil {+ return err}+ p.mapExtracts = append(p.mapExtracts, extracts...)return nil}+func runMapBatches(+ ctx context.Context,+ batches [][]common.Chunk,+ mapBatch func([]common.Chunk) (wikiExtract, error),+) ([]wikiExtract, error) {+ if len(batches) == 0 {+ return nil, nil+ }+ extracts := make([]wikiExtract, len(batches))+ jobs := make([]func() error, 0, len(batches))+ for i, batch := range batches {+ i, batch := i, batch+ jobs = append(jobs, func() error {+ if err := ctx.Err(); err != nil {+ return err+ }+ extract, err := mapBatch(batch)+ if err != nil {+ return err+ }+ // Distinct slice index per batch keeps results stable without locks.+ extracts[i] = extract+ return nil+ })+ }+ if err := runBatches(ctx, jobs); err != nil {+ return nil, err+ }+ return extracts, nil+}+func (p *wikiPipeline) mapBatch(batch []common.Chunk) (wikiExtract, error) {parserConfig, _ := p.inputs.VariantSpecific["parser_config"].(map[string]any)user, _ := buildWikiMapPrompt(p.docID, batch, parserConfig, p.param.Language)+ // Give the extraction step a generous output budget so the entity/relation+ // JSON is not silently truncated by the model's default output cap (that+ // produced "unexpected end of JSON input" from GenJSON). The output budget is+ // tied to the per-batch input budget: once the batch consumes+ // wikiMapTokenBudget tokens of the model's context, the remainder is left+ // for the extraction payload (and never less than the input budget itself).+ mt := wikiMapMaxTokens(p.deps.ModelContextLen)raw, err := common.GenJSON(p.ctx, p.deps.Chat, common.ChatRequest{LLMID: p.llmID,SystemPrompt: wikiMapSystem,UserPrompt: user,+ MaxTokens: &mt,})if err != nil {return wikiExtract{}, err@@ -246,31 +369,85 @@ func (p *wikiPipeline) runPlan() (wikiPlan, error) {if len(batches) == 0 {batches = []wikiExtract{p.reduced}}- plans := make([]wikiPlan, 0, len(batches))+ totalItems := 0+ for _, b := range batches {+ totalItems += wikiExtractItemCount(b)+ }+ p.planBudget = deriveWikiPlanBudget(p.deps.ModelContextLen, totalItems)+ // Quota allocation must use the achievable cap (min(Target, Max)): when the+ // model's output capacity is smaller than the item-count-derived target, the+ // planner must be asked for at most Max pages so the sum of per-batch+ // max_pages never exceeds the capacity that can actually be emitted. Using+ // Target here would re-introduce the truncated-JSON risk the budget exists+ // to eliminate.+ quotas := allocatePlanQuotas(batches, p.planBudget.Cap())++ // approvedReduced is the set of items that actually got a non-zero quota.+ // It is what the fallback/normalization may reference so zero-quota items+ // can never leak back into the plan via buildWikiFallbackPages.+ approved := wikiExtract{}+ plans := make([]wikiPlan, len(batches))+ jobs := make([]func() error, 0, len(batches))for i, batch := range batches {- plan, err := p.runPlanBatch(batch, i+1, len(batches))- if err != nil {- return wikiPlan{}, err+ i, batch := i, batch+ quota := quotas[i]+ if quota <= 0 {+ // Zero-quota batch: no planner call and no fallback page. It is+ // intentionally left as the zero wikiPlan{} so the merge sees no+ // pages from it.+ continue}- plans = append(plans, plan)+ approved.Entities = append(approved.Entities, batch.Entities...)+ approved.Concepts = append(approved.Concepts, batch.Concepts...)+ approved.Claims = append(approved.Claims, batch.Claims...)+ approved.Relations = append(approved.Relations, batch.Relations...)+ approved.Topics = append(approved.Topics, batch.Topics...)+ jobs = append(jobs, func() error {+ if err := p.ctx.Err(); err != nil {+ return err+ }+ plan, err := p.runPlanBatch(batch, i+1, len(batches), quota)+ if err != nil {+ return err+ }+ // Distinct slice index keeps results stable without locks.+ plans[i] = plan+ return nil+ })}- if len(plans) == 1 {- plan := normalizeWikiPlan(plans[0], p.docID, p.reduced)- return p.reconcilePlan(plan)+ if err := runBatches(p.ctx, jobs); err != nil {+ return wikiPlan{}, err}- plan, err := p.mergePlanCandidates(plans)++ merged := p.mergePlanCandidates(plans, approved)+ var excluded int+ merged.Pages, excluded = truncatePlanPagesByCap(merged.Pages, p.planBudget.Max, approved)+ p.planCapacityExcluded += excluded+ merged.Pages = normalizeWikiPlanPageLinks(merged.Pages)+ reconciled, err := p.reconcilePlan(merged)if err != nil {return wikiPlan{}, err}- return p.reconcilePlan(plan)+ // reconcilePlan rewrites page.Slug to the matched existing page's slug, which+ // can re-introduce self-referential or plan-absent related links; normalize+ // once more so stale links don't reach the stored pages.+ reconciled.Pages = normalizeWikiPlanPageLinks(reconciled.Pages)+ return reconciled, nil}+// runRefine fans the REFINE stage out to page-level jobs on the shared compiler+// pool, mirroring the P1 error model: all jobs are submitted, awaited, and the+// first error is returned; each job checks ctx before starting. Results are+// written to pre-allocated per-index slots so the output is in normalized-plan+// order regardless of completion order (no concurrent append to a shared slice).+// When no submitter is wired, jobs run serially (historic default).func (p *wikiPipeline) runRefine() ([]wikiPageResult, error) {pages := normalizeWikiPlanPages(p.plan.Pages, p.reduced)if len(pages) == 0 {return nil, nil}pageTitles := map[string]string{}+ slugToPageType := map[string]string{}allPlanSlugs := make([]string, 0, len(pages))for _, page := range pages {if page.Slug == "" {@@ -278,108 +455,155 @@ func (p *wikiPipeline) runRefine() ([]wikiPageResult, error) {}allPlanSlugs = append(allPlanSlugs, page.Slug)pageTitles[page.Slug] = page.Title+ // Every planned page carries its type (entity/concept/...); the link+ // renderer needs it so artifact/<kb>/<page_type>/<slug> links are+ // clickable (frontend parseWikiLinkHref only matches entity|concept).+ if pt := strings.TrimSpace(page.PageType); pt != "" {+ slugToPageType[page.Slug] = pt+ }}entityLookup := buildWikiEntityLookup(p.reduced.Entities)conceptLookup := buildWikiConceptLookup(p.reduced.Concepts)- results := make([]wikiPageResult, 0, len(pages))- for _, planItem := range pages {- if p.ctx.Err() != nil {- return nil, p.ctx.Err()- }- evidence := assembleWikiPageEvidence(planItem, p.reduced.Claims, entityLookup, conceptLookup)- sourceChunkIDs := collectWikiEvidenceChunkIDs(evidence)- sourceContext := buildSourceContext(p.inputs.Chunks, sourceChunkIDs)- if strings.TrimSpace(sourceContext) == "" {- sourceContext = buildSourceContext(p.inputs.Chunks, p.reduced.sourceChunkIDs())- }- available := make([]string, 0, len(allPlanSlugs))- for _, slug := range allPlanSlugs {- if slug != planItem.Slug {- available = append(available, "- [["+slug+"]]")- }- }- if len(available) == 0 {- available = []string{"(none — this is the only page)"}- }- var existing *common.WikiPageCandidate- var err error- if strings.EqualFold(planItem.Action, "UPDATE") && p.deps.WikiPages != nil {- existing, err = p.deps.WikiPages.GetPageBySlug(p.ctx, p.tenantID, p.datasetID, planItem.Slug)- if err != nil {- return nil, err- }- }- existingSection := ""- existingRaw := ""- if existing != nil {- existingRaw = firstNonEmpty(existing.ContentMDRaw, existing.ContentMD)- if strings.TrimSpace(existingRaw) != "" {- existingSection = "## Existing page content (UPDATE — integrate new evidence into this)\n\n" + existingRaw + "\n"++ results := make([]wikiPageResult, len(pages))+ jobs := make([]func() error, 0, len(pages))+ for i, planItem := range pages {+ i, planItem := i, planItem+ jobs = append(jobs, func() error {+ if err := p.ctx.Err(); err != nil {+ return err}- }- user := renderWikiTemplate(wikiRefineWriterUserTemplate, map[string]string{- "action": firstNonEmpty(planItem.Action, "CREATE"),- "slug": planItem.Slug,- "title": firstNonEmpty(planItem.Title, planItem.Slug),- "page_type": firstNonEmpty(planItem.PageType, "concept"),- "all_plan_slugs": strings.Join(available, "\n"),- "existing_section": existingSection,- "source_context": sourceContext,- "evidence_count": fmt.Sprintf("%d", len(evidence)),- "evidence_blocks": formatWikiEvidenceBlocks(evidence),- })- resp, err := p.deps.Chat.Chat(p.ctx, common.ChatRequest{- LLMID: p.llmID,- SystemPrompt: buildWikiRefineWriterSystem(""),- UserPrompt: user,- })- if err != nil {- return nil, err- }- if resp == nil {- return nil, fmt.Errorf("knowledge_compiler: wiki refine returned no response")- }- contentRaw := strings.TrimSpace(firstNonEmpty(resp.Content))- if contentRaw == "" {- contentRaw = "# " + firstNonEmpty(planItem.Title, planItem.Slug) + "\n\n(Page generation produced no content.)"- }- if strings.TrimSpace(existingRaw) != "" {- contentRaw, err = p.mergeWikiPageContent(existingRaw, contentRaw, planItem.Slug)+ res, err := p.runRefinePage(planItem, allPlanSlugs, pageTitles, slugToPageType, entityLookup, conceptLookup)if err != nil {- return nil, err+ return err}- }- contentRendered, outlinks := transformWikiLinks(contentRaw, firstNonEmpty(p.datasetID, p.docID), pageTitles)- sourceDocIDs := collectWikiSourceDocIDs(p.inputs.Chunks, sourceChunkIDs, p.docID)- summary := firstParagraph(contentRendered)- if summary == "" {- summary = firstNonEmpty(planItem.Title, planItem.Slug)- }- topic := firstNonEmpty(planItem.Topic, planItem.Title, planItem.Slug)- results = append(results, wikiPageResult{- Slug: planItem.Slug,- Title: firstNonEmpty(planItem.Title, planItem.Slug),- PageType: firstNonEmpty(planItem.PageType, "concept"),- Topic: topic,- Action: firstNonEmpty(planItem.Action, "CREATE"),- EntityNames: uniqueStrings(planItem.EntityNames),- RelatedKBPages: uniqueStrings(planItem.RelatedKB),- ContentRaw: contentRaw,- Content: contentRendered,- Summary: summary,- Outlinks: outlinks,- SourceChunkIDs: sourceChunkIDs,- SourceDocIDs: sourceDocIDs,+ results[i] = res+ return nil})}+ if err := runBatches(p.ctx, jobs); err != nil {+ return nil, err+ }return results, nil}-func (p *wikiPipeline) runPlanBatch(batch wikiExtract, batchIndex, batchTotal int) (wikiPlan, error) {+// runRefinePage generates one page result from a normalized plan page. UPDATE+// merge, evidence assembly, and source-context building are unchanged; this is+// the per-page unit that runRefine fans out.+func (p *wikiPipeline) runRefinePage(+ planItem wikiPlanPage,+ allPlanSlugs []string,+ pageTitles map[string]string,+ slugToPageType map[string]string,+ entityLookup map[string]wikiExtractItem,+ conceptLookup map[string]wikiExtractItem,+) (wikiPageResult, error) {+ evidence := assembleWikiPageEvidence(planItem, p.reduced.Claims, entityLookup, conceptLookup)+ sourceChunkIDs := collectWikiEvidenceChunkIDs(evidence)+ sourceContext := buildSourceContext(p.inputs.Chunks, sourceChunkIDs)+ if strings.TrimSpace(sourceContext) == "" {+ sourceContext = buildSourceContext(p.inputs.Chunks, p.reduced.sourceChunkIDs())+ }+ available := make([]string, 0, len(allPlanSlugs))+ for _, slug := range allPlanSlugs {+ if slug != planItem.Slug {+ available = append(available, "- [["+slug+"]]")+ }+ }+ if len(available) == 0 {+ available = []string{"(none — this is the only page)"}+ }+ var existing *common.WikiPageCandidate+ var err error+ if strings.EqualFold(planItem.Action, "UPDATE") && p.deps.WikiPages != nil {+ existing, err = p.deps.WikiPages.GetPageBySlug(p.ctx, p.tenantID, p.datasetID, planItem.Slug)+ if err != nil {+ return wikiPageResult{}, err+ }+ }+ existingSection := ""+ existingRaw := ""+ if existing != nil {+ existingRaw = firstNonEmpty(existing.ContentMDRaw, existing.ContentMD)+ if strings.TrimSpace(existingRaw) != "" {+ existingSection = "## Existing page content (UPDATE — integrate new evidence into this)\n\n" + existingRaw + "\n"+ }+ }+ user := renderWikiTemplate(wikiRefineWriterUserTemplate, map[string]string{+ "action": firstNonEmpty(planItem.Action, "CREATE"),+ "slug": planItem.Slug,+ "title": firstNonEmpty(planItem.Title, planItem.Slug),+ "page_type": firstNonEmpty(planItem.PageType, "concept"),+ "all_plan_slugs": strings.Join(available, "\n"),+ "existing_section": existingSection,+ "source_context": sourceContext,+ "evidence_count": fmt.Sprintf("%d", len(evidence)),+ "evidence_blocks": formatWikiEvidenceBlocks(evidence),+ })+ resp, err := p.deps.Chat.Chat(p.ctx, common.ChatRequest{+ LLMID: p.llmID,+ SystemPrompt: buildWikiRefineWriterSystem(""),+ UserPrompt: user,+ })+ if err != nil {+ return wikiPageResult{}, err+ }+ if resp == nil {+ return wikiPageResult{}, fmt.Errorf("knowledge_compiler: wiki refine returned no response")+ }+ contentRaw := strings.TrimSpace(firstNonEmpty(resp.Content))+ if contentRaw == "" {+ contentRaw = "# " + firstNonEmpty(planItem.Title, planItem.Slug) + "\n\n(Page generation produced no content.)"+ }+ if strings.TrimSpace(existingRaw) != "" {+ contentRaw, err = p.mergeWikiPageContent(existingRaw, contentRaw, planItem.Slug)+ if err != nil {+ return wikiPageResult{}, err+ }+ }+ contentRendered, outlinks := transformWikiLinks(contentRaw, firstNonEmpty(p.datasetID, p.docID), pageTitles, slugToPageType)+ sourceDocIDs := collectWikiSourceDocIDs(p.inputs.Chunks, sourceChunkIDs, p.docID)+ summary := firstParagraph(contentRendered)+ if summary == "" {+ summary = firstNonEmpty(planItem.Title, planItem.Slug)+ }+ topic := firstNonEmpty(planItem.Topic, planItem.Title, planItem.Slug)+ return wikiPageResult{+ Slug: planItem.Slug,+ Title: firstNonEmpty(planItem.Title, planItem.Slug),+ PageType: firstNonEmpty(planItem.PageType, "concept"),+ Topic: topic,+ Action: firstNonEmpty(planItem.Action, "CREATE"),+ EntityNames: uniqueStrings(planItem.EntityNames),+ RelatedKBPages: uniqueStrings(planItem.RelatedKB),+ ContentRaw: contentRaw,+ Content: contentRendered,+ Summary: summary,+ Outlinks: outlinks,+ SourceChunkIDs: sourceChunkIDs,+ SourceDocIDs: sourceDocIDs,+ }, nil+}++// maxPagesForBatch is the per-batch planner ceiling. It is the allocated quota+// (a fraction of the global target) directly: the quota is already bounded by+// target <= max <= output-token capacity, so no separate static cap is needed.+// A zero/negative quota yields 1 so runPlanBatch is never told "0 pages". The+// old static wikiPlanMaxPagesPerBatch cap is gone: capping a large quota at 8+// would make the P0 target unreachable for a single high-quota batch.+func maxPagesForBatch(quota int) int {+ if quota < 1 {+ return 1+ }+ return quota+}++func (p *wikiPipeline) runPlanBatch(batch wikiExtract, batchIndex, batchTotal, quota int) (wikiPlan, error) {user := renderWikiTemplate(wikiPlanBatchUserTemplate, map[string]string{"doc_id": p.docID,"batch_index": fmt.Sprintf("%d", batchIndex),"batch_total": fmt.Sprintf("%d", batchTotal),+ "max_pages": fmt.Sprintf("%d", maxPagesForBatch(quota)),"entities": mustJSON(batch.Entities),"concepts": mustJSON(batch.Concepts),"claims": mustJSON(batch.Claims),@@ -397,20 +621,52 @@ func (p *wikiPipeline) runPlanBatch(batch wikiExtract, batchIndex, batchTotal inreturn parseWikiPlan(raw, p.docID, batch), nil}-func (p *wikiPipeline) mergePlanCandidates(plans []wikiPlan) (wikiPlan, error) {- user := renderWikiTemplate(wikiPlanMergeUserTemplate, map[string]string{- "doc_id": p.docID,- "candidates": mustPrettyJSON(plans),- })- raw, err := common.GenJSON(p.ctx, p.deps.Chat, common.ChatRequest{- LLMID: p.llmID,- SystemPrompt: wikiPlanSystem,- UserPrompt: user,- })- if err != nil {- return wikiPlan{}, err+// mergePlanCandidates merges per-batch plans into one plan. reduced is the item+// set the merged plan is allowed to reference (fallback/normalization); in the+// quota-filtered PLAN path this is the approved (non-zero-quota) set so items+// from skipped batches can never leak back in via fallback pages.+func (p *wikiPipeline) mergePlanCandidates(plans []wikiPlan, reduced wikiExtract) wikiPlan {+ merged := wikiPlan{}+ mergedEntities := map[string]bool{}+ mergedRelated := map[string]bool{}+ for _, plan := range plans {+ if merged.Title == "" {+ merged.Title = strings.TrimSpace(plan.Title)+ }+ if merged.Slug == "" {+ merged.Slug = strings.TrimSpace(plan.Slug)+ }+ if merged.Lead == "" {+ merged.Lead = strings.TrimSpace(plan.Lead)+ }+ if merged.PageType == "" {+ merged.PageType = strings.TrimSpace(plan.PageType)+ }+ if merged.Topic == "" {+ merged.Topic = strings.TrimSpace(plan.Topic)+ }+ if len(merged.Sections) == 0 && len(plan.Sections) > 0 {+ merged.Sections = append([]wikiPlanSection(nil), plan.Sections...)+ }+ merged.Pages = append(merged.Pages, plan.Pages...)+ for _, name := range plan.Entities {+ name = strings.TrimSpace(name)+ if name != "" && !mergedEntities[name] {+ mergedEntities[name] = true+ merged.Entities = append(merged.Entities, name)+ }+ }+ for _, slug := range plan.Related {+ slug = strings.TrimSpace(slug)+ if slug != "" && !mergedRelated[slug] {+ mergedRelated[slug] = true+ merged.Related = append(merged.Related, slug)+ }+ }}- return normalizeWikiPlan(parseWikiPlan(raw, p.docID, p.reduced), p.docID, p.reduced), nil+ merged = normalizeWikiPlan(merged, p.docID, reduced)+ merged.Pages = normalizeWikiPlanPageLinks(merged.Pages)+ return merged}func (p *wikiPipeline) reconcilePlan(plan wikiPlan) (wikiPlan, error) {@@ -461,6 +717,9 @@ func (p *wikiPipeline) reconcilePlan(plan wikiPlan) (wikiPlan, error) {return plan, nil}+// reconcilePlanPage decides UPDATE / CREATE for one planned page against the+// existing wiki-page store. See the threshold block above for how the band and+// the overlap heuristic relate to Python's wiki.py contract.func (p *wikiPipeline) reconcilePlanPage(page wikiPlanPage, queryVec []float32) (*common.WikiPageCandidate, error) {if p.deps.WikiPages == nil {return nil, nil@@ -943,6 +1202,34 @@ func normalizeWikiPlanPages(pages []wikiPlanPage, reduced wikiExtract) []wikiPlareturn out}+func normalizeWikiPlanPageLinks(pages []wikiPlanPage) []wikiPlanPage {+ if len(pages) == 0 {+ return nil+ }+ valid := make(map[string]bool, len(pages))+ for _, page := range pages {+ if slug := strings.TrimSpace(page.Slug); slug != "" {+ valid[slug] = true+ }+ }+ out := make([]wikiPlanPage, 0, len(pages))+ for _, page := range pages {+ related := make([]string, 0, len(page.RelatedKB))+ seen := map[string]bool{}+ for _, slug := range page.RelatedKB {+ slug = strings.TrimSpace(slug)+ if slug == "" || slug == page.Slug || !valid[slug] || seen[slug] {+ continue+ }+ seen[slug] = true+ related = append(related, slug)+ }+ page.RelatedKB = related+ out = append(out, page)+ }+ return out+}+func normalizeWikiPlanPage(page wikiPlanPage) wikiPlanPage {page.Action = strings.ToUpper(strings.TrimSpace(page.Action))if page.Action == "" {@@ -1589,7 +1876,24 @@ var (wikiArtifactMarkdownLink = regexp.MustCompile(`\[([^\]]+)\]\(([^)]+)\)`))-func transformWikiLinks(content, kbID string, pageTitles map[string]string) (string, []string) {+// pageTypeOf resolves the page_type for a slug so rendered internal links use+// the artifact/<kb>/<page_type>/<slug> form the frontend wiki-link parser+// requires (it only matches entity|concept). Unknown links fall back to "page".+func pageTypeOf(slug string, slugToPageType map[string]string) string {+ if pt := strings.TrimSpace(slugToPageType[slug]); pt != "" {+ return pt+ }+ if strings.Contains(slug, "/") {+ // Slugs may already carry a <page_type>/<name> prefix; reuse it.+ first := strings.SplitN(slug, "/", 2)[0]+ if first == "entity" || first == "concept" || first == "topic" {+ return first+ }+ }+ return "page"+}++func transformWikiLinks(content, kbID string, pageTitles, slugToPageType map[string]string) (string, []string) {kbID = strings.TrimSpace(kbID)seen := map[string]bool{}var outlinks []string@@ -1635,13 +1939,29 @@ func transformWikiLinks(content, kbID string, pageTitles map[string]string) (str}return ""}+ // link renders artifact/<kb>/<page_type>/<slug> so the frontend wiki-link+ // parser (artifact/<kb>/<page_type>/<slug>, page_type in {entity,concept})+ // can resolve and navigate to the target page. Slugs may already carry a+ // <page_type>/<name> prefix, in which case that prefix is reused as the+ // page_type and the name is emitted as the bare slug.+ link := func(label, slug string) string {+ bareSlug := slug+ pageType := pageTypeOf(slug, slugToPageType)+ if idx := strings.Index(slug, "/"); idx >= 0 && slug[:idx] == pageType {+ bareSlug = slug[idx+1:]+ }+ if pageType == "" {+ pageType = "page"+ }+ return "[" + label + "](artifact/" + kbID + "/" + pageType + "/" + bareSlug + ")"+ }rewriteMD := func(label, href string) string {slug := artifactSlug(href)if slug == "" {return "[" + label + "](" + href + ")"}track(slug)- return "[" + displayText(label, slug) + "](artifact/" + kbID + "/" + slug + ")"+ return link(displayText(label, slug), slug)}out := wikiArtifactMarkdownLink.ReplaceAllStringFunc(content, func(match string) string {sub := wikiArtifactMarkdownLink.FindStringSubmatch(match)@@ -1657,7 +1977,7 @@ func transformWikiLinks(content, kbID string, pageTitles map[string]string) (str}slug := strings.TrimSpace(sub[1])track(slug)- return "[" + sub[2] + "](artifact/" + kbID + "/" + slug + ")"+ return link(sub[2], slug)})out = wikiWikilinkSimpleRe.ReplaceAllStringFunc(out, func(match string) string {sub := wikiWikilinkSimpleRe.FindStringSubmatch(match)@@ -1666,7 +1986,7 @@ func transformWikiLinks(content, kbID string, pageTitles map[string]string) (str}slug := strings.TrimSpace(sub[1])track(slug)- return "[" + displayText(slug, slug) + "](artifact/" + kbID + "/" + slug + ")"+ return link(displayText(slug, slug), slug)})return out, outlinks}@@ -1874,15 +2194,6 @@ func (p *wikiPipeline) maybeSourceIDs() []string {return out}-func (p *wikiPipeline) runMapBatch(batch []common.Chunk) error {- extract, err := p.mapBatch(batch)- if err != nil {- return err- }- p.mapExtracts = append(p.mapExtracts, extract)- return nil-}-// dedupHistorical drops products that are near-duplicates of existing historical// artifacts, implementing cross-run dedup for the wiki variant. This remains a// read-only historical lookup and does not store any wiki intermediate state.diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget.gonew file mode 100644index 0000000000..c1807d3b4b--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget.go@@ -0,0 +1,259 @@+package wiki++import "sort"++// This file implements the PLAN-stage page-budget controls that align the Go+// wiki variant with Python's wiki.py:+//+// - a global target_page_count derived from item count (clamp(8, total//3, 60));+// - a dynamic max_page_count derived from the model's context window that acts+// as an unbreakable hard cap (output-token capacity vs page-token estimate);+// - per-batch page quotas distributed by largest-remainder so the quota sum+// equals the global target and no batch silently multiplies the page count;+// - a deterministic, mention-grounded truncation that selects the top pages+// under the global cap and reports how many were excluded.+//+// The provider never receives an explicit output max_tokens on the wiki path+// (see P0 in tasks/2026-08-04-wiki-go-python-gap-alignment-plan.md): output+// scale is controlled purely through max_pages in the prompt + the in-code+// truncation below.++// Python alignment constants (wiki.py:1670-1674, 1783-1787).+const (+ wikiPlanMaxOutputTokens = 4096+ wikiPlanOutputSafetyTokens = 256+ wikiPlanPageTokenEstimate = 48+ wikiPlanTargetPageCountMin = 8+ wikiPlanTargetPageCountMax = 60+)++// wikiTargetPageCount mirrors Python _wiki_target_page_count:+// clamp(8, total//3, 60).+func wikiTargetPageCount(totalItems int) int {+ if totalItems <= 0 {+ return wikiPlanTargetPageCountMin+ }+ if n := totalItems / 3; n < wikiPlanTargetPageCountMin {+ return wikiPlanTargetPageCountMin+ } else if n > wikiPlanTargetPageCountMax {+ return wikiPlanTargetPageCountMax+ } else {+ return n+ }+}++// wikiPlanBudget is the resolved page budget for one planning run.+type wikiPlanBudget struct {+ // Target is the approximate page count the planner should aim for. It is+ // only approximate: batches are allocated quotas that sum to it, but the+ // merged result may differ slightly. Target is NOT a capacity guarantee.+ Target int+ // Max is the unbreakable global hard cap derived from output-token+ // capacity. The merged, slug-deduped page list is truncated to at most Max+ // pages regardless of what the batches produced. Max may be below Target+ // when the model's output capacity is smaller than the item-count-derived+ // target; that is deliberate (never ask a small-window model for more pages+ // than its output can hold).+ Max int+}++// Cap is the page budget the planner is actually allowed to emit. It is the+// achievable bound min(Target, Max): a capacity-limited model must never be+// asked for more pages than its output can hold, so the cap (not the+// item-count-derived Target) drives per-batch quota allocation.+func (b wikiPlanBudget) Cap() int {+ if b.Max < b.Target {+ return b.Max+ }+ return b.Target+}++// deriveWikiPlanBudget computes the global page budget from the model's context+// window and the reduced item count, mirroring Python's+// output_tokens / output_page_capacity / max_page_count derivation+// (wiki.py:2066-2078). modelContextLen is the chat model's context window in+// tokens (0 means unknown).+func deriveWikiPlanBudget(modelContextLen, totalItems int) wikiPlanBudget {+ target := wikiTargetPageCount(totalItems)++ if modelContextLen <= 0 {+ modelContextLen = 8192+ }+ // output_tokens = min(4096, max(1024, int(model_context * 0.4))).+ outputTokens := modelContextLen * 2 / 5 // 0.4+ if outputTokens < 1024 {+ outputTokens = 1024+ }+ if outputTokens > wikiPlanMaxOutputTokens {+ outputTokens = wikiPlanMaxOutputTokens+ }++ capacity := (outputTokens - wikiPlanOutputSafetyTokens) / wikiPlanPageTokenEstimate+ if capacity < 1 {+ capacity = 1+ }+ maxCount := capacity+ if n := target + 8; n < maxCount {+ maxCount = n+ }+ if n := target * 2; n < maxCount {+ maxCount = n+ }+ // Max is the unbreakable global hard cap. It is NOT raised back up to+ // Target when output-token capacity is small: a small-window model must+ // never be asked to emit more pages than its output capacity permits, or we+ // reintroduce truncated-JSON risk. When capacity < Target, Max simply lands+ // below Target and the achievable page count is capacity-bound.+ return wikiPlanBudget{Target: target, Max: maxCount}+}++// wikiExtractItemCount counts the planning items in one reduced extract. It is+// the unit used for proportional quota allocation.+func wikiExtractItemCount(e wikiExtract) int {+ return len(e.Entities) + len(e.Concepts) + len(e.Claims) + len(e.Relations) + len(e.Topics)+}++// allocatePlanQuotas distributes totalTarget pages across batches proportionally+// to each batch's item count using the largest-remainder method, padding by+// remainder in original batch order. When the number of batches exceeds the+// target, small batches naturally receive a zero quota (their floor rounds to+// zero and no remainder remains for them).+//+// Invariant: when len(batches) <= totalTarget, the returned quotas sum exactly+// to totalTarget; when len(batches) > totalTarget they sum to totalTarget but+// some entries are zero. In all cases no quota exceeds totalTarget, so the+// global target is never duplicated per batch.+func allocatePlanQuotas(batches []wikiExtract, totalTarget int) []int {+ if len(batches) == 0 {+ return nil+ }+ if len(batches) == 1 {+ return []int{totalTarget}+ }+ items := make([]int, len(batches))+ total := 0+ for i, b := range batches {+ items[i] = wikiExtractItemCount(b)+ total += items[i]+ }+ if total <= 0 {+ total = len(batches)+ }+ quotas := make([]int, len(batches))+ remaining := totalTarget+ for i := range batches {+ q := items[i] * totalTarget / total+ quotas[i] = q+ remaining -= q+ }+ // Largest-remainder: hand out the leftover pages to batches with the+ // largest fractional remainder, breaking ties by original index (stable+ // sort preserves first-seen order).+ type remItem struct {+ remainder int+ idx int+ }+ rems := make([]remItem, len(batches))+ for i := range batches {+ rems[i] = remItem{remainder: items[i] * totalTarget % total, idx: i}+ }+ sort.SliceStable(rems, func(a, b int) bool {+ if rems[a].remainder == rems[b].remainder {+ return rems[a].idx < rems[b].idx+ }+ return rems[a].remainder > rems[b].remainder+ })+ for i := 0; i < len(rems) && remaining > 0; i++ {+ quotas[rems[i].idx]+++ remaining--+ }+ return quotas+}++// pageMentionCount estimates how strongly a planned page is grounded in the+// reduced extract by counting the distinct source chunks that mention its+// entities, concepts, or subject claims. It is used as the deterministic+// priority when pages must be dropped to fit the global hard cap.+func pageMentionCount(page wikiPlanPage, reduced wikiExtract) int {+ names := map[string]bool{}+ for _, n := range page.EntityNames {+ if k := normKey(n); k != "" {+ names[k] = true+ }+ }+ if len(names) == 0 {+ if t := normKey(page.Title); t != "" {+ names[t] = true+ }+ if topic := normKey(page.Topic); topic != "" {+ names[topic] = true+ }+ }+ chunks := map[string]bool{}+ for _, e := range reduced.Entities {+ if !names[normKey(e.Name)] {+ continue+ }+ for _, c := range e.SourceChunkIDs {+ chunks[c] = true+ }+ }+ for _, c := range reduced.Concepts {+ if !names[normKey(c.Term)] {+ continue+ }+ for _, cid := range c.SourceChunkIDs {+ chunks[cid] = true+ }+ }+ for _, c := range reduced.Claims {+ if !names[normKey(c.Subject)] {+ continue+ }+ for _, cid := range c.SourceChunkIDs {+ chunks[cid] = true+ }+ }+ return len(chunks)+}++// truncatePlanPagesByCap keeps at most maxPageCount planned pages, selecting by+// deterministic priority (mention count descending, then priority ascending,+// then slug), and returns the number of pages excluded by the cap. The output+// preserves the input order (original priority/slug order after normalize) so+// downstream slug-dedup and link normalization stay stable. It never fabricates+// a fallback page to fill the budget.+func truncatePlanPagesByCap(pages []wikiPlanPage, maxPageCount int, reduced wikiExtract) ([]wikiPlanPage, int) {+ if maxPageCount < 0 {+ maxPageCount = 0+ }+ if len(pages) <= maxPageCount {+ return pages, 0+ }+ type scored struct {+ idx int+ pg wikiPlanPage+ mc int+ }+ scoredPages := make([]scored, len(pages))+ for i, pg := range pages {+ scoredPages[i] = scored{idx: i, pg: pg, mc: pageMentionCount(pg, reduced)}+ }+ sort.SliceStable(scoredPages, func(a, b int) bool {+ if scoredPages[a].mc != scoredPages[b].mc {+ return scoredPages[a].mc > scoredPages[b].mc+ }+ if scoredPages[a].pg.Priority != scoredPages[b].pg.Priority {+ return scoredPages[a].pg.Priority < scoredPages[b].pg.Priority+ }+ return scoredPages[a].pg.Slug < scoredPages[b].pg.Slug+ })+ selected := scoredPages[:maxPageCount]+ // Restore original order by index so output order is deterministic.+ sort.SliceStable(selected, func(a, b int) bool { return selected[a].idx < selected[b].idx })+ out := make([]wikiPlanPage, 0, maxPageCount)+ for _, s := range selected {+ out = append(out, s.pg)+ }+ return out, len(pages) - maxPageCount+}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget_test.gonew file mode 100644index 0000000000..2a26cb2c36--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget_test.go@@ -0,0 +1,559 @@+package wiki++import (+ "context"+ "errors"+ "strings"+ "sync"+ "testing"+ "time"++ "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++func TestWikiTargetPageCount_Clamp(t *testing.T) {+ cases := []struct {+ total int+ want int+ }{+ {0, 8}, // default floor+ {1, 8}, // below floor+ {24, 8}, // 24//3 = 8+ {60, 20}, // 60//3 = 20+ {180, 60}, // 180//3 = 60 (cap)+ {500, 60}, // above cap+ }+ for _, c := range cases {+ if got := wikiTargetPageCount(c.total); got != c.want {+ t.Fatalf("wikiTargetPageCount(%d) = %d, want %d", c.total, got, c.want)+ }+ }+}++// TestDeriveWikiPlanBudget_MaxReflectsOutputCapacity locks the corrected P0+// contract: Max is the unbreakable output-capacity bound and is NOT raised back+// up to Target. A small-window model must never be asked for more pages than its+// output capacity permits.+func TestDeriveWikiPlanBudget_MaxReflectsOutputCapacity(t *testing.T) {+ // Tiny window (modelLen=1024): output_tokens = max(1024, 1024*0.4=409) =+ // 1024; capacity = (1024-256)//48 = 16. For a large item count+ // (target=60), Max must stay at 16 (capacity-bound), NOT be raised to 60.+ b := deriveWikiPlanBudget(1024, 1000)+ if b.Target != 60 {+ t.Fatalf("Target = %d, want 60", b.Target)+ }+ if b.Max != 16 {+ t.Fatalf("Max = %d, want 16 (capacity-bound, must not re-raise to Target 60)", b.Max)+ }+ // A tiny item count with the same window: target = 8, max = min(16, 16, 16)+ // = 16.+ b = deriveWikiPlanBudget(1024, 1)+ if b.Max != 16 {+ t.Fatalf("Max = %d, want 16", b.Max)+ }+ // A roomy window: Max = min(capacity, target+8, target*2). For total=1000+ // (target 60) and window 8192: output=3276, capacity=62 -> max=min(62,68,120)=62.+ b = deriveWikiPlanBudget(8192, 1000)+ if b.Max != 62 {+ t.Fatalf("Max = %d, want 62", b.Max)+ }+}++func TestDeriveWikiPlanBudget_OutputCapacityBounds(t *testing.T) {+ // With a 8192 model: output_tokens = min(4096, max(1024, 8192*0.4=3276))+ // = 3276; capacity = (3276-256)//48 = 62. For total=1000 target=60,+ // max = min(62, max(68, 120)) = 62. Max must equal 62 and be >= target 60.+ b := deriveWikiPlanBudget(8192, 1000)+ if b.Target != 60 {+ t.Fatalf("Target = %d, want 60", b.Target)+ }+ want := 62+ if b.Max != want {+ t.Fatalf("Max = %d, want %d (output-token capacity)", b.Max, want)+ }+}++func TestAllocatePlanQuotas_SumsToTarget(t *testing.T) {+ batches := []wikiExtract{+ {Entities: make([]wikiEntity, 5)},+ {Concepts: make([]wikiConcept, 5)},+ {Claims: make([]wikiClaim, 5)},+ }+ quotas := allocatePlanQuotas(batches, 10)+ sum := 0+ for _, q := range quotas {+ sum += q+ }+ if sum != 10 {+ t.Fatalf("quota sum = %d, want 10 (got %v)", sum, quotas)+ }+ if len(quotas) != 3 {+ t.Fatalf("len(quotas) = %d, want 3", len(quotas))+ }+}++func TestAllocatePlanQuotas_LargestRemainderOrdered(t *testing.T) {+ // 7 items in batch0, 3 in batch1, target=10:+ // floors: 7 and 3; remainders 0 and 0 -> [7,3].+ batches := []wikiExtract{+ {Entities: make([]wikiEntity, 7)},+ {Concepts: make([]wikiConcept, 3)},+ }+ quotas := allocatePlanQuotas(batches, 10)+ if quotas[0] != 7 || quotas[1] != 3 {+ t.Fatalf("quotas = %v, want [7 3]", quotas)+ }++ // 7,2,1 target=10: floors 7,2,1 rem=0 -> [7,2,1].+ batches = []wikiExtract{+ {Entities: make([]wikiEntity, 7)},+ {Concepts: make([]wikiConcept, 2)},+ {Claims: make([]wikiClaim, 1)},+ }+ quotas = allocatePlanQuotas(batches, 10)+ if quotas[0] != 7 || quotas[1] != 2 || quotas[2] != 1 {+ t.Fatalf("quotas = %v, want [7 2 1]", quotas)+ }+}++func TestAllocatePlanQuotas_ZeroForOverflowingBatches(t *testing.T) {+ // More batches than target: some batches must get a zero quota and none may+ // exceed the target.+ target := 4+ batches := make([]wikiExtract, 8)+ for i := range batches {+ batches[i] = wikiExtract{Entities: []wikiEntity{{Name: "e"}}}+ }+ quotas := allocatePlanQuotas(batches, target)+ sum := 0+ zero := 0+ for _, q := range quotas {+ sum += q+ if q == 0 {+ zero+++ }+ }+ if sum != target {+ t.Fatalf("quota sum = %d, want %d", sum, target)+ }+ if zero == 0 {+ t.Fatalf("expected at least one zero quota with %d batches > target %d", len(batches), target)+ }+ for _, q := range quotas {+ if q > target {+ t.Fatalf("quota %d exceeds target %d", q, target)+ }+ }+}++func TestTruncatePlanPagesByCap_SelectsByMentionCount(t *testing.T) {+ reduced := wikiExtract{+ Entities: []wikiEntity{+ {Name: "High", SourceChunkIDs: []string{"a", "b", "c", "d"}},+ {Name: "Low", SourceChunkIDs: []string{"a"}},+ },+ }+ pages := []wikiPlanPage{+ {Slug: "entity/low", Title: "Low", EntityNames: []string{"Low"}, Priority: 1},+ {Slug: "entity/high", Title: "High", EntityNames: []string{"High"}, Priority: 2},+ }+ kept, excluded := truncatePlanPagesByCap(pages, 1, reduced)+ if excluded != 1 {+ t.Fatalf("excluded = %d, want 1", excluded)+ }+ if len(kept) != 1 || kept[0].Slug != "entity/high" {+ t.Fatalf("kept = %#v, want entity/high", kept)+ }+}++func TestTruncatePlanPagesByCap_NoCapNoDrop(t *testing.T) {+ pages := []wikiPlanPage{+ {Slug: "a", Priority: 1},+ {Slug: "b", Priority: 2},+ }+ kept, excluded := truncatePlanPagesByCap(pages, 5, wikiExtract{})+ if excluded != 0 || len(kept) != 2 {+ t.Fatalf("got kept=%d excluded=%d, want 2/0", len(kept), excluded)+ }+}++func TestTruncatePlanPagesByCap_PreservesInputOrder(t *testing.T) {+ reduced := wikiExtract{+ Entities: []wikiEntity{+ {Name: "X", SourceChunkIDs: []string{"a"}},+ {Name: "Y", SourceChunkIDs: []string{"a", "b"}},+ },+ }+ // Cap is large enough to keep everything; input order must be preserved.+ pages := []wikiPlanPage{+ {Slug: "z", Title: "Z", EntityNames: []string{"X"}, Priority: 2},+ {Slug: "a", Title: "A", EntityNames: []string{"Y"}, Priority: 1},+ }+ kept, _ := truncatePlanPagesByCap(pages, 5, reduced)+ if len(kept) != 2 || kept[0].Slug != "z" || kept[1].Slug != "a" {+ t.Fatalf("kept = %#v, want input order [z a]", kept)+ }+}++// TestRunPlan_PromptMaxPagesNeverExceedsCap locks the capacity-limited quota+// fix: when the model's output capacity is smaller than the item-derived target+// (e.g. ModelContextLen=1024, target 60, Max 16), the sum of the per-batch+// "at most N page entries" values placed in the planner prompts must never+// exceed Max. This prevents the truncated-JSON risk from re-appearing.+func TestRunPlan_PromptMaxPagesNeverExceedsCap(t *testing.T) {+ previous := batchSubmitter+ defer SetBatchSubmitter(previous)++ SetBatchSubmitter(func(ctx context.Context, jobs []func() error) error {+ for _, j := range jobs {+ if err := ctx.Err(); err != nil {+ return err+ }+ if err := j(); err != nil {+ return err+ }+ }+ return ctx.Err()+ })++ var mu sync.Mutex+ var maxPagesSeen []int+ big := strings.Repeat("x", 5000)+ // 12 large entities each pack as their own (or small) batch, giving multiple+ // batches. total items >= 36 => target clamps to 60; ModelContextLen=1024 =>+ // output capacity 16 => Max = min(16, 68, 120) = 16 => Cap = 16.+ entities := make([]wikiEntity, 0, 12)+ for i := 0; i < 12; i++ {+ entities = append(entities, wikiEntity{Name: "Ent " + itoa(i) + big})+ }+ p := &wikiPipeline{+ ctx: context.Background(),+ deps: common.Deps{+ ModelContextLen: 1024,+ Chat: chatFunc(func(_ context.Context, req common.ChatRequest) (*common.ChatResponse, error) {+ if n := extractMaxPages(req.UserPrompt); n >= 0 {+ mu.Lock()+ maxPagesSeen = append(maxPagesSeen, n)+ mu.Unlock()+ }+ return &common.ChatResponse{Content: `{"pages":[]}`}, nil+ }),+ },+ reduced: wikiExtract{Entities: entities},+ docID: "doc-1",+ }+ if _, err := p.runPlan(); err != nil {+ t.Fatalf("runPlan err = %v", err)+ }+ if len(maxPagesSeen) == 0 {+ t.Fatalf("no planning prompt captured max_pages")+ }+ sum := 0+ for _, n := range maxPagesSeen {+ sum += n+ }+ if sum > p.planBudget.Max {+ t.Fatalf("sum of per-batch max_pages = %d, want <= Max %d (target 60)", sum, p.planBudget.Max)+ }+}++// extractMaxPages parses the "at most N page entries" instruction from a plan+// prompt, returning -1 when absent.+func extractMaxPages(prompt string) int {+ const marker = "at most "+ idx := strings.Index(prompt, marker)+ if idx < 0 {+ return -1+ }+ rest := prompt[idx+len(marker):]+ j := 0+ for j < len(rest) && rest[j] >= '0' && rest[j] <= '9' {+ j+++ }+ if j == 0 {+ return -1+ }+ n := 0+ for _, c := range rest[:j] {+ n = n*10 + int(c-'0')+ }+ return n+}++// TestMergePlanCandidates_FallbackOnlyUsesApprovedItems locks F3: the fallback+// page set is built from the approved (non-zero-quota) item set only, so items+// from skipped zero-quota batches can never leak back into the plan.+func TestMergePlanCandidates_FallbackOnlyUsesApprovedItems(t *testing.T) {+ p := &wikiPipeline{docID: "doc-1"}+ approved := wikiExtract{+ Entities: []wikiEntity{{Name: "Approved", SourceChunkIDs: []string{"c1"}}},+ }+ // All approved batches returned no pages; the merged plan must fall back to+ // approved items only.+ merged := p.mergePlanCandidates(nil, approved)+ if len(merged.Pages) == 0 {+ t.Fatalf("expected at least one fallback page")+ }+ hasApproved := false+ for _, pg := range merged.Pages {+ for _, n := range pg.EntityNames {+ if strings.Contains(n, "Skipped") {+ t.Fatalf("fallback leaked zero-quota item %q", n)+ }+ if strings.Contains(n, "Approved") {+ hasApproved = true+ }+ }+ }+ if !hasApproved {+ t.Fatalf("fallback missing approved item")+ }+}++// TestRunPlan_TruncatesToGlobalHardCap drives runPlan through a planner that+// returns more pages than the derived global max_page_count, and asserts the+// merged page list is truncated to the hard cap with the excluded count+// recorded. This is the P0 acceptance criterion that the final page count never+// exceeds max_page_count after slug dedup + global cap.+func TestRunPlan_TruncatesToGlobalHardCap(t *testing.T) {+ previous := batchSubmitter+ defer SetBatchSubmitter(previous)++ SetBatchSubmitter(func(ctx context.Context, jobs []func() error) error {+ for _, j := range jobs {+ if err := j(); err != nil {+ return err+ }+ }+ return nil+ })++ // Planner returns 30 pages. With one entity and ModelContextLen unset,+ // target = clamp(8, 1//3, 60) = 8, and max = min(capacity=62, 16, 16) = 16.+ pages := make([]map[string]any, 0, 30)+ for i := 0; i < 30; i++ {+ pages = append(pages, map[string]any{+ "action": "CREATE",+ "slug": "entity/item-" + itoa(i),+ "title": "Item " + itoa(i),+ "page_type": "entity",+ "topic": "Item",+ "entity_names": []any{"Entity"},+ "priority": i + 1,+ })+ }+ payload := map[string]any{"pages": pages}+ p := &wikiPipeline{+ ctx: context.Background(),+ deps: common.Deps{+ Chat: reconcileChatStub{resp: mustJSON(payload)},+ },+ reduced: wikiExtract{+ Entities: []wikiEntity{{Name: "Entity", SourceChunkIDs: []string{"c1"}}},+ },+ docID: "doc-1",+ }+ plan, err := p.runPlan()+ if err != nil {+ t.Fatalf("runPlan err = %v", err)+ }+ if got := len(plan.Pages); got != 16 {+ t.Fatalf("plan pages = %d, want 16 (global hard cap)", got)+ }+ if got := p.planCapacityExcluded; got != 14 {+ t.Fatalf("planCapacityExcluded = %d, want 14", got)+ }+}++func itoa(i int) string {+ if i == 0 {+ return "0"+ }+ neg := i < 0+ if neg {+ i = -i+ }+ var b []byte+ for i > 0 {+ b = append([]byte{byte('0' + i%10)}, b...)+ i /= 10+ }+ if neg {+ b = append([]byte{'-'}, b...)+ }+ return string(b)+}++// batchPlanChatStub returns one page per planning batch based on which entity+// name is present in the batch prompt. It lets a fake submitter drive each+// batch's planner call with a distinct, deterministic result.+type batchPlanChatStub struct{}++func (batchPlanChatStub) Chat(_ context.Context, req common.ChatRequest) (*common.ChatResponse, error) {+ var title string+ switch {+ case strings.Contains(req.UserPrompt, "Alpha"):+ title = "Alpha"+ case strings.Contains(req.UserPrompt, "Beta"):+ title = "Beta"+ default:+ title = "Gamma"+ }+ return &common.ChatResponse{Content: `{"pages":[{"action":"CREATE","slug":"entity/` + slugify(title) + `","title":"` + title + `","page_type":"entity","topic":"` + title + `","entity_names":["` + title + `"],"priority":1}]}`}, nil+}++// TestRunPlan_ParallelBatchesMergeInOrder drives runPlan through a submitter+// that completes batches out of order (batch1 finishes before batch0) and+// asserts the merged plan preserves the original batch order deterministically.+// This exercises the P1 invariant that jobs write only their own index and the+// merge reads slots in order.+func TestRunPlan_ParallelBatchesMergeInOrder(t *testing.T) {+ previous := batchSubmitter+ defer SetBatchSubmitter(previous)++ SetBatchSubmitter(func(ctx context.Context, jobs []func() error) error {+ var wg sync.WaitGroup+ for i, j := range jobs {+ i, j := i, j+ wg.Add(1)+ go func() {+ defer wg.Done()+ if i == 0 {+ time.Sleep(30 * time.Millisecond) // batch0 completes last+ }+ j()+ }()+ }+ wg.Wait()+ return ctx.Err()+ })++ // Three entities sized so Alpha+Beta pack into batch1 and Gamma falls into+ // batch2 (token budget 3500).+ big := strings.Repeat("x", 7000)+ p := &wikiPipeline{+ ctx: context.Background(),+ deps: common.Deps{+ Chat: batchPlanChatStub{},+ },+ reduced: wikiExtract{+ Entities: []wikiEntity{+ {Name: "Alpha" + big},+ {Name: "Beta"},+ {Name: "Gamma" + big},+ },+ },+ docID: "doc-1",+ }+ plan, err := p.runPlan()+ if err != nil {+ t.Fatalf("runPlan err = %v", err)+ }+ // Batch1 (Alpha) must appear before batch2 (Gamma) in the merged plan.+ if len(plan.Pages) < 2 {+ t.Fatalf("plan pages = %d, want >= 2", len(plan.Pages))+ }+ if plan.Pages[0].Title != "Alpha" {+ t.Fatalf("merged pages[0].Title = %q, want Alpha (batch order preserved)", plan.Pages[0].Title)+ }+ if plan.Pages[1].Title != "Gamma" {+ t.Fatalf("merged pages[1].Title = %q, want Gamma", plan.Pages[1].Title)+ }+}++// TestRunPlan_ParallelBatchesFirstError verifies the P1 error model: the first+// batch error is returned after all submitted jobs settle.+func TestRunPlan_ParallelBatchesFirstError(t *testing.T) {+ previous := batchSubmitter+ defer SetBatchSubmitter(previous)++ SetBatchSubmitter(func(ctx context.Context, jobs []func() error) error {+ var wg sync.WaitGroup+ errs := make(chan error, len(jobs))+ for _, j := range jobs {+ j := j+ wg.Add(1)+ go func() {+ defer wg.Done()+ errs <- j()+ }()+ }+ wg.Wait()+ close(errs)+ for err := range errs {+ if err != nil {+ return err+ }+ }+ return ctx.Err()+ })++ big := strings.Repeat("x", 7000)+ boom := errors.New("planning failed")+ p := &wikiPipeline{+ ctx: context.Background(),+ deps: common.Deps{+ Chat: failPlanChatStub{err: boom},+ },+ reduced: wikiExtract{+ Entities: []wikiEntity{+ {Name: "Alpha" + big},+ {Name: "Beta"},+ {Name: "Gamma" + big},+ },+ },+ docID: "doc-1",+ }+ if _, err := p.runPlan(); err != boom {+ t.Fatalf("runPlan err = %v, want boom", err)+ }+}++// failPlanChatStub fails every planning call with a fixed error.+type failPlanChatStub struct {+ err error+}++func (f failPlanChatStub) Chat(_ context.Context, _ common.ChatRequest) (*common.ChatResponse, error) {+ return nil, f.err+}++// TestRunPlan_CancelledCtxAborts verifies that a cancelled context aborts the+// planning fan-out and surfaces the context error.+func TestRunPlan_CancelledCtxAborts(t *testing.T) {+ previous := batchSubmitter+ defer SetBatchSubmitter(previous)++ SetBatchSubmitter(func(ctx context.Context, jobs []func() error) error {+ for _, j := range jobs {+ if err := ctx.Err(); err != nil {+ return err+ }+ if err := j(); err != nil {+ return err+ }+ }+ return ctx.Err()+ })++ big := strings.Repeat("x", 7000)+ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ p := &wikiPipeline{+ ctx: ctx,+ deps: common.Deps{+ Chat: batchPlanChatStub{},+ },+ reduced: wikiExtract{+ Entities: []wikiEntity{+ {Name: "Alpha" + big},+ {Name: "Beta"},+ {Name: "Gamma" + big},+ },+ },+ docID: "doc-1",+ }+ if _, err := p.runPlan(); err == nil {+ t.Fatalf("runPlan err = nil, want context cancelled")+ }+}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_reduce.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_reduce.gonew file mode 100644index 0000000000..d506d630dd--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_reduce.go@@ -0,0 +1,185 @@+package wiki++import (+ "strings"++ "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++// This file implements the REDUCE-stage canonical-entity enhancement that+// narrows the gap with Python's wiki.py canonicalization:+//+// - entities with distinct names but high embedding similarity are treated as+// ambiguous and sent to an LLM merge decision (collapsing near-duplicates);+// - concepts keep exact-term dedup, matching Python's current semantic (any+// embedding/LLM dedup for concepts must be a separate new capability with+// its own quality bar, not an alignment claim).+//+// The exact-key merge in reduceExtracts stays the deterministic baseline; this+// step layers embedding + LLM disambiguation on top. When the embedder or chat+// seam is unavailable, entities pass through unchanged (degrade gracefully).++// wikiEntityMergeThreshold is the embedding-cosine similarity at or above which+// two distinct-name entities are considered ambiguous and sent to the LLM merge+// decision. It is deliberately high so only genuinely similar candidates reach+// the LLM.+const wikiEntityMergeThreshold = 0.85++// wikiEntityMergeMaxCalls caps how many LLM disambiguation calls a single+// REDUCE run may make, bounding the cost on entity-dense documents.+const wikiEntityMergeMaxCalls = 16++// wikiEntityMergeMaxCandidates caps how many candidate partners one entity is+// checked against to keep the pairwise scan bounded.+const wikiEntityMergeMaxCandidates = 8++// dedupeEntities returns a copy of in with ambiguous near-duplicate entities+// collapsed via LLM disambiguation. It is a no-op when fewer than two entities+// are present or when deps.Embed / deps.Chat are unavailable.+func (p *wikiPipeline) dedupeEntities(in []wikiEntity) []wikiEntity {+ if len(in) < 2 || p.deps.Embed == nil || p.deps.Chat == nil {+ return in+ }+ names := make([]string, len(in))+ for i, e := range in {+ names[i] = e.Name+ }+ vecs, err := p.deps.Embed.Encode(p.ctx, names)+ if err != nil || len(vecs) != len(in) {+ return in+ }++ // Canonical entity per input index: which index owns the final entity.+ canon := make([]int, len(in))+ for i := range canon {+ canon[i] = i+ }+ llmCalls := 0++ // Greedy best-partner scan in input order (already deterministic after+ // reduceExtracts sorts by name).+ for i := 0; i < len(in) && llmCalls < wikiEntityMergeMaxCalls; i++ {+ if canon[i] != i {+ // Already merged into another canonical entity.+ continue+ }+ bestIdx, bestSim := -1, -1.0+ checked := 0+ for j := 0; j < len(in) && checked < wikiEntityMergeMaxCandidates; j++ {+ if i == j {+ continue+ }+ if canon[j] != j {+ // Consumed by an earlier merge; never a standalone partner.+ continue+ }+ if normKey(in[i].Name) == normKey(in[j].Name) {+ // Exact-name duplicates are already merged by reduceExtracts;+ // never treat them as a pair here.+ continue+ }+ // Same-type-only candidate filtering (Python canonicalizes entities+ // within the same type). Two entities with provably different types+ // are never ambiguous regardless of embedding similarity. An empty+ // type is treated as compatible (cannot prove a difference).+ if in[i].Type != "" && in[j].Type != "" && !strings.EqualFold(in[i].Type, in[j].Type) {+ continue+ }+ checked+++ sim := cosine32(vecs[i], vecs[j])+ if sim >= wikiEntityMergeThreshold && sim > bestSim {+ bestSim = sim+ bestIdx = j+ }+ }+ if bestIdx < 0 {+ continue+ }+ // i and bestIdx are guaranteed standalone by the loop guards above.+ // Count the call BEFORE issuing it so a persistent failure cannot drive+ // unbounded external requests: the llmCalls budget is consumed even when+ // the request fails. The outer loop's `llmCalls < max` guard then stops+ // further iterations once the budget is exhausted.+ llmCalls+++ merge, err := p.llmMergeEntityDecision(in[i], in[bestIdx])+ if err != nil {+ // A failed disambiguation call should not abort the whole REDUCE;+ // keep the entities separate and move on.+ continue+ }+ if !merge {+ continue+ }+ // Merge j into i: i is canonical, j is consumed.+ canon[bestIdx] = i+ in[i].Aliases = mergeStrings(in[i].Aliases, in[bestIdx].Aliases)+ if in[i].Name != in[bestIdx].Name {+ in[i].Aliases = mergeStrings(in[i].Aliases, []string{in[bestIdx].Name})+ }+ in[i].SourceChunkIDs = mergeStrings(in[i].SourceChunkIDs, in[bestIdx].SourceChunkIDs)+ if in[i].Type == "" {+ in[i].Type = in[bestIdx].Type+ }+ }++ out := make([]wikiEntity, 0, len(in))+ for i := range in {+ if canon[i] != i {+ continue+ }+ out = append(out, in[i])+ }+ return out+}++// llmMergeEntityDecision asks the chat seam whether two distinct-name entities+// refer to the same real-world concept. Returns true to merge.+func (p *wikiPipeline) llmMergeEntityDecision(a, b wikiEntity) (bool, error) {+ raw, err := common.GenJSON(p.ctx, p.deps.Chat, common.ChatRequest{+ LLMID: p.llmID,+ SystemPrompt: wikiReduceEntityDisambiguateSystem,+ UserPrompt: renderWikiTemplate(wikiReduceEntityDisambiguateUserTemplate, map[string]string{+ "entity_a": mustPrettyJSON(a),+ "entity_b": mustPrettyJSON(b),+ }),+ })+ if err != nil {+ return false, err+ }+ v, ok := raw["merge"]+ if !ok {+ return false, nil+ }+ return toBoolValue(v), nil+}++// toBoolValue interprets a loosely-typed boolean field returned by the LLM (the+// model may emit JSON true or a string like "true"/"yes").+func toBoolValue(v any) bool {+ switch x := v.(type) {+ case bool:+ return x+ case string:+ switch strings.ToLower(strings.TrimSpace(x)) {+ case "true", "yes", "1", "same", "merge":+ return true+ }+ case float64:+ return x != 0+ }+ return false+}++// cosine32 computes the cosine similarity between two float32 vectors.+func cosine32(a, b []float32) float64 {+ na := l2Norm32(a)+ nb := l2Norm32(b)+ if na == 0 || nb == 0 {+ return 0+ }+ var dot float64+ for i := 0; i < len(a) && i < len(b); i++ {+ dot += float64(a[i]) * float64(b[i])+ }+ return dot / (na * nb)+}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_reduce_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_reduce_test.gonew file mode 100644index 0000000000..6f621535fe--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_reduce_test.go@@ -0,0 +1,225 @@+package wiki++import (+ "context"+ "errors"+ "testing"++ "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++// TestDedupeEntities_NoSeamIsNoop verifies dedupeEntities degrades to a no-op+// when the embedder or chat seam is nil (M1-style unit safety).+func TestDedupeEntities_NoSeamIsNoop(t *testing.T) {+ p := &wikiPipeline{ctx: context.Background()}+ in := []wikiEntity{+ {Name: "Alpha", SourceChunkIDs: []string{"c1"}},+ {Name: "Alpha Corp", SourceChunkIDs: []string{"c2"}},+ }+ got := p.dedupeEntities(in)+ if len(got) != 2 {+ t.Fatalf("got %d entities, want 2 (no-op without seams)", len(got))+ }+}++// TestDedupeEntities_LLMMergesAmbiguousPair verifies two distinct-name entities+// with high embedding similarity are collapsed into one canonical entity via the+// LLM merge decision, with aliases and provenance merged.+func TestDedupeEntities_LLMMergesAmbiguousPair(t *testing.T) {+ p := &wikiPipeline{+ ctx: context.Background(),+ llmID: "llm1",+ deps: common.Deps{+ Chat: reconcileChatStub{resp: `{"merge":true,"reason":"same company"}`},+ Embed: mergeEmbedStub{},+ },+ }+ in := []wikiEntity{+ {Name: "Alpha Inc", Type: "org", SourceChunkIDs: []string{"c1"}},+ {Name: "Alpha Incorporated", Type: "org", SourceChunkIDs: []string{"c2"}},+ }+ got := p.dedupeEntities(in)+ if len(got) != 1 {+ t.Fatalf("got %d entities, want 1 (LLM merge)", len(got))+ }+ if got[0].Name != "Alpha Inc" {+ t.Fatalf("canonical name = %q, want Alpha Inc", got[0].Name)+ }+ if len(got[0].SourceChunkIDs) != 2 {+ t.Fatalf("provenance = %#v, want 2 chunk ids", got[0].SourceChunkIDs)+ }+ if len(got[0].Aliases) == 0 {+ t.Fatalf("aliases not merged: %#v", got[0].Aliases)+ }+}++// TestDedupeEntities_LLMRejectsDistinct verifies the LLM rejecting a merge keeps+// both entities distinct.+func TestDedupeEntities_LLMRejectsDistinct(t *testing.T) {+ p := &wikiPipeline{+ ctx: context.Background(),+ llmID: "llm1",+ deps: common.Deps{+ Chat: reconcileChatStub{resp: `{"merge":false,"reason":"distinct products"}`},+ Embed: mergeEmbedStub{},+ },+ }+ in := []wikiEntity{+ {Name: "Alpha", SourceChunkIDs: []string{"c1"}},+ {Name: "Beta", SourceChunkIDs: []string{"c2"}},+ }+ got := p.dedupeEntities(in)+ if len(got) != 2 {+ t.Fatalf("got %d entities, want 2 (LLM rejected merge)", len(got))+ }+}++// TestDedupeEntities_ExactNameIsNotAmbiguous verifies entities with identical+// normalized names (already collapsed by reduceExtracts before this stage) are+// not treated as ambiguous: they pass through untouched and no LLM call is made+// for the exact-name pair.+func TestDedupeEntities_ExactNameIsNotAmbiguous(t *testing.T) {+ p := &wikiPipeline{+ ctx: context.Background(),+ llmID: "llm1",+ deps: common.Deps{+ Chat: reconcileChatStub{resp: `{"merge":true}`},+ Embed: mergeEmbedStub{},+ },+ }+ in := []wikiEntity{+ {Name: "Alpha", SourceChunkIDs: []string{"c1"}},+ {Name: "Alpha", SourceChunkIDs: []string{"c2"}},+ }+ got := p.dedupeEntities(in)+ // Exact-name duplicates are out of scope for the embedding step; both are+ // kept unchanged (they would already be one entity after reduceExtracts).+ if len(got) != 2 {+ t.Fatalf("got %d entities, want 2 (exact-name pairs are not ambiguous)", len(got))+ }+}++// TestDedupeEntities_ConceptStaysExact validates the REDUCE boundary: concept+// dedup must remain exact (no embedding/LLM). This test guards that the entity+// enhancement never touches concepts.+func TestReduceExtracts_ConceptsStayExact(t *testing.T) {+ reduced := reduceExtracts([]wikiExtract{+ {Concepts: []wikiConcept{{Term: "RAG", Definition: "d1", SourceChunkIDs: []string{"c1"}}}},+ {Concepts: []wikiConcept{{Term: "Retrieval Augmented Generation", Definition: "d2", SourceChunkIDs: []string{"c2"}}}},+ })+ if len(reduced.Concepts) != 2 {+ t.Fatalf("concepts = %d, want 2 (exact-term dedup must not collapse distinct terms)", len(reduced.Concepts))+ }+}++// TestDedupeEntities_FailingChatIsBudgeted locks F4: a chat seam that+// persistently fails must not drive unbounded external calls. The llmCalls+// budget is consumed before the request, so the loop stops after at most+// wikiEntityMergeMaxCalls attempts.+func TestDedupeEntities_FailingChatIsBudgeted(t *testing.T) {+ calls := 0+ p := &wikiPipeline{+ ctx: context.Background(),+ llmID: "llm1",+ deps: common.Deps{+ Chat: chatFunc(func(_ context.Context, _ common.ChatRequest) (*common.ChatResponse, error) {+ calls+++ return nil, errors.New("llm down")+ }),+ Embed: mergeEmbedStub{},+ },+ }+ // 20 entities with distinct names but identical embeddings => every pair is+ // ambiguous and would trigger an LLM call.+ in := make([]wikiEntity, 0, 20)+ for i := 0; i < 20; i++ {+ in = append(in, wikiEntity{Name: "Entity " + itoa(i), Type: "person"})+ }+ got := p.dedupeEntities(in)+ if calls > wikiEntityMergeMaxCalls {+ t.Fatalf("chat calls = %d, want <= %d despite persistent failures", calls, wikiEntityMergeMaxCalls)+ }+ // No merges happen because every call fails, so all 20 entities survive.+ if len(got) != 20 {+ t.Fatalf("entities = %d, want 20 (no merges on failure)", len(got))+ }+}++// TestDedupeEntities_CrossTypeHighSimDoesNotCallLLM locks F5: entities with+// provably different types must never be treated as ambiguous, even with+// identical embeddings, so no LLM call is made for them.+func TestDedupeEntities_CrossTypeHighSimDoesNotCallLLM(t *testing.T) {+ calls := 0+ p := &wikiPipeline{+ ctx: context.Background(),+ llmID: "llm1",+ deps: common.Deps{+ Chat: chatFunc(func(_ context.Context, _ common.ChatRequest) (*common.ChatResponse, error) {+ calls+++ return &common.ChatResponse{Content: `{"merge":true}`}, nil+ }),+ Embed: mergeEmbedStub{},+ },+ }+ // Both embed to [1,1,1] (identical vectors, cosine 1.0) but types differ.+ in := []wikiEntity{+ {Name: "Alpha", Type: "person"},+ {Name: "Beta Corp", Type: "org"},+ }+ got := p.dedupeEntities(in)+ if calls != 0 {+ t.Fatalf("chat calls = %d, want 0 (cross-type pairs must not reach the LLM)", calls)+ }+ if len(got) != 2 {+ t.Fatalf("entities = %d, want 2 (cross-type entities must stay distinct)", len(got))+ }+}++// mergeEmbedStub returns embeddings where identical names share a vector and+// distinct names are far apart (cosine ~0), so it can drive both the ambiguous+// and distinct test paths deterministically.+type mergeEmbedStub struct{}++func (mergeEmbedStub) Encode(_ context.Context, texts []string) ([][]float32, error) {+ out := make([][]float32, len(texts))+ for i, text := range texts {+ // "Alpha Inc" and "Alpha Incorporated" both contain "alpha" -> same+ // vector; "Beta" differs.+ if containsFold(text, "beta") {+ out[i] = []float32{1, 0, 0}+ continue+ }+ out[i] = []float32{1, 1, 1}+ }+ return out, nil+}++func (mergeEmbedStub) Dimensions() int { return 3 }++func containsFold(s, sub string) bool {+ return len(s) >= len(sub) && (len(sub) == 0 || indexFold(s, sub) >= 0)+}++func indexFold(s, sub string) int {+ if sub == "" {+ return 0+ }+ ls := toLowerASCII(s)+ lsub := toLowerASCII(sub)+ for i := 0; i+len(lsub) <= len(ls); i++ {+ if ls[i:i+len(lsub)] == lsub {+ return i+ }+ }+ return -1+}++func toLowerASCII(s string) string {+ b := []byte(s)+ for i := range b {+ if b[i] >= 'A' && b[i] <= 'Z' {+ b[i] += 'a' - 'A'+ }+ }+ return string(b)+}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_refine_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_refine_test.gonew file mode 100644index 0000000000..a8bf19bedf--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_refine_test.go@@ -0,0 +1,156 @@+package wiki++import (+ "context"+ "errors"+ "strings"+ "sync"+ "testing"+ "time"++ "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++// refineChatStub returns per-page markdown keyed by the page title in the+// writer prompt so each page's result is distinct and deterministic.+type refineChatStub struct{}++func (refineChatStub) Chat(_ context.Context, req common.ChatRequest) (*common.ChatResponse, error) {+ title := "Page"+ for _, cand := range []string{"Alpha", "Beta", "Gamma"} {+ if strings.Contains(req.UserPrompt, cand) {+ title = cand+ break+ }+ }+ return &common.ChatResponse{Content: "# " + title + "\n\nContent for " + title + ".\n"}, nil+}++func refinePipeline() *wikiPipeline {+ return &wikiPipeline{+ ctx: context.Background(),+ tenantID: "t1",+ datasetID: "kb1",+ llmID: "llm1",+ docID: "doc-1",+ deps: common.Deps{+ Chat: refineChatStub{},+ },+ reduced: wikiExtract{+ Entities: []wikiEntity{{Name: "Alpha", SourceChunkIDs: []string{"c1"}}},+ Claims: []wikiClaim{{Statement: "Alpha exists", Subject: "Alpha", SourceChunkIDs: []string{"c1"}}},+ },+ inputs: common.Inputs{+ Chunks: []common.Chunk{{ID: "c1", Text: "Alpha content", Meta: map[string]any{"doc_id": "doc-1"}}},+ },+ }+}++func TestRunRefine_ParallelPagesKeepPlanOrder(t *testing.T) {+ previous := batchSubmitter+ defer SetBatchSubmitter(previous)++ SetBatchSubmitter(func(ctx context.Context, jobs []func() error) error {+ var wg sync.WaitGroup+ for i, j := range jobs {+ i, j := i, j+ wg.Add(1)+ go func() {+ defer wg.Done()+ if i == 0 {+ time.Sleep(30 * time.Millisecond) // page 0 completes last+ }+ j()+ }()+ }+ wg.Wait()+ return ctx.Err()+ })++ p := refinePipeline()+ p.plan = wikiPlan{+ Pages: []wikiPlanPage{+ {Action: "CREATE", Slug: "entity/alpha", Title: "Alpha", PageType: "entity", Topic: "Alpha", EntityNames: []string{"Alpha"}, Priority: 1},+ {Action: "CREATE", Slug: "entity/beta", Title: "Beta", PageType: "entity", Topic: "Beta", EntityNames: []string{"Beta"}, Priority: 2},+ },+ }+ got, err := p.runRefine()+ if err != nil {+ t.Fatalf("runRefine err = %v", err)+ }+ if len(got) != 2 {+ t.Fatalf("got %d pages, want 2", len(got))+ }+ if got[0].Title != "Alpha" || got[1].Title != "Beta" {+ t.Fatalf("page order = [%s, %s], want [Alpha, Beta] (plan order preserved)", got[0].Title, got[1].Title)+ }+ if !strings.Contains(got[0].Content, "Content for Alpha") {+ t.Fatalf("page0 content missing: %q", got[0].Content)+ }+}++func TestRunRefine_FirstErrorAborts(t *testing.T) {+ previous := batchSubmitter+ defer SetBatchSubmitter(previous)++ boom := errors.New("refine failed")+ SetBatchSubmitter(func(ctx context.Context, jobs []func() error) error {+ var wg sync.WaitGroup+ errs := make(chan error, len(jobs))+ for _, j := range jobs {+ j := j+ wg.Add(1)+ go func() {+ defer wg.Done()+ errs <- j()+ }()+ }+ wg.Wait()+ close(errs)+ for err := range errs {+ if err != nil {+ return err+ }+ }+ return ctx.Err()+ })++ p := refinePipeline()+ p.plan = wikiPlan{Pages: []wikiPlanPage{+ {Action: "CREATE", Slug: "entity/alpha", Title: "Alpha", Priority: 1},+ }}+ p.deps.Chat = chatFunc(func(_ context.Context, _ common.ChatRequest) (*common.ChatResponse, error) {+ return nil, boom+ })+ if _, err := p.runRefine(); err != boom {+ t.Fatalf("runRefine err = %v, want boom", err)+ }+}++func TestRunRefine_CancelledCtxAborts(t *testing.T) {+ previous := batchSubmitter+ defer SetBatchSubmitter(previous)++ SetBatchSubmitter(func(ctx context.Context, jobs []func() error) error {+ for _, j := range jobs {+ if err := ctx.Err(); err != nil {+ return err+ }+ if err := j(); err != nil {+ return err+ }+ }+ return ctx.Err()+ })++ ctx, cancel := context.WithCancel(context.Background())+ cancel()+ p := refinePipeline()+ p.ctx = ctx+ p.plan = wikiPlan{Pages: []wikiPlanPage{+ {Action: "CREATE", Slug: "entity/alpha", Title: "Alpha", Priority: 1},+ }}+ if _, err := p.runRefine(); err == nil {+ t.Fatalf("runRefine err = nil, want context cancelled")+ }+}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.goindex 5f233ba648..2cf03c3b07 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go@@ -3,7 +3,9 @@ package wikiimport ("context""strings"+ "sync""testing"+ "time""ragflow/internal/ingestion/component/knowledge_compiler/common")@@ -44,6 +46,77 @@ func TestPackWikiPlanBatches_SplitsLargeInput(t *testing.T) {}}+// TestWikiMapMaxTokens_OutputBudgetTracksInputBudget locks the input/output+// budget coupling: the extraction MaxTokens must leave at least the whole+// wikiMapTokenBudget input budget of headroom and, with a roomy model, give the+// output the rest of the context window after the batch's input is reserved.+func TestWikiMapMaxTokens_OutputBudgetTracksInputBudget(t *testing.T) {+ // Unknown model context -> default window (DefaultLLMContextLength). Output+ // gets the whole window minus the input budget.+ got := wikiMapMaxTokens(0)+ if want := common.DefaultLLMContextLength - wikiMapTokenBudget; got != want {+ t.Fatalf("wikiMapMaxTokens(0) = %d, want %d", got, want)+ }+ // A model window that barely fits one batch must still grant at least the+ // input budget of output space (never starve the output).+ if got := wikiMapMaxTokens(2048); got != wikiMapTokenBudget {+ t.Fatalf("wikiMapMaxTokens(2048) = %d, want %d (floor at input budget)", got, wikiMapTokenBudget)+ }+ // A roomy model: output = window - input budget.+ if got := wikiMapMaxTokens(16384); got != 16384-wikiMapTokenBudget {+ t.Fatalf("wikiMapMaxTokens(16384) = %d, want %d", got, 16384-wikiMapTokenBudget)+ }+}++func TestRunMapBatches_PreservesBatchOrderWithSubmitter(t *testing.T) {+ previous := batchSubmitter+ defer SetBatchSubmitter(previous)++ SetBatchSubmitter(func(ctx context.Context, jobs []func() error) error {+ var wg sync.WaitGroup+ errs := make(chan error, len(jobs))+ for _, job := range jobs {+ job := job+ wg.Add(1)+ go func() {+ defer wg.Done()+ errs <- job()+ }()+ }+ wg.Wait()+ close(errs)+ for err := range errs {+ if err != nil {+ return err+ }+ }+ return ctx.Err()+ })++ batches := [][]common.Chunk{+ {{ID: "slow", Text: "slow"}},+ {{ID: "fast-1", Text: "fast-1"}},+ {{ID: "fast-2", Text: "fast-2"}},+ }+ got, err := runMapBatches(context.Background(), batches, func(batch []common.Chunk) (wikiExtract, error) {+ if batch[0].ID == "slow" {+ time.Sleep(25 * time.Millisecond)+ }+ return wikiExtract{Topics: []string{batch[0].ID}}, nil+ })+ if err != nil {+ t.Fatalf("runMapBatches err = %v", err)+ }+ if len(got) != len(batches) {+ t.Fatalf("runMapBatches len = %d, want %d", len(got), len(batches))+ }+ for i, want := range []string{"slow", "fast-1", "fast-2"} {+ if len(got[i].Topics) != 1 || got[i].Topics[0] != want {+ t.Fatalf("runMapBatches[%d] = %#v, want topic %q", i, got[i], want)+ }+ }+}+func TestBuildSourceContext_SelectsKnownChunks(t *testing.T) {ctx := buildSourceContext([]common.Chunk{{ID: "c1", Text: "alpha text"},@@ -71,6 +144,61 @@ func TestNormalizeWikiPlanPages_FallbacksToEntitiesAndConcepts(t *testing.T) {}}+func TestMergePlanCandidates_DeduplicatesWithoutLLMMerge(t *testing.T) {+ p := &wikiPipeline{+ docID: "doc-1",+ reduced: wikiExtract{+ Entities: []wikiEntity{{Name: "Alpha"}},+ },+ }+ merged := p.mergePlanCandidates([]wikiPlan{+ {+ Title: "Alpha",+ Pages: []wikiPlanPage{+ {+ Slug: "entity/alpha",+ Title: "Alpha",+ PageType: "entity",+ Topic: "Alpha",+ EntityNames: []string{"Alpha"},+ RelatedKB: []string{"entity/beta", "missing", "entity/alpha"},+ Priority: 2,+ },+ },+ },+ {+ Pages: []wikiPlanPage{+ {+ Slug: "entity/beta",+ Title: "Beta",+ PageType: "entity",+ Topic: "Beta",+ EntityNames: []string{"Beta"},+ RelatedKB: []string{"entity/alpha"},+ Priority: 1,+ },+ {+ Slug: "entity/alpha",+ Title: "Alpha duplicate",+ PageType: "entity",+ Topic: "Alpha",+ EntityNames: []string{"Alpha"},+ Priority: 3,+ },+ },+ },+ }, p.reduced)+ if len(merged.Pages) != 2 {+ t.Fatalf("merged pages = %d, want 2", len(merged.Pages))+ }+ if merged.Pages[0].Slug != "entity/beta" || merged.Pages[1].Slug != "entity/alpha" {+ t.Fatalf("merged page order = %#v", merged.Pages)+ }+ if got := merged.Pages[1].RelatedKB; len(got) != 1 || got[0] != "entity/beta" {+ t.Fatalf("alpha related links = %#v, want [entity/beta]", got)+ }+}+type reconcileChatStub struct {resp string}@@ -134,6 +262,48 @@ func TestReconcilePlanPage_MaybeUsesLLMDecision(t *testing.T) {}}+// TestReconcilePlanPage_OverlapHeuristicSkipsLLM locks the Go-only enhancement:+// a candidate whose score is inside [maybe, update) but whose topic matches the+// planned page's topic is promoted straight to UPDATE without an LLM round.+func TestReconcilePlanPage_OverlapHeuristicSkipsLLM(t *testing.T) {+ called := false+ p := &wikiPipeline{+ ctx: context.Background(),+ tenantID: "t1",+ datasetID: "kb1",+ llmID: "llm1",+ deps: common.Deps{+ Chat: chatFunc(func(_ context.Context, _ common.ChatRequest) (*common.ChatResponse, error) {+ called = true+ return &common.ChatResponse{Content: `{"action":"CREATE"}`}, nil+ }),+ Embed: reconcileEmbedStub{},+ WikiPages: wikiStoreStub{similar: []common.WikiPageCandidate{{Slug: "topic/alpha", Title: "Alpha topic", Topic: "Alpha", Score: 0.85}}},+ },+ }+ got, err := p.reconcilePlanPage(wikiPlanPage{+ Slug: "topic/alpha-new",+ Title: "Alpha Topic",+ PageType: "topic",+ Topic: "Alpha",+ }, []float32{0.1, 0.2, 0.3})+ if err != nil {+ t.Fatalf("reconcilePlanPage err = %v", err)+ }+ if got == nil || got.Slug != "topic/alpha" {+ t.Fatalf("reconcilePlanPage = %#v, want topic/alpha (topic overlap promotes to UPDATE)", got)+ }+ if called {+ t.Fatalf("overlap heuristic should not invoke the LLM")+ }+}++type chatFunc func(context.Context, common.ChatRequest) (*common.ChatResponse, error)++func (f chatFunc) Chat(ctx context.Context, req common.ChatRequest) (*common.ChatResponse, error) {+ return f(ctx, req)+}+func TestReconcilePlanPage_LowScoreSkipsLLM(t *testing.T) {p := &wikiPipeline{ctx: context.Background(),diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.goindex f8001736f6..ba91cdaf98 100644--- a/internal/ingestion/component/tokenizer.go+++ b/internal/ingestion/component/tokenizer.go@@ -82,10 +82,8 @@ import ("encoding/json""fmt""log"- "os""regexp""slices"- "strconv""strings""go.uber.org/zap"@@ -100,19 +98,6 @@ import (const ComponentNameTokenizer = "Tokenizer"-// embeddingBatchSize returns the embedding batch size, matching Python's-// settings.EMBEDDING_BATCH_SIZE. Reads TOKENIZER_EMBEDDING_BATCH_SIZE env-// var; defaults to 16. Invalid / non-positive values fall back to the-// default (diff Tokenizer Omission-3).-func embeddingBatchSize() int {- if v := os.Getenv("TOKENIZER_EMBEDDING_BATCH_SIZE"); v != "" {- if n, err := strconv.Atoi(v); err == nil && n > 0 {- return n- }- }- return 16-}-// titleExtRE strips a trailing file-extension (e.g. ".pdf") from the// upstream document name before tokenizing it. Mirrors the python// `re.sub(r"\.[a-zA-Z]+$", "", name)` in tokenizer.py:137.@@ -134,6 +119,7 @@ type EmbeddingResult struct {// Embedder is the testability seam for the embedding branch.type Embedder interface {MaxTokens() int+ BatchSize() intEncode(ctx context.Context, texts []string) ([]EmbeddingResult, error)}@@ -447,8 +433,12 @@ func (c *TokenizerComponent) embedChunks(ctx context.Context, tenantID, kbID, em}contentResults := make([]EmbeddingResult, 0, len(texts))- for start := 0; start < len(texts); start += embeddingBatchSize() {- end := start + embeddingBatchSize()+ batchSize := embedder.BatchSize()+ if batchSize <= 0 {+ return nil, 0, fmt.Errorf("tokenizer: embedder reported non-positive batch size %d", batchSize)+ }+ for start := 0; start < len(texts); start += batchSize {+ end := start + batchSizeif end > len(texts) {end = len(texts)}diff --git a/internal/ingestion/component/tokenizer_test.go b/internal/ingestion/component/tokenizer_test.goindex 1dbe579168..f6ba5acb42 100644--- a/internal/ingestion/component/tokenizer_test.go+++ b/internal/ingestion/component/tokenizer_test.go@@ -414,6 +414,8 @@ type countMismatchedEmbedder struct{ want int }func (c *countMismatchedEmbedder) MaxTokens() int { return 2048 }+func (c *countMismatchedEmbedder) BatchSize() int { return 16 }+func (c *countMismatchedEmbedder) Encode(ctx context.Context, texts []string) ([]EmbeddingResult, error) {out := make([]EmbeddingResult, c.want)for i := range out {diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.goindex f82fc434f2..5c60d96cb7 100644--- a/internal/ingestion/component/tokenizer_unit_test.go+++ b/internal/ingestion/component/tokenizer_unit_test.go@@ -30,6 +30,7 @@ import ("time""ragflow/internal/agent/runtime"+ "ragflow/internal/entity/models""ragflow/internal/ingestion/component/schema""ragflow/internal/tokenizer")@@ -56,6 +57,10 @@ func (s *stubEmbedder) MaxTokens() int {return s.maxTokens}+func (s *stubEmbedder) BatchSize() int {+ return 16+}+func (s *stubEmbedder) Encode(ctx context.Context, texts []string) ([]EmbeddingResult, error) {s.calls.Add(1)copied := append([]string(nil), texts...)@@ -436,20 +441,22 @@ func TestTruncateForEmbedding_UnconfiguredClampsToDefault(t *testing.T) {// TestEmbeddingBatchSizeEnvVar covers Tokenizer Omission-3: the batch size// must be configurable via TOKENIZER_EMBEDDING_BATCH_SIZE env var, matching-// Python's configurable settings.EMBEDDING_BATCH_SIZE.+// Python's configurable settings.EMBEDDING_BATCH_SIZE. The resolved batch size+// is now provided by models.GetEmbeddingBatchSize (env override -> provider+// capability -> default), surfaced through Embedder.BatchSize().func TestEmbeddingBatchSizeEnvVar(t *testing.T) {- if got := embeddingBatchSize(); got != 16 {- t.Errorf("embeddingBatchSize() default = %d, want 16", got)+ if got := models.GetEmbeddingBatchSize(""); got != models.DefaultEmbeddingBatchSize {+ t.Errorf("GetEmbeddingBatchSize() default = %d, want %d", got, models.DefaultEmbeddingBatchSize)}os.Setenv("TOKENIZER_EMBEDDING_BATCH_SIZE", "32")t.Cleanup(func() { os.Unsetenv("TOKENIZER_EMBEDDING_BATCH_SIZE") })- if got := embeddingBatchSize(); got != 32 {- t.Errorf("embeddingBatchSize() after env = %d, want 32", got)+ if got := models.GetEmbeddingBatchSize(""); got != 32 {+ t.Errorf("GetEmbeddingBatchSize() after env = %d, want 32", got)}// Invalid value falls back to default.os.Setenv("TOKENIZER_EMBEDDING_BATCH_SIZE", "bad")- if got := embeddingBatchSize(); got != 16 {- t.Errorf("embeddingBatchSize() invalid env = %d, want 16", got)+ if got := models.GetEmbeddingBatchSize(""); got != models.DefaultEmbeddingBatchSize {+ t.Errorf("GetEmbeddingBatchSize() invalid env = %d, want %d", got, models.DefaultEmbeddingBatchSize)}}diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex 1d8bb6c3df..2a663c7c87 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -21,8 +21,11 @@ import ("sync""time"+ "ragflow/internal/common""ragflow/internal/engine"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"++ "go.uber.org/zap")// Consumer is the dataset-level post-processing worker (§11.5). Multiple@@ -183,10 +186,19 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {// must leave the claimed batch in the backlog for reclamation/retry rather// than silently dropping it (C5: never ack what we failed to merge).if batchErr != nil {+ common.Error("knowledge_compile: batch processing failed, leaving batch for retry",+ batchErr,+ zap.String("dataset_id", datasetID),+ zap.Int("entries", len(cr.Entries)))+ if err := c.scheduler.SetError(ctx, datasetID, cr.Token, batchErr.Error()); err != nil {+ common.Warn("knowledge_compile: failed to record error_msg",+ zap.String("dataset_id", datasetID), zap.Error(err))+ }return}if _, err := c.scheduler.Ack(ctx, datasetID, cr.Token, cr.Entries); err != nil {- _ = err+ common.Warn("knowledge_compile: ack failed",+ zap.String("dataset_id", datasetID), zap.Error(err))}}@@ -195,6 +207,10 @@ func (c *Consumer) processClaim(ctx context.Context, cr ClaimResult) {// returns an error if any reader/dedup/writer step fails so the caller can// leave the batch for reclamation instead of acking dropped work.func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entries []BacklogEntry) error {+ common.Info("knowledge_compile: processing claimed batch",+ zap.String("dataset_id", kb),+ zap.String("tenant_id", tenant),+ zap.Int("entries", len(entries)))c.mu.Lock()if c.tombs == nil {c.tombs = map[string]map[string]uint64{}@@ -355,7 +371,7 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entries// fan it out across the shared global compilerPool (vCPU-sized). Output order// is irrelevant: merged rows are upserted by their idempotent dataset-level// id, and each candidate lands in exactly one group / the unmatched set.- jobs := make([]compilerJob, 0, len(candidates))+ jobs := make([]CompilerJob, 0, len(candidates))for _, cand := range candidates {cand := candjobs = append(jobs, func() error {@@ -437,5 +453,10 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entriesdelete(c.tombs[kb], docID)}c.mu.Unlock()+ common.Info("knowledge_compile: batch merge complete",+ zap.String("dataset_id", kb),+ zap.Int("completed_docs", len(completed)),+ zap.Int("deleted_docs", len(deleted)),+ zap.Int("merged_rows_written", len(mergedFinal)))return nil}diff --git a/internal/ingestion/knowledge_compile/consumer_test.go b/internal/ingestion/knowledge_compile/consumer_test.goindex 2c3c9d9035..29f0cec042 100644--- a/internal/ingestion/knowledge_compile/consumer_test.go+++ b/internal/ingestion/knowledge_compile/consumer_test.go@@ -249,3 +249,154 @@ func TestSchedulerReclaimExpired(t *testing.T) {t.Fatalf("expected 1 entry after reclaim, got %d", len(cr2.Entries))}}++// rowCounts snapshots a fake scheduling row's inflight/backlog entry counts.+type rowCounts struct {+ inflight int+ backlog int+}++func fakeRowCounts(sch *FakeScheduler, datasetID string) (rowCounts, string) {+ sch.mu.Lock()+ defer sch.mu.Unlock()+ r := sch.rows[datasetID]+ if r == nil {+ return rowCounts{}, ""+ }+ return rowCounts{inflight: len(r.inflight), backlog: len(r.backlog)}, r.state+}++// TestSchedulerStateMachineLocksStateAndCounts locks the full lifecycle state+// machine (plan v4.1 §9.2) on the FakeScheduler, which mirrors the MySQL+// scheduler's transitions:+//+// Publish -> pending; Claim -> running (+error cleared);+// SetError (failed batch left for retry) keeps running + records error;+// lease expiry -> reclaimOne -> pending (inflight moved back to backlog);+// Ack with backlog drained -> completed.+func TestSchedulerStateMachineLocksStateAndCounts(t *testing.T) {+ sch := NewFakeScheduler()++ // Publish two docs: state=pending, backlog=2, inflight=0.+ if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted), 1); err != nil {+ t.Fatalf("publish d1: %v", err)+ }+ if err := sch.Publish(context.Background(), "t1", "kb1", "d2", string(EventTypeCompleted), 2); err != nil {+ t.Fatalf("publish d2: %v", err)+ }+ if c, s := fakeRowCounts(sch, "kb1"); s != DatasetStatePending || c.backlog != 2 || c.inflight != 0 {+ t.Fatalf("after publish: want state=pending backlog=2 inflight=0, got state=%s %+v", s, c)+ }++ // Claim: state=running, backlog moves to inflight (batch=2), error cleared.+ cr, ok, err := sch.Claim(context.Background(), "kb1")+ if err != nil || !ok {+ t.Fatalf("claim: ok=%v err=%v", ok, err)+ }+ if len(cr.Entries) != 2 {+ t.Fatalf("expected 2-entry claim, got %d", len(cr.Entries))+ }+ if c, s := fakeRowCounts(sch, "kb1"); s != DatasetStateRunning || c.backlog != 0 || c.inflight != 2 {+ t.Fatalf("after claim: want state=running backlog=0 inflight=2, got state=%s %+v", s, c)+ }++ // Failed batch left for retry: SetError records a diagnostic, state stays running.+ if err := sch.SetError(context.Background(), "kb1", cr.Token, "boom"); err != nil {+ t.Fatalf("set error: %v", err)+ }+ sch.mu.Lock()+ gotErr := sch.rows["kb1"].errorMsg+ sch.mu.Unlock()+ if gotErr != "boom" {+ t.Fatalf("expected errorMsg=boom, got %q", gotErr)+ }+ if _, s := fakeRowCounts(sch, "kb1"); s != DatasetStateRunning {+ t.Fatalf("failed batch must stay running (left for retry), got state=%s", s)+ }++ // Lease expires: reclaimOne moves inflight back to backlog, clears lease -> pending.+ past := time.Now().Add(-time.Hour)+ sch.mu.Lock()+ sch.rows["kb1"].expires = &past+ sch.mu.Unlock()+ // Lock the reclaim transition in isolation (before any re-claim): the fake's+ // reclaim helper is the same code path TryClaim uses, mirroring reclaimOne.+ sch.mu.Lock()+ if id := sch.fakeReclaimExpired(time.Now()); id != "kb1" {+ sch.mu.Unlock()+ t.Fatalf("expected kb1 to be reclaimed, got %q", id)+ }+ sch.mu.Unlock()+ if c, s := fakeRowCounts(sch, "kb1"); s != DatasetStatePending || c.backlog != 2 || c.inflight != 0 {+ t.Fatalf("after reclaim: want state=pending backlog=2 inflight=0, got state=%s %+v", s, c)+ }++ // Claim again then Ack the drained batch -> completed, counts zeroed.+ cr2, ok3, err := sch.Claim(context.Background(), "kb1")+ if err != nil || !ok3 {+ t.Fatalf("re-claim: ok=%v err=%v", ok3, err)+ }+ if _, err := sch.Ack(context.Background(), "kb1", cr2.Token, cr2.Entries); err != nil {+ t.Fatalf("ack: %v", err)+ }+ if c, s := fakeRowCounts(sch, "kb1"); s != DatasetStateCompleted || c.backlog != 0 || c.inflight != 0 {+ t.Fatalf("after ack drain: want state=completed backlog=0 inflight=0, got state=%s %+v", s, c)+ }+}++// TestSchedulerSetErrorScopedToClaimToken locks the concurrency guard on+// SetError: a failed batch is diagnosed only while its own claim token is still+// live. If worker A's lease expires and is reclaimed, worker B re-claims and+// completes; a late SetError from A (stale token) must NOT overwrite the row's+// diagnostic, otherwise a completed state would be misread as failed.+func TestSchedulerSetErrorScopedToClaimToken(t *testing.T) {+ sch := NewFakeScheduler()+ if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted), 1); err != nil {+ t.Fatalf("publish: %v", err)+ }++ // A claims and begins processing (running).+ crA, ok, err := sch.Claim(context.Background(), "kb1")+ if err != nil || !ok {+ t.Fatalf("claim A: ok=%v err=%v", ok, err)+ }++ // A's lease expires before it finishes; the sweeper reclaims the inflight+ // batch back to backlog (pending) and B takes over.+ past := time.Now().Add(-time.Hour)+ sch.mu.Lock()+ sch.rows["kb1"].expires = &past+ sch.mu.Unlock()+ if id := sch.fakeReclaimExpired(time.Now()); id != "kb1" {+ t.Fatalf("expected kb1 reclaimed, got %q", id)+ }+ crB, okB, err := sch.Claim(context.Background(), "kb1")+ if err != nil || !okB {+ t.Fatalf("claim B: ok=%v err=%v", okB, err)+ }+ if crA.Token == crB.Token {+ t.Fatalf("expected distinct claim tokens, got %q", crA.Token)+ }++ // B succeeds and drains the backlog to completed.+ if _, err := sch.Ack(context.Background(), "kb1", crB.Token, crB.Entries); err != nil {+ t.Fatalf("ack B: %v", err)+ }+ if c, s := fakeRowCounts(sch, "kb1"); s != DatasetStateCompleted || c.backlog != 0 || c.inflight != 0 {+ t.Fatalf("after B ack: want completed empty, got state=%s %+v", s, c)+ }++ // A's late failure arrives with its now-stale token: it must be ignored.+ if err := sch.SetError(context.Background(), "kb1", crA.Token, "stale failure from A"); err != nil {+ t.Fatalf("stale set error: %v", err)+ }+ sch.mu.Lock()+ gotErr := sch.rows["kb1"].errorMsg+ sch.mu.Unlock()+ if gotErr != "" {+ t.Fatalf("stale SetError overwrote diagnostic: got errorMsg=%q want empty", gotErr)+ }+ if _, s := fakeRowCounts(sch, "kb1"); s != DatasetStateCompleted {+ t.Fatalf("stale SetError changed state: got %q want completed", s)+ }+}diff --git a/internal/ingestion/knowledge_compile/dedup.go b/internal/ingestion/knowledge_compile/dedup.goindex 6c0cec3993..6cc775769c 100644--- a/internal/ingestion/knowledge_compile/dedup.go+++ b/internal/ingestion/knowledge_compile/dedup.go@@ -79,7 +79,7 @@ func NewLLMDeduper(chat kccommon.ChatInvoker, embed kccommon.Embedder, llmID str// never blocks on a single job, so a stopped pool returns an error instead of// hanging DecideBatch.decider.SetSubmitter(func(ctx context.Context, fn func() error) error {- return SubmitCompilerJobs(ctx, []compilerJob{fn})+ return SubmitCompilerJobs(ctx, []CompilerJob{fn})})return &llmDeduper{group: structure.NewGroupedDeduper(decider), decider: decider, embed: embed}}diff --git a/internal/ingestion/knowledge_compile/pool.go b/internal/ingestion/knowledge_compile/pool.goindex b6f5dd31a7..6b527dffd3 100644--- a/internal/ingestion/knowledge_compile/pool.go+++ b/internal/ingestion/knowledge_compile/pool.go@@ -24,10 +24,11 @@ import ("ragflow/internal/utility")-// compilerJob is one unit of knowledge-compilation work (an I/O- or-// LLM-bounded task) executed on the shared global pool. It is a type alias for-// func() error so callers can pass plain []func() error slices without a cast.-type compilerJob = func() error+// CompilerJob is one unit of knowledge-compilation work (an I/O- or+// LLM-bounded task) executed on the shared global pool. It is an exported type+// alias for func() error so callers (including lower-level packages such as the+// knowledge_compiler wiring) can pass plain []func() error slices without a cast.+type CompilerJob = func() error// compilerPool is the process-wide bounded worker pool that drives cross-doc// concurrency for every knowledge-compilation stage: the DocEngine KNN pass in@@ -43,10 +44,10 @@ type compilerJob = func() error// (KNN / write / delete) or LLM-bounded (merge decisions) rather than// CPU-bounded, so the degree of useful parallelism is capped by the number of// available cores rather than by a hand-tuned constant.-var compilerPool = utility.NewWorkerPool[compilerJob, struct{}](+var compilerPool = utility.NewWorkerPool[CompilerJob, struct{}](compilerConcurrency(),compilerConcurrency()*4,- func(_ context.Context, j compilerJob) (struct{}, error) { return struct{}{}, j() },+ func(_ context.Context, j CompilerJob) (struct{}, error) { return struct{}{}, j() },)// compilerConcurrency resolves the global pool size. It defaults to the host@@ -82,11 +83,11 @@ func SetCompilerConcurrency(n int) {// then Wait on each in a second pass on the calling goroutine. This keeps the// fan-out bounded by the shared pool's worker count while avoiding len(jobs)// short-lived goroutines.-func runCompilerJobs(ctx context.Context, jobs []compilerJob) error {+func runCompilerJobs(ctx context.Context, jobs []CompilerJob) error {if len(jobs) == 0 {return nil}- futures := make([]utility.WorkerPoolFuture[compilerJob, struct{}], 0, len(jobs))+ futures := make([]utility.WorkerPoolFuture[CompilerJob, struct{}], 0, len(jobs))var firstErr errorfor _, j := range jobs {f, err := compilerPool.Submit(ctx, j)@@ -121,7 +122,7 @@ func runCompilerJobs(ctx context.Context, jobs []compilerJob) error {// SubmitCompilerJob runs a single job on the global pool and waits for it,// returning its error. Used to inject bounded parallelism into lower-level// packages (e.g. structure.LLMMergeDecider) without creating an import cycle.-func SubmitCompilerJob(ctx context.Context, fn compilerJob) error {+func SubmitCompilerJob(ctx context.Context, fn CompilerJob) error {f, err := compilerPool.Submit(ctx, fn)if err != nil {return err@@ -138,10 +139,10 @@ func SubmitCompilerJob(ctx context.Context, fn compilerJob) error {// the one process-wide compiler pool. Implementations must submit every job to// the shared pool, wait for all to finish, and return the first non-nil error// (without StopWait-ing the global pool).-type CompilerBatchSubmitter func(ctx context.Context, jobs []compilerJob) error+type CompilerBatchSubmitter func(ctx context.Context, jobs []CompilerJob) error// SubmitCompilerJobs fans out a batch of jobs on the global pool and returns the// first error. This is the CompilerBatchSubmitter handed to variant packages.-func SubmitCompilerJobs(ctx context.Context, jobs []compilerJob) error {+func SubmitCompilerJobs(ctx context.Context, jobs []CompilerJob) error {return runCompilerJobs(ctx, jobs)}diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.goindex 3b0332a85e..a8deeee6ae 100644--- a/internal/ingestion/knowledge_compile/reader.go+++ b/internal/ingestion/knowledge_compile/reader.go@@ -68,6 +68,16 @@ var compiledSelectFields = []string{"slug_kwd", "type",}+// wikiSelectFields are the additional columns a wiki page carries (beyond+// compiledSelectFields) that must survive the doc→merge round-trip so the+// dataset-level merged rows keep the fields the artifact API and page renderers+// depend on (page_type_kwd/topic_kwd/title_kwd/...).+var wikiSelectFields = []string{+ "page_type_kwd", "topic_kwd", "title_kwd",+ "entity_names_kwd", "summary_with_weight",+ "related_kb_pages_kwd", "outlinks_kwd", "section_level_int",+}+// loadDocProductsLimit is the per-page size used when scrolling a single// document's compiled rows. A document can compile more than this many rows, so// LoadDocProducts pages until the engine returns fewer than a full page.@@ -92,7 +102,7 @@ func (r engineReader) LoadDocProducts(ctx context.Context, tenant, kb, docID strIndexNames: []string{fmt.Sprintf("ragflow_%s", tenant)},KbIDs: []string{kb},Filter: map[string]interface{}{"doc_id": docID},- SelectFields: compiledSelectFields,+ SelectFields: append(append([]string(nil), compiledSelectFields...), wikiSelectFields...),Limit: loadDocProductsLimit,Offset: offset,})@@ -148,8 +158,36 @@ func productFromChunkMap(c map[string]interface{}, tenant string) (kccommon.Prodmeta["kind"] = "relation"}if v, ok := c["slug_kwd"].(string); ok && v != "" {+ // slug_kwd is the full "<page_type>/<slug>" form (Python writer+ // contract); reconstruct it verbatim so the round-trip stays full-form.meta["slug"] = v}+ // Restore wiki page fields so the merged product (and hence the dataset-level+ // merged row) retains the metadata the artifact API and page renderers read.+ if v, ok := c["page_type_kwd"].(string); ok && v != "" {+ meta["page_type"] = v+ }+ if v, ok := c["topic_kwd"].(string); ok && v != "" {+ meta["topic"] = v+ }+ if v, ok := c["title_kwd"].(string); ok && v != "" {+ meta["title"] = v+ }+ if v, ok := c["summary_with_weight"].(string); ok && v != "" {+ meta["summary"] = v+ }+ if v := metaStringSlice(c, "entity_names_kwd"); len(v) > 0 {+ meta["entity_names"] = v+ }+ if v := metaStringSlice(c, "related_kb_pages_kwd"); len(v) > 0 {+ meta["related_kb_pages"] = v+ }+ if v := metaStringSlice(c, "outlinks_kwd"); len(v) > 0 {+ meta["outlinks"] = v+ }+ if v, ok := metaInt(c, "section_level_int"); ok {+ meta["section_level"] = v+ }if v, ok := c["type"].(string); ok && v != "" {meta["type"] = v}@@ -196,9 +234,10 @@ func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, variIndexNames: []string{fmt.Sprintf("ragflow_%s", tenant)},KbIDs: []string{kb},Limit: topN,- SelectFields: []string{"id", "doc_id", "kb_id", "content_with_weight", "kc_payload",+ SelectFields: append([]string{"id", "doc_id", "kb_id", "content_with_weight", "kc_payload","name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd", "slug_kwd","type", "source_chunk_ids", "source_doc_ids", "kc_merged", "compile_kwd"},+ wikiSelectFields...),Filter: map[string]interface{}{"kc_merged": 1,"compile_kwd": string(variant),diff --git a/internal/ingestion/knowledge_compile/scheduler.go b/internal/ingestion/knowledge_compile/scheduler.goindex 323311877c..03e9f15be4 100644--- a/internal/ingestion/knowledge_compile/scheduler.go+++ b/internal/ingestion/knowledge_compile/scheduler.go@@ -23,9 +23,11 @@ import ("sync""time"+ "ragflow/internal/common""ragflow/internal/engine""ragflow/internal/entity"+ "go.uber.org/zap""gorm.io/gorm""gorm.io/gorm/clause")@@ -41,6 +43,15 @@ var ErrClaimTokenMismatch = errors.New("knowledge_compile: claim token mismatch"// scheduling truth.const notifySubject = "notify.kc.workers"+// Dataset compile lifecycle states. Defined in entity so the scheduler, the+// dataset service (status API) and tests share one source of truth.+const (+ DatasetStateIdle = entity.DatasetStateIdle+ DatasetStatePending = entity.DatasetStatePending+ DatasetStateRunning = entity.DatasetStateRunning+ DatasetStateCompleted = entity.DatasetStateCompleted+)+// BacklogEntry is one scheduling unit appended to a KB's backlog (Option E// §11.4). It carries the doc id plus the original event kind/seq so the// consumer can re-apply the same out-of-order / tombstone guards as the@@ -95,6 +106,14 @@ type Claimer interface {// only when backlog is also empty.Ack(ctx context.Context, datasetID, token string, batch []BacklogEntry) (backlogRemaining int, err error)+ // SetError records a diagnostic message for a failed batch without changing+ // the lifecycle state (the batch is left for retry, so state stays running).+ // It is best-effort for observability. token must be the claim token of the+ // batch that failed: the update only applies while that exact claim is still+ // live, so a stale worker whose lease was reclaimed cannot overwrite the+ // status of the worker that took over.+ SetError(ctx context.Context, datasetID, token, errMsg string) error+// SubscribeNotify returns a channel of dataset ids pushed by Publish, or nil// when the implementation has no push wake-up (callers fall back to polling).SubscribeNotify(ctx context.Context) (<-chan string, error)@@ -197,11 +216,23 @@ func (s *mysqlScheduler) Publish(ctx context.Context, tenantID, datasetID, docIDbacklog := parseEntries(row.BacklogDocIDs)backlog = append(backlog, entry)row.BacklogDocIDs = marshalEntries(backlog)+ // Surface a pending state unless a worker is already running (a live lease+ // means the consumer is mid-merge on this KB; a pending transition would+ // wrongly hide that). completed -> pending when new work arrives.+ if row.State != DatasetStateRunning {+ row.State = DatasetStatePending+ }return tx.Save(&row).Error})if err != nil {return fmt.Errorf("knowledge_compile: publish backlog %s: %w", datasetID, err)}+ common.Info("knowledge_compile: published backlog entry",+ zap.String("dataset_id", datasetID),+ zap.String("tenant_id", tenantID),+ zap.String("doc_id", docID),+ zap.String("event_type", eventType),+ zap.Uint64("seq", seq))return s.notify(ctx, datasetID)}@@ -238,9 +269,17 @@ func (s *mysqlScheduler) claimRow(ctx context.Context, tx *gorm.DB, datasetID strow.ClaimToken = generateHolder()exp := now.Add(s.leaseTTL)row.ClaimExpiresAt = &exp+ row.State = DatasetStateRunning+ row.ErrorMsg = ""if err := tx.Save(&row).Error; err != nil {return ClaimResult{}, false, err}+ common.Info("knowledge_compile: claimed dataset batch",+ zap.String("dataset_id", row.DatasetID),+ zap.String("tenant_id", row.TenantID),+ zap.Int("batch_size", len(batch)),+ zap.Int("backlog_remaining", len(backlog)-n),+ zap.String("token", row.ClaimToken))return ClaimResult{DatasetID: row.DatasetID, TenantID: row.TenantID, Entries: batch, Token: row.ClaimToken}, true, nil}@@ -286,18 +325,61 @@ func (s *mysqlScheduler) Ack(ctx context.Context, datasetID, token string, batchrow.ClaimToken = ""row.ClaimExpiresAt = nil}+ remaining = len(parseEntries(row.BacklogDocIDs))+ // Backlog still has work -> pending; drained to empty -> completed.+ if remaining > 0 {+ row.State = DatasetStatePending+ } else {+ row.State = DatasetStateCompleted+ now := time.Now()+ row.LastCompletedAt = &now+ }if err := tx.Save(&row).Error; err != nil {return err}- remaining = len(parseEntries(row.BacklogDocIDs))return nil})if err != nil {return 0, err}+ common.Info("knowledge_compile: acked dataset batch",+ zap.String("dataset_id", datasetID),+ zap.Int("batch_size", len(batch)),+ zap.Int("backlog_remaining", remaining))return remaining, nil}+// SetError records a best-effort diagnostic on the dataset row when a batch+// merge fails (consumer failure path). It does not change the lifecycle state:+// a failed batch is left in backlog for retry, so the row stays running/pending+// and the error message is surfaced by the status API for diagnosis.+//+// token scopes the write to the exact claim that failed: the update only lands+// while that claim token is still live on the row. If the original worker's+// lease expired and another worker took over (a new claim token), a stale+// SetError from the old worker is a no-op and cannot overwrite the new worker's+// status/diagnostic.+func (s *mysqlScheduler) SetError(ctx context.Context, datasetID, token, errMsg string) error {+ if s.db == nil {+ return nil+ }+ msg := errMsg+ if len(msg) > 4000 {+ msg = msg[:4000]+ }+ res := s.db.WithContext(ctx).Model(&entity.KnowledgeCompileDataset{}).+ Where("dataset_id = ? AND claim_token = ?", datasetID, token).+ Update("error_msg", msg)+ if res.Error != nil {+ return res.Error+ }+ if res.RowsAffected == 0 {+ common.Warn("knowledge_compile: set_error ignored (claim token mismatch / lease reclaimed)",+ zap.String("dataset_id", datasetID))+ }+ return nil+}+func (s *mysqlScheduler) TouchClaim(ctx context.Context, datasetID, token string, ttl time.Duration) (bool, error) {if s.db == nil {return false, nil@@ -308,6 +390,10 @@ func (s *mysqlScheduler) TouchClaim(ctx context.Context, datasetID, token stringif res.Error != nil {return false, res.Error}+ if res.RowsAffected == 0 {+ common.Warn("knowledge_compile: claim touch failed (lease lost or token mismatch)",+ zap.String("dataset_id", datasetID), zap.String("token", token))+ }return res.RowsAffected > 0, nil}@@ -405,9 +491,14 @@ func (s *mysqlScheduler) reclaimOne(ctx context.Context, tx *gorm.DB, now time.Tcur.ClaimOwner = ""cur.ClaimToken = ""cur.ClaimExpiresAt = nil+ cur.State = DatasetStatePendingif err := tx.Save(&cur).Error; err != nil {return "", false, err}+ common.Warn("knowledge_compile: reclaimed expired inflight lease",+ zap.String("dataset_id", cur.DatasetID),+ zap.String("tenant_id", cur.TenantID),+ zap.Int("reclaimed_entries", len(inflight)))return cur.DatasetID, true, nil}return "", false, nil@@ -420,7 +511,13 @@ func (s *mysqlScheduler) notify(ctx context.Context, datasetID string) error {return nil}payload, _ := json.Marshal(map[string]string{"dataset_id": datasetID})- return s.mq.PublishKnowledgeCompile(notifySubject, payload)+ if err := s.mq.PublishKnowledgeCompile(notifySubject, payload); err != nil {+ common.Warn("knowledge_compile: publish notify failed (workers will poll)",+ zap.String("dataset_id", datasetID), zap.Error(err))+ return err+ }+ common.Info("knowledge_compile: published worker notify", zap.String("dataset_id", datasetID))+ return nil}func (s *mysqlScheduler) SubscribeNotify(ctx context.Context) (<-chan string, error) {@@ -461,6 +558,8 @@ type fakeRow struct {owner stringtoken stringexpires *time.Time+ state string+ errorMsg string}// FakeScheduler is an in-memory Publisher + Claimer used by tests. It mirrors@@ -498,6 +597,10 @@ func (f *FakeScheduler) Publish(_ context.Context, tenantID, datasetID, docID, er.tenant = tenantID}r.backlog = append(r.backlog, BacklogEntry{DocID: docID, EventType: eventType, Seq: seq})+ // Mirror the MySQL scheduler: surface pending unless a worker is running.+ if r.state != DatasetStateRunning {+ r.state = DatasetStatePending+ }select {case f.notifyCh <- datasetID:default:@@ -528,6 +631,8 @@ func (f *FakeScheduler) Claim(_ context.Context, datasetID string) (ClaimResult,r.token = generateHolder()exp := now.Add(f.leaseTTL)r.expires = &exp+ r.state = DatasetStateRunning+ r.errorMsg = ""return ClaimResult{DatasetID: datasetID, TenantID: r.tenant, Entries: batch, Token: r.token}, true, nil}@@ -545,6 +650,12 @@ func (f *FakeScheduler) Ack(_ context.Context, datasetID, token string, batch []if len(r.inflight) == 0 {r.owner, r.token, r.expires = "", "", nil}+ // Mirror the MySQL scheduler: backlog still has work -> pending; drained -> completed.+ if len(r.backlog) > 0 {+ r.state = DatasetStatePending+ } else {+ r.state = DatasetStateCompleted+ }return len(r.backlog), nil}@@ -560,6 +671,39 @@ func (f *FakeScheduler) TouchClaim(_ context.Context, datasetID, token string, _return true, nil}+func (f *FakeScheduler) SetError(_ context.Context, datasetID, token, errMsg string) error {+ f.mu.Lock()+ defer f.mu.Unlock()+ r, ok := f.rows[datasetID]+ if !ok || r.token != token {+ return nil+ }+ r.errorMsg = errMsg+ return nil+}++// fakeReclaimExpired finds one dataset with an expired lease and non-empty+// inflight, moves the inflight batch back to backlog, clears the lease, and+// marks the row pending — mirroring the MySQL scheduler's reclaimOne. It returns+// the reclaimed dataset id, or "" when nothing is expired. The lock must already+// be held by the caller.+func (f *FakeScheduler) fakeReclaimExpired(now time.Time) string {+ for id, r := range f.rows {+ if r.owner != "" && r.expires != nil && r.expires.After(now) {+ continue+ }+ if len(r.inflight) > 0 {+ r.backlog = append(r.backlog, r.inflight...)+ r.inflight = nil+ r.owner, r.token, r.expires = "", "", nil+ // reclaimOne: inflight -> backlog, lease cleared -> pending.+ r.state = DatasetStatePending+ return id+ }+ }+ return ""+}+// TryClaim mirrors the production flow: claim a ready dataset, otherwise reclaim// an expired lease and claim it.func (f *FakeScheduler) TryClaim(ctx context.Context) (ClaimResult, bool, error) {@@ -573,18 +717,7 @@ func (f *FakeScheduler) TryClaim(ctx context.Context) (ClaimResult, bool, error)}}if readyID == "" {- for id, r := range f.rows {- if r.owner != "" && r.expires != nil && r.expires.After(now) {- continue- }- if len(r.inflight) > 0 {- r.backlog = append(r.backlog, r.inflight...)- r.inflight = nil- r.owner, r.token, r.expires = "", "", nil- expiredID = id- break- }- }+ expiredID = f.fakeReclaimExpired(now)}f.mu.Unlock()if readyID != "" {diff --git a/internal/ingestion/knowledge_compile/service.go b/internal/ingestion/knowledge_compile/service.goindex 7faedcd0f4..8141605b9c 100644--- a/internal/ingestion/knowledge_compile/service.go+++ b/internal/ingestion/knowledge_compile/service.go@@ -101,7 +101,7 @@ func defaultDeduperFactory(tenant string) (Deduper, error) {if err != nil {return nil, err}- return NewLLMDeduper(deps.Chat, deps.Embed, defaultLLMID, 0.99, deps.LLMMaxLength), nil+ return NewLLMDeduper(deps.Chat, deps.Embed, defaultLLMID, 0.99, deps.ModelContextLen), nil}func generateHolder() string {diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.goindex 126e13d71c..45e6fded28 100644--- a/internal/ingestion/knowledge_compile/writer.go+++ b/internal/ingestion/knowledge_compile/writer.go@@ -20,6 +20,7 @@ import ("crypto/sha256""encoding/hex""fmt"+ "strings""ragflow/internal/engine""ragflow/internal/engine/types"@@ -70,7 +71,7 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, producbaseName := fmt.Sprintf("ragflow_%s", tenant)// Shard the rows and drive the inserts through the shared global pool// (docengine-bounded) instead of one monolithic InsertChunks call.- jobs := make([]compilerJob, 0, (len(products)+writeMergedBatchSize-1)/writeMergedBatchSize)+ jobs := make([]CompilerJob, 0, (len(products)+writeMergedBatchSize-1)/writeMergedBatchSize)for start := 0; start < len(products); start += writeMergedBatchSize {end := start + writeMergedBatchSizeif end > len(products) {@@ -108,6 +109,47 @@ func mergedChunkMap(tenant, kb string, p kccommon.Product) map[string]interface{"source_doc_ids": srcDocIDs,"source_chunk_ids": srcChunkIDs,}+ // Carry the wiki page metadata onto the merged row so the dataset-level+ // products keep the fields the artifact API (ListArtifacts/ListWikiTopics)+ // and page renderers read. Without this the merged rows lose page_type_kwd /+ // topic_kwd / title_kwd and the compilation page would show no wiki pages+ // even though per-document products carry them.+ //+ // slug_kwd follows the Python writer contract (api/db/db_models.py): it is+ // stored as the full "<page_type>/<slug>" form so GetWikiPage's filter+ // (page_type + "/" + slug) matches directly.+ pageType := metaString(p.Meta, "page_type")+ if slug := metaString(p.Meta, "slug"); slug != "" {+ // Normalize to the full "<page_type>/<slug>" form (Python writer+ // contract). Idempotent: slugs that already carry the prefix are kept.+ fullSlug := slug+ if pageType != "" && !strings.Contains(slug, "/") {+ fullSlug = pageType + "/" + slug+ }+ m["slug_kwd"] = fullSlug+ m["artifact_slug_kwd"] = fullSlug+ }+ if v := metaString(p.Meta, "title"); v != "" {+ m["title_kwd"] = v+ }+ if pageType != "" {+ m["page_type_kwd"] = pageType+ }+ if v := metaString(p.Meta, "topic"); v != "" {+ m["topic_kwd"] = v+ }+ if v := metaString(p.Meta, "summary"); v != "" {+ m["summary_with_weight"] = v+ }+ if v := metaStringSlice(p.Meta, "entity_names"); len(v) > 0 {+ m["entity_names_kwd"] = v+ }+ if v := metaStringSlice(p.Meta, "related_kb_pages"); len(v) > 0 {+ m["related_kb_pages_kwd"] = v+ }+ if v := metaStringSlice(p.Meta, "outlinks"); len(v) > 0 {+ m["outlinks_kwd"] = v+ }// Persist the merged product's embedding under the dimension-suffixed column// used elsewhere in the index, so dataset-level rows remain vector-searchable// and the Reader can reconstruct them (otherwise the vector is silently@@ -166,7 +208,7 @@ func (w engineWriter) StripMergedSources(ctx context.Context, tenant, kb string,const batchSize = 2000var toDeleteIDs []string- var jobs []compilerJob+ var jobs []CompilerJoboffset := 0for {res, err := eng.Search(ctx, &types.SearchRequest{@@ -267,6 +309,15 @@ func hashStr(s string) string {return hex.EncodeToString(sum[:])}+// metaString extracts a string from a map value, tolerating a missing or+// non-string entry.+func metaString(m map[string]any, key string) string {+ if v, ok := m[key].(string); ok {+ return v+ }+ return ""+}+func metaStringSlice(m map[string]any, key string) []string {switch v := m[key].(type) {case []string:@@ -282,3 +333,23 @@ func metaStringSlice(m map[string]any, key string) []string {}return nil}++// metaInt extracts an integer from a map value that may be boxed as float64+// (JSON number), int64, string, or a typed int — the engine/JSON round-trip does+// not guarantee a single numeric type.+func metaInt(m map[string]any, key string) (int64, bool) {+ switch v := m[key].(type) {+ case int64:+ return v, true+ case int:+ return int64(v), true+ case float64:+ return int64(v), true+ case string:+ var n int64+ if _, err := fmt.Sscanf(v, "%d", &n); err == nil {+ return n, true+ }+ }+ return 0, false+}diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.gonew file mode 100644index 0000000000..8f7e9bfe42--- /dev/null+++ b/internal/ingestion/knowledge_compile/writer_test.go@@ -0,0 +1,98 @@+package knowledge_compile++import (+ "testing"++ kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++// TestMergedChunkMapKeepsWikiFields locks the fix for the merged-row metadata+// gap: the dataset-level merged row written by mergedChunkMap must carry the+// wiki page fields (page_type_kwd/topic_kwd/title_kwd/slug_kwd/...) that the+// artifact API (ListArtifacts/ListWikiTopics) and page renderers read. Without+// them the compilation page surfaces no wiki pages from the merged products.+func TestMergedChunkMapKeepsWikiFields(t *testing.T) {+ p := kccommon.Product{+ ID: "merged-1", DocID: "kb1", TenantID: "t1", Variant: kccommon.VariantWiki,+ Content: "# Alpha\n\nBody",+ Vector: []float32{0.1, 0.2, 0.3},+ Meta: map[string]any{+ "slug": "entity/alpha",+ "title": "Alpha",+ "page_type": "entity",+ "topic": "Alpha",+ "summary": "A page about Alpha",+ "entity_names": []string{"Alpha"},+ "related_kb_pages": []string{"entity/beta"},+ "source_doc_ids": []string{"d1"},+ "source_chunk_ids": []string{"c1"},+ },+ }+ m := mergedChunkMap("t1", "kb1", p)++ cases := map[string]string{+ "slug_kwd": "entity/alpha",+ "artifact_slug_kwd": "entity/alpha",+ "title_kwd": "Alpha",+ "page_type_kwd": "entity",+ "topic_kwd": "Alpha",+ "summary_with_weight": "A page about Alpha",+ }+ for k, want := range cases {+ if got, _ := m[k].(string); got != want {+ t.Errorf("merged row[%q] = %q, want %q", k, got, want)+ }+ }+ if v, _ := m["entity_names_kwd"].([]string); len(v) != 1 || v[0] != "Alpha" {+ t.Errorf("entity_names_kwd = %#v, want [Alpha]", m["entity_names_kwd"])+ }+ if m["doc_id"] != "kb1" || m["kc_merged"] != 1 || m["available_int"] != 1 {+ t.Errorf("merged flags wrong: doc_id=%v kc_merged=%v available_int=%v", m["doc_id"], m["kc_merged"], m["available_int"])+ }+ if m["q_3_vec"] == nil {+ t.Errorf("vector column missing")+ }+}++// TestProductFromChunkMapRestoresWikiFields locks the reader side: the wiki page+// columns must be reconstructed into the product Meta so the merge step can carry+// them onto the merged row.+func TestProductFromChunkMapRestoresWikiFields(t *testing.T) {+ c := map[string]interface{}{+ "id": "wiki/1",+ "doc_id": "d1",+ "compile_kwd": "wiki_page",+ "content_with_weight": "# Alpha",+ "kc_payload": "# Alpha\n\nBody",+ "slug_kwd": "entity/alpha",+ "page_type_kwd": "entity",+ "topic_kwd": "Alpha",+ "title_kwd": "Alpha",+ "summary_with_weight": "A page about Alpha",+ "entity_names_kwd": []interface{}{"Alpha"},+ "related_kb_pages_kwd": []interface{}{"entity/beta"},+ "section_level_int": float64(2),+ }+ p, ok := productFromChunkMap(c, "t1")+ if !ok {+ t.Fatalf("productFromChunkMap returned not-ok")+ }+ want := map[string]string{+ "slug": "entity/alpha", "page_type": "entity", "topic": "Alpha",+ "title": "Alpha", "summary": "A page about Alpha",+ }+ for k, v := range want {+ if got, _ := p.Meta[k].(string); got != v {+ t.Errorf("meta[%q] = %q, want %q", k, got, v)+ }+ }+ if v, _ := p.Meta["section_level"].(int64); v != 2 {+ t.Errorf("section_level = %v, want 2", p.Meta["section_level"])+ }+ if v, _ := p.Meta["entity_names"].([]string); len(v) != 1 || v[0] != "Alpha" {+ t.Errorf("entity_names = %#v, want [Alpha]", p.Meta["entity_names"])+ }+ if p.Merged {+ t.Errorf("per-doc row must not be marked merged")+ }+}diff --git a/internal/ingestion/pipeline/pipeline_knowledge_compiler_dsl_test.go b/internal/ingestion/pipeline/pipeline_knowledge_compiler_dsl_test.goindex 66b76f356d..7a23c9e57f 100644--- a/internal/ingestion/pipeline/pipeline_knowledge_compiler_dsl_test.go+++ b/internal/ingestion/pipeline/pipeline_knowledge_compiler_dsl_test.go@@ -95,10 +95,12 @@ func TestKnowledgeCompilerDSL_FixtureDecodesAndBindsParams(t *testing.T) {t.Errorf("fixture unexpectedly sets variant; frontend Compiler DSL omits it")}- // Authored as a single string group id in the frontend form.- gid, ok := params["compilation_template_group_id"].(string)- if !ok || gid != "c3aa748c8b2111f191f3047c16ec874f" {- t.Fatalf("compilation_template_group_id = %v, want single string id", params["compilation_template_group_id"])+ // Authored as a single string group id in the frontend form. The shipped+ // template leaves it empty by design (the user selects the template group at+ // runtime), so we only assert the DSL shape is a plain string (and that the+ // node carries no variant).+ if _, ok := params["compilation_template_group_id"].(string); !ok {+ t.Fatalf("compilation_template_group_id = %v, want a single string id", params["compilation_template_group_id"])}}@@ -137,8 +139,11 @@ func TestKnowledgeCompilerDSL_FrontendDSLDecodesAndConstructs(t *testing.T) {t.Fatal("default runtime factory not installed")}- // The fixture params (single group id, no variant) construct fine.- comp, err := f("KnowledgeCompiler", params)+ // The fixture params carry no variant and a string group id. The shipped+ // template leaves the group id empty (selected at runtime), so give it a+ // concrete id here to verify the DSL constructs once configured.+ params["compilation_template_group_id"] = "tpl-group"+ comp, err := f("Compiler", params)if err != nil {t.Fatalf("construct from fixture params: %v", err)}@@ -154,16 +159,15 @@ func TestKnowledgeCompilerDSL_FrontendDSLDecodesAndConstructs(t *testing.T) {}// TestKnowledgeCompilerDSL_RegisteredAndConstructible confirms the Go runtime-// registers the component under the canonical name "KnowledgeCompiler" and that-// the runtime factory can build a component instance from a DSL params map that-// carries either compilation_template_id or compilation_template_group_id (the-// variant is no longer part of the DSL surface). The frontend label "Compiler"-// (see the fixture tests) maps to this runtime name at the API/canvas layer,-// not inside the pipeline DSL decoder.+// registers the knowledge-compiler component under the unified name "Compiler"+// (matching the Python side rag/flow/compiler/compiler.py) and that the runtime+// factory can build a component instance from a DSL params map that carries+// either compilation_template_id or compilation_template_group_id (the variant+// is no longer part of the DSL surface).func TestKnowledgeCompilerDSL_RegisteredAndConstructible(t *testing.T) {runtime.InstallDefaultRegistryFactory()- if _, _, _, ok := runtime.DefaultRegistry.Lookup("KnowledgeCompiler"); !ok {- t.Fatal("KnowledgeCompiler not registered in the runtime factory")+ if _, _, _, ok := runtime.DefaultRegistry.Lookup("Compiler"); !ok {+ t.Fatal("Compiler not registered in the runtime factory")}f := runtime.DefaultFactory()if f == nil {@@ -177,7 +181,7 @@ func TestKnowledgeCompilerDSL_RegisteredAndConstructible(t *testing.T) {{"compilation_template_id": "t1", "compilation_template_group_id": "g1"},}for i, params := range cases {- comp, err := f("KnowledgeCompiler", params)+ comp, err := f("Compiler", params)if err != nil {t.Fatalf("case %d construct: %v", i, err)}@@ -187,7 +191,7 @@ func TestKnowledgeCompilerDSL_RegisteredAndConstructible(t *testing.T) {}// Param map with neither id resolves to a parse error.- if _, err := f("KnowledgeCompiler", map[string]any{}); err == nil {+ if _, err := f("Compiler", map[string]any{}); err == nil {t.Fatal("construct with no template spec: expected error")}}@@ -203,6 +207,8 @@ func TestKnowledgeCompilerDSL_ParamBinding(t *testing.T) {"compilation_template_group_id": "g1","llm_id": "llm-1","embedding_model": "emb-1",+ "tenant_id": "tenant-1",+ "dataset_id": "kb-1","language": "Chinese","extra": map[string]any{"prompt": "summarize"},})@@ -218,6 +224,8 @@ func TestKnowledgeCompilerDSL_ParamBinding(t *testing.T) {if p.Variant != "" {t.Errorf("Variant should be empty after ParseParam (derived from kind later), got %q", p.Variant)}+ // TenantID/DatasetID are injected at runtime by the component (not via the+ // DSL/ParseParam), so they are not asserted here.if p.LLMID != "llm-1" || p.EmbeddingModel != "emb-1" || p.Language != "Chinese" {t.Errorf("scalar fields = %+v", p)}diff --git a/internal/ingestion/pipeline/pipeline_knowledge_compiler_test.go b/internal/ingestion/pipeline/pipeline_knowledge_compiler_test.goindex 2c7fd5d797..a2422f837d 100644--- a/internal/ingestion/pipeline/pipeline_knowledge_compiler_test.go+++ b/internal/ingestion/pipeline/pipeline_knowledge_compiler_test.go@@ -57,7 +57,7 @@ func TestKnowledgeCompilerTemplate_RegisteredAndDecodable(t *testing.T) {if runtime.DefaultFactory() == nil {t.Fatal("default runtime factory not installed")}- for _, name := range []string{"File", "Parser", "TokenChunker", "KnowledgeCompiler"} {+ for _, name := range []string{"File", "Parser", "TokenChunker", "Compiler"} {if _, _, _, ok := runtime.DefaultRegistry.Lookup(name); !ok {t.Errorf("component %q referenced by template is not registered in the runtime factory", name)}diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.json b/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.jsonindex 6b9bf3f5da..364bc66b09 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_knowledge_compiler.json@@ -6,9 +6,9 @@"zh": "知识编译"},"description": {- "en": "Compiles parsed chunks into structured knowledge units (graph/wiki/raptor/mindmap/datasetnav) via the KnowledgeCompiler component, emitting them as chunks merged into the upstream chunk stream. Ideal for building a retrievable knowledge layer on top of chunked documents.",- "de": "Kompiliert geparste Chunks über die KnowledgeCompiler-Komponente in strukturierte Wissenseinheiten und gibt sie als Chunks im upstream-Chunk-Strom zurück.",- "zh": "通过 KnowledgeCompiler 组件将解析后的分块编译为结构化知识单元(图谱/百科/RAPTOR/思维导图/数据集导航),以 chunks 形式合并进上游分块流输出,适合在分块文档之上构建可检索的知识层。"+ "en": "Compiles parsed chunks into structured knowledge units (graph/wiki/raptor/mindmap/datasetnav) via the Compiler component, emitting them as chunks merged into the upstream chunk stream. Ideal for building a retrievable knowledge layer on top of chunked documents.",+ "de": "Kompiliert geparste Chunks über die Compiler-Komponente in strukturierte Wissenseinheiten und gibt sie als Chunks im upstream-Chunk-Strom zurück.",+ "zh": "通过 Compiler 组件将解析后的分块编译为结构化知识单元(图谱/百科/RAPTOR/思维导图/数据集导航),以 chunks 形式合并进上游分块流输出,适合在分块文档之上构建可检索的知识层。"},"canvas_type": "Ingestion Pipeline","canvas_category": "dataflow_canvas",@@ -150,7 +150,7 @@},"TokenChunker:SixApplesFall": {"downstream": [- "KnowledgeCompiler:KnownSwiftLions"+ "Compiler:KnownSwiftLions"],"obj": {"component_name": "TokenChunker",@@ -182,11 +182,12 @@"Parser:HipSignsRhyme"]},- "KnowledgeCompiler:KnownSwiftLions": {+ "Compiler:KnownSwiftLions": {"downstream": [],"obj": {- "component_name": "KnowledgeCompiler",+ "component_name": "Compiler","params": {+ "llm_id": "","variant": "structure","language": "English"}@@ -216,10 +217,10 @@"targetHandle": "end"},{- "id": "xy-edge__TokenChunker:SixApplesFallstart-KnowledgeCompiler:KnownSwiftLionsend",+ "id": "xy-edge__TokenChunker:SixApplesFallstart-Compiler:KnownSwiftLionsend","source": "TokenChunker:SixApplesFall","sourceHandle": "start",- "target": "KnowledgeCompiler:KnownSwiftLions",+ "target": "Compiler:KnownSwiftLions","targetHandle": "end"}],@@ -280,10 +281,10 @@},{"data": {- "label": "KnowledgeCompiler",+ "label": "Compiler","name": "Knowledge Compiler_0"},- "id": "KnowledgeCompiler:KnownSwiftLions",+ "id": "Compiler:KnownSwiftLions","measured": {"height": 74,"width": 200diff --git a/internal/ingestion/pipeline/template_integration_test.go b/internal/ingestion/pipeline/template_integration_test.goindex bee1d14632..e5e43a2a21 100644--- a/internal/ingestion/pipeline/template_integration_test.go+++ b/internal/ingestion/pipeline/template_integration_test.go@@ -821,8 +821,8 @@ func TestPipelineRun_AllIngestionTemplates_RealComponentsSmoke(t *testing.T) {if templateUsesComponent(t, templateBytes, "TagChunker") {t.Skip("template uses TagChunker which requires tag-structured content and parser setups not available for generic .md input; covered separately")}- if templateUsesComponent(t, templateBytes, "KnowledgeCompiler") {- t.Skip("template uses KnowledgeCompiler which requires LLM/embedder/ES wiring not available in the headless smoke run; covered by the knowledge_compiler component E2E tests")+ if templateUsesComponent(t, templateBytes, "Compiler") {+ t.Skip("template uses Compiler which requires LLM/embedder/ES wiring not available in the headless smoke run; covered by the knowledge_compiler component E2E tests")}terminalIDs := terminalComponentIDsFromTemplate(t, templateBytes)if len(terminalIDs) != 1 {diff --git a/internal/ingestion/task/embedder.go b/internal/ingestion/task/embedder.goindex 6e0c1ec97a..8bd4ed0a97 100644--- a/internal/ingestion/task/embedder.go+++ b/internal/ingestion/task/embedder.go@@ -38,6 +38,13 @@ func (e *embedder) MaxTokens() int {return e.model.MaxTokens}+func (e *embedder) BatchSize() int {+ if e == nil || e.model == nil {+ return models.DefaultEmbeddingBatchSize+ }+ return e.model.ResolveBatchSize()+}+func (e *embedder) Encode(ctx context.Context, texts []string) ([]componentpkg.EmbeddingResult, error) {if e.model.ModelDriver == nil {return nil, fmt.Errorf("embedder: embedding model driver is nil for model %v", e.model.ModelName)diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.goindex 311e4e1970..a67d7266d8 100644--- a/internal/ingestion/task/knowledge_compiler_wiring.go+++ b/internal/ingestion/task/knowledge_compiler_wiring.go@@ -28,7 +28,9 @@ import (enginetypes "ragflow/internal/engine/types""ragflow/internal/entity""ragflow/internal/entity/models"+ _ "ragflow/internal/ingestion/component/knowledge_compiler"kc "ragflow/internal/ingestion/component/knowledge_compiler/common"+ "ragflow/internal/ingestion/knowledge_compile""ragflow/internal/service""gorm.io/gorm"@@ -97,12 +99,15 @@ func newKnowledgeCompilerDepsResolver() kc.DepsResolver {}// Resolve the chat model's context window so RAPTOR can truncate each// cluster's texts to fit the LLM context (mirrors Python self._llm_model.max_length).+ // This uses content_length (PR #17839) — the total context window — not+ // max_output. max_output is only the generation cap; using it as the+ // budget source would collapse per-chunk input quotas.llmMax := kc.DefaultLLMContextLength// Bound the model-config lookup so a stalled provider/instance DB read// cannot block document ingestion indefinitely.ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)defer cancel()- if _, _, _, ml, merr := svc.ResolveModelConfig(ctx, tenantID, entity.ModelTypeChat, llmID); merr == nil && ml > 0 {+ if ml, merr := svc.ResolveModelContextLength(ctx, tenantID, llmID); merr == nil && ml > 0 {llmMax = ml}@@ -113,7 +118,7 @@ func newKnowledgeCompilerDepsResolver() kc.DepsResolver {// HistoricalKNN / Redis are optional (wiki historical dedup,// datasetnav lock). They are wired separately when the// surrounding pipeline supplies the backing services.- LLMMaxLength: llmMax,+ ModelContextLen: llmMax,}, nil}}@@ -174,33 +179,93 @@ func (e *kcEmbedder) Encode(ctx context.Context, texts []string) ([][]float32, eif len(texts) == 0 {return nil, nil}- embdID := strings.TrimSpace(e.embdID)- if embdID == "" {- return nil, fmt.Errorf("knowledge_compiler: embedding_model is required for production embedding")- }- mdl, err := e.svc.GetEmbeddingModel(ctx, e.tenantID, embdID)+ mdl, err := e.resolveModel(ctx)if err != nil {- return nil, fmt.Errorf("knowledge_compiler: resolve embedding model: %w", err)- }- if mdl == nil || mdl.ModelDriver == nil {- return nil, fmt.Errorf("knowledge_compiler: embedding model %q is unavailable", embdID)+ return nil, err}config := &models.EmbeddingConfig{}- // Embed expects *string for the model name; nil ModelUsage (not tracked here).- embeds, err := mdl.ModelDriver.Embed(ctx, mdl.ModelName, texts, mdl.APIConfig, config, nil)- if err != nil {- return nil, fmt.Errorf("knowledge_compiler: embed: %w", err)+ // Slice inputs into per-provider batches: providers cap the per-request input+ // count and reject larger batches rather than chunking internally. The batch+ // size is resolved from the model's capability (all_models.json batch_size,+ // added by #17877/#17878) via EmbeddingModel.ResolveBatchSize, which falls+ // back to a conservative default. Batches are fanned out on the shared compiler+ // pool and concatenated back in input order.+ batchSize := mdl.ResolveBatchSize()+ numBatches := (len(texts) + batchSize - 1) / batchSize+ slots := make([][][]float32, numBatches) // per-batch vector lists, distinct indices => no race+ jobs := make([]knowledge_compile.CompilerJob, 0, numBatches)+ for b := 0; b < numBatches; b++ {+ b := b+ start := b * batchSize+ end := start + batchSize+ if end > len(texts) {+ end = len(texts)+ }+ batchTexts := texts[start:end]+ jobs = append(jobs, func() error {+ if err := ctx.Err(); err != nil {+ return err+ }+ embeds, err := mdl.ModelDriver.Embed(ctx, mdl.ModelName, batchTexts, mdl.APIConfig, config, nil)+ if err != nil {+ return fmt.Errorf("knowledge_compiler: embed: %w", err)+ }+ vecs := make([][]float32, len(embeds))+ for i, v := range embeds {+ vecs[i] = float64sToFloat32(v.Embedding)+ }+ slots[b] = vecs+ return nil+ })}- out := make([][]float32, len(embeds))- for i, v := range embeds {- out[i] = float64sToFloat32(v.Embedding)+ if err := knowledge_compile.SubmitCompilerJobs(ctx, jobs); err != nil {+ return nil, err}- if len(out) > 0 {- e.dim.CompareAndSwap(0, int64(len(out[0])))+ // Flatten in input order and derive the vector dimension from the first+ // batch's first vector.+ out := make([][]float32, 0, len(texts))+ var batchDim int+ for _, slot := range slots {+ for _, vec := range slot {+ out = append(out, vec)+ if batchDim == 0 {+ batchDim = len(vec)+ }+ }+ }+ if batchDim > 0 {+ e.dim.CompareAndSwap(0, int64(batchDim))}return out, nil}+// resolveModel returns the embedding model to embed with. It prefers the+// explicitly configured embedding_model; when the caller left it unset, it falls+// back to the tenant's default embedding model (mirrors Python, which uses the+// KB/tenant's configured embedding model for wiki compilation). A clear error is+// returned only when neither is available, so a KB with no embedding model fails+// loudly instead of silently producing empty vectors.+func (e *kcEmbedder) resolveModel(ctx context.Context) (*models.EmbeddingModel, error) {+ if embdID := strings.TrimSpace(e.embdID); embdID != "" {+ mdl, err := e.svc.GetEmbeddingModel(ctx, e.tenantID, embdID)+ if err != nil {+ return nil, fmt.Errorf("knowledge_compiler: resolve embedding model: %w", err)+ }+ if mdl == nil || mdl.ModelDriver == nil {+ return nil, fmt.Errorf("knowledge_compiler: embedding model %q is unavailable", embdID)+ }+ return mdl, nil+ }+ driver, name, apiConfig, _, err := e.svc.GetTenantDefaultModelByType(ctx, e.tenantID, entity.ModelTypeEmbedding)+ if err != nil {+ return nil, fmt.Errorf("knowledge_compiler: embedding_model is required and no tenant default embedding model is set: %w", err)+ }+ if driver == nil || name == "" {+ return nil, fmt.Errorf("knowledge_compiler: embedding_model is required (tenant default embedding model unavailable)")+ }+ return &models.EmbeddingModel{ModelDriver: driver, ModelName: &name, APIConfig: apiConfig}, nil+}+func (e *kcEmbedder) Dimensions() int { return int(e.dim.Load()) }// float64sToFloat32 converts an embedding vector to the product schema's@@ -230,9 +295,12 @@ func (s *kcWikiPageStore) FindSimilarPages(ctx context.Context, tenantID, dataseKbIDs: []string{datasetID},Limit: k,SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "summary_with_weight", "content_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "kc_content_md_raw", "_score"},+ // compile_kwd="wiki_page" is the schema-backed discriminator for wiki+ // pages (sections carry compile_kwd="wiki_section"); there is no+ // "kc_kind" column in the chunk schema, so filtering on it would return+ // empty on Infinity.Filter: map[string]interface{}{- "compile_kwd": "artifact_page",- "kc_kind": "page",+ "compile_kwd": "wiki_page",},MatchExprs: []interface{}{&enginetypes.MatchDenseExpr{VectorColumnName: fmt.Sprintf("q_%d_vec", len(vec)),@@ -264,9 +332,8 @@ func (s *kcWikiPageStore) GetPageBySlug(ctx context.Context, tenantID, datasetIDLimit: 1,SelectFields: []string{"id", "slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "summary_with_weight", "content_with_weight", "entity_names_kwd", "related_kb_pages_kwd", "outlinks_kwd", "kc_content_md_raw", "_score"},Filter: map[string]interface{}{- "compile_kwd": "artifact_page",+ "compile_kwd": "wiki_page","slug_kwd": slug,- "kc_kind": "page",},}res, err := s.docEngine.Search(ctx, req)diff --git a/internal/ingestion/task/knowledge_compiler_wiring_test.go b/internal/ingestion/task/knowledge_compiler_wiring_test.gonew file mode 100644index 0000000000..ab4a81abf4--- /dev/null+++ b/internal/ingestion/task/knowledge_compiler_wiring_test.go@@ -0,0 +1,43 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package task++import (+ "testing"++ "ragflow/internal/agent/runtime"+)++// TestKnowledgeCompilerRegisteredByWiring locks the composition-root contract:+// the task package imports the knowledge_compiler root package (via blank+// import in knowledge_compiler_wiring.go) so its init() registers the+// knowledge-compilation component under the unified name "Compiler" in the+// production runtime registry. Without this blank import the component is never+// registered and an ingestor consuming a canvas with a Compiler node fails with+// "unknown component".+func TestKnowledgeCompilerRegisteredByWiring(t *testing.T) {+ factory, category, _, ok := runtime.DefaultRegistry.Lookup("Compiler")+ if !ok {+ t.Fatal("knowledge-compiler component \"Compiler\" is not registered; the task package blank-import must be present for its init() to run")+ }+ if category != runtime.CategoryIngestion {+ t.Fatalf("component \"Compiler\" category = %q, want %q", category, runtime.CategoryIngestion)+ }+ if factory == nil {+ t.Fatal("component \"Compiler\" registered with a nil factory")+ }+}diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex c0cbfbc994..4a77724867 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -418,14 +418,6 @@ func (s *PipelineExecutor) runPipelineWithDSL(ctx context.Context, dsl string) (// injected in place below without a nil-map assignment panic.parserConfig = map[string]interface{}{}}- common.InjectExtractorLLMID(parserConfig, s.taskCtx.Tenant.LLMID)- // When the dataset enables auto-metadata, ensure the Extractor node(s)- // carry the enable_metadata mode + field schema so the LLM extraction fires- // (mirrors Python task_executor.py:519 enabling gen_metadata_task). The- // dataset flag is authoritative: a node that already has enable_metadata- // turned on keeps its own config, but a shipped DSL defaulting it to 0 is- // still overridden so auto-metadata can activate.- common.InjectExtractorEnableMetadata(parserConfig)// Surface component params whose cpnID is absent from the DSL. The// runtime merge (override_params) silently drops such entries;diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.goindex b7cb5ba9dd..b1eca5024f 100644--- a/internal/ingestion/task/pipeline_executor_defaults_test.go+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go@@ -51,7 +51,7 @@ var builtinComponentParamsGolden = map[string]string{"qa": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"Tokenizer:ColdCloudsDream\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"QAChunker:TidyCloudsThink\": {}}","resume": "{\"Extractor:ThreeDrinksAct\": {\"field_name\": \"metadata\", \"frequencyPenaltyEnabled\": true, \"frequency_penalty\": 0.7, \"llm_id\": \"THUDM/GLM-4.1V-9B-Thinking@SILICONFLOW\", \"maxTokensEnabled\": false, \"max_tokens\": 256, \"presencePenaltyEnabled\": true, \"presence_penalty\": 0.4, \"prompts\": [{\"content\": \"Content: {TitleChunker:FlatMiceFix@chunks}\", \"role\": \"user\"}], \"sys_prompt\": \"Act as a precise resume metadata extractor. Extract stable, chunk-supported metadata from the provided resume content.\\n\\nRules:\\n1. Use only information explicitly stated in the content. Do not infer, guess, normalize, or add missing facts.\\n2. The input may be only one chunk of a resume. Extract only what this content directly supports.\\n3. Use only these field names:\\ncandidate_name, gender, phone, email, city, location, nationality, linkedin, github, website, highest_degree, degree_levels, school_names, majors, graduation_years, work_experience_years, current_job_title, job_titles, company_names, job_experience, industries, target_job_titles, target_locations, employment_types, skills, certificates, awards, summary_tags\\n4. Ignore detailed responsibilities, project descriptions, achievement narratives, self-evaluation, and other low-value local details.\\n5. Keep values in the same language as the source text whenever possible.\\n6. Remove duplicates and keep only concise, high-value metadata.\\n7. Return only fields that are explicitly supported by the content. Do not return empty or unsupported fields.\\n\\nField guidance:\\n- highest_degree: highest explicit degree level mentioned\\n- degree_levels: all explicit degree levels mentioned\\n- school_names: explicit school, college, or university names\\n- majors: explicit fields of study\\n- graduation_years: explicit graduation years only\\n- work_experience_years: only if explicitly stated\\n- current_job_title: only if explicitly current or most recent\\n- job_titles: explicit role titles\\n- company_names: explicit employer names\\n- job_experience: concise structured work entries explicitly supported by the content, preferably including title, company, and time information when available\\n- industries: explicit industry names only\\n- target_job_titles: explicit desired roles only\\n- target_locations: explicit desired work locations only\\n- skills: concise, core, search-useful skills explicitly mentioned\\n- certificates: explicit certificate names only\\n- awards: explicit award names only\\n- summary_tags: short, high-value tags strictly supported by the content\\n\\nReturn only the extracted metadata. Do not output explanatory text.\", \"temperature\": 0.1, \"temperatureEnabled\": true, \"tenant_llm_id\": 29, \"topPEnabled\": true, \"top_p\": 0.3, \"auto_keywords\": 0, \"auto_questions\": 0, \"auto_tags\": 0, \"enable_metadata\": 0, \"metadata\": [], \"tag_file_id\": \"\"}, \"File\": {}, \"Parser:HipSignsRhyme\": {\"docx\": {\"flatten_media_to_text\": true, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": true, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:FlatMiceFix\": {\"hierarchy\": 1, \"include_heading_content\": false, \"levels\": [[\"^\\\\s*(?i:(?:\\\\d+[\\\\.\\\\)]\\\\s*)?(?:EDUCATION|ACADEMIC\\\\s*BACKGROUND|ACADEMIC\\\\s*HISTORY|EDUCATIONAL\\\\s*BACKGROUND|RELEVANT\\\\s*COURSEWORK|COURSEWORK|EXPERIENCE|WORK\\\\s*EXPERIENCE|PROFESSIONAL\\\\s*EXPERIENCE|RELEVANT\\\\s*EXPERIENCE|EMPLOYMENT\\\\s*HISTORY|CAREER\\\\s*HISTORY|INTERNSHIP\\\\s*EXPERIENCE|PROJECTS|PROJECT\\\\s*EXPERIENCE|ACADEMIC\\\\s*PROJECTS|PROFESSIONAL\\\\s*PROJECTS|SKILLS|TECHNICAL\\\\s*SKILLS|CORE\\\\s*COMPETENCIES|COMPETENCIES|QUALIFICATIONS|SUMMARY\\\\s*OF\\\\s*QUALIFICATIONS|CERTIFICATIONS|LICENSES|CERTIFICATES|AWARDS|HONORS|HONOURS|ACHIEVEMENTS|PUBLICATIONS|RESEARCH|RESEARCH\\\\s*EXPERIENCE|LEADERSHIP|LEADERSHIP\\\\s*EXPERIENCE|ACTIVITIES|EXTRACURRICULAR\\\\s*ACTIVITIES|ACTIVITIES\\\\s*(?:&|AND)\\\\s*SKILLS|INVOLVEMENT|CAMPUS\\\\s*INVOLVEMENT|VOLUNTEER\\\\s*EXPERIENCE|VOLUNTEERING|COMMUNITY\\\\s*SERVICE|LANGUAGES|INTERESTS|HOBBIES|PROFILE|PROFESSIONAL\\\\s*PROFILE|SUMMARY|PROFESSIONAL\\\\s*SUMMARY|CAREER\\\\s*SUMMARY|OBJECTIVE|CAREER\\\\s*OBJECTIVE|PERSONAL\\\\s*INFORMATION|CONTACT\\\\s*INFORMATION|ADDITIONAL\\\\s*INFORMATION|TRAINING))\\\\s*[::]?\\\\s*$\"], [\"^\\\\s*(?:\\\\d+[\\\\.、\\\\)]\\\\s*)?(?:教育背景|教育经历|学历背景|学术背景|技术背景|工作经历|工作经验|实习经历|项目经历|项目经验|科研经历|研究经历|校园经历|实践经历|专业经历|职业经历|技能|专业技能|技能特长|核心技能|技术栈|个人技能|工作技能|职业技能|技能与评价|技能与自我评价|工作技能与自我评价|职业技能与自我评价|证书|资格证书|职业资格|资质证书|获奖情况|获奖经历|荣誉|荣誉奖项|奖项|科研成果|论文发表|发表论文|领导经历|学生工作|校园活动|社团经历|活动经历|志愿经历|志愿服务|社会实践|语言能力|语言|自我评价|个人评价|自我总结|个人总结|个人优势|个人简介|个人信息|基本信息|联系方式|求职意向|应聘意向|职业目标|求职目标|兴趣爱好|兴趣特长|培训经历|其他信息|附加信息)\\\\s*[::]?\\\\s*$\"]], \"method\": \"hierarchy\"}, \"Tokenizer:KindHandsWin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}","table": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}, \"column_mode\": \"auto\", \"column_roles\": {}, \"column_names\": []}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TableChunker:FastFoxesJump\": {}, \"Tokenizer:DeepLakesShine\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}}",- "knowledge_compiler": "{\"File\": {}, \"KnowledgeCompiler:KnownSwiftLions\": {\"language\": \"English\", \"variant\": \"structure\"}, \"Parser:HipSignsRhyme\": {\"setups\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"token_size\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}}",+ "knowledge_compiler": "{\"File\": {}, \"Compiler:KnownSwiftLions\": {\"language\": \"English\", \"llm_id\": \"\", \"variant\": \"structure\"}, \"Parser:HipSignsRhyme\": {\"setups\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"token_size\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}}",}// Per-template test methods. Each resolves default component params from adiff --git a/internal/ingestion/task/pipeline_executor_test.go b/internal/ingestion/task/pipeline_executor_test.goindex b366e90466..e5ccd059ff 100644--- a/internal/ingestion/task/pipeline_executor_test.go+++ b/internal/ingestion/task/pipeline_executor_test.go@@ -32,7 +32,7 @@ func TestMarkCompiledProductsHidden(t *testing.T) {chunks := []map[string]any{{"id": "src-1", "content_with_weight": "ordinary source chunk"},{"id": "struct-1", "compile_kwd": "structure", "content_with_weight": "entity A"},- {"id": "wiki-1", "compile_kwd": "artifact_page", "content_with_weight": "page X"},+ {"id": "wiki-1", "compile_kwd": "wiki_page", "content_with_weight": "page X"},{"id": "src-2", "content_with_weight": "another source chunk"},}markCompiledProductsHidden(chunks)diff --git a/internal/router/router.go b/internal/router/router.goindex 1d679c4a79..fd2dc699c2 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -351,9 +351,9 @@ func (r *Router) Setup(engine *gin.Engine) {datasets.DELETE("/:dataset_id/tags", r.datasetsHandler.RemoveTags)datasets.POST("/:dataset_id/embedding/check", r.datasetsHandler.CheckEmbedding)datasets.POST("/:dataset_id/documents/batch-update-status", r.documentHandler.BatchUpdateDocumentStatus)- datasets.GET("/:dataset_id/index", r.datasetsHandler.TraceIndex)- datasets.POST("/:dataset_id/index", r.datasetsHandler.RunIndex)- datasets.DELETE("/:dataset_id/index", r.datasetsHandler.DeleteIndex)+ // Scheduler compile-status contract (API_PROXY_SCHEME=go/hybrid);+ // replaces the retired RunIndex/TraceIndex/DeleteIndex /index routes.+ datasets.GET("/:dataset_id/compilation/status", r.datasetsHandler.GetCompilationStatus)// Knowledge-compilation wiki artifactsdatasets.HEAD("/:dataset_id/artifacts", r.datasetArtifactHandler.AnyArtifact)@@ -380,8 +380,6 @@ func (r *Router) Setup(engine *gin.Engine) {datasets.GET("/:dataset_id/skills/:skill_kwd", r.datasetArtifactHandler.GetSkillPage)datasets.DELETE("/:dataset_id/skills/:skill_kwd", r.datasetArtifactHandler.DeleteSkill)- datasets.DELETE("/:dataset_id/:index_type", r.datasetsHandler.DeleteIndex)- //datasets.DELETE("/:dataset_id/graph", r.datasetsHandler.DeleteKnowledgeGraph)datasets.POST("", r.datasetsHandler.CreateDataset)datasets.DELETE("", r.datasetsHandler.DeleteDatasets)datasets.POST("/search", r.datasetsHandler.SearchDatasets)diff --git a/internal/server/config/base.go b/internal/server/config/base.goindex 7a769b1636..dfbb624240 100644--- a/internal/server/config/base.go+++ b/internal/server/config/base.go@@ -30,8 +30,8 @@ type Config struct {admin AdminConfigapiServer APIServerConfig- ingestor IngestorConfigsyncer SyncerConfig+ ingestor IngestorConfiglog LogConfigsmtp common.SMTPConfigdiff --git a/internal/server/config/ingestor_config.go b/internal/server/config/ingestor_config.goindex 7ea6edb0e5..684e289e06 100644--- a/internal/server/config/ingestor_config.go+++ b/internal/server/config/ingestor_config.go@@ -12,19 +12,26 @@// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.// See the License for the specific language governing permissions and// limitations under the License.-//package configimport "github.com/spf13/viper"type IngestorConfig struct {+ // MaxConcurrentWorkers bounds how many ingestion tasks the ingestor runs in+ // parallel (the task channel width and dataset-level compile worker count+ // default to this value). 0/negative falls back to runtime.NumCPU().MaxConcurrentWorkers int `mapstructure:"max_concurrent_workers"`+ // CompilerPoolSize bounds the process-wide knowledge-compilation worker+ // pool that drives the cross-doc KNN / LLM-merge / write stages. 0/negative+ // falls back to runtime.NumCPU() (or KC_COMPILE_CONCURRENCY if set).+ CompilerPoolSize int `mapstructure:"compiler_pool_size"`}func (c *Config) ParseIngestorConfig(v *viper.Viper) error {// Default Ingestor configc.ingestor.MaxConcurrentWorkers = 1+ c.ingestor.CompilerPoolSize = 0if !v.IsSet("ingestor") {return nil@@ -38,5 +45,13 @@ func (c *Config) ParseIngestorConfig(v *viper.Viper) error {c.ingestor.MaxConcurrentWorkers = sub.GetInt("max_concurrent_workers")}+ if sub.IsSet("compiler_pool_size") {+ c.ingestor.CompilerPoolSize = sub.GetInt("compiler_pool_size")+ }+return nil}++func (c *Config) GetIngestorConfig() *IngestorConfig {+ return &c.ingestor+}diff --git a/internal/service/component_scoped_parser_config.go b/internal/service/component_scoped_parser_config.gonew file mode 100644index 0000000000..55ca470129--- /dev/null+++ b/internal/service/component_scoped_parser_config.go@@ -0,0 +1,118 @@+package service++import (+ "strings"++ "ragflow/internal/entity"+)++// ApplyComponentScopedParserConfig fills dataset-scoped component params onto a+// parser_config map without reintroducing top-level flat fields. It mutates the+// provided map in place and returns it for convenience.+func ApplyComponentScopedParserConfig(+ parserConfig entity.JSONMap,+ llmID string,+) entity.JSONMap {+ if parserConfig == nil {+ parserConfig = entity.JSONMap{}+ }++ enableMetadata := parserConfigTruthy(parserConfig["enable_metadata"])+ metadataFields := mergeMetadataFields(parserConfig)+ hasMetadataConfig := hasMetadataConfigShape(parserConfig)++ for cpnID, raw := range parserConfig {+ params, ok := raw.(map[string]any)+ if !ok {+ continue+ }++ cpnLower := strings.ToLower(cpnID)+ switch {+ case strings.HasPrefix(cpnLower, "extractor:") || strings.HasPrefix(cpnLower, "extractor_"):+ if value, _ := params["llm_id"].(string); strings.TrimSpace(value) == "" && strings.TrimSpace(llmID) != "" {+ params["llm_id"] = llmID+ }+ if enableMetadata && len(metadataFields) > 0 {+ params["enable_metadata"] = 1+ params["metadata"] = metadataFields+ } else if hasMetadataConfig {+ params["enable_metadata"] = 0+ params["metadata"] = []any{}+ }+ case strings.HasPrefix(cpnLower, "compiler:") || strings.HasPrefix(cpnLower, "compiler_"):+ if value, _ := params["llm_id"].(string); strings.TrimSpace(value) == "" && strings.TrimSpace(llmID) != "" {+ params["llm_id"] = llmID+ }+ }+ }++ return parserConfig+}++func mergeMetadataFields(parserConfig entity.JSONMap) []any {+ var out []any+ for _, key := range []string{"metadata", "built_in_metadata"} {+ for _, item := range anySlice(parserConfig[key]) {+ field, ok := item.(map[string]any)+ if !ok {+ continue+ }+ name, _ := field["key"].(string)+ if strings.TrimSpace(name) == "" {+ continue+ }+ out = append(out, field)+ }+ }+ return out+}++func anySlice(value any) []any {+ switch typed := value.(type) {+ case []any:+ return typed+ case []map[string]any:+ out := make([]any, 0, len(typed))+ for _, item := range typed {+ out = append(out, item)+ }+ return out+ default:+ return nil+ }+}++func hasMetadataConfigShape(parserConfig entity.JSONMap) bool {+ if parserConfig == nil {+ return false+ }+ if _, ok := parserConfig["enable_metadata"]; ok {+ return true+ }+ for _, key := range []string{"metadata", "built_in_metadata"} {+ if anySlice(parserConfig[key]) != nil {+ return true+ }+ }+ return false+}++func parserConfigTruthy(value any) bool {+ switch typed := value.(type) {+ case bool:+ return typed+ case string:+ switch typed {+ case "true", "True", "TRUE", "1":+ return true+ }+ case float64:+ return typed > 0+ case int:+ return typed > 0+ case int64:+ return typed > 0+ }+ return false+}diff --git a/internal/service/component_scoped_parser_config_test.go b/internal/service/component_scoped_parser_config_test.gonew file mode 100644index 0000000000..bce355d2be--- /dev/null+++ b/internal/service/component_scoped_parser_config_test.go@@ -0,0 +1,147 @@+package service++import (+ "reflect"+ "testing"++ "ragflow/internal/entity"+)++func TestApplyComponentScopedParserConfig_SyncsExtractorAndCompiler(t *testing.T) {+ parserConfig := entity.JSONMap{+ "enable_metadata": true,+ "metadata": []any{+ map[string]any{"key": "author", "type": "string"},+ },+ "built_in_metadata": []any{+ map[string]any{"key": "document_name", "type": "string"},+ },+ "Extractor:AutoExtractDefault": map[string]any{},+ "Compiler:KnownSwiftLions": map[string]any{},+ }++ got := ApplyComponentScopedParserConfig(+ parserConfig,+ "llm-default",+ )++ extractor := got["Extractor:AutoExtractDefault"].(map[string]any)+ if extractor["llm_id"] != "llm-default" {+ t.Fatalf("extractor llm_id = %#v, want llm-default", extractor["llm_id"])+ }+ if extractor["enable_metadata"] != 1 {+ t.Fatalf("extractor enable_metadata = %#v, want 1", extractor["enable_metadata"])+ }+ wantFields := []any{+ map[string]any{"key": "author", "type": "string"},+ map[string]any{"key": "document_name", "type": "string"},+ }+ if !reflect.DeepEqual(extractor["metadata"], wantFields) {+ t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantFields)+ }++ compiler := got["Compiler:KnownSwiftLions"].(map[string]any)+ if compiler["llm_id"] != "llm-default" {+ t.Fatalf("compiler llm_id = %#v, want llm-default", compiler["llm_id"])+ }+ if _, ok := compiler["embedding_model"]; ok {+ t.Fatalf("compiler embedding_model = %#v, want absent", compiler["embedding_model"])+ }+ if _, ok := compiler["tenant_id"]; ok {+ t.Fatalf("compiler tenant_id = %#v, want absent", compiler["tenant_id"])+ }+ if _, ok := compiler["dataset_id"]; ok {+ t.Fatalf("compiler dataset_id = %#v, want absent", compiler["dataset_id"])+ }+}++func TestApplyComponentScopedParserConfig_PreservesExplicitExtractorLLMID(t *testing.T) {+ parserConfig := entity.JSONMap{+ "Extractor:Custom": map[string]any{+ "llm_id": "custom-llm",+ },+ }++ got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")+ extractor := got["Extractor:Custom"].(map[string]any)+ if extractor["llm_id"] != "custom-llm" {+ t.Fatalf("extractor llm_id = %#v, want custom-llm", extractor["llm_id"])+ }+}++func TestApplyComponentScopedParserConfig_AcceptsTypedMetadataSlices(t *testing.T) {+ parserConfig := entity.JSONMap{+ "enable_metadata": true,+ "metadata": []map[string]interface{}{+ {"key": "author", "type": "string"},+ },+ "built_in_metadata": []map[string]interface{}{+ {"key": "document_name", "type": "string"},+ },+ "Extractor:AutoExtractDefault": map[string]any{},+ }++ got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")+ extractor := got["Extractor:AutoExtractDefault"].(map[string]any)++ wantFields := []any{+ map[string]interface{}{"key": "author", "type": "string"},+ map[string]interface{}{"key": "document_name", "type": "string"},+ }+ if !reflect.DeepEqual(extractor["metadata"], wantFields) {+ t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], wantFields)+ }+}++func TestApplyComponentScopedParserConfig_ClearsExtractorMetadataWhenDisabled(t *testing.T) {+ parserConfig := entity.JSONMap{+ "enable_metadata": false,+ "metadata": []map[string]interface{}{},+ "built_in_metadata": []map[string]interface{}{+ {"key": "document_name", "type": "string"},+ },+ "Extractor:AutoExtractDefault": map[string]any{+ "enable_metadata": 1,+ "metadata": []any{+ map[string]any{"key": "stale", "type": "string"},+ },+ },+ }++ got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")+ extractor := got["Extractor:AutoExtractDefault"].(map[string]any)++ if extractor["enable_metadata"] != 0 {+ t.Fatalf("extractor enable_metadata = %#v, want 0", extractor["enable_metadata"])+ }+ if !reflect.DeepEqual(extractor["metadata"], []any{}) {+ t.Fatalf("extractor metadata = %#v, want empty list", extractor["metadata"])+ }+}++func TestApplyComponentScopedParserConfig_DoesNotTreatDocumentMetadataValuesAsSchema(t *testing.T) {+ parserConfig := entity.JSONMap{+ "metadata": map[string]interface{}{+ "author": "Alice",+ },+ "Extractor:AutoExtractDefault": map[string]any{+ "enable_metadata": 1,+ "metadata": []any{+ map[string]any{"key": "author", "type": "string"},+ },+ },+ }++ got := ApplyComponentScopedParserConfig(parserConfig, "tenant-llm")+ extractor := got["Extractor:AutoExtractDefault"].(map[string]any)++ if extractor["enable_metadata"] != 1 {+ t.Fatalf("extractor enable_metadata = %#v, want 1", extractor["enable_metadata"])+ }+ want := []any{+ map[string]any{"key": "author", "type": "string"},+ }+ if !reflect.DeepEqual(extractor["metadata"], want) {+ t.Fatalf("extractor metadata = %#v, want %#v", extractor["metadata"], want)+ }+}diff --git a/internal/service/dataset/compilation_status.go b/internal/service/dataset/compilation_status.gonew file mode 100644index 0000000000..0c996d565b--- /dev/null+++ b/internal/service/dataset/compilation_status.go@@ -0,0 +1,84 @@+package dataset++import (+ "context"+ "encoding/json"+ "errors"+ "time"++ "gorm.io/gorm"++ "ragflow/internal/common"+ "ragflow/internal/dao"+ "ragflow/internal/entity"+)++// CompilationStatus is the dataset-level knowledge-compile lifecycle state+// surfaced by GET /datasets/:id/compilation/status. It is the Go scheduler+// contract that replaces the Python-era RunIndex/TraceIndex task progress for+// API_PROXY_SCHEME=go / hybrid.+//+// State only takes one of idle/pending/running/completed. Error is NOT a fifth+// state: it is a diagnostic attached to a pending/running batch left for retry,+// so the frontend should test `error != ""` on its own (and hide the counts)+// rather than treating it as a peer of state.+type CompilationStatus struct {+ State string `json:"state"` // idle | pending | running | completed+ Error string `json:"error,omitempty"` // most recent batch diagnostic (empty when none)+ Inflight int `json:"inflight"` // entries currently claimed (in-flight batch)+ Backlog int `json:"backlog"` // entries still waiting to be claimed+ LastCompletedAt *time.Time `json:"last_completed_at,omitempty"` // last backlog drain+ UpdatedAt time.Time `json:"updated_at"` // last scheduling-row activity+}++// GetDatasetCompilationStatus returns the scheduling-row lifecycle state for a+// dataset after verifying the calling user owns it. When no row exists the+// dataset has never had any compile work, so the state is idle.+func (d *DatasetService) GetDatasetCompilationStatus(ctx context.Context, userID, datasetID string) (CompilationStatus, common.ErrorCode, error) {+ if datasetID == "" {+ return CompilationStatus{}, common.CodeDataError, errors.New("dataset_id is required")+ }+ if !d.kbDAO.Accessible(ctx, dao.DB, datasetID, userID) {+ return CompilationStatus{}, common.CodeDataError, errors.New("no authorization")+ }+ st := CompilationStatus{State: entity.DatasetStateIdle}+ db := dao.GetDB()+ if db == nil {+ return st, common.CodeSuccess, nil+ }+ var row entity.KnowledgeCompileDataset+ err := db.WithContext(ctx).+ Where("dataset_id = ?", datasetID).+ First(&row).Error+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return st, common.CodeSuccess, nil // never compiled -> idle+ }+ if err != nil {+ return st, common.CodeServerError, err+ }+ st.State = row.State+ if st.State == "" {+ st.State = entity.DatasetStateIdle+ }+ st.Error = row.ErrorMsg+ st.Inflight = jsonArrayLen(row.InflightDocIDs)+ st.Backlog = jsonArrayLen(row.BacklogDocIDs)+ st.LastCompletedAt = row.LastCompletedAt+ st.UpdatedAt = row.UpdatedAt+ return st, common.CodeSuccess, nil+}++// jsonArrayLen counts the top-level elements of a JSON array stored as TEXT.+// The *_doc_ids columns hold a `[]BacklogEntry` array ({doc_id,event_type,seq}),+// so each element is one scheduling entry (NOT a deduplicated doc). Empty or+// malformed strings count as 0.+func jsonArrayLen(s string) int {+ if s == "" {+ return 0+ }+ var arr []json.RawMessage+ if err := json.Unmarshal([]byte(s), &arr); err != nil {+ return 0+ }+ return len(arr)+}diff --git a/internal/service/dataset/compilation_status_test.go b/internal/service/dataset/compilation_status_test.gonew file mode 100644index 0000000000..9b910fbc4b--- /dev/null+++ b/internal/service/dataset/compilation_status_test.go@@ -0,0 +1,197 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package dataset++import (+ "context"+ "testing"+ "time"++ "gorm.io/gorm"++ "ragflow/internal/common"+ "ragflow/internal/dao"+ "ragflow/internal/entity"+)++// TestJSONArrayLen locks the inflight/backlog count derivation (plan v4.1+// §9.3): counts are BacklogEntry array lengths, not deduplicated doc counts.+func TestJSONArrayLen(t *testing.T) {+ cases := []struct {+ name string+ in string+ want int+ }{+ {name: "empty", in: "", want: 0},+ {name: "empty array", in: "[]", want: 0},+ {name: "single entry", in: `[{"doc_id":"d1","event_type":"completed","seq":1}]`, want: 1},+ {name: "two entries same doc", in: `[{"doc_id":"d1","event_type":"completed","seq":1},{"doc_id":"d1","event_type":"deleted","seq":2}]`, want: 2},+ {name: "malformed", in: "{not json", want: 0},+ {name: "null", in: "null", want: 0},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ if got := jsonArrayLen(tc.in); got != tc.want {+ t.Fatalf("jsonArrayLen(%q) = %d, want %d", tc.in, got, tc.want)+ }+ })+ }+}++// setupCompilationStatusTestDB migrates the minimal schema for+// GetDatasetCompilationStatus (Knowledgebase for the Accessible check plus the+// KnowledgeCompileDataset scheduling row) and pushes it onto dao.DB.+func setupCompilationStatusTestDB(t *testing.T) *gorm.DB {+ t.Helper()+ db := setupServiceTestDB(t)+ if err := db.AutoMigrate(&entity.KnowledgeCompileDataset{}); err != nil {+ t.Fatalf("migrate knowledge_compile_docs: %v", err)+ }+ pushServiceDB(t, db)+ return db+}++// insertCompilationOwnerKB inserts a valid KB owned by userID (TenantID ==+// userID, so Accessible returns true).+func insertCompilationOwnerKB(t *testing.T, kbID, userID string) {+ t.Helper()+ status := string(entity.StatusValid)+ kb := &entity.Knowledgebase{+ ID: kbID,+ TenantID: userID,+ Name: "compile-status-kb",+ EmbdID: "BAAI/bge-large-zh-v1.5@Builtin",+ CreatedBy: userID,+ Permission: string(entity.TenantPermissionMe),+ Status: &status,+ }+ if err := dao.DB.Create(kb).Error; err != nil {+ t.Fatalf("insert kb: %v", err)+ }+}++func testCompilationStatusService() *DatasetService {+ return &DatasetService{kbDAO: dao.NewKnowledgebaseDAO()}+}++// TestGetDatasetCompilationStatus_NoRowIsIdle verifies a dataset with no+// scheduling row reports the idle state with zero counts.+func TestGetDatasetCompilationStatus_NoRowIsIdle(t *testing.T) {+ setupCompilationStatusTestDB(t)+ insertCompilationOwnerKB(t, "kb-no-row", "user-1")++ st, code, err := testCompilationStatusService().GetDatasetCompilationStatus(+ t.Context(), "user-1", "kb-no-row")+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("code=%d want %d", code, common.CodeSuccess)+ }+ if st.State != entity.DatasetStateIdle {+ t.Fatalf("state=%q want idle", st.State)+ }+ if st.Inflight != 0 || st.Backlog != 0 {+ t.Fatalf("expected zero counts for idle, got inflight=%d backlog=%d", st.Inflight, st.Backlog)+ }+ if st.Error != "" {+ t.Fatalf("expected empty error, got %q", st.Error)+ }+}++// TestGetDatasetCompilationStatus_FullOutput locks the complete response+// mapping from the MySQL row: state, inflight/backlog counts, error diagnostic+// and last_completed_at.+func TestGetDatasetCompilationStatus_FullOutput(t *testing.T) {+ db := setupCompilationStatusTestDB(t)+ insertCompilationOwnerKB(t, "kb-full", "user-1")++ // A running row with 2 inflight + 1 backlog entries and a retained error+ // diagnostic (error is NOT a fifth state: state stays running).+ lastDone := time.Now().Add(-time.Hour).UTC()+ row := entity.KnowledgeCompileDataset{+ DatasetID: "kb-full",+ TenantID: "user-1",+ BacklogDocIDs: `[{"doc_id":"d3","event_type":"completed","seq":3}]`,+ InflightDocIDs: `[{"doc_id":"d1","event_type":"completed","seq":1},{"doc_id":"d2","event_type":"completed","seq":2}]`,+ State: entity.DatasetStateRunning,+ ErrorMsg: "merge failed: boom",+ LastCompletedAt: &lastDone,+ }+ if err := db.Create(&row).Error; err != nil {+ t.Fatalf("insert scheduling row: %v", err)+ }++ st, code, err := testCompilationStatusService().GetDatasetCompilationStatus(+ t.Context(), "user-1", "kb-full")+ if err != nil {+ t.Fatalf("unexpected error: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("code=%d want %d", code, common.CodeSuccess)+ }+ if st.State != entity.DatasetStateRunning {+ t.Fatalf("state=%q want running", st.State)+ }+ if st.Inflight != 2 || st.Backlog != 1 {+ t.Fatalf("want inflight=2 backlog=1, got inflight=%d backlog=%d", st.Inflight, st.Backlog)+ }+ if st.Error != "merge failed: boom" {+ t.Fatalf("error=%q want %q", st.Error, "merge failed: boom")+ }+ if st.LastCompletedAt == nil || !st.LastCompletedAt.Equal(lastDone) {+ t.Fatalf("last_completed_at=%v want %v", st.LastCompletedAt, lastDone)+ }+}++// TestGetDatasetCompilationStatus_Unauthorized verifies a user who does not+// own the dataset is rejected before reading the scheduling row.+func TestGetDatasetCompilationStatus_Unauthorized(t *testing.T) {+ setupCompilationStatusTestDB(t)+ // Owner is user-1; a different user-2 must be denied.+ insertCompilationOwnerKB(t, "kb-other", "user-1")+ if err := dao.DB.Create(&entity.KnowledgeCompileDataset{+ DatasetID: "kb-other",+ TenantID: "user-1",+ BacklogDocIDs: "[]",+ InflightDocIDs: "[]",+ State: entity.DatasetStatePending,+ }).Error; err != nil {+ t.Fatalf("insert scheduling row: %v", err)+ }++ st, code, err := testCompilationStatusService().GetDatasetCompilationStatus(+ t.Context(), "user-2", "kb-other")+ if err == nil {+ t.Fatalf("expected authorization error, got nil (status=%+v)", st)+ }+ if code != common.CodeDataError {+ t.Fatalf("code=%d want %d", code, common.CodeDataError)+ }+}++// TestGetDatasetCompilationStatus_EmptyID validates the required-field guard.+func TestGetDatasetCompilationStatus_EmptyID(t *testing.T) {+ setupCompilationStatusTestDB(t)+ _, code, err := testCompilationStatusService().GetDatasetCompilationStatus(+ context.Background(), "user-1", "")+ if err == nil {+ t.Fatal("expected error for empty dataset_id")+ }+ if code != common.CodeDataError {+ t.Fatalf("code=%d want %d", code, common.CodeDataError)+ }+}diff --git a/internal/service/dataset/create_test.go b/internal/service/dataset/create_test.goindex 6132d0631d..4305065bb2 100644--- a/internal/service/dataset/create_test.go+++ b/internal/service/dataset/create_test.go@@ -85,6 +85,54 @@ func TestCreateDataset_ComponentParamsPopulated(t *testing.T) {if !ok || len(parserConfig) == 0 {t.Fatal("expected non-empty parser_config for general pipeline")}+ extractor, ok := parserConfig["Extractor:AutoExtractDefault"].(map[string]interface{})+ if !ok {+ t.Fatalf("expected extractor component params, got %#v", parserConfig["Extractor:AutoExtractDefault"])+ }+ if extractor["llm_id"] != "llm-default" {+ t.Fatalf("extractor llm_id = %#v, want llm-default", extractor["llm_id"])+ }+}++func TestCreateDataset_KnowledgeCompilerParamsPopulated(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertCreateDatasetTenant(t, "tenant-1")+ ctx := t.Context()++ parserID := "knowledge_compiler"+ parseType := 1+ result, code, err := testDatasetCreateService(t).CreateDataset(ctx, &service.CreateDatasetRequest{+ Name: "ds-kc-cp",+ ParserID: &parserID,+ ParseType: &parseType,+ }, "tenant-1")+ if err != nil {+ t.Fatalf("CreateDataset failed: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("expected success code, got %d", code)+ }+ parserConfig, ok := result["parser_config"].(entity.JSONMap)+ if !ok || len(parserConfig) == 0 {+ t.Fatal("expected non-empty parser_config for knowledge_compiler pipeline")+ }+ compiler, ok := parserConfig["Compiler:KnownSwiftLions"].(map[string]interface{})+ if !ok {+ t.Fatalf("expected compiler component params, got %#v", parserConfig["Compiler:KnownSwiftLions"])+ }+ if compiler["llm_id"] != "llm-default" {+ t.Fatalf("compiler llm_id = %#v, want llm-default", compiler["llm_id"])+ }+ if _, ok := compiler["embedding_model"]; ok {+ t.Fatalf("compiler embedding_model = %#v, want absent", compiler["embedding_model"])+ }+ if _, ok := compiler["tenant_id"]; ok {+ t.Fatalf("compiler tenant_id = %#v, want absent", compiler["tenant_id"])+ }+ if _, ok := compiler["dataset_id"]; ok {+ t.Fatalf("compiler dataset_id = %#v, want absent", compiler["dataset_id"])+ }}func TestCreateDataset_ParseTypeBuiltinClearsPipelineID(t *testing.T) {diff --git a/internal/service/dataset/crud.go b/internal/service/dataset/crud.goindex af10883cac..79d1081ac8 100644--- a/internal/service/dataset/crud.go+++ b/internal/service/dataset/crud.go@@ -121,6 +121,11 @@ func (d *DatasetService) CreateDataset(ctx context.Context, req *service.CreateD// unique within the tenant.name = d.dedupeDatasetName(ctx, name, tenantID)+ parserConfig = service.ApplyComponentScopedParserConfig(+ parserConfig,+ tenant.LLMID,+ )+kb := &entity.Knowledgebase{ID: kbID,Name: name,@@ -521,6 +526,21 @@ func stringPtrIfNotEmpty(s string) *string {}// extractDocIDs returns the document IDs from a slice of documents.+// datasetIndexTaskIDs returns the deduplicated set of dataset-level index task+// ids recorded on the KB (graphrag/raptor/mindmap legacy task fields). It is+// used by deleteDataset to clear residual entity.Task rows when a KB is deleted.+// Kept here because it belongs to the dataset delete lifecycle, not the retired+// RunIndex scheduling path.+func datasetIndexTaskIDs(kb *entity.Knowledgebase) []string {+ taskIDs := make([]string, 0, 3)+ for _, taskID := range []*string{kb.GraphragTaskID, kb.RaptorTaskID, kb.MindmapTaskID} {+ if taskID != nil && *taskID != "" {+ taskIDs = append(taskIDs, *taskID)+ }+ }+ return common.Deduplicate(taskIDs)+}+func extractDocIDs(docs []entity.Document) []string {ids := make([]string, 0, len(docs))for _, doc := range docs {diff --git a/internal/service/dataset/embedding.go b/internal/service/dataset/embedding.gonew file mode 100644index 0000000000..6149976640--- /dev/null+++ b/internal/service/dataset/embedding.go@@ -0,0 +1,285 @@+package dataset++import (+ "context"+ "errors"+ "fmt"+ "math/rand"+ "sort"+ "strings"++ "ragflow/internal/common"+ "ragflow/internal/dao"+ "ragflow/internal/entity"+ "ragflow/internal/entity/models"+ "ragflow/internal/service"++ enginetypes "ragflow/internal/engine/types"+)++// embeddingCheckSample is one sampled chunk with its stored vector, used by the+// embedding availability check.+type embeddingCheckSample struct {+ ChunkID string+ KbID string+ DocID string+ DocName string+ VectorField string+ Vector []float64+ PageNum interface{}+ Position interface{}+ Top interface{}+ ContentWithWeight string+ QuestionKeywords []string+}++// CheckEmbedding verifies that a candidate embedding model is compatible with a+// dataset's existing vectors (the standard "switch embedding model" validation).+// It is independent of the retired RunIndex/graph_rag_queue scheduling path.+func (d *DatasetService) CheckEmbedding(ctx context.Context, userID, datasetID string, req *service.CheckEmbeddingRequest) (*service.EmbeddingCheckResponse, common.ErrorCode, error) {+ if datasetID == "" {+ return nil, common.CodeDataError, errors.New(`lack of "Dataset ID"`)+ }+ if !d.kbDAO.Accessible(ctx, dao.DB, datasetID, userID) {+ return nil, common.CodeDataError, errors.New("no authorization")+ }++ kb, err := d.kbDAO.GetByID(ctx, dao.DB, datasetID)+ if err != nil {+ if dao.IsNotFoundErr(err) {+ return nil, common.CodeDataError, errors.New("invalid Dataset ID")+ }+ return nil, common.CodeServerError, errors.New("internal server error")+ }++ if req == nil || strings.TrimSpace(req.EmbeddingID) == "" {+ return nil, common.CodeDataError, errors.New("`embd_id` is required")+ }+ embeddingID := strings.TrimSpace(req.EmbeddingID)+ if d.docEngine == nil {+ return nil, common.CodeServerError, errors.New("doc engine not initialized")+ }++ driver, modelName, apiConfig, maxTokens, err := service.NewModelProviderService().ResolveModelConfig(ctx, kb.TenantID, entity.ModelTypeEmbedding, embeddingID)+ if err != nil {+ return nil, common.CodeDataError, err+ }+ embeddingModel := models.NewEmbeddingModel(driver, &modelName, apiConfig, maxTokens)++ checkNum := defaultEmbeddingCheckNum+ if req.CheckNum != nil {+ checkNum = *req.CheckNum+ }+ if checkNum <= 0 {+ checkNum = defaultEmbeddingCheckNum+ }++ samples, err := d.sampleRandomChunksWithVectors(ctx, kb.TenantID, datasetID, checkNum)+ if err != nil {+ return nil, common.CodeServerError, err+ }+ if len(samples) == 0 {+ return &service.EmbeddingCheckResponse{+ Summary: datasetEmbeddingCheckSummary(datasetID, embeddingID, 0, nil, ""),+ Results: nil,+ }, common.CodeSuccess, nil+ }++ results := make([]service.EmbeddingCheckResult, 0, len(samples))+ effectiveSimilarities := make([]float64, 0, len(samples))+ sawTitleAndContent := false+ for _, sample := range samples {+ if sample.Vector == nil || len(sample.Vector) == 0 {+ continue+ }++ rawChunk, err := d.docEngine.GetChunk(ctx, fmt.Sprintf("ragflow_%s", kb.TenantID), sample.ChunkID, []string{datasetID})+ if err != nil {+ continue+ }+ chunkMap := datasetMap(rawChunk)+ if len(chunkMap) == 0 {+ continue+ }++ title := datasetString(chunkMap["title_tks"])+ content := datasetString(chunkMap["content_ltks"])++ var titleVector [][]float64+ if title != "" {+ titleVector, err = datasetEncodeEmbedding(ctx, embeddingModel, []string{title})+ if err != nil {+ return nil, common.CodeServerError, err+ }+ }+ var contentVector [][]float64+ if content != "" {+ contentVector, err = datasetEncodeEmbedding(ctx, embeddingModel, []string{content})+ if err != nil {+ return nil, common.CodeServerError, err+ }+ }++ var vectors [][]float64+ if len(titleVector) > 0 && len(contentVector) > 0 {+ vectors = [][]float64{titleVector[0], contentVector[0]}+ sawTitleAndContent = true+ } else if len(titleVector) > 0 {+ vectors = titleVector+ } else if len(contentVector) > 0 {+ vectors = contentVector+ } else {+ continue+ }++ if len(vectors[0]) != len(sample.Vector) {+ return nil, common.CodeDataError, fmt.Errorf("Embedding failure. The dimension (%d) of given embedding model is different from the original (%d)", len(vectors[0]), len(sample.Vector))+ }++ var sim float64+ if len(vectors) == 2 {+ simContent := datasetCosSim(vectors[1], sample.Vector)+ simMix := datasetCosSim(datasetMixVectors(vectors[0], vectors[1], 0.1), sample.Vector)+ sim = simContent+ if simMix > sim {+ sim = simMix+ sawTitleAndContent = true+ }+ } else {+ sim = datasetCosSim(vectors[0], sample.Vector)+ }+ sim = datasetRoundFloat(sim, 6)++ effectiveSimilarities = append(effectiveSimilarities, sim)+ results = append(results, service.EmbeddingCheckResult{+ ChunkID: sample.ChunkID,+ DocID: sample.DocID,+ DocName: sample.DocName,+ VectorField: sample.VectorField,+ VectorDim: len(sample.Vector),+ CosSim: sim,+ })+ }++ // Aggregate the batch mode explicitly: title_and_content when any sample was+ // matched against the title+content mix, content_only otherwise.+ matchMode := "content_only"+ if sawTitleAndContent {+ matchMode = "title_and_content"+ }+ summary := datasetEmbeddingCheckSummary(datasetID, embeddingID, len(samples), effectiveSimilarities, matchMode)+ response := &service.EmbeddingCheckResponse{Summary: summary, Results: results}+ if len(effectiveSimilarities) == 0 {+ return nil, common.CodeDataError, errors.New("No embedded chunks are available to compare.")+ }+ if summary.AvgCosSim >= 0.9 {+ return response, common.CodeSuccess, nil+ }+ return response, common.CodeNotEffective, errors.New("Embedding model switch failed: the average similarity between old and new vectors is below 0.9, indicating incompatible vector spaces.")+}++func (d *DatasetService) sampleRandomChunksWithVectors(ctx context.Context, tenantID, datasetID string, n int) ([]embeddingCheckSample, error) {+ indexName := fmt.Sprintf("ragflow_%s", tenantID)+ totalResult, err := d.docEngine.Search(ctx, &enginetypes.SearchRequest{+ IndexNames: []string{indexName},+ KbIDs: []string{datasetID},+ Offset: 0,+ Limit: 1,+ Filter: map[string]interface{}{+ "kb_id": datasetID,+ "available_int": 1,+ },+ })+ if err != nil {+ return nil, err+ }+ if totalResult == nil || totalResult.Total <= 0 {+ return []embeddingCheckSample{}, nil+ }++ total := int(totalResult.Total)+ // Each sampled offset costs an engine Search + GetChunk plus up to two+ // provider calls, so bound the client-controlled sample count server-side.+ const maxEmbeddingSamples = 32+ if n < 0 {+ return nil, fmt.Errorf("invalid sample size: %d", n)+ }+ if n > maxEmbeddingSamples {+ n = maxEmbeddingSamples+ }+ if n > total {+ n = total+ }+ limit := total+ if limit > 1000 {+ limit = 1000+ }+ if n > limit {+ n = limit+ }+ offsets := rand.Perm(limit)+ offsets = offsets[:n]+ sort.Ints(offsets)++ baseFields := []string{"docnm_kwd", "doc_id", "content_with_weight", "page_num_int", "position_int", "top_int"}+ samples := make([]embeddingCheckSample, 0, n)+ for _, offset := range offsets {+ searchResult, err := d.docEngine.Search(ctx, &enginetypes.SearchRequest{+ IndexNames: []string{indexName},+ KbIDs: []string{datasetID},+ Offset: offset,+ Limit: 1,+ SelectFields: baseFields,+ Filter: map[string]interface{}{+ "kb_id": datasetID,+ "available_int": 1,+ },+ })+ if err != nil {+ return nil, err+ }+ if searchResult == nil || len(searchResult.Chunks) == 0 {+ continue+ }+ chunkID := datasetChunkID(searchResult.Chunks[0])+ if chunkID == "" {+ continue+ }+ fullChunk, err := d.docEngine.GetChunk(ctx, indexName, chunkID, []string{datasetID})+ if err != nil {+ return nil, err+ }+ chunkMap := datasetMap(fullChunk)+ if len(chunkMap) == 0 {+ continue+ }+ vectorField := datasetGuessVecField(chunkMap)+ vector := datasetAsFloatVec(chunkMap[vectorField])+ samples = append(samples, embeddingCheckSample{+ ChunkID: chunkID,+ KbID: datasetID,+ DocID: datasetString(chunkMap["doc_id"]),+ DocName: datasetString(chunkMap["docnm_kwd"]),+ VectorField: vectorField,+ Vector: vector,+ PageNum: chunkMap["page_num_int"],+ Position: chunkMap["position_int"],+ Top: chunkMap["top_int"],+ ContentWithWeight: datasetString(chunkMap["content_with_weight"]),+ QuestionKeywords: datasetStringSlice(chunkMap["question_keywords"]),+ })+ }++ if len(samples) == 0 {+ return nil, errors.New("no valid chunks with vectors found")+ }+ return samples, nil+}++func (d *DatasetService) verifyEmbeddingAvailability(ctx context.Context, embdID string, tenantID string) (bool, string) {+ _, _, _, _, err := service.NewModelProviderService().ResolveModelConfig(ctx, tenantID, entity.ModelTypeEmbedding, embdID)+ if err != nil {+ return false, err.Error()+ }+ return true, ""+}diff --git a/internal/service/dataset/helpers.go b/internal/service/dataset/helpers.goindex d1559b594d..1dee37048b 100644--- a/internal/service/dataset/helpers.go+++ b/internal/service/dataset/helpers.go@@ -269,6 +269,49 @@ func preserveDatasetParserConfigMetadata(next, existing entity.JSONMap, incomingreturn next}+func parserConfigJSONMap(value interface{}) entity.JSONMap {+ switch typed := value.(type) {+ case nil:+ return nil+ case entity.JSONMap:+ return typed+ case map[string]interface{}:+ return entity.JSONMap(typed)+ default:+ return nil+ }+}++func cloneJSONMap(source entity.JSONMap) entity.JSONMap {+ if source == nil {+ return nil+ }+ cloned := make(entity.JSONMap, len(source))+ for key, value := range source {+ cloned[key] = cloneJSONValue(value)+ }+ return cloned+}++func cloneJSONValue(value interface{}) interface{} {+ switch typed := value.(type) {+ case map[string]interface{}:+ nested := make(map[string]interface{}, len(typed))+ for key, item := range typed {+ nested[key] = cloneJSONValue(item)+ }+ return nested+ case []interface{}:+ nested := make([]interface{}, len(typed))+ for idx, item := range typed {+ nested[idx] = cloneJSONValue(item)+ }+ return nested+ default:+ return typed+ }+}+func normalizeDatasetUpdateExt(ext map[string]interface{}) map[string]interface{} {if ext == nil {return nildiff --git a/internal/service/dataset/index.go b/internal/service/dataset/index.godeleted file mode 100644index 8f0eda7ca7..0000000000--- a/internal/service/dataset/index.go+++ /dev/null@@ -1,743 +0,0 @@-package dataset--import (- "context"- "encoding/json"- "errors"- "fmt"- "math/rand"- "sort"- "strings"- "time"-- "ragflow/internal/common"- "ragflow/internal/dao"- redisengine "ragflow/internal/engine/redis"- enginetypes "ragflow/internal/engine/types"- "ragflow/internal/entity"- modelModule "ragflow/internal/entity/models"- "ragflow/internal/service"- "ragflow/internal/utility"-- "github.com/cespare/xxhash/v2"- "go.uber.org/zap"- "gorm.io/gorm"- "gorm.io/gorm/clause"-)--func checkType(indexType string) bool {- haveType := false- for _, t := range validIndexTypes {- if indexType == t {- haveType = true- }- }- return haveType-}--func (d *DatasetService) newRaptorOrGraphRagTask(ctx context.Context, sampleDoc *entity.Document, taskType string, taskDocID string, queueDocID string, docIDs []string) (*entity.Task, map[string]interface{}, error) {- if docIDs == nil || len(docIDs) == 0 {- docIDs = make([]string, 0)- }- if !checkIndexTaskType(taskType) {- return nil, nil, errors.New("type should be graphrag, raptor or mindmap")- }-- chunkingConfig, err := d.documentDAO.GetChunkingConfig(ctx, dao.DB, sampleDoc.ID)- if err != nil {- return nil, nil, err- }-- hasher := xxhash.New()- keys := make([]string, 0, len(chunkingConfig))- for key := range chunkingConfig {- keys = append(keys, key)- }- sort.Strings(keys)- for _, key := range keys {- _, _ = hasher.Write([]byte(key))- _, _ = hasher.Write([]byte{0})- v, mErr := json.Marshal(chunkingConfig[key])- if mErr != nil {- return nil, nil, mErr- }- _, _ = hasher.Write(v)- _, _ = hasher.Write([]byte{0})- }-- taskID := utility.GenerateUUID()- beginAt := time.Now().Truncate(time.Second)- progressMsg := beginAt.Format("15:04:05") + " created task " + taskType-- for _, field := range []interface{}{taskDocID, maximumTaskPageNumber, maximumTaskPageNumber, taskType} {- _, _ = hasher.Write([]byte(fmt.Sprint(field)))- }- digest := fmt.Sprintf("%016x", hasher.Sum64())- task := &entity.Task{- ID: taskID,- DocID: taskDocID,- FromPage: maximumTaskPageNumber,- ToPage: maximumTaskPageNumber,- TaskType: taskType,- ProgressMsg: &progressMsg,- BeginAt: &beginAt,- Digest: &digest,- }-- queueMessage := map[string]interface{}{- "id": taskID,- "doc_id": queueDocID,- "from_page": maximumTaskPageNumber,- "to_page": maximumTaskPageNumber,- "task_type": taskType,- "progress_msg": progressMsg,- "begin_at": beginAt.Format("2006-01-02 15:04:05"),- "digest": digest,- "doc_ids": docIDs,- }-- return task, queueMessage, nil-}--func createDatasetIndexTaskInTx(tx *gorm.DB, task *entity.Task, queueDocID string) (*entity.Document, error) {- if task == nil {- return nil, errors.New("task is required")- }- if err := tx.Create(task).Error; err != nil {- return nil, err- }-- if queueDocID == "" {- return nil, nil- }-- var document entity.Document- err := tx.Select("id", "progress_msg", "process_begin_at").Where("id = ?", queueDocID).First(&document).Error- if err != nil {- if errors.Is(err, gorm.ErrRecordNotFound) {- return nil, nil- }- return nil, err- }-- beginAt := time.Now().Truncate(time.Second)- if task.BeginAt != nil {- beginAt = *task.BeginAt- }- if err = tx.Model(&entity.Document{}).Where("id = ?", queueDocID).Updates(map[string]interface{}{- "progress_msg": "Task is queued...",- "process_begin_at": beginAt,- }).Error; err != nil {- return nil, err- }-- return &document, nil-}--func enqueueDatasetIndexTask(ctx context.Context, priority int, queueMessage map[string]interface{}) error {- redisClient := redisengine.Get()- if redisClient == nil || !redisClient.QueueProduct(ctx, datasetIndexQueueName(priority), queueMessage) {- return errors.New("can't access Redis. Please check the Redis' status")- }- return nil-}--func cleanupFailedDatasetIndexTask(taskID string, updatedDocument *entity.Document, kbID string, indexType string) error {- return dao.DB.Transaction(func(tx *gorm.DB) error {- if err := tx.Unscoped().Where("id = ?", taskID).Delete(&entity.Task{}).Error; err != nil {- return fmt.Errorf("delete task %s: %w", taskID, err)- }-- if column := datasetIndexTaskIDColumn(indexType); kbID != "" && column != "" {- if err := tx.Model(&entity.Knowledgebase{}).Where("id = ? AND "+column+" = ?", kbID, taskID).Update(column, nil).Error; err != nil {- return fmt.Errorf("clear dataset task id %s: %w", taskID, err)- }- }-- if updatedDocument == nil {- return nil- }-- return tx.Model(&entity.Document{}).Where("id = ?", updatedDocument.ID).Updates(map[string]interface{}{- "progress_msg": updatedDocument.ProgressMsg,- "process_begin_at": updatedDocument.ProcessBeginAt,- }).Error- })-}--func datasetIndexTaskIDColumn(indexType string) string {- switch indexType {- case "graph":- return "graphrag_task_id"- case "raptor":- return "raptor_task_id"- case "mindmap":- return "mindmap_task_id"- default:- return ""- }-}--func datasetIndexTaskFinishAtColumn(indexType string) string {- switch indexType {- case "graph":- return "graphrag_task_finish_at"- case "raptor":- return "raptor_task_finish_at"- case "mindmap":- return "mindmap_task_finish_at"- default:- return ""- }-}--func checkIndexTaskType(taskType string) bool {- switch taskType {- case "graphrag", "raptor", "mindmap":- return true- default:- return false- }-}--func datasetIndexTaskID(kb *entity.Knowledgebase, indexType string) string {- if kb == nil {- return ""- }- switch indexType {- case "graph":- if kb.GraphragTaskID != nil {- return *kb.GraphragTaskID- }- case "raptor":- if kb.RaptorTaskID != nil {- return *kb.RaptorTaskID- }- case "mindmap":- if kb.MindmapTaskID != nil {- return *kb.MindmapTaskID- }- }- return ""-}--func datasetIndexTaskIDUpdate(indexType, taskID string) map[string]interface{} {- switch indexType {- case "graph":- return map[string]interface{}{"graphrag_task_id": taskID}- case "raptor":- return map[string]interface{}{"raptor_task_id": taskID}- case "mindmap":- return map[string]interface{}{"mindmap_task_id": taskID}- default:- return map[string]interface{}{}- }-}--func datasetIndexTaskIDs(kb *entity.Knowledgebase) []string {- if kb == nil {- return nil- }- taskIDs := make([]string, 0, 3)- for _, taskID := range []*string{kb.GraphragTaskID, kb.RaptorTaskID, kb.MindmapTaskID} {- if taskID != nil && *taskID != "" {- taskIDs = append(taskIDs, *taskID)- }- }- return common.Deduplicate(taskIDs)-}--func datasetIndexQueueName(priority int) string {- return fmt.Sprintf("%s.%d.common", serverQueueNamePrefix, priority)-}--func clearGraphPhaseMarkers(ctx context.Context, redisClient *redisengine.Client, datasetID string) {- if redisClient == nil || datasetID == "" {- return- }- for _, phase := range []string{graphPhaseResolutionDone, graphPhaseCommunityDone} {- if !redisClient.Delete(ctx, fmt.Sprintf("graphrag:phase:%s:%s", datasetID, phase)) {- common.Warn("Failed to clear GraphRAG phase marker", zap.String("dataset_id", datasetID), zap.String("phase", phase))- }- }-}--func (d *DatasetService) RunIndex(ctx context.Context, userID, datasetID, indexType string) (map[string]interface{}, common.ErrorCode, error) {- if !checkType(indexType) {- return nil, common.CodeDataError, fmt.Errorf("invalid index type '%s'. Must be one of %v", indexType, validIndexTypes)- }-- if datasetID == "" {- return nil, common.CodeDataError, errors.New(`lack of "Dataset ID"`)- }- if !d.kbDAO.Accessible(ctx, dao.DB, datasetID, userID) {- return nil, common.CodeDataError, errors.New("no authorization")- }-- kb, err := d.kbDAO.GetByID(ctx, dao.DB, datasetID)- if err != nil {- if dao.IsNotFoundErr(err) {- return nil, common.CodeDataError, errors.New("invalid Dataset ID")- }- return nil, common.CodeDataError, errors.New("internal server error")- }-- taskType := indexTypeToTaskType[indexType]- displayName := indexTypeToDisplayName[indexType]-- documents, code, err := d.getDocumentsByDatasetForIndex(ctx, datasetID)- if err != nil {- return nil, code, err- }- _ = documents-- sampleDocument := documents[0]- documentIDs := make([]string, len(documents))-- for i, doc := range documents {- documentIDs[i] = doc.ID- }-- task, queueMessage, err := d.newRaptorOrGraphRagTask(ctx, sampleDocument, taskType, sampleDocument.ID, graphRaptorQueueDocID, documentIDs)- if err != nil {- common.Warn("Failed to build dataset index task", zap.String("dataset_id", datasetID), zap.String("task_type", taskType), zap.Error(err))- return nil, common.CodeDataError, errors.New("internal server error")- }-- var updatedDocument *entity.Document- var dataErr error- err = dao.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- var lockedKB entity.Knowledgebase- if err = tx.Clauses(clause.Locking{Strength: "UPDATE"}).- Where("id = ? AND status = ?", kb.ID, string(entity.StatusValid)).- First(&lockedKB).Error; err != nil {- return err- }-- existingTaskID := datasetIndexTaskID(&lockedKB, indexType)- if existingTaskID != "" {- var existingTask entity.Task- taskErr := tx.Where("id = ?", existingTaskID).First(&existingTask).Error- if taskErr != nil {- if errors.Is(taskErr, gorm.ErrRecordNotFound) {- } else {- return taskErr- }- } else if existingTask.Progress != 1 && existingTask.Progress != -1 {- dataErr = fmt.Errorf("task %s in progress with status %v. A %s Task is already running", existingTaskID, existingTask.Progress, displayName)- return dataErr- }- }-- updatedDocument, err = createDatasetIndexTaskInTx(tx, task, graphRaptorQueueDocID)- if err != nil {- return err- }- return tx.Model(&entity.Knowledgebase{}).Where("id = ?", lockedKB.ID).Updates(datasetIndexTaskIDUpdate(indexType, task.ID)).Error- })- if err != nil {- if dataErr != nil {- return nil, common.CodeDataError, dataErr- }- common.Warn("Failed to create dataset index task", zap.String("dataset_id", datasetID), zap.String("task_type", taskType), zap.Error(err))- return nil, common.CodeDataError, errors.New("internal server error")- }-- if err = enqueueDatasetIndexTask(ctx, 0, queueMessage); err != nil {- if cleanupErr := cleanupFailedDatasetIndexTask(task.ID, updatedDocument, kb.ID, indexType); cleanupErr != nil {- err = errors.Join(err, cleanupErr)- }- common.Warn("Failed to queue dataset index task", zap.String("dataset_id", datasetID), zap.String("task_type", taskType), zap.Error(err))- return nil, common.CodeDataError, errors.New("internal server error")- }-- return map[string]interface{}{"task_id": task.ID}, common.CodeSuccess, nil-}--func (d *DatasetService) getDocumentsByDatasetForIndex(ctx context.Context, datasetID string) ([]*entity.Document, common.ErrorCode, error) {- documents, _, err := d.documentDAO.GetByKBID(ctx, dao.DB, datasetID)- if err != nil {- common.Warn("Failed to load dataset documents for index", zap.String("dataset_id", datasetID), zap.Error(err))- return nil, common.CodeDataError, errors.New("internal server error")- }- if len(documents) == 0 {- return nil, common.CodeDataError, fmt.Errorf("no documents in Dataset %s", datasetID)- }- return documents, common.CodeSuccess, nil-}--func (d *DatasetService) TraceIndex(ctx context.Context, datasetID, userID, indexType string) (*entity.Task, common.ErrorCode, error) {- if !checkType(indexType) {- return nil, common.CodeDataError, fmt.Errorf("invalid index type '%s'. Must be one of %v", indexType, validIndexTypes)- }-- if datasetID == "" {- return nil, common.CodeDataError, errors.New(`lack of "Dataset ID"`)- }- if !d.kbDAO.Accessible(ctx, dao.DB, datasetID, userID) {- return nil, common.CodeDataError, errors.New("no authorization")- }-- kb, err := d.kbDAO.GetByID(ctx, dao.DB, datasetID)- if err != nil {- if dao.IsNotFoundErr(err) {- return nil, common.CodeDataError, errors.New("invalid Dataset ID")- }- return nil, common.CodeDataError, errors.New("internal server error")- }-- taskID := datasetIndexTaskID(kb, indexType)-- var task *entity.Task- if taskID != "" {- task, err = d.taskDAO.GetByID(ctx, dao.DB, taskID)- if err != nil {- if dao.IsNotFoundErr(err) {- return nil, common.CodeSuccess, nil- }- return nil, common.CodeServerError, errors.New("internal server error")- }- if task == nil {- return nil, common.CodeSuccess, nil- }- }-- return task, common.CodeSuccess, nil-}--type embeddingCheckSample struct {- ChunkID string- KbID string- DocID string- DocName string- VectorField string- Vector []float64- PageNum interface{}- Position interface{}- Top interface{}- ContentWithWeight string- QuestionKeywords []string-}--func (d *DatasetService) CheckEmbedding(ctx context.Context, userID, datasetID string, req *service.CheckEmbeddingRequest) (*service.EmbeddingCheckResponse, common.ErrorCode, error) {- if datasetID == "" {- return nil, common.CodeDataError, errors.New(`lack of "Dataset ID"`)- }- if !d.kbDAO.Accessible(ctx, dao.DB, datasetID, userID) {- return nil, common.CodeDataError, errors.New("no authorization")- }-- kb, err := d.kbDAO.GetByID(ctx, dao.DB, datasetID)- if err != nil {- if dao.IsNotFoundErr(err) {- return nil, common.CodeDataError, errors.New("invalid Dataset ID")- }- return nil, common.CodeServerError, errors.New("internal server error")- }-- if req == nil || strings.TrimSpace(req.EmbeddingID) == "" {- return nil, common.CodeDataError, errors.New("`embd_id` is required")- }- embeddingID := strings.TrimSpace(req.EmbeddingID)- if ok, message := d.verifyEmbeddingAvailability(ctx, embeddingID, kb.TenantID); !ok {- return nil, common.CodeDataError, errors.New(message)- }- if d.docEngine == nil {- return nil, common.CodeServerError, errors.New("doc engine not initialized")- }-- driver, modelName, apiConfig, maxTokens, err := service.NewModelProviderService().ResolveModelConfig(ctx, kb.TenantID, entity.ModelTypeEmbedding, embeddingID)- if err != nil {- return nil, common.CodeDataError, err- }- embeddingModel := modelModule.NewEmbeddingModel(driver, &modelName, apiConfig, maxTokens)-- checkNum := defaultEmbeddingCheckNum- if req.CheckNum != nil {- checkNum = *req.CheckNum- }- if checkNum <= 0 {- checkNum = defaultEmbeddingCheckNum- }-- samples, err := d.sampleRandomChunksWithVectors(ctx, kb.TenantID, datasetID, checkNum)- if err != nil {- return nil, common.CodeServerError, err- }- if len(samples) == 0 {- return &service.EmbeddingCheckResponse{- Summary: datasetEmbeddingCheckSummary(datasetID, embeddingID, 0, nil, ""),- Results: nil,- }, common.CodeSuccess, nil- }-- results := make([]service.EmbeddingCheckResult, 0, len(samples))- effectiveSimilarities := make([]float64, 0, len(samples))- matchMode := "content_only"- for _, sample := range samples {- if sample.Vector == nil || len(sample.Vector) == 0 {- continue- }-- rawChunk, err := d.docEngine.GetChunk(ctx, fmt.Sprintf("ragflow_%s", kb.TenantID), sample.ChunkID, []string{datasetID})- if err != nil {- continue- }- chunkMap := datasetMap(rawChunk)- if len(chunkMap) == 0 {- continue- }-- title := datasetString(chunkMap["title_tks"])- content := datasetString(chunkMap["content_ltks"])-- var titleVector [][]float64- if title != "" {- titleVector, err = datasetEncodeEmbedding(ctx, embeddingModel, []string{title})- if err != nil {- return nil, common.CodeServerError, err- }- }- var contentVector [][]float64- if content != "" {- contentVector, err = datasetEncodeEmbedding(ctx, embeddingModel, []string{content})- if err != nil {- return nil, common.CodeServerError, err- }- }-- var vectors [][]float64- if len(titleVector) > 0 && len(contentVector) > 0 {- vectors = [][]float64{titleVector[0], contentVector[0]}- matchMode = "title_and_content"- } else if len(titleVector) > 0 {- vectors = titleVector- } else if len(contentVector) > 0 {- vectors = contentVector- } else {- continue- }-- if len(vectors[0]) != len(sample.Vector) {- return nil, common.CodeDataError, fmt.Errorf("Embedding failure. The dimension (%d) of given embedding model is different from the original (%d)", len(vectors[0]), len(sample.Vector))- }-- var sim float64- if len(vectors) == 2 {- simContent := datasetCosSim(vectors[1], sample.Vector)- simMix := datasetCosSim(datasetMixVectors(vectors[0], vectors[1], 0.1), sample.Vector)- sim = simContent- if simMix > sim {- sim = simMix- matchMode = "title+content"- }- } else {- sim = datasetCosSim(vectors[0], sample.Vector)- }- sim = datasetRoundFloat(sim, 6)-- effectiveSimilarities = append(effectiveSimilarities, sim)- results = append(results, service.EmbeddingCheckResult{- ChunkID: sample.ChunkID,- DocID: sample.DocID,- DocName: sample.DocName,- VectorField: sample.VectorField,- VectorDim: len(sample.Vector),- CosSim: sim,- })- }-- summary := datasetEmbeddingCheckSummary(datasetID, embeddingID, len(samples), effectiveSimilarities, matchMode)- response := &service.EmbeddingCheckResponse{Summary: summary, Results: results}- if len(effectiveSimilarities) == 0 {- return nil, common.CodeDataError, errors.New("No embedded chunks are available to compare.")- }- if summary.AvgCosSim >= 0.9 {- return response, common.CodeSuccess, nil- }- return response, common.CodeNotEffective, errors.New("Embedding model switch failed: the average similarity between old and new vectors is below 0.9, indicating incompatible vector spaces.")-}--func (d *DatasetService) sampleRandomChunksWithVectors(ctx context.Context, tenantID, datasetID string, n int) ([]embeddingCheckSample, error) {- indexName := fmt.Sprintf("ragflow_%s", tenantID)- totalResult, err := d.docEngine.Search(ctx, &enginetypes.SearchRequest{- IndexNames: []string{indexName},- KbIDs: []string{datasetID},- Offset: 0,- Limit: 1,- Filter: map[string]interface{}{- "kb_id": datasetID,- "available_int": 1,- },- })- if err != nil {- return nil, err- }- if totalResult == nil || totalResult.Total <= 0 {- return []embeddingCheckSample{}, nil- }-- total := int(totalResult.Total)- const maxEmbeddingSamples = 1024- if n < 0 {- return nil, fmt.Errorf("invalid sample size: %d", n)- }- if n > maxEmbeddingSamples {- n = maxEmbeddingSamples- }- if n > total {- n = total- }- limit := total- if limit > 1000 {- limit = 1000- }- if n > limit {- n = limit- }- offsets := rand.Perm(limit)- offsets = offsets[:n]- sort.Ints(offsets)-- baseFields := []string{"docnm_kwd", "doc_id", "content_with_weight", "page_num_int", "position_int", "top_int"}- samples := make([]embeddingCheckSample, 0, n)- for _, offset := range offsets {- searchResult, err := d.docEngine.Search(ctx, &enginetypes.SearchRequest{- IndexNames: []string{indexName},- KbIDs: []string{datasetID},- Offset: offset,- Limit: 1,- SelectFields: baseFields,- Filter: map[string]interface{}{- "kb_id": datasetID,- "available_int": 1,- },- })- if err != nil {- return nil, err- }- if searchResult == nil || len(searchResult.Chunks) == 0 {- continue- }- chunkID := datasetChunkID(searchResult.Chunks[0])- if chunkID == "" {- continue- }- fullChunk, err := d.docEngine.GetChunk(ctx, indexName, chunkID, []string{datasetID})- if err != nil {- return nil, err- }- chunkMap := datasetMap(fullChunk)- if len(chunkMap) == 0 {- continue- }- vectorField := datasetGuessVecField(chunkMap)- vector := datasetAsFloatVec(chunkMap[vectorField])- samples = append(samples, embeddingCheckSample{- ChunkID: chunkID,- KbID: datasetID,- DocID: datasetString(chunkMap["doc_id"]),- DocName: datasetString(chunkMap["docnm_kwd"]),- VectorField: vectorField,- Vector: vector,- PageNum: chunkMap["page_num_int"],- Position: chunkMap["position_int"],- Top: chunkMap["top_int"],- ContentWithWeight: datasetString(chunkMap["content_with_weight"]),- QuestionKeywords: datasetStringSlice(chunkMap["question_keywords"]),- })- }-- if len(samples) == 0 {- return nil, errors.New("no valid chunks with vectors found")- }- return samples, nil-}--func (d *DatasetService) verifyEmbeddingAvailability(ctx context.Context, embdID string, tenantID string) (bool, string) {- _, _, _, _, err := service.NewModelProviderService().ResolveModelConfig(ctx, tenantID, entity.ModelTypeEmbedding, embdID)- if err != nil {- return false, err.Error()- }- return true, ""-}--func (d *DatasetService) DeleteIndex(ctx context.Context, userID, datasetID, indexType string, wipe bool) (common.ErrorCode, error) {- if !checkType(indexType) {- return common.CodeArgumentError, fmt.Errorf("invalid index type '%s'", indexType)- }-- if datasetID == "" {- return common.CodeDataError, errors.New(`lack of "Dataset ID"`)- }-- if !d.kbDAO.Accessible(ctx, dao.DB, datasetID, userID) {- return common.CodeDataError, errors.New("no authorization")- }-- kb, err := d.kbDAO.GetByID(ctx, dao.DB, datasetID)- if err != nil {- if dao.IsNotFoundErr(err) {- return common.CodeDataError, errors.New("invalid Dataset ID")- }- return common.CodeDataError, errors.New("internal server error")- }-- taskFinishAtField := datasetIndexTaskFinishAtColumn(indexType)- taskID := datasetIndexTaskID(kb, indexType)-- common.Info("delete_index", zap.String("dataset_id", datasetID), zap.String("index_type", indexType), zap.Bool("wipe", wipe))-- if taskID != "" {- redisClient := redisengine.Get()- if redisClient == nil || !redisClient.Set(ctx, fmt.Sprintf("%s-cancel", taskID), "x", time.Hour) {- common.Warn("Failed to set dataset index cancellation marker", zap.String("dataset_id", datasetID), zap.String("task_id", taskID))- }- if err := dao.DB.Unscoped().Where("id = ?", taskID).Delete(&entity.Task{}).Error; err != nil {- common.Warn("Failed to delete dataset index task", zap.String("dataset_id", datasetID), zap.String("task_id", taskID), zap.Error(err))- return common.CodeDataError, errors.New("internal server error")- }- }-- if wipe && indexType == "graph" {- if d.docEngine == nil {- return common.CodeServerError, errors.New("document engine is not initialized")- }- indexName := fmt.Sprintf("ragflow_%s", kb.TenantID)- _, err = d.docEngine.DeleteChunks(ctx, map[string]interface{}{- "knowledge_graph_kwd": []interface{}{"graph", "subgraph", "entity", "relation", "community_report"},- "kb_id": datasetID,- }, indexName, datasetID)- if err != nil {- common.Warn("Failed to delete GraphRAG artefacts", zap.String("dataset_id", datasetID), zap.Error(err))- return common.CodeDataError, errors.New("internal server error")- }- clearGraphPhaseMarkers(ctx, redisengine.Get(), datasetID)- common.Info("delete_index: cleared GraphRAG artefacts and phase markers", zap.String("dataset_id", datasetID))- } else if wipe && indexType == "raptor" {- if d.docEngine == nil {- return common.CodeServerError, errors.New("document engine is not initialized")- }- indexName := fmt.Sprintf("ragflow_%s", kb.TenantID)- _, err = d.docEngine.DeleteChunks(ctx, map[string]interface{}{- "raptor_kwd": []interface{}{"raptor"},- "kb_id": datasetID,- }, indexName, datasetID)- if err != nil {- common.Warn("Failed to delete RAPTOR artefacts", zap.String("dataset_id", datasetID), zap.Error(err))- return common.CodeDataError, errors.New("internal server error")- }- }-- updates := datasetIndexTaskIDUpdate(indexType, "")- if taskFinishAtField != "" {- updates[taskFinishAtField] = nil- }- if len(updates) > 0 {- if err = d.kbDAO.UpdateByID(ctx, dao.DB, kb.ID, updates); err != nil {- common.Warn("Failed to clear KB index task refs", zap.String("dataset_id", datasetID), zap.Error(err))- }- }-- return common.CodeSuccess, nil-}diff --git a/internal/service/dataset/index_delete_test.go b/internal/service/dataset/index_delete_test.godeleted file mode 100644index a4bb1cbdae..0000000000--- a/internal/service/dataset/index_delete_test.go+++ /dev/null@@ -1,272 +0,0 @@-//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.-//--package dataset--import (- "context"- "errors"- "testing"- "time"-- "gorm.io/gorm"-- "ragflow/internal/common"- "ragflow/internal/dao"- "ragflow/internal/entity"-)--type deleteIndexDocEngine struct {- fakeChatDocEngine- deleteCalls []deleteIndexDocEngineCall-}--type deleteIndexDocEngineCall struct {- condition map[string]interface{}- indexName string- datasetID string-}--func (e *deleteIndexDocEngine) DeleteChunks(_ context.Context, condition map[string]interface{}, indexName string, datasetID string) (int64, error) {- e.deleteCalls = append(e.deleteCalls, deleteIndexDocEngineCall{- condition: condition,- indexName: indexName,- datasetID: datasetID,- })- return 1, nil-}--func testDatasetServiceForDeleteIndex(docEngine *deleteIndexDocEngine) *DatasetService {- return &DatasetService{- kbDAO: dao.NewKnowledgebaseDAO(),- taskDAO: dao.NewTaskDAO(),- docEngine: docEngine,- }-}--func insertDeleteIndexKB(t *testing.T, indexType string, taskID string) {- t.Helper()-- finishAt := time.Date(2026, 6, 23, 10, 0, 0, 0, time.UTC)- kb := &entity.Knowledgebase{- ID: "kb-1",- TenantID: "user-1",- Name: "test-kb",- EmbdID: "embedding@OpenAI",- CreatedBy: "user-1",- Permission: string(entity.TenantPermissionMe),- ParserID: "naive",- ParserConfig: entity.JSONMap{},- Status: sptr("1"),- }-- switch indexType {- case "graph":- kb.GraphragTaskID = &taskID- kb.GraphragTaskFinishAt = &finishAt- case "raptor":- kb.RaptorTaskID = &taskID- kb.RaptorTaskFinishAt = &finishAt- case "mindmap":- kb.MindmapTaskID = &taskID- kb.MindmapTaskFinishAt = &finishAt- }-- if err := dao.DB.Create(kb).Error; err != nil {- t.Fatalf("insert kb: %v", err)- }- if taskID != "" {- if err := dao.DB.Create(&entity.Task{ID: taskID, DocID: "doc-1", TaskType: indexTypeToTaskType[indexType]}).Error; err != nil {- t.Fatalf("insert task: %v", err)- }- }-}--func TestDatasetServiceDeleteIndexGraphWipeFalseOnlyCancelsTask(t *testing.T) {- db := setupServiceTestDB(t)- pushServiceDB(t, db)- insertDeleteIndexKB(t, "graph", "graph-task")- ctx := t.Context()-- docEngine := &deleteIndexDocEngine{}- code, err := testDatasetServiceForDeleteIndex(docEngine).DeleteIndex(ctx, "user-1", "kb-1", "graph", false)- if err != nil {- t.Fatalf("DeleteIndex failed: %v", err)- }- if code != common.CodeSuccess {- t.Fatalf("expected success code, got %d", code)- }- if len(docEngine.deleteCalls) != 0 {- t.Fatalf("wipe=false should not delete doc-store artefacts, got %#v", docEngine.deleteCalls)- }-- assertDeleteIndexTaskDeleted(t, "graph-task")- kb := getDeleteIndexKB(t)- if kb.GraphragTaskID == nil || *kb.GraphragTaskID != "" {- t.Fatalf("expected graphrag_task_id to be cleared to empty string, got %#v", kb.GraphragTaskID)- }- if kb.GraphragTaskFinishAt != nil {- t.Fatalf("expected graphrag_task_finish_at to be cleared, got %#v", kb.GraphragTaskFinishAt)- }-}--func TestDatasetServiceDeleteIndexGraphWipeTrueDeletesArtefacts(t *testing.T) {- db := setupServiceTestDB(t)- pushServiceDB(t, db)- insertDeleteIndexKB(t, "graph", "graph-task")- ctx := t.Context()-- docEngine := &deleteIndexDocEngine{}- code, err := testDatasetServiceForDeleteIndex(docEngine).DeleteIndex(ctx, "user-1", "kb-1", "graph", true)- if err != nil {- t.Fatalf("DeleteIndex failed: %v", err)- }- if code != common.CodeSuccess {- t.Fatalf("expected success code, got %d", code)- }- if len(docEngine.deleteCalls) != 1 {- t.Fatalf("expected one doc-store delete call, got %#v", docEngine.deleteCalls)- }-- call := docEngine.deleteCalls[0]- if call.indexName != "ragflow_user-1" || call.datasetID != "kb-1" {- t.Fatalf("unexpected delete target: %#v", call)- }- if call.condition["kb_id"] != "kb-1" {- t.Fatalf("delete condition must include kb_id, got %#v", call.condition)- }- assertStringSet(t, call.condition["knowledge_graph_kwd"], []string{"graph", "subgraph", "entity", "relation", "community_report"})- assertDeleteIndexTaskDeleted(t, "graph-task")-}--func TestDatasetServiceDeleteIndexRaptorWipeTrueDeletesRaptorArtefacts(t *testing.T) {- db := setupServiceTestDB(t)- pushServiceDB(t, db)- insertDeleteIndexKB(t, "raptor", "raptor-task")- ctx := t.Context()-- docEngine := &deleteIndexDocEngine{}- code, err := testDatasetServiceForDeleteIndex(docEngine).DeleteIndex(ctx, "user-1", "kb-1", "raptor", true)- if err != nil {- t.Fatalf("DeleteIndex failed: %v", err)- }- if code != common.CodeSuccess {- t.Fatalf("expected success code, got %d", code)- }- if len(docEngine.deleteCalls) != 1 {- t.Fatalf("expected one doc-store delete call, got %#v", docEngine.deleteCalls)- }- call := docEngine.deleteCalls[0]- if call.condition["kb_id"] != "kb-1" {- t.Fatalf("delete condition must include kb_id, got %#v", call.condition)- }- assertStringSet(t, call.condition["raptor_kwd"], []string{"raptor"})- assertDeleteIndexTaskDeleted(t, "raptor-task")-- kb := getDeleteIndexKB(t)- if kb.RaptorTaskID == nil || *kb.RaptorTaskID != "" {- t.Fatalf("expected raptor_task_id to be cleared to empty string, got %#v", kb.RaptorTaskID)- }- if kb.RaptorTaskFinishAt != nil {- t.Fatalf("expected raptor_task_finish_at to be cleared, got %#v", kb.RaptorTaskFinishAt)- }-}--func TestDatasetServiceDeleteIndexMindmapDoesNotDeleteDocStore(t *testing.T) {- db := setupServiceTestDB(t)- pushServiceDB(t, db)- insertDeleteIndexKB(t, "mindmap", "mindmap-task")- ctx := t.Context()-- docEngine := &deleteIndexDocEngine{}- code, err := testDatasetServiceForDeleteIndex(docEngine).DeleteIndex(ctx, "user-1", "kb-1", "mindmap", true)- if err != nil {- t.Fatalf("DeleteIndex failed: %v", err)- }- if code != common.CodeSuccess {- t.Fatalf("expected success code, got %d", code)- }- if len(docEngine.deleteCalls) != 0 {- t.Fatalf("mindmap delete should not delete doc-store artefacts, got %#v", docEngine.deleteCalls)- }- assertDeleteIndexTaskDeleted(t, "mindmap-task")-- kb := getDeleteIndexKB(t)- if kb.MindmapTaskID == nil || *kb.MindmapTaskID != "" {- t.Fatalf("expected mindmap_task_id to be cleared to empty string, got %#v", kb.MindmapTaskID)- }- if kb.MindmapTaskFinishAt != nil {- t.Fatalf("expected mindmap_task_finish_at to be cleared, got %#v", kb.MindmapTaskFinishAt)- }-}--func TestDatasetServiceDeleteIndexRejectsInvalidType(t *testing.T) {- db := setupServiceTestDB(t)- pushServiceDB(t, db)- ctx := t.Context()-- code, err := testDatasetServiceForDeleteIndex(&deleteIndexDocEngine{}).DeleteIndex(ctx, "user-1", "kb-1", "invalid", true)- if err == nil {- t.Fatal("expected invalid index type error")- }- if code != common.CodeArgumentError {- t.Fatalf("expected argument error code, got %d", code)- }-}--func assertDeleteIndexTaskDeleted(t *testing.T, taskID string) {- t.Helper()- var task entity.Task- err := dao.DB.Where("id = ?", taskID).First(&task).Error- if !errors.Is(err, gorm.ErrRecordNotFound) {- t.Fatalf("expected task %s to be deleted, got err=%v task=%#v", taskID, err, task)- }-}--func getDeleteIndexKB(t *testing.T) entity.Knowledgebase {- t.Helper()- var kb entity.Knowledgebase- if err := dao.DB.Where("id = ?", "kb-1").First(&kb).Error; err != nil {- t.Fatalf("fetch kb: %v", err)- }- return kb-}--func assertStringSet(t *testing.T, actual interface{}, expected []string) {- t.Helper()-- items, ok := actual.([]interface{})- if !ok {- t.Fatalf("expected []interface{}, got %#v", actual)- }- if len(items) != len(expected) {- t.Fatalf("expected %d items, got %#v", len(expected), items)- }-- seen := make(map[string]bool, len(items))- for _, item := range items {- value, ok := item.(string)- if !ok {- t.Fatalf("expected string item, got %#v", item)- }- seen[value] = true- }- for _, item := range expected {- if !seen[item] {- t.Fatalf("missing %q in %#v", item, items)- }- }-}diff --git a/internal/service/dataset/metadata.go b/internal/service/dataset/metadata.goindex a5e7676cdd..49b25d0e0e 100644--- a/internal/service/dataset/metadata.go+++ b/internal/service/dataset/metadata.go@@ -38,6 +38,14 @@ func (d *DatasetService) UpdateDocumentMetadataConfig(ctx context.Context, userIparserConfig = entity.JSONMap{}}parserConfig["metadata"] = metadata+ if kb, kbErr := d.kbDAO.GetByID(ctx, dao.DB, datasetID); kbErr == nil && kb != nil {+ if tenant, tenantErr := d.tenantDAO.GetByID(ctx, dao.DB, kb.TenantID); tenantErr == nil && tenant != nil {+ parserConfig = service.ApplyComponentScopedParserConfig(+ parserConfig,+ tenant.LLMID,+ )+ }+ }if err = d.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{"parser_config": parserConfig}); err != nil {return nil, common.CodeServerError, errors.New("database operation failed")@@ -114,6 +122,12 @@ func (d *DatasetService) UpdateMetadataConfig(ctx context.Context, datasetID, te}parserConfig["metadata"] = metadataparserConfig["built_in_metadata"] = builtInMetadata+ if tenant, tenantErr := d.tenantDAO.GetByID(ctx, dao.DB, kb.TenantID); tenantErr == nil && tenant != nil {+ parserConfig = service.ApplyComponentScopedParserConfig(+ parserConfig,+ tenant.LLMID,+ )+ }if err = d.kbDAO.UpdateByID(ctx, dao.DB, kb.ID, map[string]interface{}{"parser_config": parserConfig}); err != nil {return nil, common.CodeServerError, errors.New("update auto-metadata error.(Database error)")diff --git a/internal/service/dataset/metadata_config_test.go b/internal/service/dataset/metadata_config_test.goindex 5553d28461..ddc0f3f248 100644--- a/internal/service/dataset/metadata_config_test.go+++ b/internal/service/dataset/metadata_config_test.go@@ -22,13 +22,30 @@ import ("ragflow/internal/common""ragflow/internal/dao""ragflow/internal/entity"+ "ragflow/internal/service")+func metadataFlagInt(t *testing.T, value interface{}) int {+ t.Helper()+ switch typed := value.(type) {+ case int:+ return typed+ case int64:+ return int(typed)+ case float64:+ return int(typed)+ default:+ t.Fatalf("unexpected metadata flag type %T (%#v)", value, value)+ return 0+ }+}+func testDatasetServiceForDocumentMetadataConfig(t *testing.T) *DatasetService {t.Helper()return &DatasetService{kbDAO: dao.NewKnowledgebaseDAO(),documentDAO: dao.NewDocumentDAO(),+ tenantDAO: dao.NewTenantDAO(),}}@@ -207,3 +224,116 @@ func TestDatasetServiceUpdateDocumentMetadataConfigAllowsTeamMember(t *testing.Tt.Fatalf("metadata was not updated: %#v", doc.ParserConfig)}}++func TestDatasetServiceUpdateMetadataConfigSyncsExtractorSchema(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertCreateDatasetTenant(t, "tenant-1")+ insertDatasetMetadataConfigKB(t, "kb-1", "tenant-1")+ if err := dao.DB.Model(&entity.Knowledgebase{}).+ Where("id = ?", "kb-1").+ Update("parser_config", entity.JSONMap{+ "enable_metadata": false,+ "Extractor:AutoExtractDefault": map[string]any{+ "enable_metadata": 1,+ "metadata": []any{+ map[string]any{"key": "stale", "type": "string"},+ },+ },+ }).Error; err != nil {+ t.Fatalf("seed parser_config: %v", err)+ }++ ctx := t.Context()+ result, code, err := (&DatasetService{+ kbDAO: dao.NewKnowledgebaseDAO(),+ tenantDAO: dao.NewTenantDAO(),+ }).UpdateMetadataConfig(ctx, "kb-1", "tenant-1", &service.MetadataConfigRequest{+ Metadata: []service.MetadataConfigField{+ {Key: "author", Type: "string"},+ },+ BuiltInMetadata: []service.MetadataConfigField{+ {Key: "document_name", Type: "string"},+ },+ })+ if err != nil {+ t.Fatalf("UpdateMetadataConfig failed: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("expected success code, got %d", code)+ }+ if result["metadata"] == nil {+ t.Fatalf("metadata response missing: %#v", result)+ }++ persisted, err := dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")+ if err != nil {+ t.Fatalf("failed to fetch persisted dataset: %v", err)+ }+ extractor, ok := persisted.ParserConfig["Extractor:AutoExtractDefault"].(map[string]interface{})+ if !ok {+ t.Fatalf("expected extractor component params, got %#v", persisted.ParserConfig["Extractor:AutoExtractDefault"])+ }+ if got := metadataFlagInt(t, extractor["enable_metadata"]); got != 0 {+ t.Fatalf("extractor enable_metadata = %#v, want 0 when top-level flag stays disabled", extractor["enable_metadata"])+ }++ if err := dao.DB.Model(&entity.Knowledgebase{}).+ Where("id = ?", "kb-1").+ Update("parser_config", entity.JSONMap{+ "enable_metadata": true,+ "Extractor:AutoExtractDefault": map[string]any{},+ }).Error; err != nil {+ t.Fatalf("reset parser_config: %v", err)+ }++ _, code, err = (&DatasetService{+ kbDAO: dao.NewKnowledgebaseDAO(),+ tenantDAO: dao.NewTenantDAO(),+ }).UpdateMetadataConfig(ctx, "kb-1", "tenant-1", &service.MetadataConfigRequest{+ Metadata: []service.MetadataConfigField{+ {Key: "author", Type: "string"},+ },+ BuiltInMetadata: []service.MetadataConfigField{+ {Key: "document_name", Type: "string"},+ },+ })+ if err != nil {+ t.Fatalf("UpdateMetadataConfig with enabled metadata failed: %v", err)+ }+ if code != common.CodeSuccess {+ t.Fatalf("expected success code, got %d", code)+ }++ persisted, err = dao.NewKnowledgebaseDAO().GetByID(ctx, db, "kb-1")+ if err != nil {+ t.Fatalf("failed to fetch persisted dataset: %v", err)+ }+ extractor, ok = persisted.ParserConfig["Extractor:AutoExtractDefault"].(map[string]interface{})+ if !ok {+ t.Fatalf("expected extractor component params, got %#v", persisted.ParserConfig["Extractor:AutoExtractDefault"])+ }+ if got := metadataFlagInt(t, extractor["enable_metadata"]); got != 1 {+ t.Fatalf("extractor enable_metadata = %#v, want 1", extractor["enable_metadata"])+ }+ gotFields, ok := extractor["metadata"].([]interface{})+ if !ok {+ t.Fatalf("extractor metadata = %#v, want []interface{}", extractor["metadata"])+ }+ wantFields := []map[string]interface{}{+ {"key": "author", "type": "string"},+ {"key": "document_name", "type": "string"},+ }+ if len(gotFields) != len(wantFields) {+ t.Fatalf("extractor metadata len = %d, want %d (%#v)", len(gotFields), len(wantFields), gotFields)+ }+ for i, want := range wantFields {+ field, ok := gotFields[i].(map[string]interface{})+ if !ok {+ t.Fatalf("extractor metadata[%d] = %#v, want map[string]interface{}", i, gotFields[i])+ }+ if field["key"] != want["key"] || field["type"] != want["type"] {+ t.Fatalf("extractor metadata[%d] = %#v, want key/type %#v", i, field, want)+ }+ }+}diff --git a/internal/service/dataset/task_cleanup_test.go b/internal/service/dataset/task_cleanup_test.godeleted file mode 100644index 5c40f5190a..0000000000--- a/internal/service/dataset/task_cleanup_test.go+++ /dev/null@@ -1,111 +0,0 @@-//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.-//--package dataset--import (- "errors"- "testing"- "time"-- "gorm.io/gorm"-- "ragflow/internal/dao"- "ragflow/internal/entity"-)--func TestCleanupFailedDatasetIndexTaskDeletesTaskAndRestoresDocument(t *testing.T) {- db := setupServiceTestDB(t)- pushServiceDB(t, db)-- previousMsg := "previous progress"- previousBeginAt := time.Date(2026, 6, 18, 10, 0, 0, 0, time.UTC)- queuedMsg := "Task is queued..."- queuedBeginAt := previousBeginAt.Add(time.Hour)- taskID := "task-1"-- kb := &entity.Knowledgebase{- ID: "kb-1",- TenantID: "user-1",- Name: "test-kb",- EmbdID: "embedding@OpenAI",- CreatedBy: "user-1",- Permission: string(entity.TenantPermissionMe),- ParserID: "naive",- ParserConfig: entity.JSONMap{},- GraphragTaskID: &taskID,- Status: sptr("1"),- }- if err := dao.DB.Create(kb).Error; err != nil {- t.Fatalf("insert kb: %v", err)- }-- doc := &entity.Document{- ID: "doc-1",- KbID: "kb-1",- ParserID: "naive",- ParserConfig: entity.JSONMap{},- SourceType: "local",- Type: "pdf",- CreatedBy: "user-1",- Suffix: ".pdf",- ProgressMsg: &queuedMsg,- ProcessBeginAt: &queuedBeginAt,- }- if err := dao.DB.Create(doc).Error; err != nil {- t.Fatalf("insert document: %v", err)- }-- task := &entity.Task{ID: taskID, DocID: doc.ID, TaskType: "graphrag"}- if err := dao.DB.Create(task).Error; err != nil {- t.Fatalf("insert task: %v", err)- }-- snapshot := &entity.Document{- ID: doc.ID,- ProgressMsg: &previousMsg,- ProcessBeginAt: &previousBeginAt,- }- if err := cleanupFailedDatasetIndexTask(task.ID, snapshot, kb.ID, "graph"); err != nil {- t.Fatalf("cleanup failed: %v", err)- }-- var persistedTask entity.Task- err := dao.DB.Where("id = ?", task.ID).First(&persistedTask).Error- if !errors.Is(err, gorm.ErrRecordNotFound) {- t.Fatalf("expected task to be deleted, got err=%v task=%#v", err, persistedTask)- }-- ctx := t.Context()- persistedDoc, err := dao.NewDocumentDAO().GetByID(ctx, db, doc.ID)- if err != nil {- t.Fatalf("fetch document: %v", err)- }- if persistedDoc.ProgressMsg == nil || *persistedDoc.ProgressMsg != previousMsg {- t.Fatalf("expected progress_msg %q, got %#v", previousMsg, persistedDoc.ProgressMsg)- }- if persistedDoc.ProcessBeginAt == nil || !persistedDoc.ProcessBeginAt.Equal(previousBeginAt) {- t.Fatalf("expected process_begin_at %v, got %#v", previousBeginAt, persistedDoc.ProcessBeginAt)- }-- var persistedKB entity.Knowledgebase- if err = dao.DB.Where("id = ?", kb.ID).First(&persistedKB).Error; err != nil {- t.Fatalf("fetch kb: %v", err)- }- if persistedKB.GraphragTaskID != nil {- t.Fatalf("expected graphrag_task_id to be cleared, got %#v", *persistedKB.GraphragTaskID)- }-}diff --git a/internal/service/dataset/update.go b/internal/service/dataset/update.goindex a2fbef107b..388ccd58f5 100644--- a/internal/service/dataset/update.go+++ b/internal/service/dataset/update.go@@ -287,6 +287,23 @@ func (d *DatasetService) UpdateDataset(ctx context.Context, datasetID, tenantIDupdates["parser_config"] = preserveDatasetParserConfigMetadata(cpDefaults, lockedKB.ParserConfig, req.ParserConfig)}}++ effectiveParserConfig := parserConfigJSONMap(updates["parser_config"])+ if effectiveParserConfig == nil && embdIDProvided {+ effectiveParserConfig = cloneJSONMap(lockedKB.ParserConfig)+ }+ if effectiveParserConfig != nil {+ llmID := ""+ if ownerTenant, tenantErr := d.tenantDAO.GetByID(ctx, tx, lockedKB.TenantID); tenantErr == nil && ownerTenant != nil {+ llmID = ownerTenant.LLMID+ }+ effectiveParserConfig = service.ApplyComponentScopedParserConfig(+ effectiveParserConfig,+ llmID,+ )+ updates["parser_config"] = effectiveParserConfig+ }+if len(updates) > 0 {if err = tx.Model(&entity.Knowledgebase{}).Where("id = ?", lockedKB.ID).Updates(updates).Error; err != nil {if dao.IsDuplicateKeyErr(err) {diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.goindex 1e91220679..a094f4b2ae 100644--- a/internal/service/dataset_artifact_service.go+++ b/internal/service/dataset_artifact_service.go@@ -18,6 +18,7 @@ import ("encoding/json""fmt""sort"+ "strings""ragflow/internal/engine""ragflow/internal/engine/types"@@ -165,7 +166,15 @@ type WikiPageItem struct {// ListWikiPages lists wiki pages for a dataset with optional page_type/topic// filters and pagination.func (s *DatasetArtifactService) ListWikiPages(ctx context.Context, tenantID, datasetID, pageType, topic string, page, pageSize int) ([]WikiPageItem, int64, error) {- filter := map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage}}+ // Only surface the merged dataset-level pages. Each unique (page_type, slug)+ // can also have a per-document source row (available_int=0); without this+ // filter the same entity/concept would appear once per source doc. Python's+ // list_wiki_pages has no such duplication because its writer emits one row+ // per page, so mirror that by selecting the merged rows (available_int=1).+ filter := map[string]interface{}{+ "compile_kwd": []string{CompileKwdWikiPage},+ "available_int": 1, // merged dataset-level rows only (see engine available_int handling)+ }if pageType != "" {filter["page_type_kwd"] = []string{pageType}}@@ -181,10 +190,18 @@ func (s *DatasetArtifactService) ListWikiPages(ctx context.Context, tenantID, da}items := make([]WikiPageItem, 0, len(chunks))for _, c := range chunks {+ pageType := firstStringValue(c["page_type_kwd"])+ // slug_kwd is stored as the full "<page_type>/<slug>" form (Python+ // contract); expose the bare slug to the frontend so it can be placed in+ // a single URL path segment (gin :slug does not match '/').+ bareSlug := firstStringValue(c["slug_kwd"])+ if pageType != "" {+ bareSlug = strings.TrimPrefix(bareSlug, pageType+"/")+ }items = append(items, WikiPageItem{- Slug: firstStringValue(c["slug_kwd"]),+ Slug: bareSlug,Title: firstStringValue(c["title_kwd"]),- PageType: firstStringValue(c["page_type_kwd"]),+ PageType: pageType,Topic: firstStringValue(c["topic_kwd"]),Summary: firstStringValue(c["summary_with_weight"]),})@@ -192,13 +209,15 @@ func (s *DatasetArtifactService) ListWikiPages(ctx context.Context, tenantID, dareturn items, total, nil}-// WikiPageDetail is the full wiki page payload.+// WikiPageDetail is the full wiki page payload. The content field is exposed as+// content_md_rendered to match the frontend IArtifactPage contract (and Python's+// get_wiki_page), which renders it directly.type WikiPageDetail struct {Slug string `json:"slug"`Title string `json:"title"`PageType string `json:"page_type"`Topic string `json:"topic"`- ContentMd string `json:"content_md"`+ ContentMd string `json:"content_md_rendered"`Summary string `json:"summary"`EntityNames []string `json:"entity_names"`Outlinks []string `json:"outlinks"`@@ -209,16 +228,21 @@ type WikiPageDetail struct {// GetWikiPage returns a single wiki page by page_type and slug.func (s *DatasetArtifactService) GetWikiPage(ctx context.Context, tenantID, datasetID, pageType, slug string) (*WikiPageDetail, error) {+ // Match Python's get_wiki_page contract (dataset_api_service.py): slug_kwd+ // is stored as the full "<page_type>/<slug>" form, and list_wiki_pages+ // returns the bare slug. Reconstruct the full form deterministically so the+ // filter matches the stored value exactly.slugKwd := pageType + "/" + slugfilter := map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage},"page_type_kwd": []string{pageType},"slug_kwd": []string{slugKwd},+ "available_int": 1, // merged dataset-level page, not the per-doc source row}chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter,- []string{"slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "content_with_weight",- "summary_with_weight", "entity_names_kwd", "outlinks_kwd", "related_kb_pages_kwd",- "source_chunk_ids", "source_doc_ids"},+ []string{"slug_kwd", "title_kwd", "page_type_kwd", "topic_kwd", "md_with_weight",+ "content_with_weight", "summary_with_weight", "entity_names_kwd", "outlinks_kwd",+ "related_kb_pages_kwd", "source_chunk_ids", "source_doc_ids"},0, 1, nil)if err != nil {return nil, err@@ -227,12 +251,26 @@ func (s *DatasetArtifactService) GetWikiPage(ctx context.Context, tenantID, datareturn nil, nil}c := chunks[0]+ // Python stores the page body in md_with_weight (incremental writer), falling+ // back to content_with_weight for legacy rows; mirror that here.+ content := firstStringValue(c["md_with_weight"])+ if content == "" {+ content = firstStringValue(c["content_with_weight"])+ }+ // slug_kwd is the full "<page_type>/<slug>" form; expose the bare slug so a+ // client can pass it straight back to GetWikiPage/UpdateWikiPage without the+ // "<page_type>/" prefix being doubled (matches ListWikiPages).+ detailPageType := firstStringValue(c["page_type_kwd"])+ detailSlug := firstStringValue(c["slug_kwd"])+ if detailPageType != "" {+ detailSlug = strings.TrimPrefix(detailSlug, detailPageType+"/")+ }detail := &WikiPageDetail{- Slug: firstStringValue(c["slug_kwd"]),+ Slug: detailSlug,Title: firstStringValue(c["title_kwd"]),- PageType: firstStringValue(c["page_type_kwd"]),+ PageType: detailPageType,Topic: firstStringValue(c["topic_kwd"]),- ContentMd: firstStringValue(c["content_with_weight"]),+ ContentMd: content,Summary: firstStringValue(c["summary_with_weight"]),EntityNames: toStringSlice(c["entity_names_kwd"]),Outlinks: toStringSlice(c["outlinks_kwd"]),@@ -251,11 +289,14 @@ func (s *DatasetArtifactService) UpdateWikiPage(ctx context.Context, tenantID, dif docEngine == nil {return nil, fmt.Errorf("document engine is not initialized")}+ // Python contract: slug_kwd is stored as "page_type/slug"; reconstruct it+ // deterministically from the bare slug (see GetWikiPage).slugKwd := pageType + "/" + slugfilter := map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage},"page_type_kwd": []string{pageType},"slug_kwd": []string{slugKwd},+ "available_int": 1, // merged dataset-level page only}chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter, []string{"id"}, 0, 1, nil)if err != nil {@@ -270,6 +311,9 @@ func (s *DatasetArtifactService) UpdateWikiPage(ctx context.Context, tenantID, d}update := map[string]interface{}{}if contentMd != "" {+ // GetWikiPage prefers md_with_weight and falls back to content_with_weight,+ // so write both to keep the edit readable regardless of the row's writer.+ update["md_with_weight"] = contentMdupdate["content_with_weight"] = contentMd}if title != "" {@@ -300,9 +344,10 @@ func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, dfilter := map[string]interface{}{"compile_kwd": []string{CompileKwdWikiPage},"page_type_kwd": []string{"concept", "entity"},+ "available_int": 1, // count only merged pages, not per-doc source rows}chunks, _, err := s.searchCompiled(ctx, tenantID, datasetID, filter,- []string{"topic_kwd", "title_kwd", "slug_kwd"}, 0, 1000, nil)+ []string{"topic_kwd", "title_kwd", "slug_kwd", "page_type_kwd"}, 0, 1000, nil)if err != nil {return nil, 0, err}@@ -313,12 +358,17 @@ func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, dif t == "" {continue}+ pageType := firstStringValue(c["page_type_kwd"])+ bareSlug := firstStringValue(c["slug_kwd"])+ if pageType != "" {+ bareSlug = strings.TrimPrefix(bareSlug, pageType+"/")+ }counts[t]++if _, ok := metas[t]; !ok {metas[t] = WikiTopicItem{Topic: t,Title: firstStringValue(c["title_kwd"]),- Slug: firstStringValue(c["slug_kwd"]),+ Slug: bareSlug,}}}diff --git a/internal/service/dataset_types.go b/internal/service/dataset_types.goindex cbcc7364d7..0864d66aa1 100644--- a/internal/service/dataset_types.go+++ b/internal/service/dataset_types.go@@ -1,10 +1,5 @@package service-// TraceIndexRequest is the request structure for tracing an index task.-type TraceIndexRequest struct {- Type string `json:"type" binding:"required"`-}-// CheckEmbeddingRequest is the request structure for checking embedding compatibility.type CheckEmbeddingRequest struct {EmbeddingID string `json:"embd_id" binding:"required"`diff --git a/internal/service/document/document_dataset_update.go b/internal/service/document/document_dataset_update.goindex 9d3cc1b0cf..40ab76b5c1 100644--- a/internal/service/document/document_dataset_update.go+++ b/internal/service/document/document_dataset_update.go@@ -166,6 +166,13 @@ func (s *DocumentService) UpdateDatasetDocument(ctx context.Context, userID, dat}} else {cleaned := pipelinepkg.BuildParserConfig(dslJSON, req.ParserConfig)+ tenant, tenantErr := dao.NewTenantDAO().GetByID(ctx, dao.DB, kb.TenantID)+ if tenantErr == nil && tenant != nil {+ cleaned = service.ApplyComponentScopedParserConfig(+ cleaned,+ tenant.LLMID,+ )+ }if err = s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{"parser_config": cleaned,}); err != nil {diff --git a/internal/service/model_service.go b/internal/service/model_service.goindex 37b35f52c1..4ff4b0ed0c 100644--- a/internal/service/model_service.go+++ b/internal/service/model_service.go@@ -3533,6 +3533,66 @@ func (m *ModelProviderService) ResolveModelConfig(ctx context.Context, tenantIDreturn m.GetModelConfigFromProviderInstance(ctx, tenantID, modelType, modelRef)}+// ResolveModelContextLength returns the chat model's context window+// (content_length) in tokens, or 0 when unknown. After the all_models.json+// migration (PR #17839) content_length is the total context window and+// max_output is the generation cap; the knowledge_compiler prompt-budget logic+// needs the context window, not the output cap. modelRef accepts either a+// tenant model UUID or a "model@instance@provider" composite name.+func (m *ModelProviderService) ResolveModelContextLength(ctx context.Context, tenantID string, modelRef string) (int, error) {+ if strings.TrimSpace(modelRef) == "" {+ return 0, fmt.Errorf("model ref is required")+ }+ if modelObj, err := m.modelDAO.GetByID(ctx, dao.DB, modelRef); err == nil {+ return m.modelContextLengthByID(ctx, modelObj)+ } else if !errors.Is(err, gorm.ErrRecordNotFound) {+ return 0, err+ }+ pureName, _, providerName, err := parseModelName(modelRef)+ if err != nil {+ return 0, err+ }+ return m.modelContextLengthByName(providerName, pureName)+}++// modelContextLengthByID reads content_length from the factory catalog for a+// tenant model row (by id).+func (m *ModelProviderService) modelContextLengthByID(ctx context.Context, modelObj *entity.TenantModel) (int, error) {+ if modelObj.Status != "active" {+ return 0, fmt.Errorf("tenant model id=%s is disabled", modelObj.ID)+ }+ provider, err := m.modelProviderDAO.GetByID(ctx, dao.DB, modelObj.ProviderID)+ if err != nil {+ return 0, err+ }+ if provider == nil {+ return 0, fmt.Errorf("provider id=%s not found for model id=%s", modelObj.ProviderID, modelObj.ID)+ }+ if mi, _ := dao.GetModelProviderManager().GetModelByName(provider.ProviderName, modelObj.ModelName); mi != nil && mi.ContentLength != nil {+ return *mi.ContentLength, nil+ }+ return 0, nil+}++// modelContextLengthByName reads content_length from the factory catalog for a+// "model@provider" style reference. It is best-effort: an unknown provider or+// model returns 0 (caller falls back to a default context length).+func (m *ModelProviderService) modelContextLengthByName(providerName, pureName string) (int, error) {+ targetProvider := dao.GetModelProviderManager().FindProvider(providerName)+ if targetProvider == nil {+ return 0, fmt.Errorf("model provider config not found: %s", providerName)+ }+ for i := range targetProvider.Models {+ if strings.EqualFold(targetProvider.Models[i].Name, pureName) {+ if targetProvider.Models[i].ContentLength != nil {+ return *targetProvider.Models[i].ContentLength, nil+ }+ return 0, nil+ }+ }+ return 0, nil+}+func (m *ModelProviderService) ResolveModelID(ctx context.Context, tenantID string, modelType entity.ModelType, modelName string) (string, error) {if modelObj, err := m.modelDAO.GetByID(ctx, dao.DB, modelName); err == nil {if modelObj.Status != "active" {diff --git a/internal/service/model_service_test.go b/internal/service/model_service_test.goindex 86e81f0607..8059a63772 100644--- a/internal/service/model_service_test.go+++ b/internal/service/model_service_test.go@@ -201,6 +201,63 @@ func TestModelProviderServiceGetModelConfigByID(t *testing.T) {}}+func TestModelProviderServiceResolveModelContextLength(t *testing.T) {+ db := setupModelProviderServiceTestDB(t)+ useModelProviderServiceTestDB(t, db)+ // Seed a tenant chat model that maps to a real factory-catalog model+ // (Anthropic / claude-opus-4-8 has content_length=1000000, max_output=128000).+ activeStatus := "1"+ rows := []interface{}{+ &entity.UserTenant{ID: "user-tenant-cl", UserID: "user-1", TenantID: "tenant-cl", Role: "owner", InvitedBy: "user-1", Status: &activeStatus},+ &entity.TenantModelProvider{ID: "provider-anthropic", TenantID: "tenant-cl", ProviderName: "Anthropic"},+ &entity.TenantModelInstance{ID: "instance-anthropic", ProviderID: "provider-anthropic", InstanceName: "default", APIKey: "sk-anthropic", Status: "active", Extra: "{}"},+ &entity.TenantModel{ID: "model-claude", ProviderID: "provider-anthropic", InstanceID: "instance-anthropic", ModelName: "claude-opus-4-8", ModelType: int(entity.ModelTypeChat), Status: "active"},+ }+ for _, row := range rows {+ if err := db.Create(row).Error; err != nil {+ t.Fatalf("failed to seed %T: %v", row, err)+ }+ }++ svc := NewModelProviderService()+ ctx := t.Context()++ // UUID path: resolves content_length (context window) from the factory+ // catalog, NOT max_output.+ got, err := svc.ResolveModelContextLength(ctx, "user-1", "model-claude")+ if err != nil {+ t.Fatalf("ResolveModelContextLength(uuid) error = %v", err)+ }+ if got != 1000000 {+ t.Fatalf("uuid content_length = %d, want 1000000 (must be the context window, not max_output=128000)", got)+ }++ // Composite "model@instance@provider" path resolves the same value.+ got2, err := svc.ResolveModelContextLength(ctx, "user-1", "claude-opus-4-8@default@Anthropic")+ if err != nil {+ t.Fatalf("ResolveModelContextLength(composite) error = %v", err)+ }+ if got2 != 1000000 {+ t.Fatalf("composite content_length = %d, want 1000000", got2)+ }+}++func TestModelProviderServiceResolveModelContextLengthUnknownModel(t *testing.T) {+ db := setupModelProviderServiceTestDB(t)+ useModelProviderServiceTestDB(t, db)++ // A model that does not exist in the factory catalog resolves to 0 so the+ // caller falls back to its default context length instead of failing.+ got, err := NewModelProviderService().ResolveModelContextLength(+ t.Context(), "user-1", "gpt-no-such-model@default@OpenAI")+ if err != nil {+ t.Fatalf("ResolveModelContextLength(unknown) error = %v", err)+ }+ if got != 0 {+ t.Fatalf("unknown model content_length = %d, want 0", got)+ }+}+func TestModelProviderServiceAlterModelRejectsInvalidStatus(t *testing.T) {ctx := t.Context()code, err := NewModelProviderService().AlterModel(ctx, "OpenAI", "default", "", "user-1", "model-1", map[string]interface{}{"status": "disabled"})diff --git a/internal/service/wikisearch/engine_service.go b/internal/service/wikisearch/engine_service.gonew file mode 100644index 0000000000..9eac92b799--- /dev/null+++ b/internal/service/wikisearch/engine_service.go@@ -0,0 +1,222 @@+package wikisearch++import (+ "context"+ "fmt"+ "strings"++ "ragflow/internal/engine"+ "ragflow/internal/engine/types"+)++// compileKWDWikiPage is the canonical compile_kwd for compiled wiki pages. It+// must match Python's WIKI_PAGE_COMPILE_KWD ("wiki_page", wiki.py:1661) AND the+// Go compiler's variantCompileKWD[VariantWiki] (component.go), so both Python-+// and Go-produced wiki pages are surfaced by this service.+const compileKWDWikiPage = "wiki_page"++// tenantIndexName returns the tenant-scoped chunk index name+// ("ragflow_<tenantID>"), matching how the rest of the stack derives the index+// (internal/handler/dataset.go). The dataset IDs are passed as KB filters, NOT+// used as index names.+func tenantIndexName(tenantID string) string {+ return fmt.Sprintf("ragflow_%s", tenantID)+}++// engineWikiService is the concrete compiled-wiki search service, backed+// directly by the document engine.+//+// - Every operation derives the chunk index from the tenantID+// (ragflow_<tenantID>) and passes dataset IDs only as KB filters, so scope+// can never cross tenants.+// - QueryPages issues an engine Search restricted to compile_kwd="wiki_page"+// (+ supported kinds) so ordinary source chunks are never relabeled as wiki+// pages; the raw rows also carry source_chunk_ids, which are emitted for+// P7/R3 evidence backfill.+// - BackfillChunks fetches original chunks by id via GetChunk, scoped to the+// tenant index + dataset IDs.+type engineWikiService struct {+ engine engine.DocEngine // may be nil -> degrade+}++// NewEngineService builds the concrete wiki search service from the document+// engine (engine.Get() in production; nil disables all operations gracefully).+func NewEngineService(docEngine engine.DocEngine) Service {+ return &engineWikiService{engine: docEngine}+}++func (s *engineWikiService) AvailableFor(ctx context.Context, tenantID string, datasetIDs []string) bool {+ if s.engine == nil || tenantID == "" || len(datasetIDs) == 0 {+ return false+ }+ // "Wiki available" must mean the bound KBs actually carry wiki pages, not+ // merely that a chunk store exists (a non-wiki KB would otherwise trigger a+ // needless empty wiki-query/fallback round trip). Do a bounded existence+ // search (Limit=1) filtered to compile_kwd="wiki_page", returning true if+ // any page row exists.+ res, err := s.engine.Search(ctx, &types.SearchRequest{+ IndexNames: []string{tenantIndexName(tenantID)},+ KbIDs: datasetIDs,+ Limit: 1,+ Filter: map[string]interface{}{+ "compile_kwd": compileKWDWikiPage,+ "available_int": 1,+ },+ })+ if err != nil {+ return false+ }+ return res != nil && len(res.Chunks) > 0+}++func (s *engineWikiService) QueryPages(ctx context.Context, tenantID string, datasetIDs []string, query, keywords string, topN int) (SearchResult, error) {+ if s.engine == nil || tenantID == "" || len(datasetIDs) == 0 || strings.TrimSpace(query) == "" {+ return SearchResult{Chunks: []map[string]interface{}{}, DocAggs: []map[string]interface{}{}}, nil+ }+ if topN <= 0 {+ topN = 12+ }+ text := query+ if kw := strings.TrimSpace(keywords); kw != "" {+ text = query + " " + kw+ }+ req := &types.SearchRequest{+ IndexNames: []string{tenantIndexName(tenantID)},+ KbIDs: datasetIDs,+ Limit: topN,+ // Select the exact projection we consume. Infinity's default projection+ // omits slug_kwd and source_chunk_ids (chunk.go:736-748); without them+ // the page slug is blank and P7 evidence backfill has no provenance.+ SelectFields: []string{+ "id", "kb_id", "doc_id", "docnm_kwd", "content_with_weight",+ "slug_kwd", "source_chunk_ids",+ },+ // Discriminate wiki pages by compile_kwd="wiki_page". There is NO+ // "kc_kind" column in the chunk schema (infinity_mapping.json:47-56), so+ // it must not be used as a filter. Sections (page.go kind:"section") are+ // stamped compile_kwd="wiki_section", so this filter returns pages only.+ Filter: map[string]interface{}{+ "compile_kwd": compileKWDWikiPage,+ "available_int": 1,+ },+ MatchExprs: []interface{}{+ &types.MatchTextExpr{+ Fields: []string{"content_with_weight^2", "title_tks^5", "content_ltks"},+ MatchingText: text,+ TopN: topN,+ },+ },+ }+ res, err := s.engine.Search(ctx, req)+ if err != nil || res == nil || len(res.Chunks) == 0 {+ return SearchResult{Chunks: []map[string]interface{}{}, DocAggs: []map[string]interface{}{}}, nil+ }+ out := SearchResult{Chunks: []map[string]interface{}{}, DocAggs: []map[string]interface{}{}}+ seenDoc := map[string]bool{}+ for _, row := range res.Chunks {+ // Engine raw rows carry the chunk id under "id" (shimmed to _id) and the+ // dataset under "kb_id". Normalize explicitly to the agent chunk shape+ // (chunk_id/dataset_id) so a stable id and KB scope are never lost.+ id := firstString(row["id"])+ datasetID := firstString(row["kb_id"])+ content := firstString(row["content_with_weight"])+ if id == "" && content == "" {+ continue+ }+ docID := firstString(row["doc_id"])+ chunk := map[string]interface{}{+ "chunk_id": id, "content_with_weight": content,+ "doc_id": docID, "docnm_kwd": firstString(row["docnm_kwd"]),+ "dataset_id": datasetID,+ "wiki_slug_kwd": firstString(row["slug_kwd"]),+ }+ if src := stringArray(row["source_chunk_ids"]); len(src) > 0 {+ chunk["source_chunk_ids"] = src+ }+ out.Chunks = append(out.Chunks, chunk)+ if docID != "" && !seenDoc[docID] {+ seenDoc[docID] = true+ out.DocAggs = append(out.DocAggs, map[string]interface{}{"doc_id": docID, "doc_name": firstString(row["docnm_kwd"])})+ }+ }+ return out, nil+}++func (s *engineWikiService) BackfillChunks(ctx context.Context, tenantID string, datasetIDs []string, chunkIDs []string) ([]map[string]interface{}, error) {+ if s.engine == nil || tenantID == "" || len(chunkIDs) == 0 {+ return nil, nil+ }+ const maxBackfill = 16+ seen := map[string]bool{}+ ids := make([]string, 0, len(chunkIDs))+ for _, id := range chunkIDs {+ if id == "" || seen[id] {+ continue+ }+ seen[id] = true+ ids = append(ids, id)+ if len(ids) >= maxBackfill {+ break+ }+ }+ if len(ids) == 0 {+ return nil, nil+ }+ out := make([]map[string]interface{}, 0, len(ids))+ for _, id := range ids {+ raw, err := s.engine.GetChunk(ctx, tenantIndexName(tenantID), id, datasetIDs)+ if err != nil {+ continue+ }+ m, ok := raw.(map[string]interface{})+ if !ok {+ continue+ }+ // GetChunk rows also carry id/kb_id (ES shims id to _id; source["id"]+ // set in chunk.go:1991). Normalize to the agent chunk shape.+ out = append(out, map[string]interface{}{+ "chunk_id": firstString(m["id"]), "content_with_weight": firstString(m["content_with_weight"]),+ "doc_id": firstString(m["doc_id"]), "docnm_kwd": firstString(m["docnm_kwd"]),+ "dataset_id": firstString(m["kb_id"]),+ })+ }+ return out, nil+}++// firstString returns the first string of a possibly array-shaped engine field+// value (the document engine surfaces keyword fields as arrays), matching the+// firstStringValue helper used by the artifact service.+func firstString(v interface{}) string {+ switch t := v.(type) {+ case string:+ return t+ case []string:+ if len(t) > 0 {+ return t[0]+ }+ case []interface{}:+ if len(t) > 0 {+ if s, ok := t[0].(string); ok {+ return s+ }+ }+ }+ return ""+}++func stringArray(v interface{}) []string {+ if raw, ok := v.([]string); ok {+ return raw+ }+ arr, ok := v.([]interface{})+ if !ok {+ return nil+ }+ out := make([]string, 0, len(arr))+ for _, item := range arr {+ if s, ok := item.(string); ok {+ out = append(out, s)+ }+ }+ return out+}diff --git a/internal/service/wikisearch/engine_service_test.go b/internal/service/wikisearch/engine_service_test.gonew file mode 100644index 0000000000..57b4939d2f--- /dev/null+++ b/internal/service/wikisearch/engine_service_test.go@@ -0,0 +1,265 @@+package wikisearch++import (+ "context"+ "reflect"+ "testing"++ "ragflow/internal/engine"+ "ragflow/internal/engine/types"+)++// fakeDocEngine embeds engine.DocEngine (all methods nil) and overrides only the+// chunk-store existence probe, by-id chunk fetch, and search used by the service.+// It records the index/dataset params so tests can assert tenant-scoped index and+// dataset-scoped filters.+type fakeDocEngine struct {+ engine.DocEngine+ // existsDatasets lists dataset ids whose table exists; empty means all probe+ // results follow `exists` (when false, nothing exists).+ exists bool+ existsDatasets map[string]bool+ chunks map[string]interface{} // chunk id -> raw row (GetChunk)+ searchRows []map[string]interface{}+ searchReq *types.SearchRequest+ gotChunkArgs [][3]interface{} // [indexName, chunkID, datasetIDs]+ existsArgs [][2]string // [indexName, datasetID]+}++func (f *fakeDocEngine) ChunkStoreExists(_ context.Context, indexName, datasetID string) (bool, error) {+ f.existsArgs = append(f.existsArgs, [2]string{indexName, datasetID})+ if f.existsDatasets != nil {+ return f.existsDatasets[datasetID], nil+ }+ return f.exists, nil+}++func (f *fakeDocEngine) GetChunk(_ context.Context, indexName, chunkID string, datasetIDs []string) (interface{}, error) {+ f.gotChunkArgs = append(f.gotChunkArgs, [3]interface{}{indexName, chunkID, datasetIDs})+ return f.chunks[chunkID], nil+}++func (f *fakeDocEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {+ f.searchReq = req+ rows := f.searchRows+ // Honor the compile_kwd filter so availability semantics are exercised+ // faithfully (only rows carrying the requested compile_kwd match).+ if kwd, ok := req.Filter["compile_kwd"].(string); ok {+ filtered := make([]map[string]interface{}, 0, len(rows))+ for _, r := range rows {+ if r["compile_kwd"] == kwd {+ filtered = append(filtered, r)+ }+ }+ rows = filtered+ }+ return &types.SearchResult{Chunks: rows, Total: int64(len(rows))}, nil+}++// realEngineRow returns a wiki page row in the engine's actual shape: id+// (shimmed _id), kb_id (dataset), compile_kwd, doc_id, docnm_kwd, slug_kwd,+// source_chunk_ids.+func realEngineRow(id, kbID, docID, name, slug string, source []string) map[string]interface{} {+ row := map[string]interface{}{+ "id": id, "kb_id": kbID, "compile_kwd": "wiki_page",+ "doc_id": docID, "docnm_kwd": name,+ "content_with_weight": "content of " + id, "slug_kwd": slug,+ }+ if len(source) > 0 {+ row["source_chunk_ids"] = source+ }+ return row+}++func TestEngineService_QueryPages_NormalizesEngineRowShape(t *testing.T) {+ eng := &fakeDocEngine{searchRows: []map[string]interface{}{+ realEngineRow("wiki/alpha", "kb1", "d1", "Alpha", "entity/alpha", []string{"c1", "c2"}),+ }}+ svc := NewEngineService(eng)+ res, err := svc.QueryPages(context.Background(), "t1", []string{"kb1"}, "alpha", "", 5)+ if err != nil {+ t.Fatalf("QueryPages err = %v", err)+ }+ if len(res.Chunks) != 1 {+ t.Fatalf("chunks = %d, want 1", len(res.Chunks))+ }+ // id -> chunk_id, kb_id -> dataset_id must be normalized (Finding 2).+ if res.Chunks[0]["chunk_id"] != "wiki/alpha" {+ t.Errorf("chunk_id = %v, want wiki/alpha (from engine id)", res.Chunks[0]["chunk_id"])+ }+ if res.Chunks[0]["dataset_id"] != "kb1" {+ t.Errorf("dataset_id = %v, want kb1 (from engine kb_id)", res.Chunks[0]["dataset_id"])+ }+ if res.Chunks[0]["wiki_slug_kwd"] != "entity/alpha" {+ t.Errorf("wiki_slug_kwd = %v, want entity/alpha", res.Chunks[0]["wiki_slug_kwd"])+ }+ src, ok := res.Chunks[0]["source_chunk_ids"].([]string)+ if !ok || len(src) != 2 {+ t.Fatalf("source_chunk_ids missing: %#v", res.Chunks[0])+ }+ // The engine Search must be scoped to the tenant index + dataset filter and+ // filtered to compiled wiki pages.+ if eng.searchReq == nil {+ t.Fatalf("no SearchRequest issued")+ }+ if !reflect.DeepEqual(eng.searchReq.IndexNames, []string{"ragflow_t1"}) {+ t.Errorf("IndexNames = %v, want [ragflow_t1]", eng.searchReq.IndexNames)+ }+ if !reflect.DeepEqual(eng.searchReq.KbIDs, []string{"kb1"}) {+ t.Errorf("KbIDs = %v, want [kb1]", eng.searchReq.KbIDs)+ }+ if f, ok := eng.searchReq.Filter["compile_kwd"].(string); !ok || f != "wiki_page" {+ t.Errorf("compile_kwd filter = %v, want wiki_page", eng.searchReq.Filter["compile_kwd"])+ }+ // SelectFields must project slug_kwd + source_chunk_ids (Infinity's default+ // projection omits them), otherwise the page slug and P7 provenance are lost.+ projected := map[string]bool{}+ for _, f := range eng.searchReq.SelectFields {+ projected[f] = true+ }+ for _, f := range []string{"id", "kb_id", "doc_id", "docnm_kwd", "content_with_weight", "slug_kwd", "source_chunk_ids"} {+ if !projected[f] {+ t.Errorf("SelectFields missing %q: %v", f, eng.searchReq.SelectFields)+ }+ }+}++// TestEngineService_QueryPages_ArrayShapedFields locks firstString handling of+// array-shaped engine keyword fields: the document engine returns slug_kwd /+// docnm_kwd as arrays, and those must not be dropped.+func TestEngineService_QueryPages_ArrayShapedFields(t *testing.T) {+ eng := &fakeDocEngine{searchRows: []map[string]interface{}{+ {+ "id": "wiki/alpha", "kb_id": "kb1", "compile_kwd": "wiki_page",+ "doc_id": "d1", "docnm_kwd": []string{"Alpha"},+ "content_with_weight": "content of wiki/alpha",+ "slug_kwd": []string{"entity/alpha"},+ "source_chunk_ids": []string{"c1"},+ },+ }}+ svc := NewEngineService(eng)+ res, err := svc.QueryPages(context.Background(), "t1", []string{"kb1"}, "alpha", "", 5)+ if err != nil {+ t.Fatalf("QueryPages err = %v", err)+ }+ if len(res.Chunks) != 1 {+ t.Fatalf("chunks = %d, want 1", len(res.Chunks))+ }+ if res.Chunks[0]["wiki_slug_kwd"] != "entity/alpha" {+ t.Errorf("wiki_slug_kwd = %v, want entity/alpha (array-shaped slug_kwd)", res.Chunks[0]["wiki_slug_kwd"])+ }+ if res.Chunks[0]["docnm_kwd"] != "Alpha" {+ t.Errorf("docnm_kwd = %v, want Alpha (array-shaped docnm_kwd)", res.Chunks[0]["docnm_kwd"])+ }+ if len(res.DocAggs) != 1 || res.DocAggs[0]["doc_name"] != "Alpha" {+ t.Errorf("DocAggs doc_name = %v, want Alpha", res.DocAggs)+ }+}++func TestEngineService_QueryPages_DegradesEmpty(t *testing.T) {+ svc := NewEngineService(nil)+ res, err := svc.QueryPages(context.Background(), "t1", []string{"kb1"}, "alpha", "", 5)+ if err != nil || len(res.Chunks) != 0 {+ t.Fatalf("got chunks=%d err=%v, want empty/noerr (no engine)", len(res.Chunks), err)+ }+ svc2 := NewEngineService(&fakeDocEngine{searchRows: []map[string]interface{}{+ realEngineRow("x", "kb1", "d", "D", "s", nil),+ }})+ res2, _ := svc2.QueryPages(context.Background(), "t1", []string{"kb1"}, " ", "", 5)+ if len(res2.Chunks) != 0 {+ t.Fatalf("chunks = %d, want 0 for blank query", len(res2.Chunks))+ }+}++func TestEngineService_BackfillChunks_ByIDScoped(t *testing.T) {+ eng := &fakeDocEngine{chunks: map[string]interface{}{+ "c1": map[string]interface{}{"id": "c1", "content_with_weight": "raw 1", "doc_id": "d1", "docnm_kwd": "D", "kb_id": "kb1"},+ "c2": map[string]interface{}{"id": "c2", "content_with_weight": "raw 2", "doc_id": "d1", "docnm_kwd": "D", "kb_id": "kb1"},+ }}+ svc := NewEngineService(eng)+ out, err := svc.BackfillChunks(context.Background(), "t1", []string{"kb1", "kb2"}, []string{"c1", "c2", "c1", "missing"})+ if err != nil {+ t.Fatalf("BackfillChunks err = %v", err)+ }+ if len(out) != 2 {+ t.Fatalf("backfill = %d, want 2 (deduped c1,c2; missing skipped)", len(out))+ }+ if out[0]["chunk_id"] != "c1" || out[1]["chunk_id"] != "c2" {+ t.Errorf("backfill ids = %#v, want c1,c2", out)+ }+ // kb_id -> dataset_id must be normalized on evidence rows too (Low finding).+ if out[0]["dataset_id"] != "kb1" || out[1]["dataset_id"] != "kb1" {+ t.Errorf("backfill dataset_id = %v / %v, want kb1 for both (from engine kb_id)",+ out[0]["dataset_id"], out[1]["dataset_id"])+ }+ // Every GetChunk must be against the tenant index and the dataset scope.+ for _, args := range eng.gotChunkArgs {+ if args[0] != "ragflow_t1" {+ t.Errorf("GetChunk index = %v, want ragflow_t1", args[0])+ }+ ds, _ := args[2].([]string)+ if !reflect.DeepEqual(ds, []string{"kb1", "kb2"}) {+ t.Errorf("GetChunk datasetIDs = %v, want [kb1 kb2]", ds)+ }+ }+}++func TestEngineService_BackfillChunks_DegradesNoEngine(t *testing.T) {+ svc := NewEngineService(nil)+ out, err := svc.BackfillChunks(context.Background(), "t1", []string{"kb1"}, []string{"c1"})+ if err != nil || len(out) != 0 {+ t.Fatalf("got %d err=%v, want empty/noerr (no engine)", len(out), err)+ }+}++func TestEngineService_AvailableFor_BoundedExistenceSearch(t *testing.T) {+ // A KB carrying wiki pages => AvailableFor true, and the request must be a+ // bounded (Limit=1) search on the tenant index, filtered to+ // compile_kwd="wiki_page" and the dataset KBs.+ eng := &fakeDocEngine{searchRows: []map[string]interface{}{+ realEngineRow("wiki/p1", "kb1", "d1", "P", "p1", nil),+ }}+ svc := NewEngineService(eng)+ if !svc.AvailableFor(context.Background(), "t1", []string{"kb1"}) {+ t.Errorf("AvailableFor should be true when a wiki page row exists")+ }+ if eng.searchReq == nil {+ t.Fatalf("no existence Search issued")+ }+ if !reflect.DeepEqual(eng.searchReq.IndexNames, []string{"ragflow_t1"}) {+ t.Errorf("existence IndexNames = %v, want [ragflow_t1]", eng.searchReq.IndexNames)+ }+ if eng.searchReq.Limit != 1 {+ t.Errorf("existence Limit = %d, want 1 (bounded)", eng.searchReq.Limit)+ }+ if f, ok := eng.searchReq.Filter["compile_kwd"].(string); !ok || f != "wiki_page" {+ t.Errorf("existence compile_kwd filter = %v, want wiki_page", eng.searchReq.Filter["compile_kwd"])+ }+ if !reflect.DeepEqual(eng.searchReq.KbIDs, []string{"kb1"}) {+ t.Errorf("existence KbIDs = %v, want [kb1]", eng.searchReq.KbIDs)+ }+ // No engine / empty tenant / no datasets -> false.+ if NewEngineService(nil).AvailableFor(context.Background(), "t1", []string{"kb1"}) {+ t.Errorf("AvailableFor should be false with no engine")+ }+ if NewEngineService(eng).AvailableFor(context.Background(), "", []string{"kb1"}) {+ t.Errorf("AvailableFor should be false with empty tenant")+ }+ if NewEngineService(eng).AvailableFor(context.Background(), "t1", nil) {+ t.Errorf("AvailableFor should be false with no datasets")+ }+ // No wiki page rows (only ordinary chunks) -> false, even though a chunk+ // store exists. This is the "wiki pages exist, not just a table" gate.+ svcNoWiki := NewEngineService(&fakeDocEngine{searchRows: []map[string]interface{}{+ {"id": "c1", "kb_id": "kb1", "content_with_weight": "plain chunk"}, // no compile_kwd+ }})+ if svcNoWiki.AvailableFor(context.Background(), "t1", []string{"kb1"}) {+ t.Errorf("AvailableFor should be false for a KB with no wiki page rows")+ }+}++func TestTenantIndexName(t *testing.T) {+ if got := tenantIndexName("t1"); got != "ragflow_t1" {+ t.Errorf("tenantIndexName(t1) = %q, want ragflow_t1", got)+ }+}diff --git a/internal/service/wikisearch/registry.go b/internal/service/wikisearch/registry.gonew file mode 100644index 0000000000..d29d0bccb2--- /dev/null+++ b/internal/service/wikisearch/registry.go@@ -0,0 +1,26 @@+package wikisearch++import "sync"++var (+ svcMu sync.RWMutex+ svcInst Service+)++// SetService installs the (production or test) wiki-search service singleton.+// Passing nil reverts to "not configured" (GetService returns nil), which is the+// pre-wiring state: the wiki_query tool then returns an empty result so the agent+// falls back to hybrid search.+func SetService(s Service) {+ svcMu.Lock()+ defer svcMu.Unlock()+ svcInst = s+}++// GetService returns the installed wiki-search service. It may be nil until+// SetService is called during server bootstrap.+func GetService() Service {+ svcMu.RLock()+ defer svcMu.RUnlock()+ return svcInst+}diff --git a/internal/service/wikisearch/wikisearch.go b/internal/service/wikisearch/wikisearch.gonew file mode 100644index 0000000000..b7e9a756ed--- /dev/null+++ b/internal/service/wikisearch/wikisearch.go@@ -0,0 +1,52 @@+// Package wikisearch defines the compiled-wiki search service contract. It is a+// dependency-light leaf package so that both the agent tool layer+// (internal/agent/tool) and any concrete engine-backed implementation can depend+// on it without an import cycle — mirroring the internal/service/nav pattern.+//+// The interface abstracts two independent concerns:+//+// - QueryPages: hybrid search over compiled wiki/artifact pages for a query,+// returning rendered page content + slug/title + stable doc aggregation.+// - BackfillChunks: fetch the ORIGINAL source chunks a compiled page was built+// from, by chunk id, scoped to tenant + datasets. This is how the harness P7+// evidence expansion gets the raw evidence (the general retrieval Search API+// cannot fetch by chunk id).+// - AvailableFor: whether a dataset actually has wiki artifacts the tool may+// search, so the production runner only selects the wiki path when the bound+// KBs carry the artifact (Python's compilation_available gate).+//+// The concrete implementation lives in the same package+// (engineWikiService, see engine_service.go) and is backed by the document+// engine. It is installed at server bootstrap via SetService.+package wikisearch++import "context"++// SearchResult is the normalized wiki_query return shape (mirrors Python's+// {"answer":"", "chunks":[...], "doc_aggs":[...]}).+//+// Each chunk is a map with at least: chunk_id, content_with_weight, doc_id,+// docnm_kwd, dataset_id, and (for a compiled page) wiki_slug_kwd.+type SearchResult struct {+ Chunks []map[string]interface{}+ DocAggs []map[string]interface{}+}++// Service is the single read entrypoint the wiki_query agent tool and the+// production runner use.+type Service interface {+ // AvailableFor reports whether any of the given datasets carries searchable+ // wiki/artifact pages (Python's compilation_available gate). Returning false+ // means the wiki tool should not be selected for those datasets.+ AvailableFor(ctx context.Context, tenantID string, datasetIDs []string) bool+ // QueryPages hybrid-searches compiled wiki/artifact pages across the given+ // datasets and returns rendered page chunks. Returns an empty result (never+ // an error) when nothing matches or the backend is unavailable, so the agent+ // can fall back to hybrid search.+ QueryPages(ctx context.Context, tenantID string, datasetIDs []string, query, keywords string, topN int) (SearchResult, error)+ // BackfillChunks fetches original source chunks by their ids, scoped to the+ // tenant and datasets. It returns only chunks it could resolve, in the given+ // order, deduped by id; unresolvable ids are skipped (never an error). Empty+ // when the backend is unavailable or none resolve.+ BackfillChunks(ctx context.Context, tenantID string, datasetIDs []string, chunkIDs []string) ([]map[string]interface{}, error)+}diff --git a/rag/advanced_rag/knowlege_compile/wiki.py b/rag/advanced_rag/knowlege_compile/wiki.pyindex f354028f01..b18758dfbe 100644--- a/rag/advanced_rag/knowlege_compile/wiki.py+++ b/rag/advanced_rag/knowlege_compile/wiki.py@@ -821,9 +821,10 @@ async def _wiki_extract_one_batch(language: str,llm_timeout: int,parser_config: Optional[dict] = None,-) -> dict:+) -> Optional[dict]:"""Single LLM call for one packed batch. Returns the raw (label-tagged)- extract dict.+ extract dict, or ``None`` on a transient LLM timeout/error so the caller+ can avoid persisting a poisoned empty result.The entity / relation schemas and the extra rules sections of theprompt are rendered from ``parser_config`` when supplied (mirroring@@ -851,10 +852,10 @@ async def _wiki_extract_one_batch()except asyncio.TimeoutError:logging.warning("wiki_map: batch extraction timed out after %ds (%d chunks)", llm_timeout, len(packed))- return _wiki_empty_extract()+ return Noneexcept Exception:logging.exception("wiki_map: batch extraction failed (%d chunks)", len(packed))- return _wiki_empty_extract()+ return None_ = language # reserved for future localizationreturn _wiki_unwrap_extract(res)@@ -881,6 +882,12 @@ async def _wiki_process_batch(the top of ``wiki_map_from_chunks``; threaded through so thepersisted resume rows record the right hash and the nextincremental run can compare cleanly.++ On a transient LLM failure/timeout (``_wiki_extract_one_batch`` returns+ ``None``) the batch is NOT persisted with a resume hash. The next+ incremental run then sees those chunks as ``new`` and retries, instead of+ replaying a permanently cached empty extract. Only a genuine LLM response+ (even one with zero items) is persisted."""if not packed:return _wiki_empty_extract()@@ -896,6 +903,10 @@ async def _wiki_process_batch(llm_timeout,parser_config=parser_config,)+ if raw_extract is None:+ # LLM call failed/timed out: leave no resume hash so the next run+ # re-extracts these chunks instead of locking in an empty result.+ return _wiki_empty_extract()merged, per_chunk = _wiki_resolve_chunk_ids(raw_extract, label_to_id)await _wiki_persist_extracts(per_chunk,diff --git a/test/testcases/restful_api/conftest.py b/test/testcases/restful_api/conftest.pyindex ad3030287f..36e2b73516 100644--- a/test/testcases/restful_api/conftest.py+++ b/test/testcases/restful_api/conftest.py@@ -25,6 +25,13 @@ from utils import wait_forGO_ONLY_SKIPS = {"Go route is not implemented": {+ # Dataset-level graph/raptor/mindmap indexing via POST/GET/DELETE+ # /datasets/:id/index is a Python-era RunIndex/TraceIndex/DeleteIndex+ # contract; the Go port schedules dataset compilation through the+ # knowledge_compile scheduler instead and does not serve /index.+ "test_dataset_index_endpoints",+ "test_dataset_index_trace_and_delete_type_contract",+ "test_dataset_index_run_with_document_creates_task","test_document_download_by_id_invalid_id_contract","test_llm_factories_live_auth_contract","test_llm_list_live_auth_contract",diff --git a/uv.lock b/uv.lockindex 9d25eb4487..9b8fcc6eda 100644--- a/uv.lock+++ b/uv.lock@@ -8167,7 +8167,6 @@ dependencies = [{ name = "webdav4" },{ name = "webdriver-manager" },{ name = "wechatpy" },- { name = "werkzeug" },{ name = "wikipedia" },{ name = "word2number" },{ name = "xgboost" },@@ -8324,7 +8323,6 @@ requires-dist = [{ name = "webdav4", specifier = ">=0.10.0,<0.11.0" },{ name = "webdriver-manager", specifier = "==4.0.1" },{ name = "wechatpy", specifier = ">=1.8.18" },- { name = "werkzeug", specifier = ">=3.1.7,<4" },{ name = "wikipedia", specifier = "==1.4.0" },{ name = "word2number", specifier = "==1.1" },{ name = "xgboost", specifier = "==1.6.0" },@@ -9922,14 +9920,14 @@ wheels = [[[package]]name = "werkzeug"-version = "3.1.8"+version = "3.1.5"source = { registry = "https://mirrors.aliyun.com/pypi/simple" }dependencies = [{ name = "markupsafe" },]-sdist = { url = "https://mirrors.aliyun.com/pypi/packages/dd/b2/381be8cfdee792dd117872481b6e378f85c957dd7c5bca38897b08f765fd/werkzeug-3.1.8.tar.gz", hash = "sha256:9bad61a4268dac112f1c5cd4630a56ede601b6ed420300677a869083d70a4c44" }+sdist = { url = "https://mirrors.aliyun.com/pypi/packages/5a/70/1469ef1d3542ae7c2c7b72bd5e3a4e6ee69d7978fa8a3af05a38eca5becf/werkzeug-3.1.5.tar.gz", hash = "sha256:6a548b0e88955dd07ccb25539d7d0cc97417ee9e179677d22c7041c8f078ce67" }wheels = [- { url = "https://mirrors.aliyun.com/pypi/packages/93/8c/2e650f2afeb7ee576912636c23ddb621c91ac6a98e66dc8d29c3c69446e1/werkzeug-3.1.8-py3-none-any.whl", hash = "sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50" },+ { url = "https://mirrors.aliyun.com/pypi/packages/ad/e4/8d97cca767bcc1be76d16fb76951608305561c6e056811587f36cb1316a8/werkzeug-3.1.5-py3-none-any.whl", hash = "sha256:5111e36e91086ece91f93268bb39b4a35c1e6f1feac762c9c822ded0a4e322dc" },][[package]]diff --git a/web/src/components/large-model-form-field.tsx b/web/src/components/large-model-form-field.tsxindex 76da460131..f5040f84dc 100644--- a/web/src/components/large-model-form-field.tsx+++ b/web/src/components/large-model-form-field.tsx@@ -39,9 +39,15 @@ export const LargeModelFilterFormSchema = {llm_filter: z.string().optional(),};-type LargeModelFormFieldProps = Pick<NextInnerLLMSelectProps, 'ownerTenantId'>;+type LargeModelFormFieldProps = Pick<+ NextInnerLLMSelectProps,+ 'ownerTenantId'+> & {+ name?: string;+};export function LargeModelFormField({ownerTenantId,+ name = 'llm_id',}: LargeModelFormFieldProps) {const form = useFormContext();const { t } = useTranslation();@@ -51,7 +57,7 @@ export function LargeModelFormField({<><FormFieldcontrol={form.control}- name="llm_id"+ name={name}render={({ field }) => (<FormItem><FormLabel tooltip={t('chat.modelTip')}>diff --git a/web/src/components/pipeline-operator-tabs/pipeline-operator-form.tsx b/web/src/components/pipeline-operator-tabs/pipeline-operator-form.tsxindex bb133e7a98..221a599fe8 100644--- a/web/src/components/pipeline-operator-tabs/pipeline-operator-form.tsx+++ b/web/src/components/pipeline-operator-tabs/pipeline-operator-form.tsx@@ -1,5 +1,6 @@import { Operator } from '@/constants/agent';import { RAGFlowNodeType } from '@/interfaces/database/agent';+import CompilationForm from '@/pages/agent/form/compilation-form';import ExtractorForm from '@/pages/agent/form/extractor-form';import ParserForm from '@/pages/agent/form/parser-form';import TitleChunkerForm from '@/pages/agent/form/title-chunker-form';@@ -61,6 +62,14 @@ const PipelineOperatorForm = ({hideOutputs/>);+ case Operator.Compiler:+ return (+ <CompilationForm+ node={node}+ onValuesChange={handleValuesChange}+ hideOutputs+ />+ );case Operator.Tokenizer:return (<TokenizerFormdiff --git a/web/src/hooks/__tests__/use-dataset-generate-status.test.tsx b/web/src/hooks/__tests__/use-dataset-generate-status.test.tsxnew file mode 100644index 0000000000..8e275d57e2--- /dev/null+++ b/web/src/hooks/__tests__/use-dataset-generate-status.test.tsx@@ -0,0 +1,110 @@+import { QueryClient, QueryClientProvider } from '@tanstack/react-query';+import { renderHook, waitFor } from '@testing-library/react';+import React from 'react';+import { GenerateType } from '@/constants/knowledge';++import { useTraceRunData } from '../use-dataset-generate';++jest.mock('react-router', () => ({+ useParams: jest.fn(() => ({ id: 'kb1' })),+}));++jest.mock('@/utils/api-proxy-scheme', () => ({+ isGoDatasetBackend: jest.fn(() => true),+}));++jest.mock('@/services/knowledge-service', () => ({+ getDatasetCompilationStatus: jest.fn(),+}));++// use-dataset-generate imports agent-service (and transitively register-server /+// next-request / locales config that touch import.meta.env). Mock it so the+// Go status path under test doesn't pull in that module graph.+jest.mock('@/services/agent-service', () => ({+ __esModule: true,+ default: { cancelDataflow: jest.fn(), deletePipelineTask: jest.fn() },+}));++jest.mock('react-i18next', () => ({+ useTranslation: () => ({ t: (key: string) => key }),+}));++import { getDatasetCompilationStatus } from '@/services/knowledge-service';+import { isGoDatasetBackend } from '@/utils/api-proxy-scheme';++const mockStatus = jest.mocked(getDatasetCompilationStatus);++function makeWrapper() {+ const queryClient = new QueryClient({+ defaultOptions: {+ queries: { retry: false },+ },+ });+ // esbuild-jest config here loads .ts with the "tsx" loader but .tsx with the+ // plain "ts" loader, so JSX in this test file would not transform. Build the+ // provider element with createElement instead.+ // eslint-disable-next-line @typescript-eslint/no-explicit-any+ const Wrapper = (props: { children: any }) =>+ React.createElement(+ QueryClientProvider,+ { client: queryClient },+ props.children,+ );+ return Wrapper;+}++describe('useTraceRunData (Go/hybrid compile-status contract)', () => {+ beforeEach(() => {+ jest.clearAllMocks();+ (isGoDatasetBackend as jest.Mock).mockReturnValue(true);+ });++ it('maps a successful status to the scheduler contract fields', async () => {+ mockStatus.mockResolvedValue({+ data: {+ code: 0,+ data: { state: 'running', inflight: 2, backlog: 1, error: '' },+ },+ } as never);++ const { result } = renderHook(+ () => useTraceRunData(GenerateType.Artifact),+ {+ wrapper: makeWrapper(),+ },+ );++ await waitFor(() => expect(result.current.isPending).toBe(false), {+ timeout: 5000,+ });++ const info = result.current.data;+ expect(info?.compilationState).toBe('running');+ expect(info?.inflight).toBe(2);+ expect(info?.backlog).toBe(1);+ expect(info?.compilationError).toBe('');+ });++ it('rejects a non-zero business code instead of mapping to idle', async () => {+ mockStatus.mockResolvedValue({+ data: { code: 1, message: 'no authorization', data: undefined },+ } as never);++ const { result } = renderHook(+ () => useTraceRunData(GenerateType.Artifact),+ {+ wrapper: makeWrapper(),+ },+ );++ // The query configures retry: 3 with a 1s delay, so the terminal error state+ // arrives after the retries drain.+ await waitFor(() => expect(result.current.isError).toBe(true), {+ timeout: 10000,+ });++ expect(result.current.error).toBeTruthy();+ // The error must not be silently treated as an idle/empty status.+ expect(result.current.data).toBeUndefined();+ });+});diff --git a/web/src/hooks/use-dataset-generate.ts b/web/src/hooks/use-dataset-generate.tsindex a0092f9ae4..8d568cef7c 100644--- a/web/src/hooks/use-dataset-generate.ts+++ b/web/src/hooks/use-dataset-generate.ts@@ -9,9 +9,11 @@ import {import agentService from '@/services/agent-service';import {deletePipelineTask,+ getDatasetCompilationStatus,runIndex,traceIndex,} from '@/services/knowledge-service';+import { isGoDatasetBackend } from '@/utils/api-proxy-scheme';import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';import { useMemo } from 'react';import { useTranslation } from 'react-i18next';@@ -48,6 +50,15 @@ export interface ITraceInfo {to_page: number;update_date: string;update_time: number;+ // Go scheduler compile-status contract (API_PROXY_SCHEME=go/hybrid). These+ // replace the legacy task percentage for the Go backend: state is the raw+ // dataset-level lifecycle (idle/pending/running/completed), inflight/backlog+ // are the MySQL scheduling-entry counts, and error is the batch diagnostic+ // (NOT a peer state). Only populated by the Go/hybrid branch of useTraceQuery.+ compilationState?: string;+ inflight?: number;+ backlog?: number;+ compilationError?: string;}const useTraceQuery = (@@ -61,6 +72,15 @@ const useTraceQuery = (gcTime: 0,refetchInterval: (query) => {const progress = query.state.data?.progress;+ // Go/hybrid: keep polling while the dataset compile is pending/running+ // (a failed batch is left for retry, so the row stays running/pending and+ // we keep polling until it drains to completed).+ if (isGoDatasetBackend()) {+ const state = query.state.data?.compilationState;+ return state === 'pending' || state === 'running'+ ? PollIntervalMs+ : false;+ }return progress != null && progress >= 0 && progress < 1? PollIntervalMs: false;@@ -69,6 +89,35 @@ const useTraceQuery = (retryDelay: 1000,enabled: open && !!id,queryFn: async () => {+ if (isGoDatasetBackend()) {+ // Scheduler compile-status contract (dataset-level, variant-agnostic).+ // The status is NOT a task percentage: we carry the raw state, the+ // MySQL inflight/backlog entry counts and the error diagnostic, and+ // derive the display status in useGenerateStatus. progress is only set+ // so the shared refetch/status helpers keep their contract (idle->0,+ // running/pending->0, completed->1, error->0).+ const res = await getDatasetCompilationStatus(id!);+ const data = res?.data;+ // The handler returns HTTP 200 with a non-zero business code for+ // authorization/business errors (e.g. "no authorization"). The request+ // interceptor only shows a toast and does not reject, so without this+ // explicit check a failed read would be mapped to a misleading idle+ // state. Reject so the query surfaces the error instead.+ if (!data || data.code !== 0) {+ throw new Error(data?.message || 'Failed to read compilation status');+ }+ const st = data.data ?? {};+ const state: string = st.state ?? 'idle';+ const error: string = st.error ?? '';+ return {+ progress: state === 'completed' ? 1 : state === 'idle' ? 0 : 0,+ progress_msg: error || state,+ compilationState: state,+ inflight: st.inflight ?? 0,+ backlog: st.backlog ?? 0,+ compilationError: error,+ } as ITraceInfo;+ }const { data } = await traceIndex(id!, traceType);return data?.data ?? {};},@@ -131,6 +180,14 @@ export const useDatasetGenerate = () => {} = useMutation({mutationKey: [DatasetKey.generate],mutationFn: async ({ type }: { type: GenerateType }) => {+ // Go/hybrid: dataset compilation is driven automatically by the scheduler+ // on document completion; there is no manual RunIndex trigger, so a manual+ // "generate" must NOT pretend to succeed. The UI hides/disables the+ // control; if it is ever invoked, reject loudly so callers don't mistake+ // it for a real run (plan v4.1 §4.2).+ if (isGoDatasetBackend()) {+ throw new Error(t('message.compileNotSupported'));+ }const { data } = await runIndex(id!, TraceTypeMap[type]);if (data.code === 0) {message.success(t('message.operated'));@@ -151,6 +208,12 @@ export const useDatasetGenerate = () => {task_id: string;type: GenerateType;}) => {+ // Go/hybrid: the scheduler has no task-level cancel; dataset compilation+ // is auto-driven. There is no pause to perform, so reject rather than+ // report success (the UI hides the pause control — plan v4.1 §4.2).+ if (isGoDatasetBackend()) {+ throw new Error(t('message.compileNotSupported'));+ }const { data } = await agentService.cancelDataflow(task_id);// For GraphRAG, pause must preserve partial progress (subgraphs,@@ -177,6 +240,19 @@ export function useGenerateStatus(data?: ITraceInfo) {if (!data) {return GenerateStatus.Start;}+ if (isGoDatasetBackend()) {+ // Go/hybrid: derive from the scheduler contract, not a fake task+ // percentage. Error diagnostic takes priority; otherwise map the raw+ // dataset-level state (completed->Completed, idle->Start, running/pending+ // ->Running).+ if (data.compilationError) {+ return GenerateStatus.Failed;+ }+ const st = data.compilationState;+ if (st === 'completed') return GenerateStatus.Completed;+ if (st === 'running' || st === 'pending') return GenerateStatus.Running;+ return GenerateStatus.Start;+ }if (data.progress >= 1) {return GenerateStatus.Completed;} else if (!data.progress && data.progress !== 0) {@@ -190,6 +266,13 @@ export function useGenerateStatus(data?: ITraceInfo) {}, [data]);const percent = useMemo(() => {+ if (isGoDatasetBackend()) {+ // No stable terminal state and no DocTotal/DocProcessed, so there is no+ // meaningful percentage. Failures render as a full error marker; active+ // runs show the inflight/backlog counts instead of a percent (see the+ // UpdateRunProgress / EmptyState components).+ return status === GenerateStatus.Failed ? 100 : 0;+ }if (status === GenerateStatus.Failed) {return 100;} else if (status === GenerateStatus.Running) {diff --git a/web/src/hooks/use-knowledge-request.ts b/web/src/hooks/use-knowledge-request.tsindex 138f05f19f..d33797d88f 100644--- a/web/src/hooks/use-knowledge-request.ts+++ b/web/src/hooks/use-knowledge-request.ts@@ -1,5 +1,6 @@import { useHandleFilterSubmit } from '@/components/list-filter-bar/use-handle-filter-submit';import message from '@/components/ui/message';+import { isGoDatasetBackend } from '@/utils/api-proxy-scheme';import { GenerateType, ParseType } from '@/constants/knowledge';import { ResponsePostType, ResponseType } from '@/interfaces/database/base';import {@@ -964,6 +965,13 @@ export const useRunArtifactIndex = (kind: string) => {} = useMutation({mutationKey: [KnowledgeApiAction.RunArtifactIndex],mutationFn: async () => {+ // Go/hybrid: wiki compilation is auto-driven by the scheduler; there is no+ // legacy RunIndex endpoint. Reject instead of reporting success so a wiki+ // update can't be mistaken for a real re-merge (the UI hides/disables the+ // update control — plan v4.1 §4.2).+ if (isGoDatasetBackend()) {+ throw new Error(i18n.t('message.compileNotSupported'));+ }const { data } = await runIndex(knowledgeBaseId, 'artifact');if (data?.code === 0) {message.success(i18n.t('message.operated'));diff --git a/web/src/locales/en.ts b/web/src/locales/en.tsindex 38754fe3d3..dc02289284 100644--- a/web/src/locales/en.ts+++ b/web/src/locales/en.ts@@ -476,6 +476,10 @@ Example: A 1 KB message with 1024-dim embedding uses ~9 KB. The 5 MB default limlog: 'Log',noSkills: 'No skills yet',generate: 'Generate',+ compiling: 'Compiling…',+ compilingCounts: '{{inflight}} processing / {{backlog}} queued',+ autoCompiled: 'Compiled automatically when documents are parsed.',+raptor: 'RAPTOR',artifact: 'Artifact',toSkills: 'To skills',@@ -2412,6 +2416,10 @@ Example: Virtual Hosted Style`,noLangfuseConfigToDelete: 'No Langfuse configuration to delete',renamed: 'Renamed',operated: 'Operated',+ compileAutoGenerated:+ 'Compilation runs automatically when documents are parsed; no manual trigger is needed.',+ compileNotSupported:+ 'Manual compilation is not supported here; it runs automatically when documents are parsed.',updated: 'Updated',uploaded: 'Uploaded',200: 'The server successfully returns the requested data.',diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.tsindex e8a93a1d3f..ae7a36cf5c 100644--- a/web/src/locales/zh.ts+++ b/web/src/locales/zh.ts@@ -412,6 +412,9 @@ export default {generateToSkills:'从该数据集构建分层技能树,并存储生成的技能页面以供搜索和复用。',noWikiPages: '暂无 Wiki 页面',+ compiling: '编译中…',+ compilingCounts: '处理中 {{inflight}} / 待处理 {{backlog}}',+ autoCompiled: '文档解析时自动编译。',clearWikiTitle: '清空 Wiki',clearWikiDescription:'确定要清空该数据集下的所有 Wiki 页面吗?此操作无法撤销。',@@ -2057,6 +2060,9 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取实体和关系noLangfuseConfigToDelete: '没有可删除的 Langfuse 配置',renamed: '重命名成功',operated: '操作成功',+ compileAutoGenerated: '知识编译会在文档解析时自动进行,无需手动触发。',+ compileNotSupported:+ '此处不支持手动编译,知识编译会在文档解析时自动进行。',updated: '更新成功',uploaded: '上传成功',200: '服务器成功返回请求的数据。',diff --git a/web/src/pages/agent/canvas/node/compilation-node.tsx b/web/src/pages/agent/canvas/node/compilation-node.tsxindex b86659d51f..bb5f0aaf2e 100644--- a/web/src/pages/agent/canvas/node/compilation-node.tsx+++ b/web/src/pages/agent/canvas/node/compilation-node.tsx@@ -2,7 +2,7 @@ import { useCompilationTemplateGroupOptions } from '@/hooks/use-compilation-tempimport { IRagNode } from '@/interfaces/database/agent';import { NodeProps } from '@xyflow/react';import { get } from 'lodash';-import { LabelCard } from './card';+import { LabelCard, LLMLabelCard } from './card';import { RagNode } from './index';import { useTranslation } from 'react-i18next';@@ -11,12 +11,14 @@ export function CompilationNode({ ...props }: NodeProps<IRagNode>) {const { t } = useTranslation();const options = useCompilationTemplateGroupOptions();const groupId = get(data, 'form.compilation_template_group_id');+ const llmId = get(data, 'form.llm_id');const groupName =options.find((option) => option.value === groupId)?.label ?? groupId;return (<RagNode {...props}><section className="flex flex-col gap-2">+ <LLMLabelCard llmId={llmId}></LLMLabelCard><LabelCard className="text-text-primary flex justify-between flex-col gap-1"><span className="text-text-secondary">{t('knowledgeConfiguration.compilationTemplate')}diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsxindex f09cc15686..03dbb7e6f5 100644--- a/web/src/pages/agent/constant/pipeline.tsx+++ b/web/src/pages/agent/constant/pipeline.tsx@@ -362,6 +362,7 @@ export const initialExtractorValues = {export const initialCompilationValues = {compilation_template_group_id: '',+ llm_id: '',outputs: {chunks: { type: 'Array<Object>', value: [] },},diff --git a/web/src/pages/agent/form/compilation-form/index.tsx b/web/src/pages/agent/form/compilation-form/index.tsxindex 3e2a7d3fdf..1cdbf0fc02 100644--- a/web/src/pages/agent/form/compilation-form/index.tsx+++ b/web/src/pages/agent/form/compilation-form/index.tsx@@ -1,10 +1,13 @@import { CompilationTemplateFormField } from '@/components/compilation-template-form-field';+import { LargeModelFormField } from '@/components/large-model-form-field';import { Form } from '@/components/ui/form';import { zodResolver } from '@hookform/resolvers/zod';import { memo } from 'react';import { useForm } from 'react-hook-form';import { z } from 'zod';import { initialCompilationValues } from '../../constant/pipeline';+import { useOwnerTenantId } from '../../context';+import { useFormChangeCallback } from '../../hooks/use-form-change-callback';import { useFormValues } from '../../hooks/use-form-values';import { useWatchFormChange } from '../../hooks/use-watch-form-change';import { INextOperatorForm } from '../../interface';@@ -14,28 +17,44 @@ import { Output } from '../components/output';export const FormSchema = z.object({compilation_template_group_id: z.string().optional(),+ llm_id: z.string().optional(),});export type CompilationFormSchemaType = z.infer<typeof FormSchema>;const outputList = buildOutputList(initialCompilationValues.outputs);-const CompilationForm = ({ node }: INextOperatorForm) => {+const CompilationForm = ({+ node,+ onValuesChange,+ hideOutputs,+}: INextOperatorForm) => {const defaultValues = useFormValues(initialCompilationValues, node);+ const ownerTenantId = useOwnerTenantId();const form = useForm<CompilationFormSchemaType>({defaultValues,resolver: zodResolver(FormSchema),+ mode: 'onChange',});useWatchFormChange(node?.id, form);+ useFormChangeCallback(form, onValuesChange);return (<Form {...form}><FormWrapper><CompilationTemplateFormField name="compilation_template_group_id"></CompilationTemplateFormField>- <Output list={outputList}></Output>+ <LargeModelFormField+ name="llm_id"+ ownerTenantId={ownerTenantId}+ ></LargeModelFormField></FormWrapper>+ {!hideOutputs && (+ <div className="p-5">+ <Output list={outputList}></Output>+ </div>+ )}</Form>);};diff --git a/web/src/pages/agent/hooks/use-add-node.ts b/web/src/pages/agent/hooks/use-add-node.tsindex 82491da75e..1cbe828239 100644--- a/web/src/pages/agent/hooks/use-add-node.ts+++ b/web/src/pages/agent/hooks/use-add-node.ts@@ -185,7 +185,7 @@ export const useInitializeOperatorParams = () => {sys_prompt: t('flow.prompts.system.summary'),prompts: t('flow.prompts.user.summary'),},- [Operator.Compiler]: initialCompilationValues,+ [Operator.Compiler]: { ...initialCompilationValues, llm_id: llmId },[Operator.DataOperations]: initialDataOperationsValues,[Operator.ListOperations]: initialListOperationsValues,[Operator.VariableAssigner]: initialVariableAssignerValues,diff --git a/web/src/pages/dataset/compilation/empty-state.tsx b/web/src/pages/dataset/compilation/empty-state.tsxindex 1c6556cf68..056cb08331 100644--- a/web/src/pages/dataset/compilation/empty-state.tsx+++ b/web/src/pages/dataset/compilation/empty-state.tsx@@ -9,6 +9,7 @@ import {useDatasetGenerate,useGenerateStatus,} from '@/hooks/use-dataset-generate';+import { isGoDatasetBackend } from '@/utils/api-proxy-scheme';import {GenerableViewMode,@@ -58,29 +59,66 @@ export function CompilationEmptyState({}, [pauseGenerate, data?.id, generateType]);const showProgress = status === 'running' || status === 'failed';+ const isGo = isGoDatasetBackend();return (<div className="flex-1 min-h-0 flex flex-col items-center justify-center border border-dashed border-border-button rounded-xl">{!showProgress ? (<div className="flex flex-col items-center gap-4"><p className="text-text-secondary text-lg">{t(TitleKeyMap[type])}</p>- <Button- variant="outline"- onClick={handleGenerate}- disabled={disabled}- >- <WandSparkles className="mr-2 size-4" />- {t('knowledgeDetails.generate')}- </Button>+ {!isGo && (+ <Button+ variant="outline"+ onClick={handleGenerate}+ disabled={disabled}+ >+ <WandSparkles className="mr-2 size-4" />+ {t('knowledgeDetails.generate')}+ </Button>+ )}+ {isGo && (+ <p className="text-sm text-text-secondary">+ {t('knowledgeDetails.autoCompiled')}+ </p>+ )}</div>) : (<div className="grid h-full w-full grid-cols-[1fr_auto_1fr] items-center gap-8 p-6"><div /><div className="flex flex-col items-center gap-5">- <ProgressRing percent={percent} failed={status === 'failed'} />+ {isGo ? (+ // Go/hybrid: no stable percentage and no scheduler cancel, so show+ // the MySQL inflight/backlog counts (or the error diagnostic).+ status === 'failed' ? (+ <div className="flex flex-col items-center gap-2 text-state-error">+ <IconFontFill name="reparse" className="size-8" />+ <span className="text-text-primary">+ {data?.compilationError || t('message.operated')}+ </span>+ </div>+ ) : (+ <div className="flex flex-col items-center gap-2 text-text-secondary">+ <span className="text-4xl font-medium text-accent-primary">+ {t('knowledgeDetails.compiling', {+ defaultValue: 'Compiling…',+ })}+ </span>+ <span>+ {t('knowledgeDetails.compilingCounts', {+ inflight: data?.inflight ?? 0,+ backlog: data?.backlog ?? 0,+ defaultValue:+ '{{inflight}} processing / {{backlog}} queued',+ })}+ </span>+ </div>+ )+ ) : (+ <ProgressRing percent={percent} failed={status === 'failed'} />+ )}<div className="flex items-center gap-2 text-text-primary"><span>{t(ViewModeLabelKeyMap[type])}</span>- {status === 'failed' && (+ {!isGo && status === 'failed' && (<span className="cursor-pointer" onClick={handleGenerate}><IconFontFillname="reparse"@@ -88,7 +126,7 @@ export function CompilationEmptyState({/></span>)}- {status !== 'failed' && (+ {!isGo && status !== 'failed' && (<spanclassName="text-state-error cursor-pointer"onClick={handlePause}diff --git a/web/src/pages/dataset/compilation/update-button.tsx b/web/src/pages/dataset/compilation/update-button.tsxindex 66dfcdb78a..6737add545 100644--- a/web/src/pages/dataset/compilation/update-button.tsx+++ b/web/src/pages/dataset/compilation/update-button.tsx@@ -11,6 +11,7 @@ import {} from '@/components/ui/tooltip';import { GenerateStatus, GenerateType } from '@/constants/knowledge';import { ITraceInfo, useGenerateStatus } from '@/hooks/use-dataset-generate';+import { isGoDatasetBackend } from '@/utils/api-proxy-scheme';import { UpdateRunProgress } from './update-run-progress';@@ -40,6 +41,13 @@ export function CompilationUpdateButton({const isRunning = status === GenerateStatus.Running;const isGenerating = isRunning || status === GenerateStatus.Failed;+ // Go/hybrid: compilation is auto-driven by the scheduler with no manual+ // re-merge, so hide the update control rather than offer a trigger that+ // cannot work (plan v4.1 §4.2).+ if (isGoDatasetBackend()) {+ return null;+ }+// A failed trace persists (progress stays < 0) until the next run, so it// must not keep the button visible on its own — only real changes or a// live run should.diff --git a/web/src/pages/dataset/compilation/update-run-progress.tsx b/web/src/pages/dataset/compilation/update-run-progress.tsxindex ff087351e8..4003decbb1 100644--- a/web/src/pages/dataset/compilation/update-run-progress.tsx+++ b/web/src/pages/dataset/compilation/update-run-progress.tsx@@ -1,5 +1,6 @@import { CircleX } from 'lucide-react';import { useCallback, type MouseEvent } from 'react';+import { useTranslation } from 'react-i18next';import { IconFontFill } from '@/components/icon-font';import { GenerateType } from '@/constants/knowledge';@@ -8,6 +9,7 @@ import {useDatasetGenerate,useGenerateStatus,} from '@/hooks/use-dataset-generate';+import { isGoDatasetBackend } from '@/utils/api-proxy-scheme';import { cn } from '@/lib/utils';import { toFixed } from '@/utils/common-util';@@ -20,8 +22,10 @@ export function UpdateRunProgress({data,generateType,}: UpdateRunProgressProps) {+ const { t } = useTranslation();const { pauseGenerate } = useDatasetGenerate();const { status, percent } = useGenerateStatus(data);+ const isGo = isGoDatasetBackend();const handlePause = useCallback((e: MouseEvent) => {@@ -33,6 +37,39 @@ export function UpdateRunProgress({[pauseGenerate, data?.id, generateType],);+ // Go/hybrid: no scheduler task-level cancel, and no stable terminal state, so+ // show the MySQL inflight/backlog entry counts instead of a percentage and no+ // pause button. Error diagnostic takes priority (see plan v4.1 §4.2).+ if (isGo && status === 'running') {+ return (+ <span className="flex items-center gap-2 text-sm text-text-secondary">+ <span className="size-2 rounded-full bg-accent-primary" />+ {data?.compilationError+ ? data.compilationError+ : t('knowledgeDetails.compiling', {+ defaultValue: 'Compiling…',+ })}+ {!data?.compilationError && (+ <span>+ {t('knowledgeDetails.compilingCounts', {+ inflight: data?.inflight ?? 0,+ backlog: data?.backlog ?? 0,+ defaultValue: '{{inflight}} processing / {{backlog}} queued',+ })}+ </span>+ )}+ </span>+ );+ }+ if (isGo && status === 'failed') {+ return (+ <span className="flex items-center gap-2 text-sm text-state-error">+ <IconFontFill name="reparse" className="text-accent-primary" />+ {data?.compilationError || t('message.operated')}+ </span>+ );+ }+return (<span className="flex items-center gap-2"><span className="bg-border-button h-1 w-16 rounded-full">diff --git a/web/src/pages/dataset/compilation/utils/parse-wiki-link.ts b/web/src/pages/dataset/compilation/utils/parse-wiki-link.tsindex 29e79d841c..7accc647b8 100644--- a/web/src/pages/dataset/compilation/utils/parse-wiki-link.ts+++ b/web/src/pages/dataset/compilation/utils/parse-wiki-link.ts@@ -1,4 +1,4 @@-export type WikiPageType = 'concept' | 'entity';+export type WikiPageType = 'concept' | 'entity' | 'topic';/*** Parse an internal wiki link href into pageType and slug.@@ -9,7 +9,7 @@ export type WikiPageType = 'concept' | 'entity';* {pageType}/{slug}* /{pageType}/{slug}*- * Only entity/ and concept/ links are considered wiki navigation links.+ * entity/, concept/ and topic/ links are all considered wiki navigation links.*/export function parseWikiLinkHref(href: string,@@ -19,7 +19,7 @@ export function parseWikiLinkHref(// Prefer the artifact/{datasetId}/{pageType}/{slug} form.const artifactMatch = normalized.match(- /(?:^|\/)artifact\/[^/]+\/(entity|concept)\/([^/\s"']+)/,+ /(?:^|\/)artifact\/[^/]+\/(entity|concept|topic)\/([^/\s"']+)/,);if (artifactMatch) {return {@@ -29,7 +29,9 @@ export function parseWikiLinkHref(}// Fallback to a plain {pageType}/{slug} form.- const simpleMatch = normalized.match(/(?:^|\/)(entity|concept)\/([^/\s"']+)/);+ const simpleMatch = normalized.match(+ /(?:^|\/)(entity|concept|topic)\/([^/\s"']+)/,+ );if (simpleMatch) {return {pageType: simpleMatch[1] as WikiPageType,diff --git a/web/src/pages/dataset/compilation/wiki-left-panel/hooks/use-wiki-navigation.ts b/web/src/pages/dataset/compilation/wiki-left-panel/hooks/use-wiki-navigation.tsindex c37fa40208..8c3adf511b 100644--- a/web/src/pages/dataset/compilation/wiki-left-panel/hooks/use-wiki-navigation.ts+++ b/web/src/pages/dataset/compilation/wiki-left-panel/hooks/use-wiki-navigation.ts@@ -6,7 +6,7 @@ import { IArtifactTopic } from '@/interfaces/database/dataset';import { useDebounce } from 'ahooks';import { useCallback, useMemo, useRef, useState } from 'react';-export type WikiPageType = 'concept' | 'entity';+export type WikiPageType = 'concept' | 'entity' | 'topic';export function useWikiNavigation() {const scrollRef = useRef<HTMLDivElement>(null);diff --git a/web/src/pages/files/files-table.tsx b/web/src/pages/files/files-table.tsxindex f25c814155..4c1268ccf5 100644--- a/web/src/pages/files/files-table.tsx+++ b/web/src/pages/files/files-table.tsx@@ -52,8 +52,7 @@ import { LinkToDatasetDialog } from './link-to-dataset-dialog';import { UseMoveDocumentShowType } from './use-move-file';import { useNavigateToOtherFolder } from './use-navigate-to-folder';import { isFolderType, isKnowledgeBaseType } from './util';--declare const __API_PROXY_SCHEME__: string;+import { isGoDatasetBackend } from '../../utils/api-proxy-scheme';type FilesTableProps = Pick<ReturnType<typeof useFetchFileList>,@@ -104,13 +103,7 @@ export function FilesTable({} = useRenameCurrentFile();// Check if skills feature is enabled (only in hybrid or go mode)- const isSkillsEnabled = useMemo(() => {- const scheme =- typeof __API_PROXY_SCHEME__ !== 'undefined'- ? __API_PROXY_SCHEME__- : 'python';- return scheme === 'hybrid' || scheme === 'go';- }, []);+ const isSkillsEnabled = useMemo(() => isGoDatasetBackend(), []);// Sort files with skills folder first, then by time// Filter out skills folder if not in hybrid/go modediff --git a/web/src/services/knowledge-service.ts b/web/src/services/knowledge-service.tsindex 62dabd62ba..91dba1b196 100644--- a/web/src/services/knowledge-service.ts+++ b/web/src/services/knowledge-service.ts@@ -10,7 +10,7 @@ import {} from '@/interfaces/request/knowledge';import api from '@/utils/api';import nextRequest from '@/utils/next-request';-import registerServer from '@/utils/register-server';+import registerServer, { registerNextServer } from '@/utils/register-server';import request from '@/utils/request';const {@@ -287,6 +287,20 @@ export const runIndex = (datasetId: string, indexType: string) =>export const traceIndex = (datasetId: string, indexType: string) =>request.get(api.traceIndex(datasetId, indexType));+// getDatasetCompilationStatus reads the Go scheduler compile-status contract+// (GET /datasets/:id/compilation/status), used by API_PROXY_SCHEME=go/hybrid to+// replace the legacy traceIndex task-progress endpoint. Route it through the+// service-layer proxy (registerNextServer -> next-request) like the rest of the+// *-service.ts HTTP proxies.+const compilationStatusProxy = registerNextServer({+ getDatasetCompilationStatus: {+ url: (datasetId: string) => api.compilationStatus(datasetId),+ method: 'get',+ },+} as const);+export const getDatasetCompilationStatus = (datasetId: string) =>+ compilationStatusProxy.getDatasetCompilationStatus(datasetId);+// Using RESTful API: GET /api/v1/datasets/{dataset_id}/documentsexport const listDocument = (params?: IFetchKnowledgeListRequestParams,diff --git a/web/src/utils/api-proxy-scheme.ts b/web/src/utils/api-proxy-scheme.tsnew file mode 100644index 0000000000..9f247da6a3--- /dev/null+++ b/web/src/utils/api-proxy-scheme.ts@@ -0,0 +1,48 @@+/**+ * Shared helper for the Vite-injected `API_PROXY_SCHEME` runtime value.+ *+ * Vite (web/vite.config.ts) injects `__API_PROXY_SCHEME__` from+ * `import.meta.env.API_PROXY_SCHEME`. It selects which backend serves the+ * dataset compilation / index APIs:+ *+ * - `python`: all /api requests go to the Python backend (9380); the legacy+ * `/datasets/:id/index` run/trace endpoints remain valid.+ * - `go`: all /api requests go to the Go backend (9384); the legacy+ * `/datasets/:id/index` routes are removed, so UI must use the scheduler+ * compile-status contract instead.+ * - `hybrid`: Vite routes `/api/v1/datasets` to the Go backend (9384);+ * dataset compilation logic behaves like `go`.+ *+ * Default is `python` for safety (matches Vite's fallback).+ *+ * All pages must read the scheme through these predicates instead of scattering+ * `typeof __API_PROXY_SCHEME__` checks (see files-table.tsx, which previously+ * inlined this logic).+ */++declare const __API_PROXY_SCHEME__: string;++export type ApiProxyScheme = 'python' | 'go' | 'hybrid';++export function getApiProxyScheme(): ApiProxyScheme {+ const raw = typeof __API_PROXY_SCHEME__ !== 'undefined' ? __API_PROXY_SCHEME__ : '';+ switch (raw) {+ case 'go':+ case 'hybrid':+ case 'python':+ return raw;+ default:+ return 'python';+ }+}++/** True when the dataset compilation APIs are served by the Go backend. */+export function isGoDatasetBackend(): boolean {+ const s = getApiProxyScheme();+ return s === 'go' || s === 'hybrid';+}++/** True when the legacy Python run/trace `/index` endpoints are in use. */+export function isPythonDatasetBackend(): boolean {+ return getApiProxyScheme() === 'python';+}diff --git a/web/src/utils/api.ts b/web/src/utils/api.tsindex 83c8371b6b..b06f4bd1d6 100644--- a/web/src/utils/api.ts+++ b/web/src/utils/api.ts@@ -215,6 +215,9 @@ export default {`${restAPIv1}/datasets/${datasetId}/index?type=${indexType.toLowerCase()}`,traceIndex: (datasetId: string, indexType: string) =>`${restAPIv1}/datasets/${datasetId}/index?type=${indexType.toLowerCase()}`,+ // Go scheduler compile-status contract (API_PROXY_SCHEME=go/hybrid).+ compilationStatus: (datasetId: string) =>+ `${restAPIv1}/datasets/${datasetId}/compilation/status`,unbindPipelineTask: (datasetId: string, indexType: string, wipe?: boolean) =>`${restAPIv1}/datasets/${datasetId}/${indexType.toLowerCase()}${wipe === false ? '?wipe=false' : ''}`,pipelineRerun: `${restAPIv1}/agents/rerun`,