RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(knowledge_compile): materialize wiki page graph (wiki_entity/wiki_relation) (#17976)
Zhichang Yuf12c0ec08ac5
diff --git a/go.mod b/go.modindex 7de8df30d6..698c7aabb1 100644--- a/go.mod+++ b/go.mod@@ -20,7 +20,7 @@ require (github.com/browserbase/stagehand-go/v3 v3.21.0github.com/cenkalti/backoff/v5 v5.0.3github.com/cespare/xxhash/v2 v2.3.0- github.com/cloudwego/eino v0.9.12+ github.com/cloudwego/eino v0.9.13github.com/denisenkom/go-mssqldb v0.12.3github.com/elastic/go-elasticsearch/v8 v8.19.1github.com/eric642/e2b-go-sdk v0.1.3diff --git a/go.sum b/go.sumindex d893f700d7..10f65eb35c 100644--- a/go.sum+++ b/go.sum@@ -167,8 +167,8 @@ github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tngithub.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=github.com/cloudwego/base64x v0.1.7 h1:NppS+Fgzg5ovhn4NkUXaDT3x9jldgH5ToMCqzBSi2zI=github.com/cloudwego/base64x v0.1.7/go.mod h1:Cu1PV9zfrSf7ET2tIbWbbEy7jO7HHJ13q4X2SQ8aWYg=-github.com/cloudwego/eino v0.9.12 h1:mHAMo5k7GdvnVD8Lc2sLyfpkxEm0S/y3PkEMhsSYt78=-github.com/cloudwego/eino v0.9.12/go.mod h1:OBD1mrkfkt/pJa4rkg1P0VnaMeOVl7l8IAdEqY//3IQ=+github.com/cloudwego/eino v0.9.13 h1:iD/ETS+lxnNp1VeNPqWVGPWdND6Dbf4LyINbLUlDRcM=+github.com/cloudwego/eino v0.9.13/go.mod h1:OBD1mrkfkt/pJa4rkg1P0VnaMeOVl7l8IAdEqY//3IQ=github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik=github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4=diff --git a/internal/agent/canvas/node_body.go b/internal/agent/canvas/node_body.goindex d01ec37b9f..7146cdf80a 100644--- a/internal/agent/canvas/node_body.go+++ b/internal/agent/canvas/node_body.go@@ -40,6 +40,8 @@ import ("time""ragflow/internal/agent/runtime"++ "go.uber.org/zap")// nodeBodyFn is the plain function shape compose.InvokableLambda accepts.@@ -154,12 +156,10 @@ func buildNodeBodyWithOptions(ctx context.Context, cpnID, name string, params maif comp == nil {return nil, fmt.Errorf("canvas: component %q (%s): factory returned nil component", cpnID, name)}- // Pass the class name through to the body so the per-class- // timeout resolver (resolveTimeout) can pick the right- // timeout without the runtime.Component interface needing- // to expose Name(). The factory returns the class name as- // the DSL's `component_name` field, which is also what- // ComponentBase.Name() would have returned.+ // Pass the class name through to the body for structured logging+ // without the runtime.Component interface needing to expose Name().+ // The factory returns the class name as the DSL's `component_name`+ // field, which is also what ComponentBase.Name() would have returned.return realComponentBodyWithOptions(cpnID, name, comp, opts), nil}// Fallback: no factory registered. This path is only exercised by@@ -221,11 +221,11 @@ func componentTimeout() time.Duration {// reach components here. Cross-cutting concerns therefore belong here,// not inside each component's Invoke://-// - per-class timeout: context.WithTimeout from resolveTimeout-// (4-level: per-class env → per-class defaults table → uniform env-// → 600s fallback). The lookup is per-invocation (not per-body) so-// operators can tune COMPONENT_EXEC_TIMEOUT[_<CLASS>] at runtime-// without rebuilding graphs.+// - execution context: derived from the parent with cancellation only — no+// framework-level wall-clock deadline is imposed on any component, so a+// long knowledge-compilation step (many LLM calls) is not cut short by a+// fixed ceiling. A cancelled task still interrupts via the parent cancel;+// per-call model deadlines stay at the model driver.// - progress: runtime.TrackProgress, with the callback pulled from// ctx (nil ⇒ no observer). This makes progress a framework-level// concern — components no longer wrap themselves.@@ -248,8 +248,14 @@ func realComponentBody(cpnID, componentClass string, comp runtime.Component) nodfunc realComponentBodyWithOptions(cpnID, componentClass string, comp runtime.Component, opts runtime.ComponentExecutionOptions) nodeBodyFn {return func(ctx context.Context, in map[string]any) (map[string]any, error) {- timeout := resolveTimeoutFromContext(ctx, componentClass)- cctx, cancel := context.WithTimeout(ctx, timeout)+ // The framework imposes no wall-clock deadline on component execution.+ // Long-running components (notably knowledge compilation, which fans out+ // over many model calls) exceed any fixed ceiling and would otherwise+ // fail with "context deadline exceeded". We derive a cancellation-only+ // context so a cancelled task (parent cancel) still interrupts the+ // component mid-run, while no arbitrary timeout is applied. Per-call+ // model deadlines remain enforced at the model driver.+ cctx, cancel := context.WithCancel(ctx)defer cancel()var out map[string]any@@ -261,10 +267,16 @@ func realComponentBodyWithOptions(cpnID, componentClass string, comp runtime.Comreturn e})if invokeErr != nil {+ // Surface the failure as a structured log line. The wrapped error+ // already carries the full cause chain (e.g. deepseek DNS/timeout),+ // but without this the failure only showed up as a generic+ // "Task ... failed" line with no clear cause in the logs.+ common.Error("canvas: component invoke failed", invokeErr,+ zap.String("component_id", cpnID),+ zap.String("component_class", componentClass))switch {case errors.Is(invokeErr, context.DeadlineExceeded):- return nil, fmt.Errorf("canvas: component %q invoke: timeout after %s: %w",- cpnID, timeout, invokeErr)+ return nil, fmt.Errorf("canvas: component %q invoke: context deadline exceeded: %w", cpnID, invokeErr)case errors.Is(invokeErr, context.Canceled):return nil, fmt.Errorf("canvas: component %q invoke: cancelled: %w", cpnID, invokeErr)}diff --git a/internal/agent/canvas/node_body_per_class_timeout_integration_test.go b/internal/agent/canvas/node_body_per_class_timeout_integration_test.godeleted file mode 100644index 1479785096..0000000000--- a/internal/agent/canvas/node_body_per_class_timeout_integration_test.go+++ /dev/null@@ -1,270 +0,0 @@-//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.-//--// node_body_per_class_timeout_integration_test.go — pins the-// per-class timeout integration.-//-// Verifies that buildNodeBody's runtime.Component path actually-// plumbs the per-class timeout (canvas/timeout.go's-// componentDefaults table) end-to-end. The per-class table is-// real; realComponentBody must call resolveTimeout(class) instead-// of the uniform componentTimeout() — this test pins that-// contract so a future refactor cannot silently regress to a-// uniform timeout.--package canvas--import (- "context"- "errors"- "sync/atomic"- "testing"- "time"-- "ragflow/internal/agent/runtime"-- "gorm.io/gorm"-)--// captureCtxComponent is a runtime.Component that records the-// deadline of the context it was invoked with. The integration-// test then asserts the deadline matches the per-class timeout-// (ExeSQL=3s, TavilySearch=12s, LLM=600s, ...). The Component-// returns immediately on success; on timeout it surfaces a-// context.DeadlineExceeded error so the body can wrap and surface-// it as a timeout error.-type captureCtxComponent struct {- name string- deadline atomic.Int64 // unix-nano deadline observed- timeoutOK atomic.Bool // true if ctx had a non-zero deadline-}--func (c *captureCtxComponent) Name() string { return c.name }-func (c *captureCtxComponent) Invoke(ctx context.Context, _ *gorm.DB, _ map[string]any) (map[string]any, error) {- if dl, ok := ctx.Deadline(); ok {- c.deadline.Store(dl.UnixNano())- c.timeoutOK.Store(true)- } else {- c.timeoutOK.Store(false)- }- // Block until ctx times out so the body can wrap and surface- // the timeout error; that is the path real components hit- // when they overrun the configured ceiling.- <-ctx.Done()- return nil, ctx.Err()-}-func (c *captureCtxComponent) Stream(_ context.Context, _ map[string]any) (<-chan map[string]any, error) {- return nil, nil-}-func (c *captureCtxComponent) Inputs() map[string]string { return nil }-func (c *captureCtxComponent) Outputs() map[string]string { return nil }--var _ runtime.Component = (*captureCtxComponent)(nil)--// TestBuildNodeBody_PerClassTimeout_ExeSQL_3s asserts that an-// ExeSQL-class component body wraps the Invoke context with a-// 3-second deadline (per the componentDefaults table in-// timeout.go), not the uniform 600s fallback or the-// COMPONENT_EXEC_TIMEOUT value (which we leave at the default).-//-// Steps:-//-// 1. Register a factory that returns a captureCtxComponent for-// the class "ExeSQL".-// 2. Call buildNodeBody("cpn", "ExeSQL", nil) — must NOT take-// the legacyNoOp / placeholder / UserFillUp branch.-// 3. Invoke the body with a fresh context, capture the-// component-observed deadline via the captureCtxComponent.-// 4. Assert the deadline is within ±500ms of 3s, AND the body-// surfaces a wrapped context.DeadlineExceeded (the-// realComponentBody timeout-wrap contract).-func TestBuildNodeBody_PerClassTimeout_ExeSQL_3s(t *testing.T) {- // Make sure uniform env is unset so the per-class table- // is the only authority; the per-class table is the one- // that defines "ExeSQL → 3s".- t.Setenv("COMPONENT_EXEC_TIMEOUT", "")- t.Setenv("COMPONENT_EXEC_TIMEOUT_EXESQL", "")-- cap := &captureCtxComponent{name: "ExeSQL"}- origFactory := runtime.DefaultFactory()- runtime.ResetDefaultFactoryForTesting()- runtime.SetDefaultFactory(func(class string, _ map[string]any) (runtime.Component, error) {- if class == "ExeSQL" {- return cap, nil- }- return nil, errors.New("not used in this test: " + class)- })- t.Cleanup(func() {- runtime.ResetDefaultFactoryForTesting()- if origFactory != nil {- runtime.SetDefaultFactory(origFactory)- }- })-- body, err := buildNodeBody(context.Background(), "cpn-exe", "ExeSQL", nil)- if err != nil {- t.Fatalf("buildNodeBody: %v", err)- }- start := time.Now()- _, err = body(context.Background(), nil)- if !errors.Is(err, context.DeadlineExceeded) {- t.Fatalf("body err = %v, want context.DeadlineExceeded (per-class timeout should fire)", err)- }- if !cap.timeoutOK.Load() {- t.Fatal("component never observed a deadline — realComponentBody did not wrap with context.WithTimeout")- }- // Semantic assertion: the deadline the body wired into the- // component's ctx must be ~3s after the call started. This is- // the load-bearing property — proving the per-class table- // reached realComponentBody. The previous `elapsed > 5s` upper- // bound conflated this with wall-clock scheduler jitter; if the- // deadline is wrong (e.g. 600s fallback), sinceDeadline catches- // it without depending on a fragile absolute threshold.- deadline := time.Unix(0, cap.deadline.Load())- sinceDeadline := deadline.Sub(start)- if sinceDeadline < 2500*time.Millisecond || sinceDeadline > 3500*time.Millisecond {- t.Errorf("ExeSQL deadline offset = %s, want ~3s", sinceDeadline)- }-}--// TestBuildNodeBody_PerClassTimeout_TavilySearch_12s asserts the-// 12-second web-search class. A Tavily call that hangs on the-// upstream API should give up after 12s and let the agent-// branch / retry, not consume 600s of wall-clock budget.-func TestBuildNodeBody_PerClassTimeout_TavilySearch_12s(t *testing.T) {- t.Setenv("COMPONENT_EXEC_TIMEOUT", "")- t.Setenv("COMPONENT_EXEC_TIMEOUT_TAVILYSEARCH", "")-- cap := &captureCtxComponent{name: "TavilySearch"}- origFactory := runtime.DefaultFactory()- runtime.ResetDefaultFactoryForTesting()- runtime.SetDefaultFactory(func(class string, _ map[string]any) (runtime.Component, error) {- if class == "TavilySearch" {- return cap, nil- }- return nil, errors.New("not used in this test: " + class)- })- t.Cleanup(func() {- runtime.ResetDefaultFactoryForTesting()- if origFactory != nil {- runtime.SetDefaultFactory(origFactory)- }- })-- body, err := buildNodeBody(context.Background(), "cpn-tav", "TavilySearch", nil)- if err != nil {- t.Fatalf("buildNodeBody: %v", err)- }- start := time.Now()- _, err = body(context.Background(), nil)- if !errors.Is(err, context.DeadlineExceeded) {- t.Fatalf("body err = %v, want context.DeadlineExceeded", err)- }- deadline := time.Unix(0, cap.deadline.Load())- sinceDeadline := deadline.Sub(start)- // Semantic: deadline the body wired is ~12s after start.- if sinceDeadline < 11*time.Second || sinceDeadline > 13*time.Second {- t.Errorf("TavilySearch deadline offset = %s, want ~12s", sinceDeadline)- }-}--// TestBuildNodeBody_PerClassTimeout_UnknownClass_UniformFallback-// asserts that a class NOT in the componentDefaults table (e.g.-// "CustomComponent") falls through to the uniform-// COMPONENT_EXEC_TIMEOUT env var. This is the 4-level resolution-// contract from timeout.go: per-class env → per-class table →-// uniform env → 600s fallback.-//-// We test the uniform-env fallback path (level 3) by setting-// COMPONENT_EXEC_TIMEOUT to a non-default value and registering-// a class that is NOT in componentDefaults.-func TestBuildNodeBody_PerClassTimeout_UnknownClass_UniformFallback(t *testing.T) {- t.Setenv("COMPONENT_EXEC_TIMEOUT", "5")- t.Setenv("COMPONENT_EXEC_TIMEOUT_CUSTOMCOMPONENT", "")-- cap := &captureCtxComponent{name: "CustomComponent"}- origFactory := runtime.DefaultFactory()- runtime.ResetDefaultFactoryForTesting()- runtime.SetDefaultFactory(func(class string, _ map[string]any) (runtime.Component, error) {- if class == "CustomComponent" {- return cap, nil- }- return nil, errors.New("not used in this test: " + class)- })- t.Cleanup(func() {- runtime.ResetDefaultFactoryForTesting()- if origFactory != nil {- runtime.SetDefaultFactory(origFactory)- }- })-- body, err := buildNodeBody(context.Background(), "cpn-cust", "CustomComponent", nil)- if err != nil {- t.Fatalf("buildNodeBody: %v", err)- }- start := time.Now()- _, err = body(context.Background(), nil)- if !errors.Is(err, context.DeadlineExceeded) {- t.Fatalf("body err = %v, want context.DeadlineExceeded", err)- }- deadline := time.Unix(0, cap.deadline.Load())- sinceDeadline := deadline.Sub(start)- // 5s uniform env should win for an unknown class. Allow a- // 1s slack on each side.- if sinceDeadline < 4*time.Second || sinceDeadline > 6*time.Second {- t.Errorf("CustomComponent deadline offset = %s, want ~5s (uniform env fallback)", sinceDeadline)- }-}--// TestBuildNodeBody_PerClassTimeout_PerClassEnvOverride asserts-// that COMPONENT_EXEC_TIMEOUT_<CLASS> wins over the table.-// ExeSQL's table entry is 3s; setting the per-class env to 7-// should make the body's deadline ~7s, not 3s.-func TestBuildNodeBody_PerClassTimeout_PerClassEnvOverride(t *testing.T) {- t.Setenv("COMPONENT_EXEC_TIMEOUT", "")- t.Setenv("COMPONENT_EXEC_TIMEOUT_EXESQL", "7")-- cap := &captureCtxComponent{name: "ExeSQL"}- origFactory := runtime.DefaultFactory()- runtime.ResetDefaultFactoryForTesting()- runtime.SetDefaultFactory(func(class string, _ map[string]any) (runtime.Component, error) {- if class == "ExeSQL" {- return cap, nil- }- return nil, errors.New("not used in this test: " + class)- })- t.Cleanup(func() {- runtime.ResetDefaultFactoryForTesting()- if origFactory != nil {- runtime.SetDefaultFactory(origFactory)- }- })-- body, err := buildNodeBody(context.Background(), "cpn-exe-ovr", "ExeSQL", nil)- if err != nil {- t.Fatalf("buildNodeBody: %v", err)- }- start := time.Now()- _, err = body(context.Background(), nil)- if !errors.Is(err, context.DeadlineExceeded) {- t.Fatalf("body err = %v, want context.DeadlineExceeded", err)- }- deadline := time.Unix(0, cap.deadline.Load())- sinceDeadline := deadline.Sub(start)- if sinceDeadline < 6500*time.Millisecond || sinceDeadline > 7500*time.Millisecond {- t.Errorf("ExeSQL deadline offset = %s, want ~7s (per-class env override)", sinceDeadline)- }-}diff --git a/internal/agent/canvas/node_body_timeout_test.go b/internal/agent/canvas/node_body_timeout_test.goindex 34a74db353..03926e8cc1 100644--- a/internal/agent/canvas/node_body_timeout_test.go+++ b/internal/agent/canvas/node_body_timeout_test.go@@ -14,22 +14,27 @@// limitations under the License.//+// node_body_timeout_test.go — pins the cancellation-only execution context+// contract of realComponentBody. The framework imposes no wall-clock deadline+// on component execution (see realComponentBodyWithOptions), so a component+// only stops when its parent context is cancelled; there is no timeout wrap to+// test any more. These tests assert cancellation propagation and the+// pass-through of fast components.+package canvasimport ("context""errors""testing"- "time""ragflow/internal/agent/runtime""gorm.io/gorm")-// blockingComponent is a runtime.Component whose Invoke blocks until ctx-// is cancelled. Used to test the per-component timeout wrapper in-// realComponentBody.+// blockingComponent is a runtime.Component whose Invoke blocks until ctx is+// cancelled.type blockingComponent struct{}func (b *blockingComponent) Name() string { return "blocking" }@@ -46,49 +51,11 @@ func (b *blockingComponent) Stream(_ context.Context, _ map[string]any) (<-chanfunc (b *blockingComponent) Inputs() map[string]string { return nil }func (b *blockingComponent) Outputs() map[string]string { return nil }-// TestRealComponentBody_RespectsTimeout verifies that a component whose-// Invoke blocks longer than the configured timeout causes the body to-// return a wrapped context.DeadlineExceeded error.-//-// The call itself runs under context.Background(); a separate watchdog-// goroutine fails the test if the inner timeout never fires. That keeps-// the assertion semantic (the returned error must wrap-// context.DeadlineExceeded) without letting an outer test context-// manufacture the same error type and create a false positive.-func TestRealComponentBody_RespectsTimeout(t *testing.T) {- t.Setenv("COMPONENT_EXEC_TIMEOUT", "1")-- comp := &blockingComponent{}- body := realComponentBody("test-cpn", "TestBlocking", comp)- if body == nil {- t.Fatalf("realComponentBody returned nil")- }-- done := make(chan error, 1)- go func() {- _, err := body(context.Background(), map[string]any{"x": 1})- done <- err- }()-- select {- case err := <-done:- if err == nil {- t.Fatalf("expected error, got nil")- }- if !errors.Is(err, context.DeadlineExceeded) {- t.Errorf("expected context.DeadlineExceeded wrapped error, got: %v", err)- }- case <-time.After(15 * time.Second):- t.Fatal("body did not return within 15s — timeout wrap is broken or call is hanging")- }-}--// TestRealComponentBody_RespectsParentCancellation verifies that when-// the parent context is already cancelled, the body surfaces a wrapped-// context.Canceled error rather than a timeout (or a generic wrap).+// TestRealComponentBody_RespectsParentCancellation verifies that when the+// parent context is already cancelled, the body surfaces a wrapped+// context.Canceled error. With no framework deadline, parent cancellation is+// the only mechanism that stops a long-running component.func TestRealComponentBody_RespectsParentCancellation(t *testing.T) {- t.Setenv("COMPONENT_EXEC_TIMEOUT", "60")-comp := &blockingComponent{}body := realComponentBody("test-cpn", "TestBlocking", comp)@@ -104,13 +71,9 @@ func TestRealComponentBody_RespectsParentCancellation(t *testing.T) {}}-// TestRealComponentBody_NoTimeoutWhenFast verifies that a component-// returning immediately does not incur any timeout-induced latency or-// error wrapping.+// TestRealComponentBody_NoTimeoutWhenFast verifies that a component returning+// immediately does not incur any timeout-induced latency or error wrapping.func TestRealComponentBody_NoTimeoutWhenFast(t *testing.T) {- t.Setenv("COMPONENT_EXEC_TIMEOUT", "60")-- // Stub component that returns immediately.comp := &echoComponent{}body := realComponentBody("test-cpn", "TestEcho", comp)@@ -126,38 +89,8 @@ func TestRealComponentBody_NoTimeoutWhenFast(t *testing.T) {}}-// TestComponentTimeout_Default verifies the default is 600s when the env-// var is unset.-func TestComponentTimeout_Default(t *testing.T) {- t.Setenv("COMPONENT_EXEC_TIMEOUT", "")- if got := componentTimeout(); got != 600*time.Second {- t.Errorf("default timeout: got %s, want 600s", got)- }-}--// TestComponentTimeout_HonoursEnv verifies a valid env value is parsed.-func TestComponentTimeout_HonoursEnv(t *testing.T) {- t.Setenv("COMPONENT_EXEC_TIMEOUT", "42")- if got := componentTimeout(); got != 42*time.Second {- t.Errorf("env timeout: got %s, want 42s", got)- }-}--// TestComponentTimeout_InvalidEnvFallsBack verifies that non-numeric or-// non-positive env values fall back to the default — invalid input must-// never widen the timeout silently.-func TestComponentTimeout_InvalidEnvFallsBack(t *testing.T) {- for _, v := range []string{"abc", "0", "-5"} {- t.Setenv("COMPONENT_EXEC_TIMEOUT", v)- if got := componentTimeout(); got != 600*time.Second {- t.Errorf("invalid env %q: got %s, want default 600s", v, got)- }- }-}--// echoComponent is a minimal runtime.Component used by the no-timeout test.-// It returns the input map unchanged plus a __cpn_id__ tag (the body will-// overwrite the tag, but that's fine).+// echoComponent is a minimal runtime.Component used by the no-timeout test. It+// returns the input map unchanged plus a __cpn_id__ tag.type echoComponent struct{}func (e *echoComponent) Name() string { return "echo" }diff --git a/internal/agent/canvas/timeout.go b/internal/agent/canvas/timeout.godeleted file mode 100644index f283b2a7f2..0000000000--- a/internal/agent/canvas/timeout.go+++ /dev/null@@ -1,157 +0,0 @@-//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.-//--// timeout.go — per-component-class Invoke timeout resolution.-//-// Background. The Python port decorates every component's _invoke with-//-// @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10 * 60)))-//-// which applies a uniform 600s ceiling across all components. The Go-// port initially did the same. Operators asked for finer granularity-// because external HTTP lookups (Tavily, DuckDuckGo, ArXiv, ...) and-// fast in-process helpers (ExeSQL, Invoke) blow up the wall-clock budget-// quickly when 10 minutes is the default. The resolution here is:-//-// 1. COMPONENT_EXEC_TIMEOUT_<CLASS>=<seconds> wins for that class.-// 2. Per-class default from componentDefaults.-// 3. COMPONENT_EXEC_TIMEOUT=<seconds> uniform override (kept for-// back-compat — operators who only set the uniform env var see no-// change).-// 4. Hard fallback of 600s (Python base.py default).-//-// All env values are parsed as seconds-suffixed durations; invalid or-// non-positive input falls through to the next step. Invalid input must-// never silently widen the timeout.-package canvas--import (- "context"- "ragflow/internal/common"- "strconv"- "strings"- "time"-)--type timeoutContextKey struct{}--// WithComponentTimeoutOverride forces all component invokes under ctx to use-// the provided timeout. This lets callers with stricter execution contracts-// reuse canvas execution without mutating the global timeout policy.-func WithComponentTimeoutOverride(ctx context.Context, d time.Duration) context.Context {- if d <= 0 {- return ctx- }- return context.WithValue(ctx, timeoutContextKey{}, d)-}--// componentDefaults lists the per-class timeout (in seconds) used when-// no per-class env override is set. The class keys are the PascalCase-// names returned by Component.Name() (e.g. "LLM", "Message",-// "TavilySearch", "Retrieval"). Names not in this table fall through to-// the uniform env var or the hard fallback.-//-// Values match the Python port's per-class expectations:-//-// - LLM / Message / Agent — 600s (long LLM-bound work, matches base.py)-// - Retrieval — 60s (vector + BM25 + rerank pipeline)-// - TavilySearch / DuckDuckGo / ArXiv / GoogleScholar / SearXNG / PubMed — 12s-// (external HTTP — fail fast and let the agent retry/branch)-// - ExeSQL / Invoke — 3s (in-process / single HTTP hop)-//-// Note: the Go port registers web search as "TavilySearch" (the-// fixture_stubs constant) rather than Python's "Tavily". We list both-// spellings so an operator can target the Go name OR a class name that-// matches Python's.-var componentDefaults = map[string]time.Duration{- "LLM": 600 * time.Second,- "Message": 600 * time.Second,- "Agent": 600 * time.Second,- "Retrieval": 60 * time.Second,- "TavilySearch": 12 * time.Second,- "Tavily": 12 * time.Second, // Python spelling alias- "DuckDuckGo": 12 * time.Second,- "ArXiv": 12 * time.Second,- "GoogleScholar": 12 * time.Second,- "SearXNG": 12 * time.Second,- "PubMed": 12 * time.Second,- "ExeSQL": 3 * time.Second,- "Invoke": 3 * time.Second,-}--// defaultComponentTimeout is the final fallback when neither the-// per-class env, the per-class default table, nor the uniform env var-// yields a value. Matches Python base.py: 10 * 60 = 600s.-const defaultComponentTimeout = 600 * time.Second--// resolveTimeout returns the Invoke timeout for a component of the-// given class (the PascalCase name from Component.Name()).-//-// Resolution order:-//-// 1. COMPONENT_EXEC_TIMEOUT_<UPPERCASE_CLASS> env var, parsed as-// "<seconds>s". Invalid or non-positive values are ignored.-// 2. Per-class default from componentDefaults.-// 3. COMPONENT_EXEC_TIMEOUT env var (uniform override, seconds),-// parsed as "<seconds>s".-// 4. defaultComponentTimeout (600s).-//-// Passing an empty class name still honours steps 3 and 4 — useful for-// callers that don't know the class (e.g. a generic dispatcher).-func resolveTimeout(componentClass string) time.Duration {- upper := strings.ToUpper(strings.TrimSpace(componentClass))- if upper != "" {- if d, ok := parseSecondsEnv("COMPONENT_EXEC_TIMEOUT_" + upper); ok {- return d- }- }- if upper != "" {- // componentDefaults uses PascalCase keys; try the original- // spelling (an uppercased lookup would never match).- if d, ok := componentDefaults[componentClass]; ok {- return d- }- }- if d, ok := parseSecondsEnv("COMPONENT_EXEC_TIMEOUT"); ok {- return d- }- return defaultComponentTimeout-}--func resolveTimeoutFromContext(ctx context.Context, componentClass string) time.Duration {- if ctx != nil {- if d, ok := ctx.Value(timeoutContextKey{}).(time.Duration); ok && d > 0 {- return d- }- }- return resolveTimeout(componentClass)-}--// parseSecondsEnv reads an env var and parses it as seconds ("42" →-// 42s). Returns (d, true) on success, (0, false) if the env var is-// unset / empty / non-numeric / non-positive. Invalid input must-// never widen the timeout silently.-func parseSecondsEnv(name string) (time.Duration, bool) {- v := common.GetEnv(name)- if v == "" {- return 0, false- }- secs, err := strconv.Atoi(strings.TrimSpace(v))- if err != nil || secs <= 0 {- return 0, false- }- return time.Duration(secs) * time.Second, true-}diff --git a/internal/common/logger.go b/internal/common/logger.goindex 925ddfe93a..74b322919d 100644--- a/internal/common/logger.go+++ b/internal/common/logger.go@@ -157,9 +157,15 @@ func InitLogger(level string, file FileOutput, serviceName string) error {)if serviceName != "" {- Logger = zap.New(core, zap.AddCallerSkip(1)).Named(serviceName)+ Logger = zap.New(core,+ zap.Fields(zap.Int("pid", os.Getpid())),+ zap.AddCallerSkip(1),+ ).Named(serviceName)} else {- Logger = zap.New(core, zap.AddCallerSkip(1))+ Logger = zap.New(core,+ zap.Fields(zap.Int("pid", os.Getpid())),+ zap.AddCallerSkip(1),+ )}Sugar = Logger.Sugar()diff --git a/internal/common/retry.go b/internal/common/retry.goindex 757cfe37fe..a4de4be11b 100644--- a/internal/common/retry.go+++ b/internal/common/retry.go@@ -17,7 +17,9 @@ package commonimport ("context"+ "errors""fmt"+ "strings""time")@@ -74,3 +76,47 @@ func RetryWithBackoff(ctx context.Context, maxRetries int, initialDelay time.Dur}return fmt.Errorf("failed after %d retries: %w", maxRetries, lastErr)}++// IsTransientError reports whether err is a transient provider failure worth+// retrying, as opposed to a permanent configuration/client error. It is the+// shared shouldRetry predicate for provider HTTP calls.+//+// Classification:+// - context.Canceled: permanent (caller cancelled; do not retry)+// - context.DeadlineExceeded: transient (a slow provider can succeed on retry)+// - an embedded HTTP status parsed from the error message: 5xx and 429 are+// transient (server / rate limit); other 4xx are permanent (auth, unknown+// model, invalid request)+// - anything else (network reset, connection refused, timeout): transient+//+// Provider drivers surface HTTP statuses as "API request failed with status+// <code>: <body>" (see BaseModel.doRequest), so parsing that prefix is enough.+func IsTransientError(err error) bool {+ if err == nil {+ return false+ }+ if errors.Is(err, context.Canceled) {+ return false+ }+ if errors.Is(err, context.DeadlineExceeded) {+ return true+ }+ msg := strings.ToLower(err.Error())+ if idx := strings.Index(msg, "status "); idx >= 0 {+ code := 0+ for _, r := range msg[idx+len("status "):] {+ if r < '0' || r > '9' {+ break+ }+ code = code*10 + int(r-'0')+ }+ if code >= 500 || code == 429 {+ return true+ }+ if code >= 400 && code < 500 {+ return false+ }+ }+ // No recognizable permanent marker: treat as transient (network, timeout).+ return true+}diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.goindex 119da87611..c4dbe77b12 100644--- a/internal/engine/elasticsearch/chunk.go+++ b/internal/engine/elasticsearch/chunk.go@@ -818,6 +818,16 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfac"terms": map[string]interface{}{"id": ids},})}+ case []string:+ // A typed []string must be handled here; the generic loop below+ // skips the "id" key, so without this branch a caller passing+ // map[string]interface{}{"id": []string{...}} would build a query+ // with no id filter and DeleteChunks could match every document.+ if len(v) > 0 {+ mustClauses = append(mustClauses, map[string]interface{}{+ "terms": map[string]interface{}{"id": v},+ })+ }case string:mustClauses = append(mustClauses, map[string]interface{}{"term": map[string]interface{}{"id": v},@@ -856,6 +866,10 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfacmustClauses = append(mustClauses, map[string]interface{}{"terms": map[string]interface{}{k: listVal},})+ } else if listVal, ok := v.([]string); ok {+ mustClauses = append(mustClauses, map[string]interface{}{+ "terms": map[string]interface{}{k: listVal},+ })} else if _, ok := v.(string); ok {mustClauses = append(mustClauses, map[string]interface{}{"term": map[string]interface{}{k: v},@@ -1275,8 +1289,6 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.SallResults = sortByScore(allResults, limit)}- common.Info("ES Search completed", zap.Int("returnedRows", len(allResults)), zap.Int64("totalHits", totalHits))-return &types.SearchResult{Chunks: allResults,Total: totalHits,diff --git a/internal/engine/elasticsearch/chunk_helpers_test.go b/internal/engine/elasticsearch/chunk_helpers_test.goindex 261f4aaaf9..4959aecf23 100644--- a/internal/engine/elasticsearch/chunk_helpers_test.go+++ b/internal/engine/elasticsearch/chunk_helpers_test.go@@ -2,6 +2,7 @@ package elasticsearchimport ("context"+ "encoding/json""net/http""net/http/httptest""reflect"@@ -71,6 +72,122 @@ func TestUpdateSingleMemoryMessageWaitsForRefresh(t *testing.T) {}}+func TestDeleteChunksPreservesStringSliceCondition(t *testing.T) {+ var deleteQuery map[string]interface{}+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("X-Elastic-Product", "Elasticsearch")+ switch r.Method {+ case http.MethodHead:+ w.WriteHeader(http.StatusOK)+ case http.MethodPost:+ if r.URL.Path != "/ragflow_tenant/_delete_by_query" {+ t.Errorf("path=%s, want /ragflow_tenant/_delete_by_query", r.URL.Path)+ http.Error(w, "unexpected request path", http.StatusNotFound)+ return+ }+ if err := json.NewDecoder(r.Body).Decode(&deleteQuery); err != nil {+ t.Errorf("decode delete query: %v", err)+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"deleted":3}`))+ default:+ t.Errorf("method=%s", r.Method)+ http.Error(w, "unexpected method", http.StatusMethodNotAllowed)+ }+ }))+ defer server.Close()++ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})+ if err != nil {+ t.Fatalf("new elasticsearch client: %v", err)+ }+ engine := &Engine{client: client}+ deleted, err := engine.DeleteChunks(context.Background(), map[string]interface{}{+ "kb_id": "kb-1",+ "compile_kwd": []string{"wiki_entity", "wiki_relation"},+ }, "ragflow_tenant", "kb-1")+ if err != nil {+ t.Fatalf("DeleteChunks: %v", err)+ }+ if deleted != 3 {+ t.Fatalf("deleted=%d, want 3", deleted)+ }++ query, ok := deleteQuery["query"].(map[string]interface{})+ if !ok {+ t.Fatalf("delete query missing query: %#v", deleteQuery)+ }+ boolQuery, ok := query["bool"].(map[string]interface{})+ if !ok {+ t.Fatalf("delete query missing bool: %#v", query)+ }+ must, ok := boolQuery["must"].([]interface{})+ if !ok || len(must) != 1 {+ t.Fatalf("delete query must=%#v, want one terms clause", boolQuery["must"])+ }+ terms := must[0].(map[string]interface{})["terms"].(map[string]interface{})+ assertEqual(t, terms["compile_kwd"], []interface{}{"wiki_entity", "wiki_relation"})+}++// TestDeleteChunksIDStringSlice guards against the id filter being dropped when+// the caller passes a typed []string. Without a dedicated []string branch in+// the id handling, the generic loop skips the "id" key and DeleteChunks would+// build a match_all query and delete every document in the index.+func TestDeleteChunksIDStringSlice(t *testing.T) {+ var deleteQuery map[string]interface{}+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("X-Elastic-Product", "Elasticsearch")+ switch r.Method {+ case http.MethodHead:+ w.WriteHeader(http.StatusOK)+ case http.MethodPost:+ if r.URL.Path != "/ragflow_tenant/_delete_by_query" {+ t.Errorf("path=%s, want /ragflow_tenant/_delete_by_query", r.URL.Path)+ http.Error(w, "unexpected request path", http.StatusNotFound)+ return+ }+ if err := json.NewDecoder(r.Body).Decode(&deleteQuery); err != nil {+ t.Errorf("decode delete query: %v", err)+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"deleted":2}`))+ default:+ t.Errorf("method=%s", r.Method)+ http.Error(w, "unexpected method", http.StatusMethodNotAllowed)+ }+ }))+ defer server.Close()++ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})+ if err != nil {+ t.Fatalf("new elasticsearch client: %v", err)+ }+ engine := &Engine{client: client}+ if _, err = engine.DeleteChunks(context.Background(), map[string]interface{}{+ "id": []string{"doc-a", "doc-b"},+ }, "ragflow_tenant", "kb-1"); err != nil {+ t.Fatalf("DeleteChunks: %v", err)+ }++ query, ok := deleteQuery["query"].(map[string]interface{})+ if !ok {+ t.Fatalf("delete query missing query: %#v", deleteQuery)+ }+ boolQuery, ok := query["bool"].(map[string]interface{})+ if !ok {+ t.Fatalf("delete query missing bool: %#v", query)+ }+ must, ok := boolQuery["must"].([]interface{})+ if !ok || len(must) != 1 {+ t.Fatalf("delete query must=%#v, want one id terms clause (match_all would be a bug)", boolQuery["must"])+ }+ terms, ok := must[0].(map[string]interface{})["terms"].(map[string]interface{})+ if !ok {+ t.Fatalf("must[0] missing terms clause: %#v", must[0])+ }+ assertEqual(t, terms["id"], []interface{}{"doc-a", "doc-b"})+}+func TestElasticsearchGetFieldsFiltersAndUsesIDFallback(t *testing.T) {engine := &Engine{}chunks := []map[string]interface{}{diff --git a/internal/engine/types/types.go b/internal/engine/types/types.goindex e96cee1b1d..ec6c53d493 100644--- a/internal/engine/types/types.go+++ b/internal/engine/types/types.go@@ -20,8 +20,6 @@ import ("errors""fmt"- "go.uber.org/zap"-"ragflow/internal/common")@@ -137,8 +135,6 @@ type FusionExpr struct {// LogSearchRequest logs SearchRequest in debug modefunc LogSearchRequest(engineName string, req *SearchRequest) {- common.Info(fmt.Sprintf("Search in %s started", engineName), zap.Any("indexNames", req.IndexNames))-if !common.IsDebugEnabled() {return}diff --git a/internal/entity/models/response_handler.go b/internal/entity/models/response_handler.goindex ee6023645b..ea537ccb1e 100644--- a/internal/entity/models/response_handler.go+++ b/internal/entity/models/response_handler.go@@ -56,7 +56,7 @@ func HandleNonStreamingResponse(body []byte, modelUsage *common.ModelUsage, chatif chatConfig != nil {chatConfig.UsageResult = usagemodel, _ := result["model"].(string)- common.Info("StreamUsage", zap.String("model", model), zap.Int("prompt", usage.PromptTokens), zap.Int("completion", usage.CompletionTokens), zap.Int("total", usage.TotalTokens))+ common.Info("NonStreamUsage", zap.String("model", model), zap.Int("prompt", usage.PromptTokens), zap.Int("completion", usage.CompletionTokens), zap.Int("total", usage.TotalTokens))}}diff --git a/internal/entity/models/xai.go b/internal/entity/models/xai.goindex 87d9379d74..3d26b8a1f0 100644--- a/internal/entity/models/xai.go+++ b/internal/entity/models/xai.go@@ -33,7 +33,7 @@ import ()var (- nonStreamCallTimeout = 120 * time.Second+ nonStreamCallTimeout = 300 * time.SecondstreamCallTimeout = 10 * time.MinutelongOpCallTimeout = 10 * time.Minute)diff --git a/internal/ingestion/component/knowledge_compiler/common/jsonchat.go b/internal/ingestion/component/knowledge_compiler/common/jsonchat.goindex d5937d0a41..4509ef4953 100644--- a/internal/ingestion/component/knowledge_compiler/common/jsonchat.go+++ b/internal/ingestion/component/knowledge_compiler/common/jsonchat.go@@ -4,11 +4,26 @@ import ("context""encoding/json""fmt"- "log"+ "math/rand""regexp""strings"+ "time"++ appcommon "ragflow/internal/common"++ "go.uber.org/zap")+// jsonRetryMax is how many times a non-JSON (or otherwise transiently failed)+// LLM reply is retried before GenJSON gives up. The highest-frequency LLM+// integration must not drop a knowledge unit on a single formatting hiccup, so+// a one-off malformed reply triggers a fresh LLM call instead of an immediate+// failure.+const jsonRetryMax = 5++// jsonRetryDelay is the initial exponential-backoff delay between retries.+const jsonRetryDelay = 2 * time.Second+// fencedJSONRE matches a ```json ... ``` or ``` ... ``` fenced block. Models// frequently wrap JSON in such fences even when JSONMode is requested, which// previously slipped through as an unparseable "_raw" payload and was silently@@ -24,25 +39,68 @@ var fencedJSONRE = regexp.MustCompile("(?s)```(?:json)?\\s*(.*?)\\s*```")// or retries rather than silently dropping the extraction — the// highest-frequency LLM integration must not lose knowledge units on a// formatting hiccup.-func GenJSON(ctx context.Context, chat ChatInvoker, req ChatRequest) (map[string]any, error) {- req.JSONMode = true- resp, err := chat.Chat(ctx, req)- if err != nil {- return nil, err+// GenJSON asks the model for a JSON reply and parses it. retryMax is an+// optional override for jsonRetryMax: pass 0 to disable retries entirely+// (used where the caller already budgets external calls itself, e.g. the+// entity-merge disambiguator — retrying there would blow through the budget).+func GenJSON(ctx context.Context, chat ChatInvoker, req ChatRequest, retryMax ...int) (map[string]any, error) {+ maxRetries := jsonRetryMax+ if len(retryMax) > 0 {+ maxRetries = retryMax[0]}- for _, candidate := range jsonCandidates(resp.Content) {- if m, ok := tryUnmarshalJSON(candidate); ok {- return m, nil+ req.JSONMode = true+ var lastErr error+ delay := jsonRetryDelay+ for attempt := 0; attempt <= maxRetries; attempt++ {+ resp, err := chat.Chat(ctx, req)+ if err != nil {+ // Permanent chat errors (auth, unknown model, context-length,+ // cancelled ctx) cannot succeed on a retry; escape immediately.+ if !appcommon.IsTransientError(err) {+ return nil, err+ }+ // Transient chat failure (timeout / transport / provider); retry+ // with a fresh LLM call.+ lastErr = err+ } else {+ candidates := jsonCandidates(resp.Content)+ for _, candidate := range candidates {+ if m, ok := tryUnmarshalJSON(candidate); ok {+ return m, nil+ }+ }+ // A non-JSON reply must NOT be persisted/reused; discard it and+ // immediately re-issue the call so a formatting hiccup does not+ // abort the whole compile. Log candidate length and the unmarshal+ // error only — the raw body may carry customer-derived content+ // (PII), so it is deliberately excluded from the log.+ for i, candidate := range candidates {+ _, perr := tryUnmarshalJSONErr(candidate)+ appcommon.Info("knowledge_compiler: GenJSON unparseable candidate",+ zap.Int("attempt", attempt), zap.Int("candidate", i),+ zap.Int("len", len(candidate)), zap.Error(perr))+ }+ lastErr = fmt.Errorf("knowledge_compiler: LLM response is not parseable JSON (%d bytes)", len(resp.Content))+ }+ if attempt == maxRetries {+ break+ }+ appcommon.Info("knowledge_compiler: GenJSON attempt failed, retrying",+ zap.Int("attempt", attempt), zap.Duration("delay", delay),+ zap.Error(lastErr))+ select {+ case <-ctx.Done():+ return nil, ctx.Err()+ case <-time.After(delay + time.Duration(rand.Int63n(int64(delay/2)+1))):+ // Jittered backoff so concurrent GenJSON jobs (parallel wiki/plan+ // batches) do not back off in lockstep and pile up on the provider.+ }+ delay *= 2+ if delay > time.Minute {+ delay = time.Minute}}- // Diagnostic: surface how far parsing got so a formatting failure is not- // opaque. Each candidate is reported with its own unmarshal error so we can- // tell an unfenced/truncated payload from a genuine syntax error.- for i, candidate := range jsonCandidates(resp.Content) {- _, err := tryUnmarshalJSONErr(candidate)- log.Printf("knowledge_compiler: GenJSON candidate[%d] len=%d parse_err=%v body=%q", i, len(candidate), err, truncate(candidate, 300))- }- return nil, fmt.Errorf("knowledge_compiler: LLM response is not parseable JSON: %q", truncate(resp.Content, 200))+ return nil, lastErr}// jsonCandidates yields progressively "cleaned" versions of an LLM reply thatdiff --git a/internal/ingestion/component/knowledge_compiler/common/types.go b/internal/ingestion/component/knowledge_compiler/common/types.goindex 7c0635566e..04d68a4849 100644--- a/internal/ingestion/component/knowledge_compiler/common/types.go+++ b/internal/ingestion/component/knowledge_compiler/common/types.go@@ -122,8 +122,8 @@ type Product struct {ParentID stringMeta map[string]any// Merged marks rows that already went through dataset-level dedup- // (kc_merged=1, doc_id=kb). The consumer distinguishes these from the- // per-document compiled rows (doc_id=<source doc>, no kc_merged) so it can+ // (available_int=1, doc_id=kb). The consumer distinguishes these from the+ // per-document compiled rows (doc_id=<source doc>, available_int=0) so it can// KNN against only the merged set instead of re-deduping the whole KB.Merged bool}diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.goindex e1e1f25ac9..fa44d109cb 100644--- a/internal/ingestion/component/knowledge_compiler/component.go+++ b/internal/ingestion/component/knowledge_compiler/component.go@@ -13,6 +13,7 @@ import ("strings""ragflow/internal/agent/runtime"+ clog "ragflow/internal/common""ragflow/internal/ingestion/component/globals""ragflow/internal/ingestion/component/knowledge_compiler/common""ragflow/internal/ingestion/component/knowledge_compiler/mindmap"@@ -23,6 +24,7 @@ import ("ragflow/internal/service/nav""ragflow/internal/tokenizer"+ "go.uber.org/zap""gorm.io/gorm")@@ -94,7 +96,15 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in// being lost when the upstream output map narrows it, which would otherwise// leave the template-group lookup with an empty tenant and fail loudly.tenantID := globals.GlobalOrInput(ctx, inputs, "tenant_id", "")+ // The ingestion pipeline seeds kb_id (not dataset_id) into the canvas+ // globals/inputs. In RAGFlow the knowledge base id IS the dataset id, so+ // fall back to kb_id when dataset_id is absent. This keeps the compiler's+ // dataset-scoped writes (merged wiki_page rows, wiki graph) keyed to the+ // correct dataset instead of an empty string.datasetID := globals.GlobalOrInput(ctx, inputs, "dataset_id", "")+ if datasetID == "" {+ datasetID = globals.GlobalOrInput(ctx, inputs, "kb_id", "")+ }// Resolve the compilation template spec(s). Priority:// compilation_template_id > compilation_template_group_id. The variant is@@ -105,6 +115,17 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, inreturn nil, err}+ // The eino wiring does not auto-merge run-level metadata into every+ // component's input map, so doc_id may arrive only via CanvasState.Globals+ // (seeded by the pipeline at run start) rather than inputs["doc_id"].+ // Mirror the dataset_id/kb_id resolution above: fall back to globals so the+ // compiler (and its wiki sub-logs) see the real doc_id instead of "unknown".+ if d, ok := inputs["doc_id"].(string); !ok || d == "" {+ if g := globals.GlobalOrInput(ctx, inputs, "doc_id", ""); g != "" {+ inputs["doc_id"] = g+ }+ }+in, err := buildInputs(inputs, param)if err != nil {return nil, err@@ -202,6 +223,32 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, inif err != nil {return nil, err}+ // Per-doc telemetry: confirm how many compiled rows (and specifically+ // wiki_page wiki sections vs pages) this single document produced, so a+ // missing dataset-level wiki_page can be traced to "never generated" vs+ // "generated then dropped".+ var pageCount, sectionCount, entityCount, relationCount int+ for _, p := range out.Products {+ switch {+ case p.Variant == common.VariantWiki && metaString(p.Meta, "kind") == "page":+ pageCount+++ case p.Variant == common.VariantWiki && metaString(p.Meta, "kind") == "section":+ sectionCount+++ case p.Variant == common.VariantWiki && metaString(p.Meta, "kind") == "entity":+ entityCount+++ case p.Variant == common.VariantWiki && metaString(p.Meta, "kind") == "relation":+ relationCount+++ }+ }+ clog.Info("knowledge_compiler: per-doc products generated",+ zap.String("doc_id", in.DocID),+ zap.Int("total_products", len(out.Products)),+ zap.Int("wiki_page", pageCount),+ zap.Int("wiki_section", sectionCount),+ zap.Int("wiki_entity", entityCount),+ zap.Int("wiki_relation", relationCount),+ zap.Int("chunk_docs", len(compiled)),+ )return mergeChunks(inputs, compiled), nil}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/page.go b/internal/ingestion/component/knowledge_compiler/wiki/page.goindex e8cd10ea5a..d9e9df45bd 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/page.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/page.go@@ -7,7 +7,10 @@ import ("strings""unicode"+ appcommon "ragflow/internal/common""ragflow/internal/ingestion/component/knowledge_compiler/common"++ "go.uber.org/zap")type wikiPageResult struct {@@ -171,8 +174,15 @@ func buildPageProducts(tenantID, docID, page string, sourceChunkIDs []string) []// per heading in its markdown body.func buildWikiPageProducts(tenantID, docID string, pages []wikiPageResult) []common.Product {if len(pages) == 0 {+ appcommon.Info("wiki: buildWikiPageProducts skipped (no refined pages)",+ zap.String("tenant_id", tenantID),+ zap.String("doc_id", docID))return nil}+ appcommon.Info("wiki: buildWikiPageProducts start",+ zap.String("tenant_id", tenantID),+ zap.String("doc_id", docID),+ zap.Int("pages", len(pages)))out := make([]common.Product, 0, len(pages)*2)for _, page := range pages {slug := strings.TrimSpace(page.Slug)@@ -180,6 +190,10 @@ func buildWikiPageProducts(tenantID, docID string, pages []wikiPageResult) []comslug = slugify(page.Title)}if slug == "" {+ appcommon.Warn("wiki: page dropped (empty slug)",+ zap.String("tenant_id", tenantID),+ zap.String("doc_id", docID),+ zap.String("title", page.Title))continue}title := strings.TrimSpace(page.Title)@@ -310,3 +324,24 @@ func slugify(s string) string {}return b.String()}++// normalizeWikiSlugHyphens canonicalizes a wiki slug to the hyphen style used+// everywhere downstream (slug_kwd, outlinks, artifact links): the last path+// segment's underscores become hyphens, while any "<page_type>/" prefix is+// preserved. The LLM freely emits both "dong_zhuo" and "dong-zhuo", so this is+// the single choke point that makes them equivalent without ad-hoc conversions+// scattered through transformWikiLinks / the graph writer.+func normalizeWikiSlugHyphens(slug string) string {+ s := strings.TrimSpace(slug)+ if s == "" {+ return ""+ }+ prefix := ""+ body := s+ if idx := strings.LastIndex(s, "/"); idx >= 0 {+ prefix = s[:idx+1]+ body = s[idx+1:]+ }+ body = strings.ReplaceAll(body, "_", "-")+ return prefix + body+}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/prompt.go b/internal/ingestion/component/knowledge_compiler/wiki/prompt.goindex 43f9cac2c9..033f745824 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/prompt.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/prompt.go@@ -155,8 +155,19 @@ Return a JSON compilation plan with one or more page entries:Rules:- Return at most {max_pages} page entries for this batch.-- Prefer one page per high-signal entity or concept when the batch supports it.-- Merge minor or weakly-supported facts into broader topic pages instead of emitting tiny standalone pages.+- Entity/concept identity is one-to-one with pages: every extracted entity and+ concept must be represented by exactly one canonical page and may appear in+ only that page's entity_names. Never split one identity across multiple+ pages, page types, thematic sections, aliases, language transliterations, or+ alternate slug spellings. Put all supported sections for that identity on+ its single canonical page.+- A page may represent several closely related low-signal entities/concepts,+ but list every represented identity in entity_names and do not repeat any of+ them on another page. Merge minor or weakly-supported facts into such a page+ instead of emitting tiny standalone pages.+- Identity ownership does not limit linking: related_kb_pages should include+ every directly related canonical page supported by the input (up to the+ available-page budget), and links must target canonical slugs only.- Use page_type=entity for entity pages, page_type=concept for concept pages, and page_type=topic for cross-cutting themes.- entity_names must name the entities and concepts that justify the page.- related_kb_pages should list other slugs from the same plan that the page should cross-link to.diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.goindex 1837d390e9..e5b03fc154 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go@@ -17,8 +17,11 @@ import ("sort""strings"+ appcommon "ragflow/internal/common""ragflow/internal/ingestion/component/knowledge_compiler/common""ragflow/internal/ingestion/component/knowledge_compiler/structure"++ "go.uber.org/zap")// batchSubmitter fans out the MAP-stage extraction jobs on the process-wide@@ -70,6 +73,16 @@ func wikiMapMaxTokens(modelContextLen int) int {return max(modelContextLen-wikiMapTokenBudget, wikiMapTokenBudget)}+// wikiRefineMaxTokens gives the page writer (REFINE step) a generous but+// bounded output cap so a long page body is not cut off mid-stream by a small+// default completion limit. It reuses the map/extraction budget derivation:+// once the input budget is consumed, the rest of the model window is handed to+// the output. modelContextLen is the model's total context window in tokens (0+// means unknown).+func wikiRefineMaxTokens(modelContextLen int) int {+ return wikiMapMaxTokens(modelContextLen)+}+type wikiPipeline struct {ctx context.Contextdeps common.Deps@@ -196,6 +209,12 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commoif docID == "" {docID = "unknown"}+ appcommon.Info("wiki: Run start",+ zap.String("dataset_id", deps.DatasetID),+ zap.String("doc_id", docID),+ zap.Int("chunks", len(inputs.Chunks)),+ zap.Bool("chat_ready", deps.Chat != nil),+ zap.Bool("embed_ready", deps.Embed != nil))p := &wikiPipeline{ctx: ctx,deps: deps,@@ -207,10 +226,23 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commollmID: llmID,}if err := p.run(); err != nil {+ appcommon.Error("wiki: Run pipeline failed", err,+ zap.String("dataset_id", deps.DatasetID),+ zap.String("doc_id", docID))return common.Outputs{}, err}+ appcommon.Info("wiki: pipeline done",+ zap.String("dataset_id", deps.DatasetID),+ zap.String("doc_id", docID),+ zap.Int("extracted_entities", len(p.reduced.Entities)),+ zap.Int("plan_pages", len(p.plan.Pages)),+ zap.Int("refined_pages", len(p.pages)))products := buildWikiPageProducts(p.tenantID, p.docID, p.pages)+ appcommon.Info("wiki: built page products",+ zap.String("dataset_id", deps.DatasetID),+ zap.String("doc_id", docID),+ zap.Int("products", len(products)))for i := range products {if products[i].Meta == nil {products[i].Meta = map[string]any{}@@ -271,6 +303,21 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commoreturn out, nil}+// runKey returns a stable identity for this wiki run's log lines. In a+// per-document run docID is populated; in a dataset-level run (the+// knowledge_compile aggregator) docID is empty, so fall back to a dataset+// scope prefix. This avoids masking which dataset a MAP/REDUCE pass ran for+// behind an uninformative "unknown".+func (p *wikiPipeline) runKey() string {+ if p.docID != "" {+ return p.docID+ }+ if p.datasetID != "" {+ return "dataset:" + p.datasetID+ }+ return "unknown"+}+func (p *wikiPipeline) run() error {if len(p.inputs.Chunks) == 0 {return nil@@ -278,24 +325,52 @@ func (p *wikiPipeline) run() error {if err := p.runMap(); err != nil {return err}+ appcommon.Info("wiki: MAP done",+ zap.String("dataset_id", p.datasetID),+ zap.String("doc_id", p.runKey()),+ zap.Int("batches", len(p.mapExtracts)),+ zap.Int("raw_entities", countExtractEntities(p.mapExtracts)))p.reduced = reduceExtracts(p.mapExtracts)// Layer embedding + LLM disambiguation onto the exact-merged entities// (REDUCE enhancement; concepts keep exact dedup). Degrades to a no-op when// the embedder/chat seams are unavailable.p.reduced.Entities = p.dedupeEntities(p.reduced.Entities)+ appcommon.Info("wiki: REDUCE done",+ zap.String("dataset_id", p.datasetID),+ zap.String("doc_id", p.runKey()),+ zap.Int("entities", len(p.reduced.Entities)),+ zap.Int("claims", len(p.reduced.Claims)))plan, err := p.runPlan()if err != nil {return err}p.plan = plan+ appcommon.Info("wiki: PLAN done",+ zap.String("dataset_id", p.datasetID),+ zap.String("doc_id", p.runKey()),+ zap.Int("plan_pages", len(plan.Pages)),+ zap.Int("capacity_excluded", p.planCapacityExcluded))pages, err := p.runRefine()if err != nil {return err}p.pages = pages+ appcommon.Info("wiki: REFINE done",+ zap.String("dataset_id", p.datasetID),+ zap.String("doc_id", p.runKey()),+ zap.Int("refined_pages", len(pages)))return nil}+// countExtractEntities sums the entity count across a list of map extracts.+func countExtractEntities(extracts []wikiExtract) int {+ n := 0+ for _, e := range extracts {+ n += len(e.Entities)+ }+ return n+}+func (p *wikiPipeline) runMap() error {// Keep each batch small enough that the LLM's entity/relation JSON output// for the batch stays well under the model's output-token limit. 2048 input@@ -406,7 +481,7 @@ func (p *wikiPipeline) runPlan() (wikiPlan, error) {if err := p.ctx.Err(); err != nil {return err}- plan, err := p.runPlanBatch(batch, i+1, len(batches), quota)+ plan, err := p.runPlanBatch(batch, i+1, len(batches), quota, p.planBudget.MaxTokens)if err != nil {return err}@@ -540,10 +615,15 @@ func (p *wikiPipeline) runRefinePage("evidence_count": fmt.Sprintf("%d", len(evidence)),"evidence_blocks": formatWikiEvidenceBlocks(evidence),})+ // wikiRefineMaxTokens gives the page writer a generous but bounded output+ // cap so a long page is not cut off mid-stream by a small default completion+ // limit (which would yield an unusable/cut page body).+ rmt := wikiRefineMaxTokens(p.deps.ModelContextLen)resp, err := p.deps.Chat.Chat(p.ctx, common.ChatRequest{LLMID: p.llmID,SystemPrompt: buildWikiRefineWriterSystem(""),UserPrompt: user,+ MaxTokens: &rmt,})if err != nil {return wikiPageResult{}, err@@ -598,7 +678,7 @@ func maxPagesForBatch(quota int) int {return quota}-func (p *wikiPipeline) runPlanBatch(batch wikiExtract, batchIndex, batchTotal, quota int) (wikiPlan, error) {+func (p *wikiPipeline) runPlanBatch(batch wikiExtract, batchIndex, batchTotal, quota, maxTokens int) (wikiPlan, error) {user := renderWikiTemplate(wikiPlanBatchUserTemplate, map[string]string{"doc_id": p.docID,"batch_index": fmt.Sprintf("%d", batchIndex),@@ -614,6 +694,7 @@ func (p *wikiPipeline) runPlanBatch(batch wikiExtract, batchIndex, batchTotal, qLLMID: p.llmID,SystemPrompt: wikiPlanSystem,UserPrompt: user,+ MaxTokens: &maxTokens,})if err != nil {return wikiPlan{}, err@@ -1239,6 +1320,11 @@ func normalizeWikiPlanPage(page wikiPlanPage) wikiPlanPage {if page.Slug == "" {page.Slug = slugify(page.Title)}+ // Canonicalize the slug to the hyphen style used by slug_kwd / outlinks /+ // artifact links. The LLM freely mixes "dong_zhuo" and "dong-zhuo", so this+ // single choke point makes plan slugs, slugToPageType/pageTitles keys and+ // the writer's bare/full reverse index all agree.+ page.Slug = normalizeWikiSlugHyphens(page.Slug)page.Title = strings.TrimSpace(page.Title)if page.Title == "" {page.Title = page.Slug@@ -1895,10 +1981,55 @@ func pageTypeOf(slug string, slugToPageType map[string]string) string {func transformWikiLinks(content, kbID string, pageTitles, slugToPageType map[string]string) (string, []string) {kbID = strings.TrimSpace(kbID)+ // Resolve a wikitext slug (which may be a bare "name" or a full+ // "<page_type>/<slug>") to the canonical full slug used as the page+ // identifier (slug_kwd). This mirrors Python's _wiki_resolve_dead_slug:+ // plain names / titles are reverse-mapped to the full pid, and slugs that+ // cannot be resolved are dropped (dead links). Without this, bare slugs+ // written by the LLM never match the full-slug page index, so wiki+ // relations (edges) are silently lost.+ bareToSlug := map[string]string{}+ titleToSlug := map[string]string{}+ // Plan slugs are canonicalized to the hyphen style upstream+ // (normalizeWikiPlanPage -> normalizeWikiSlugHyphens), so slugToPageType /+ // pageTitles keys are already hyphen full-slugs. LLM-authored wikitext+ // links may still carry underscores (e.g. "dong_zhuo"); normalize both+ // sides to hyphens so the reverse lookup matches, and always emit the+ // resolved outlink in the canonical hyphen full-slug form.+ for fullSlug := range slugToPageType {+ bareToSlug[normalizeWikiSlugHyphens(lastPathSlug(fullSlug))] = fullSlug+ }+ for fullSlug, title := range pageTitles {+ if t := strings.TrimSpace(title); t != "" {+ titleToSlug[t] = fullSlug+ }+ }+ resolveSlug := func(slug string) string {+ slug = strings.TrimSpace(slug)+ if slug == "" {+ return ""+ }+ if _, ok := slugToPageType[slug]; ok {+ return slug+ }+ if full, ok := bareToSlug[normalizeWikiSlugHyphens(lastPathSlug(slug))]; ok {+ return full+ }+ if full, ok := titleToSlug[slug]; ok {+ return full+ }+ return ""+ }seen := map[string]bool{}var outlinks []stringtrack := func(slug string) {- slug = strings.TrimSpace(slug)+ slug = resolveSlug(slug)+ // Emit the canonical hyphen full-slug so outlinks_kwd and slug_kwd+ // always agree in format (guards against a resolved slug that still+ // carries underscores, e.g. from an old existing page).+ if slug != "" {+ slug = normalizeWikiSlugHyphens(slug)+ }if slug == "" || seen[slug] {return}@@ -1945,10 +2076,16 @@ func transformWikiLinks(content, kbID string, pageTitles, slugToPageType map[str// <page_type>/<name> prefix, in which case that prefix is reused as the// page_type and the name is emitted as the bare slug.link := func(label, slug string) string {- bareSlug := slug- pageType := pageTypeOf(slug, slugToPageType)- if idx := strings.Index(slug, "/"); idx >= 0 && slug[:idx] == pageType {- bareSlug = slug[idx+1:]+ fullSlug := resolveSlug(slug)+ if fullSlug == "" {+ // Unresolved (dead) slug: keep the label as plain text so the+ // frontend does not render a broken artifact link.+ return strings.TrimSpace(label)+ }+ bareSlug := fullSlug+ pageType := pageTypeOf(fullSlug, slugToPageType)+ if idx := strings.Index(fullSlug, "/"); idx >= 0 && fullSlug[:idx] == pageType {+ bareSlug = fullSlug[idx+1:]}if pageType == "" {pageType = "page"@@ -1988,9 +2125,25 @@ func transformWikiLinks(content, kbID string, pageTitles, slugToPageType map[strtrack(slug)return link(displayText(slug, slug), slug)})+ // rawLinks counts the model-authored wikilinks in the source content. It+ // must be counted against content, not out: by this point every [[...]] has+ // been rewritten to markdown, so counting on out would always report zero.+ rawLinks := wikiWikilinkSimpleRe.FindAllString(content, -1)+ appcommon.Info("knowledge_compiler: transformWikiLinks resolved outlinks",+ zap.Int("raw_wikilinks", len(rawLinks)),+ zap.Int("resolved_outlinks", len(outlinks)),+ zap.Strings("sample_resolved", firstN(outlinks, 5)),+ zap.Strings("sample_raw", firstN(rawLinks, 5)))return out, outlinks}+func firstN(s []string, n int) []string {+ if len(s) <= n {+ return s+ }+ return s[:n]+}+func lastPathSlug(slug string) string {slug = strings.TrimSpace(slug)if idx := strings.LastIndex(slug, "/"); idx >= 0 && idx < len(slug)-1 {diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget.goindex c1807d3b4b..c9c51b1ab2 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget.go@@ -13,10 +13,12 @@ import "sort"// - a deterministic, mention-grounded truncation that selects the top pages// under the global cap and reports how many were excluded.//-// The provider never receives an explicit output max_tokens on the wiki path-// (see P0 in tasks/2026-08-04-wiki-go-python-gap-alignment-plan.md): output-// scale is controlled purely through max_pages in the prompt + the in-code-// truncation below.+// The provider receives an explicit output max_tokens on the wiki PLAN path+// equal to outputTokens (below): without it, small-window models default to a+// tiny completion cap and the large plan JSON gets truncated mid-stream,+// producing "LLM response is not parseable JSON". The REFINE (page body)+// step uses its own cap via wikiRefineMaxTokens. capacity (max page count) and+// outputTokens (hard completion cap) both derive from the same model window.// Python alignment constants (wiki.py:1670-1674, 1783-1787).const (@@ -55,6 +57,11 @@ type wikiPlanBudget struct {// target; that is deliberate (never ask a small-window model for more pages// than its output can hold).Max int+ // MaxTokens is the explicit completion cap sent to the provider for the+ // PLAN step. It is the same outputTokens used to derive Max and prevents+ // the large page JSON from being truncated mid-stream by a small default+ // completion cap.+ MaxTokens int}// Cap is the page budget the planner is actually allowed to emit. It is the@@ -104,7 +111,7 @@ func deriveWikiPlanBudget(modelContextLen, totalItems int) wikiPlanBudget {// never be asked to emit more pages than its output capacity permits, or we// reintroduce truncated-JSON risk. When capacity < Target, Max simply lands// below Target and the achievable page count is capacity-bound.- return wikiPlanBudget{Target: target, Max: maxCount}+ return wikiPlanBudget{Target: target, Max: maxCount, MaxTokens: outputTokens}}// wikiExtractItemCount counts the planning items in one reduced extract. It isdiff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_reduce.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_reduce.goindex d506d630dd..873e71c410 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_reduce.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_reduce.go@@ -135,6 +135,10 @@ func (p *wikiPipeline) dedupeEntities(in []wikiEntity) []wikiEntity {// llmMergeEntityDecision asks the chat seam whether two distinct-name entities// refer to the same real-world concept. Returns true to merge.func (p *wikiPipeline) llmMergeEntityDecision(a, b wikiEntity) (bool, error) {+ // Pass retryMax=0: dedupeEntities budgets llmCalls itself and counts one+ // decision per call; letting GenJSON retry a failing chat seam would multiply+ // each decision by (1 + jsonRetryMax) and blow through the budget (a+ // persistent failure would issue 64 calls instead of <=16).raw, err := common.GenJSON(p.ctx, p.deps.Chat, common.ChatRequest{LLMID: p.llmID,SystemPrompt: wikiReduceEntityDisambiguateSystem,@@ -142,7 +146,7 @@ func (p *wikiPipeline) llmMergeEntityDecision(a, b wikiEntity) (bool, error) {"entity_a": mustPrettyJSON(a),"entity_b": mustPrettyJSON(b),}),- })+ }, 0)if err != nil {return false, err}diff --git a/internal/ingestion/component/schema/chunker.go b/internal/ingestion/component/schema/chunker.goindex e920950de1..782906b6ca 100644--- a/internal/ingestion/component/schema/chunker.go+++ b/internal/ingestion/component/schema/chunker.go@@ -173,9 +173,10 @@ type ChunkerOutputs struct {type TokenChunkerParam struct {// DelimiterMode selects the chunking strategy.- // Allowed value: "delimiter".- // The single-chunk "one" behavior is provided by the separate- // OneChunker component.+ // Allowed value: "delimiter". The legacy "token_size" value is accepted for+ // backward compatibility and normalized to "delimiter" (it is behaviorally+ // identical at runtime; see Validate). The single-chunk "one" behavior is+ // provided by the separate OneChunker component.DelimiterMode string `json:"delimiter_mode"`// ChunkTokenSize is the target chunk size in tokens.@@ -356,6 +357,15 @@ func (p *TokenChunkerParam) Validate() error {// common/float_utils.py:50-58 fix for #17418.p.OverlappedPercent = math.Round(v * 100)}+ // Backward-compat: Python removed the standalone "token_size" mode and+ // treats it as identical to "delimiter" at runtime, normalizing it in+ // check() (rag/flow/chunker/token_chunker.py). A dataset configured under+ // the old scheme may still carry delimiter_mode="token_size"; normalize it+ // here instead of rejecting it so those pipelines keep working (mirrors+ // the Python behaviour exactly).+ if p.DelimiterMode == "token_size" {+ p.DelimiterMode = "delimiter"+ }switch p.DelimiterMode {case "delimiter":default:diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex 2a663c7c87..fcc6bfd931 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -17,6 +17,7 @@ package knowledge_compileimport ("context"+ "fmt""sort""sync""time"@@ -47,8 +48,7 @@ type Consumer struct {mergeThreshold float64 // KNN similarity threshold for "existing merged row is a duplicate"mu sync.Mutex- seqs map[string]map[string]uint64 // dataset -> docID -> last applied seq (per-doc out-of-order guard)- tombs map[string]map[string]uint64 // dataset -> docID -> delete event seq (tombstone)+ tombs map[string]map[string]uint64 // dataset -> docID -> delete marker (tombstone)}// NewConsumer constructs a Consumer driven by the given Claimer. Tests pass a@@ -64,7 +64,6 @@ func NewConsumer(scheduler Claimer, opts ...Option) *Consumer {pollInterval: 2 * time.Second,sweepInterval: 30 * time.Second,mergeThreshold: 0.99,- seqs: map[string]map[string]uint64{},tombs: map[string]map[string]uint64{},}for _, o := range opts {@@ -215,59 +214,48 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entriesif c.tombs == nil {c.tombs = map[string]map[string]uint64{}}- if c.seqs == nil {- c.seqs = map[string]map[string]uint64{}- }tomb := c.tombs[kb]- docSeqs := c.seqs[kb]- if docSeqs == nil {- docSeqs = map[string]uint64{}- c.seqs[kb] = docSeqs- }var completed []BacklogEntryvar deleted []string- // Per-document seq advances and tombstone clears are staged locally and- // committed only after the write/delete paths below return without error,- // so a failed batch leaves c.seqs/c.tombs untouched (except tombstones- // recorded for the deletion events already present in this batch) and can- // be retried on reclaim.- var pendingSeq []BacklogEntry+ // Per-document tombstone clears are staged locally and committed only after+ // the write/delete paths below return without error, so a failed batch+ // leaves c.tombs untouched and can be retried on reclaim.var pendingTombClear []string- // Reconcile each document to its highest-sequence event before classifying- // it as deleted or completed. A completion and a deletion for the same doc- // can land in the same batch, and completions/deletions can also arrive- // out of order across batches. The last event (by per-doc seq) wins, so we- // must not delete a doc that was re-ingested after its deletion, nor accept- // a stale completion shadowed by a later deletion.+ // Reconcile each document to its last event before classifying it as+ // deleted or completed. A completion and a deletion for the same doc can+ // land in the same batch, and completions/deletions can also arrive out of+ // order across batches. The LAST event for a doc (by backlog append order)+ // decides its fate, so we must not delete a doc that was re-ingested after+ // its deletion, nor merge a completion shadowed by a later deletion. No+ // sequence number is needed: the backlog is an ordered log and the last+ // write wins, which is exactly the re-parse (completed last) vs delete+ // (deleted last) semantics we want.type docState struct {winEvent EventType- winSeq uint64}byDoc := make(map[string]*docState, len(entries))for _, e := range entries {et := EventType(e.EventType)st := byDoc[e.DocID]if st == nil {- byDoc[e.DocID] = &docState{winEvent: et, winSeq: e.Seq}+ byDoc[e.DocID] = &docState{winEvent: et}continue}- if e.Seq >= st.winSeq {- st.winEvent = et- st.winSeq = e.Seq- }+ // Later entries overwrite earlier ones: last event wins.+ st.winEvent = et}for docID, st := range byDoc {switch st.winEvent {case EventTypeDeleted:// Record the tombstone for this batch's deletion so a stale- // completion (seq <= delete seq) is skipped. The tombstone is- // committed immediately (deletions are not rolled back on a- // failed batch because re-running the delete is idempotent).+ // completion is skipped. The tombstone is committed immediately+ // (deletions are not rolled back on a failed batch because+ // re-running the delete is idempotent).if tomb == nil {tomb = map[string]uint64{}c.tombs[kb] = tomb}- tomb[docID] = st.winSeq+ tomb[docID] = 1deleted = append(deleted, docID)case EventTypeCompleted:// A re-ingest after an earlier deletion: clear the prior@@ -275,22 +263,27 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entriesif _, hadTomb := tomb[docID]; hadTomb {pendingTombClear = append(pendingTombClear, docID)}- // Seq is per-document, so the stale/duplicate check must be- // scoped to the document, not the whole dataset (C4).- if prev, ok := docSeqs[docID]; ok && st.winSeq <= prev {- continue // stale / duplicate completion for this doc- }- // The seq advance is deferred to after the batch succeeds (see- // below), so a transient reader/deduper/writer failure does not- // permanently drop the completion on the next reclaim+retry.- pendingSeq = append(pendingSeq, BacklogEntry{DocID: docID, EventType: string(EventTypeCompleted), Seq: st.winSeq})- completed = append(completed, BacklogEntry{DocID: docID, EventType: string(EventTypeCompleted), Seq: st.winSeq})+ completed = append(completed, BacklogEntry{DocID: docID, EventType: string(EventTypeCompleted)})}}// Sort for deterministic iteration in the delete/load passes below.sort.Strings(deleted)c.mu.Unlock()+ // Per-entry detail (each claim carries a frozen, closed batch of doc events;+ // log them on a single line so a batch can be reconstructed from the ingestor+ // log alone).+ entryDetails := make([]string, 0, len(entries))+ for _, e := range entries {+ entryDetails = append(entryDetails, fmt.Sprintf("doc=%s event=%s", e.DocID, e.EventType))+ }++ common.Info("knowledge_compile: batch merge start",+ zap.String("dataset_id", kb),+ zap.Int("completed_docs", len(completed)),+ zap.Int("deleted_docs", len(deleted)),+ zap.Strings("entries", entryDetails))+if len(deleted) == 0 && len(completed) == 0 {return nil}@@ -442,13 +435,21 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entriesreturn err}- // All merge and delete paths succeeded: commit the staged per-document seq- // advances and tombstone clears. These are only now persisted so a failed- // batch leaves c.seqs/c.tombs untouched and a later reclaim can retry.- c.mu.Lock()- for _, e := range pendingSeq {- c.seqs[kb][e.DocID] = e.Seq+ // Re-materialize the wiki page graph from the now-consistent merged set. This+ // runs unconditionally after a merge: the graph is a global per-dataset+ // projection rebuilt from a consistent read each time, so it must run even+ // when the current batch only prunes (deletes) wiki pages, and even for a+ // non-wiki batch where the dataset already has no wiki pages (ProjectWikiGraph+ // then just drops any stale graph rows). The graph is reconstructible, so the+ // cost of an occasional no-op reprojection is accepted.+ if err := c.writer.ProjectWikiGraph(ctx, tenant, kb); err != nil {+ return err}++ // All merge and delete paths succeeded: commit the staged tombstone+ // clears. These are only now persisted so a failed batch leaves c.tombs+ // untouched and a later reclaim can retry.+ c.mu.Lock()for _, docID := range pendingTombClear {delete(c.tombs[kb], docID)}diff --git a/internal/ingestion/knowledge_compile/consumer_test.go b/internal/ingestion/knowledge_compile/consumer_test.goindex 29f0cec042..5104ad0dae 100644--- a/internal/ingestion/knowledge_compile/consumer_test.go+++ b/internal/ingestion/knowledge_compile/consumer_test.go@@ -54,6 +54,8 @@ type fakeWriter struct {written [][]kccommon.ProductdeletedDocLevel []stringstrippedSources []string+ projected []string+ dropped []string}func (w *fakeWriter) WriteMerged(_ context.Context, _, _ string, products []kccommon.Product) error {@@ -82,6 +84,20 @@ func (w *fakeWriter) StripMergedSources(_ context.Context, _, _ string, docIDs [return nil}+func (w *fakeWriter) ProjectWikiGraph(_ context.Context, _, kb string) error {+ w.mu.Lock()+ defer w.mu.Unlock()+ w.projected = append(w.projected, kb)+ return nil+}++func (w *fakeWriter) DropWikiGraph(_ context.Context, _, kb string) error {+ w.mu.Lock()+ defer w.mu.Unlock()+ w.dropped = append(w.dropped, kb)+ return nil+}+func sampleProducts() []kccommon.Product {return []kccommon.Product{{ID: "p1", DocID: "d1", TenantID: "t1", Variant: kccommon.Variant("structure"),@@ -105,7 +121,7 @@ func TestConsumerCompletedWritesMerged(t *testing.T) {w := &fakeWriter{}c := newTestConsumer(sch, r, w, func(string) (Deduper, error) { return NewNoopDeduper(), nil })- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted), 1); err != nil {+ if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted)); err != nil {t.Fatalf("append: %v", err)}c.tryClaimAndProcess(context.Background())@@ -118,6 +134,9 @@ func TestConsumerCompletedWritesMerged(t *testing.T) {if len(w.written[0]) != 2 {t.Fatalf("expected 2 merged products, got %d", len(w.written[0]))}+ if len(w.projected) != 1 || w.projected[0] != "kb1" {+ t.Fatalf("expected 1 ProjectWikiGraph(kb1) call, got %v", w.projected)+ }// After ack, the claim row must be cleared (no live lease left behind).if _, ok, _ := sch.TryClaim(context.Background()); ok {t.Fatalf("expected no claimable row after ack")@@ -130,22 +149,22 @@ func TestConsumerTombstoneSkipsCompletedBeforeDeleted(t *testing.T) {w := &fakeWriter{}c := newTestConsumer(sch, r, w, func(string) (Deduper, error) { return NewNoopDeduper(), nil })- // completed (seq 1) before deleted (seq 2): the completion is the stale- // original, so it must be skipped and d1's per-doc products orphaned.- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted), 1); err != nil {+ // completed before deleted: the deletion is the last event for d1, so it+ // wins and d1's per-doc products are orphaned (completion skipped).+ if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted)); err != nil {t.Fatalf("append completed: %v", err)}- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeDeleted), 2); err != nil {+ if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeDeleted)); err != nil {t.Fatalf("append deleted: %v", err)}c.tryClaimAndProcess(context.Background())w.mu.Lock()defer w.mu.Unlock()- // No merged write (completed skipped). The deletion is handled entirely on- // the DocEngine: d1's per-doc products are dropped in one call and d1 is- // stripped from every dataset-level product in one call. No products are- // loaded into memory.+ // No merged write (completed skipped, since deletion is the last event).+ // The deletion is handled entirely on the DocEngine: d1's per-doc products+ // are dropped in one call and d1 is stripped from every dataset-level+ // product in one call. No products are loaded into memory.if len(w.written) != 0 {t.Fatalf("expected no merged write, got %d", len(w.written))}@@ -163,20 +182,20 @@ func TestConsumerReingestAfterDeletionWins(t *testing.T) {w := &fakeWriter{}c := newTestConsumer(sch, r, w, func(string) (Deduper, error) { return NewNoopDeduper(), nil })- // deleted (seq 1) then completed (seq 2): the completion has the higher- // sequence, so it wins — the doc is re-ingested, NOT deleted. The deletion- // must not drop its per-doc products, and the completion must be merged.- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeDeleted), 1); err != nil {+ // deleted then completed: the completion is the last event for d1, so it+ // wins — the doc is re-ingested, NOT deleted. The deletion must not drop+ // its per-doc products, and the completion must be merged.+ if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeDeleted)); err != nil {t.Fatalf("append deleted: %v", err)}- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted), 2); err != nil {+ if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted)); err != nil {t.Fatalf("append completed: %v", err)}c.tryClaimAndProcess(context.Background())w.mu.Lock()defer w.mu.Unlock()- // No deletion calls: the higher-seq completion overrides the deletion.+ // No deletion calls: the last (completion) event overrides the deletion.if len(w.deletedDocLevel) != 0 {t.Fatalf("expected no DeleteDocLevelForDocs, got %v", w.deletedDocLevel)}@@ -190,13 +209,16 @@ func TestConsumerReingestAfterDeletionWins(t *testing.T) {if len(w.written[0]) != 2 {t.Fatalf("expected 2 merged products, got %d", len(w.written[0]))}+ if len(w.projected) != 1 || w.projected[0] != "kb1" {+ t.Fatalf("expected 1 ProjectWikiGraph(kb1) call, got %v", w.projected)+ }}func TestSchedulerClaimClosedBatch(t *testing.T) {sch := NewFakeScheduler()for i := 0; i < 40; i++ {docID := "d" + string(rune('a'+i%26)) + string(rune('0'+i/26))- if err := sch.Publish(context.Background(), "t1", "kb1", docID, string(EventTypeCompleted), uint64(i)); err != nil {+ if err := sch.Publish(context.Background(), "t1", "kb1", docID, string(EventTypeCompleted)); err != nil {t.Fatalf("append: %v", err)}}@@ -229,7 +251,7 @@ func TestSchedulerClaimClosedBatch(t *testing.T) {func TestSchedulerReclaimExpired(t *testing.T) {sch := NewFakeScheduler()- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted), 1); err != nil {+ if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted)); err != nil {t.Fatalf("append: %v", err)}_, ok, err := sch.Claim(context.Background(), "kb1")@@ -278,10 +300,10 @@ func TestSchedulerStateMachineLocksStateAndCounts(t *testing.T) {sch := NewFakeScheduler()// Publish two docs: state=pending, backlog=2, inflight=0.- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted), 1); err != nil {+ if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted)); err != nil {t.Fatalf("publish d1: %v", err)}- if err := sch.Publish(context.Background(), "t1", "kb1", "d2", string(EventTypeCompleted), 2); err != nil {+ if err := sch.Publish(context.Background(), "t1", "kb1", "d2", string(EventTypeCompleted)); err != nil {t.Fatalf("publish d2: %v", err)}if c, s := fakeRowCounts(sch, "kb1"); s != DatasetStatePending || c.backlog != 2 || c.inflight != 0 {@@ -351,7 +373,7 @@ func TestSchedulerStateMachineLocksStateAndCounts(t *testing.T) {// diagnostic, otherwise a completed state would be misread as failed.func TestSchedulerSetErrorScopedToClaimToken(t *testing.T) {sch := NewFakeScheduler()- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted), 1); err != nil {+ if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted)); err != nil {t.Fatalf("publish: %v", err)}diff --git a/internal/ingestion/knowledge_compile/event.go b/internal/ingestion/knowledge_compile/event.goindex c0225ea752..9c4090615a 100644--- a/internal/ingestion/knowledge_compile/event.go+++ b/internal/ingestion/knowledge_compile/event.go@@ -54,7 +54,6 @@ type KCCompileEvent struct {DatasetID string `json:"dataset_id"` // the KB scopeDocID string `json:"doc_id"` // the contributing documentEventType string `json:"event_type"` // EventType value- Seq uint64 `json:"seq"` // per-doc monotonic sequence, for out-of-order correctionTimestamp int64 `json:"ts"`}@@ -107,18 +106,18 @@ func DefaultClaimer() Claimer { return defaultClaimer }// KB's durable MySQL backlog and wakes idle workers over NATS. It is a no-op// when no Publisher has been installed (e.g. DB unavailable). A failure is// returned so callers can log but never fail the pipeline on it.-func PublishCompleted(ctx context.Context, tenantID, datasetID, docID string, seq uint64) error {+func PublishCompleted(ctx context.Context, tenantID, datasetID, docID string) error {if defaultPublisher == nil {return nil}- return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeCompleted), seq)+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeCompleted))}// PublishDeleted records a doc_deleted event the same way (Publish handles the// append + notify pairing).-func PublishDeleted(ctx context.Context, tenantID, datasetID, docID string, seq uint64) error {+func PublishDeleted(ctx context.Context, tenantID, datasetID, docID string) error {if defaultPublisher == nil {return nil}- return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted), seq)+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted))}diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.goindex a8deeee6ae..661c191d63 100644--- a/internal/ingestion/knowledge_compile/reader.go+++ b/internal/ingestion/knowledge_compile/reader.go@@ -62,6 +62,7 @@ type engineReader struct {// stored compiled chunk document.var compiledSelectFields = []string{"id", "doc_id", "tenant_id", "compile_kwd",+ "available_int","content_with_weight", "kc_payload","source_chunk_ids", "source_doc_ids","name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd",@@ -140,7 +141,7 @@ func productFromChunkMap(c map[string]interface{}, tenant string) (kccommon.Prodid, _ := c["id"].(string)docID, _ := c["doc_id"].(string)variant, _ := c["compile_kwd"].(string)- merged := isMerged(c["kc_merged"])+ merged := isAvailable(c["available_int"])meta := map[string]any{}if v, ok := c["name_kwd"].(string); ok && v != "" {@@ -216,7 +217,7 @@ func productFromChunkMap(c map[string]interface{}, tenant string) (kccommon.Prod}, true}-// SearchSimilar runs a dense KNN over the existing merged rows (kc_merged=1,+// SearchSimilar runs a dense KNN over the existing merged rows (available_int=1,// compile_kwd=variant) of the KB and returns the closest hit above minScore.func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, variant kccommon.Variant, vector []float64, topN int, minScore float64) (kccommon.Product, float64, error) {eng := r.eng@@ -236,11 +237,11 @@ func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, variLimit: topN,SelectFields: append([]string{"id", "doc_id", "kb_id", "content_with_weight", "kc_payload","name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd", "slug_kwd",- "type", "source_chunk_ids", "source_doc_ids", "kc_merged", "compile_kwd"},+ "type", "source_chunk_ids", "source_doc_ids", "available_int", "compile_kwd"},wikiSelectFields...),Filter: map[string]interface{}{- "kc_merged": 1,- "compile_kwd": string(variant),+ "available_int": 1,+ "compile_kwd": string(variant),},MatchExprs: []interface{}{&types.MatchDenseExpr{@@ -267,10 +268,10 @@ func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, varireturn kccommon.Product{}, 0, nil}-// isMerged normalizes the boxed kc_merged field returned by the DocEngine,+// isAvailable normalizes the boxed available_int field returned by the DocEngine,// which may be stored as a string ("1"/"0"/"true"), a bool, or a numeric,// depending on backend and mapping. Returns true for any positive/true form.-func isMerged(v interface{}) bool {+func isAvailable(v interface{}) bool {switch t := v.(type) {case nil:return falsediff --git a/internal/ingestion/knowledge_compile/scheduler.go b/internal/ingestion/knowledge_compile/scheduler.goindex 03e9f15be4..479d285279 100644--- a/internal/ingestion/knowledge_compile/scheduler.go+++ b/internal/ingestion/knowledge_compile/scheduler.go@@ -53,13 +53,13 @@ const ()// BacklogEntry is one scheduling unit appended to a KB's backlog (Option E-// §11.4). It carries the doc id plus the original event kind/seq so the-// consumer can re-apply the same out-of-order / tombstone guards as the-// broker-based design without re-reading the queue.+// §11.4). It carries the doc id plus the original event kind so the consumer+// can re-apply the same tombstone handling as the broker-based design without+// re-reading the queue. The backlog is an ordered log; the last event for a+// given doc (by append order) wins, so no sequence number is needed.type BacklogEntry struct {DocID string `json:"doc_id"`EventType string `json:"event_type"`- Seq uint64 `json:"seq"`}// ClaimResult is returned by Scheduler.Claim: the KB's tenant plus the closed@@ -83,7 +83,7 @@ type Publisher interface {// Publish records one doc event in the dataset's durable backlog and wakes// idle workers. It is transactional so concurrent publishers do not clobber// each other's backlog, and it always pairs the append with a notify.- Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, seq uint64) error+ Publish(ctx context.Context, tenantID, datasetID, docID, eventType string) error}// Claimer is the consumer-side role (Option E §11.5). A cluster of competing@@ -183,11 +183,11 @@ func (s *mysqlScheduler) Provision(ctx context.Context) error {// workers. The append is transactional (concurrent publishers do not clobber// each other's backlog), and the notify is always paired with the append so a// producer never needs a separate Notify call.-func (s *mysqlScheduler) Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, seq uint64) error {+func (s *mysqlScheduler) Publish(ctx context.Context, tenantID, datasetID, docID, eventType string) error {if s.db == nil {return nil}- entry := BacklogEntry{DocID: docID, EventType: eventType, Seq: seq}+ entry := BacklogEntry{DocID: docID, EventType: eventType}// KnowledgeCompileDataset.dataset_id is the PRIMARY KEY, so the SELECT ... FOR// UPDATE below also takes an InnoDB gap lock for a not-yet-existing dataset;// concurrent publishers for the same dataset serialize on that lock and the@@ -231,8 +231,7 @@ func (s *mysqlScheduler) Publish(ctx context.Context, tenantID, datasetID, docIDzap.String("dataset_id", datasetID),zap.String("tenant_id", tenantID),zap.String("doc_id", docID),- zap.String("event_type", eventType),- zap.Uint64("seq", seq))+ zap.String("event_type", eventType))return s.notify(ctx, datasetID)}@@ -279,7 +278,8 @@ func (s *mysqlScheduler) claimRow(ctx context.Context, tx *gorm.DB, datasetID stzap.String("tenant_id", row.TenantID),zap.Int("batch_size", len(batch)),zap.Int("backlog_remaining", len(backlog)-n),- zap.String("token", row.ClaimToken))+ zap.String("token", row.ClaimToken),+ zap.Any("batch", batch))return ClaimResult{DatasetID: row.DatasetID, TenantID: row.TenantID, Entries: batch, Token: row.ClaimToken}, true, nil}@@ -345,7 +345,8 @@ func (s *mysqlScheduler) Ack(ctx context.Context, datasetID, token string, batchcommon.Info("knowledge_compile: acked dataset batch",zap.String("dataset_id", datasetID),zap.Int("batch_size", len(batch)),- zap.Int("backlog_remaining", remaining))+ zap.Int("backlog_remaining", remaining),+ zap.Any("batch", batch))return remaining, nil}@@ -528,7 +529,7 @@ func (s *mysqlScheduler) SubscribeNotify(ctx context.Context) (<-chan string, er}// removeEntries drops every entry in batch from the inflight slice (match by-// doc_id + event_type + seq). The batch is the exact set the worker claimed, so+// doc_id + event_type). The batch is the exact set the worker claimed, so// this is a precise removal, not a "clear all" — any inflight added by a// concurrent claim is preserved.func removeEntries(inflight, batch []BacklogEntry) []BacklogEntry {@@ -585,7 +586,7 @@ func NewFakeScheduler() *FakeScheduler {func (f *FakeScheduler) Provision(_ context.Context) error { return nil }// Publish appends one doc event and pushes a notify (same as the MySQL path).-func (f *FakeScheduler) Publish(_ context.Context, tenantID, datasetID, docID, eventType string, seq uint64) error {+func (f *FakeScheduler) Publish(_ context.Context, tenantID, datasetID, docID, eventType string) error {f.mu.Lock()defer f.mu.Unlock()r, ok := f.rows[datasetID]@@ -596,7 +597,7 @@ func (f *FakeScheduler) Publish(_ context.Context, tenantID, datasetID, docID, eif r.tenant == "" {r.tenant = tenantID}- r.backlog = append(r.backlog, BacklogEntry{DocID: docID, EventType: eventType, Seq: seq})+ r.backlog = append(r.backlog, BacklogEntry{DocID: docID, EventType: eventType})// Mirror the MySQL scheduler: surface pending unless a worker is running.if r.state != DatasetStateRunning {r.state = DatasetStatePendingdiff --git a/internal/ingestion/knowledge_compile/wiki_graph_test.go b/internal/ingestion/knowledge_compile/wiki_graph_test.gonew file mode 100644index 0000000000..0fc55eacaf--- /dev/null+++ b/internal/ingestion/knowledge_compile/wiki_graph_test.go@@ -0,0 +1,274 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package knowledge_compile++import (+ "context"+ "encoding/json"+ "testing"+)++// page builds a wikiPageProjection with the full "<page_type>/<slug>" identity.+func page(pageType, slug string, outlinks ...string) wikiPageProjection {+ return wikiPageProjection{+ Slug: pageType + "/" + slug,+ PageType: pageType,+ Title: slug,+ Outlinks: outlinks,+ }+}++func findEntity(rows []map[string]interface{}, slug string) map[string]interface{} {+ for _, r := range rows {+ if r["compile_kwd"] == compileKwdWikiEntity && r["slug_kwd"] == slug {+ return r+ }+ }+ return nil+}++func findRelation(rows []map[string]interface{}, from, to string) map[string]interface{} {+ for _, r := range rows {+ if r["compile_kwd"] == compileKwdWikiRelation && r["from_kwd"] == from && r["to_kwd"] == to {+ return r+ }+ }+ return nil+}++const kbForTest = "kb1"++// TestProjectWikiGraphRowsCrossBatchEdgeSurvives verifies a relation whose two+// endpoints land in different "batches" (here modeled as two pages) is still+// materialized: the projection is global over the full page set, so edges never+// dangle due to batching.+func TestProjectWikiGraphRowsCrossBatchEdgeSurvives(t *testing.T) {+ w := engineWriter{}+ pages := []wikiPageProjection{+ page("entity", "alpha", "entity/beta"),+ page("entity", "beta"),+ }+ rows, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, pages)+ if err != nil {+ t.Fatalf("project: %v", err)+ }+ if findEntity(rows, "entity/alpha") == nil {+ t.Fatalf("missing entity entity/alpha")+ }+ if findEntity(rows, "entity/beta") == nil {+ t.Fatalf("missing entity entity/beta")+ }+ if findRelation(rows, "entity/alpha", "entity/beta") == nil {+ t.Fatalf("cross-page edge entity/alpha -> entity/beta not materialized")+ }+}++// TestProjectWikiGraphRowsDropAfterReprojection verifies a page removed from the+// set is gone from the reprojected graph: no orphaned entity and no dangling+// relation remain.+func TestProjectWikiGraphRowsDropAfterReprojection(t *testing.T) {+ w := engineWriter{}+ // First projection has alpha -> beta and beta -> gamma.+ first := []wikiPageProjection{+ page("entity", "alpha", "entity/beta"),+ page("entity", "beta", "entity/gamma"),+ page("entity", "gamma"),+ }+ r1, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, first)+ if err != nil {+ t.Fatalf("project1: %v", err)+ }+ if findRelation(r1, "entity/beta", "entity/gamma") == nil {+ t.Fatalf("expected beta->gamma before drop")+ }++ // beta is deleted: reproject from alpha + gamma only.+ second := []wikiPageProjection{+ page("entity", "alpha"), // alpha no longer links anywhere+ page("entity", "gamma"),+ }+ r2, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, second)+ if err != nil {+ t.Fatalf("project2: %v", err)+ }+ if findEntity(r2, "entity/beta") != nil {+ t.Fatalf("beta should be gone after drop")+ }+ if findRelation(r2, "entity/alpha", "entity/beta") != nil {+ t.Fatalf("orphan alpha->beta must not survive after beta dropped")+ }+ if findRelation(r2, "entity/beta", "entity/gamma") != nil {+ t.Fatalf("orphan beta->gamma must not survive after beta dropped")+ }+}++// TestProjectWikiGraphRowsZeroOutlinkWeightIsZero verifies a page with no+// outlinks gets weight_int == 0 (Python parity: raw outlink count, 0 allowed).+func TestProjectWikiGraphRowsZeroOutlinkWeightIsZero(t *testing.T) {+ w := engineWriter{}+ pages := []wikiPageProjection{page("entity", "lonely")}+ rows, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, pages)+ if err != nil {+ t.Fatalf("project: %v", err)+ }+ e := findEntity(rows, "entity/lonely")+ if e == nil {+ t.Fatalf("missing entity")+ }+ if wt, ok := e["weight_int"].(int); !ok || wt != 0 {+ t.Fatalf("expected weight_int==0 for zero-outlink page, got %v", e["weight_int"])+ }+}++// TestProjectWikiGraphRowsWeightEqualsOutlinkCount verifies weight_int equals+// the raw outlink count (N outlinks -> weight N), not max(1, N).+func TestProjectWikiGraphRowsWeightEqualsOutlinkCount(t *testing.T) {+ w := engineWriter{}+ pages := []wikiPageProjection{+ page("entity", "hub", "entity/a", "entity/b", "entity/c"),+ page("entity", "a"),+ page("entity", "b"),+ page("entity", "c"),+ }+ rows, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, pages)+ if err != nil {+ t.Fatalf("project: %v", err)+ }+ e := findEntity(rows, "entity/hub")+ if e == nil {+ t.Fatalf("missing hub entity")+ }+ if wt, ok := e["weight_int"].(int); !ok || wt != 3 {+ t.Fatalf("expected weight_int==3 for 3-outlink page, got %v", e["weight_int"])+ }+}++// TestProjectWikiGraphRowsFieldMapping verifies the reader-facing display+// columns are written (aliases_kwd from entity_names_kwd, description_with_weight+// from summary_with_weight), and the id is the xxhash colon-namespaced form.+func TestProjectWikiGraphRowsFieldMapping(t *testing.T) {+ w := engineWriter{}+ pages := []wikiPageProjection{+ {+ Slug: "concept/alpha",+ PageType: "concept",+ Title: "Alpha",+ Aliases: []string{"A", "Alpha Prime"},+ Summary: "the alpha page",+ Outlinks: nil,+ SourceDocIDs: []string{"d1", "d2"},+ },+ }+ rows, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, pages)+ if err != nil {+ t.Fatalf("project: %v", err)+ }+ // Use page_type "concept" (not "entity") so the writer's entity_type_kwd+ // mapping ("wiki_"+PageType) is observable — asserting wiki_entity would+ // match by coincidence with the compileKwdWikiEntity constant.+ e := findEntity(rows, "concept/alpha")+ if e == nil {+ t.Fatalf("missing entity")+ }+ if e["aliases_kwd"] == nil {+ t.Fatalf("expected aliases_kwd (mapped from entity_names_kwd) to be set")+ }+ if e["description_with_weight"] != "the alpha page" {+ t.Fatalf("expected description_with_weight=summary, got %v", e["description_with_weight"])+ }+ if e["entity_type_kwd"] != "wiki_concept" {+ t.Fatalf("expected entity_type_kwd=wiki_concept, got %v", e["entity_type_kwd"])+ }+ // id format: wiki_entity:{kb}:{slug} xxhashed to 16 hex chars.+ id, _ := e["id"].(string)+ if len(id) != 16 {+ t.Fatalf("expected 16-char xxhash id, got %q (len %d)", id, len(id))+ }+ // Recompute and compare against wikiGraphXXHash.+ wantID := wikiGraphXXHash("wiki_entity", kbForTest, "concept/alpha")+ if id != wantID {+ t.Fatalf("id mismatch: got %q want %q", id, wantID)+ }+ // content_with_weight JSON must round-trip the slug/page_type/title/aliases/summary/weight.+ raw, _ := e["content_with_weight"].(string)+ var decoded map[string]any+ if err := json.Unmarshal([]byte(raw), &decoded); err != nil {+ t.Fatalf("content_with_weight not valid JSON: %v", err)+ }+ if decoded["slug"] != "concept/alpha" || decoded["page_type"] != "concept" {+ t.Fatalf("content_with_weight missing slug/page_type: %v", decoded)+ }+}++// TestProjectWikiGraphRowsBareSlugNoCollision verifies two pages with the same+// bare slug but different page types get distinct entities (full-slug identity).+func TestProjectWikiGraphRowsBareSlugNoCollision(t *testing.T) {+ w := engineWriter{}+ pages := []wikiPageProjection{+ page("entity", "foo"),+ page("concept", "foo"),+ }+ rows, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, pages)+ if err != nil {+ t.Fatalf("project: %v", err)+ }+ if findEntity(rows, "entity/foo") == nil || findEntity(rows, "concept/foo") == nil {+ t.Fatalf("both entity/foo and concept/foo must exist as distinct entities")+ }+ e1 := findEntity(rows, "entity/foo")+ e2 := findEntity(rows, "concept/foo")+ if e1["id"] == e2["id"] {+ t.Fatalf("entity/foo and concept/foo must have distinct ids")+ }+}++// TestProjectWikiGraphRowsDanglingEdgeSkipped verifies an outlink to a page not+// present in the projection is skipped (no dangling relation).+func TestProjectWikiGraphRowsDanglingEdgeSkipped(t *testing.T) {+ w := engineWriter{}+ pages := []wikiPageProjection{+ page("entity", "alpha", "entity/missing"),+ }+ rows, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, pages)+ if err != nil {+ t.Fatalf("project: %v", err)+ }+ if findRelation(rows, "entity/alpha", "entity/missing") != nil {+ t.Fatalf("dangling edge to missing target must be skipped")+ }+ if findEntity(rows, "entity/missing") != nil {+ t.Fatalf("missing target must not be materialized as an entity")+ }+}++// TestProjectWikiGraphRowsSelfLoopSkipped verifies a page whose outlinks include+// itself does not produce a self-edge (Python parity: src == tgt is skipped).+func TestProjectWikiGraphRowsSelfLoopSkipped(t *testing.T) {+ w := engineWriter{}+ pages := []wikiPageProjection{+ page("entity", "alpha", "entity/alpha"),+ }+ rows, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, pages)+ if err != nil {+ t.Fatalf("project: %v", err)+ }+ if findRelation(rows, "entity/alpha", "entity/alpha") != nil {+ t.Fatalf("self-loop edge must be skipped (Python parity)")+ }+ if findEntity(rows, "entity/alpha") == nil {+ t.Fatalf("the self-referencing entity must still exist")+ }+}diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.goindex 45e6fded28..6765b725de 100644--- a/internal/ingestion/knowledge_compile/writer.go+++ b/internal/ingestion/knowledge_compile/writer.go@@ -19,12 +19,20 @@ import ("context""crypto/sha256""encoding/hex"+ "encoding/json""fmt"+ "sort""strings"+ "time"+ "github.com/cespare/xxhash/v2"+ "go.uber.org/zap"++ "ragflow/internal/common""ragflow/internal/engine""ragflow/internal/engine/types"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"+ "ragflow/internal/utility")// Writer persists dataset-level merged products and removes them on document@@ -32,17 +40,26 @@ import (type Writer interface {// WriteMerged upserts the dataset-level merged products (available_int=1).WriteMerged(ctx context.Context, tenant, kb string, products []kccommon.Product) error- // DeleteDocLevelForDocs drops every per-document (doc-level, kc_merged != 1)+ // DeleteDocLevelForDocs drops every per-document (doc-level, available_int=0)// product of the deleted docs in a single DocEngine call. Dataset-level// merged rows are not targeted because their doc_id equals the kb, never a// deleted source doc id.DeleteDocLevelForDocs(ctx context.Context, tenant, kb string, deletedDocIDs []string) error// StripMergedSources removes deletedDocIDs from the source_doc_ids array of- // every dataset-level (kc_merged=1) product for the dataset. It searches the+ // every dataset-level (available_int=1) product for the dataset. It searches the// merged set once, rewrites the source array of every non-empty survivor in a// single update pass, and deletes (in one call) any product whose array// became empty.StripMergedSources(ctx context.Context, tenant, kb string, deletedDocIDs []string) error+ // ProjectWikiGraph reads every merged wiki_page product for the dataset and+ // re-materializes the wiki page graph (entities + relations) as+ // wiki_entity / wiki_relation compiled rows. It is a delete-then-insert+ // replacement: the DocEngine has no cross-delete/insert transaction boundary,+ // so the graph is rebuilt from a consistent read and the old graph is dropped+ // first. The graph is reconstructible, so the lack of atomicity is accepted.+ ProjectWikiGraph(ctx context.Context, tenant, kb string) error+ // DropWikiGraph deletes every wiki_entity / wiki_relation row for the dataset.+ DropWikiGraph(ctx context.Context, tenant, kb string) error}// engineWriter persists dataset-level merged products through the global@@ -61,6 +78,20 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, producif len(products) == 0 {return nil}+ // Dataset-level telemetry: break the merged set down by compile_kwd so a+ // missing wiki_page at query time can be traced to "WriteMerged never+ // received any wiki_page products" (generation/merge bug) rather than+ // "received then dropped" (downstream delete bug).+ byKwd := map[string]int{}+ for _, p := range products {+ byKwd[compileKwdForVariant(p.Variant)]+++ }+ common.Info("knowledge_compile: WriteMerged dataset-level products",+ zap.String("kb_id", kb),+ zap.String("tenant_id", tenant),+ zap.Int("total", len(products)),+ zap.Any("by_compile_kwd", byKwd),+ )eng := w.engif eng == nil {eng = engine.Get()@@ -69,6 +100,13 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, producreturn nil}baseName := fmt.Sprintf("ragflow_%s", tenant)+ // Source-tracing fields (tasks/2026-08-07-wiki-duplicates-research.md §5.1):+ // one runID + inputHash + timestamp per WriteMerged call, shared by every+ // shard, so rows from the same execution share plan_kwd and reruns of the+ // same plan share input_hash_kwd.+ now := time.Now()+ runID := utility.GenerateUUID()+ inputHash := mergedInputHash(products)// Shard the rows and drive the inserts through the shared global pool// (docengine-bounded) instead of one monolithic InsertChunks call.jobs := make([]CompilerJob, 0, (len(products)+writeMergedBatchSize-1)/writeMergedBatchSize)@@ -81,7 +119,19 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, producjobs = append(jobs, func() error {chunks := make([]map[string]interface{}, 0, len(batch))for _, p := range batch {- chunks = append(chunks, mergedChunkMap(tenant, kb, p))+ chunks = append(chunks, mergedChunkMap(tenant, kb, runID, inputHash, now, p))+ }+ // Telemetry: confirm the bytes actually handed to InsertChunks carry+ // compile_kwd=wiki_page (vs the WriteMerged stats that only reflect+ // the in-memory Variant). If this shows wiki_page but ES returns "",+ // the engine drops the field; if this shows "" too, the map is wrong.+ if len(chunks) > 0 {+ common.Info("knowledge_compile: WriteMerged chunk sample",+ zap.String("kb_id", kb),+ zap.String("compile_kwd", metaString(chunks[0], "compile_kwd")),+ zap.String("variant", string(batch[0].Variant)),+ zap.String("id", metaString(chunks[0], "id")),+ )}_, err := eng.InsertChunks(ctx, chunks, baseName, kb)return err@@ -93,21 +143,31 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, produc// mergedChunkMap builds the chunk-index document for a dataset-level merged// product. It uses the dataset-level idempotency key (§11.6) as `id`, never the// per-doc key, and is always available_int=1 (searchable).-func mergedChunkMap(tenant, kb string, p kccommon.Product) map[string]interface{} {+//+// runID and inputHash carry the minimal source-tracing fields (see+// tasks/2026-08-07-wiki-duplicates-research.md §5.1): runID is the execution+// identifier shared by every row of one WriteMerged call (plan_kwd), inputHash+// is the canonical input fingerprint shared by reruns of the same plan+// (input_hash_kwd), and now stamps both wall-clock audit fields. We deliberately+// do not write the full PLAN JSON into the engine.+func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccommon.Product) map[string]interface{} {srcDocIDs := metaStringSlice(p.Meta, "source_doc_ids")srcChunkIDs := metaStringSlice(p.Meta, "source_chunk_ids")m := map[string]interface{}{- "id": datasetLevelID(tenant, kb, p),- "doc_id": kb,- "tenant_id": tenant,- "kb_id": kb,- "available_int": 1,- "kc_merged": 1,- "compile_kwd": string(p.Variant),- "content_with_weight": p.Content,- "kc_payload": p.Content, // raw payload, for Reader reconstruction- "source_doc_ids": srcDocIDs,- "source_chunk_ids": srcChunkIDs,+ "id": datasetLevelID(tenant, kb, p),+ "doc_id": kb,+ "tenant_id": tenant,+ "kb_id": kb,+ "available_int": 1,+ "compile_kwd": compileKwdForVariant(p.Variant),+ "content_with_weight": p.Content,+ "kc_payload": p.Content, // raw payload, for Reader reconstruction+ "source_doc_ids": srcDocIDs,+ "source_chunk_ids": srcChunkIDs,+ "plan_kwd": runID,+ "input_hash_kwd": inputHash,+ "create_time": now.Format("2006-01-02 15:04:05"),+ "create_timestamp_flt": float64(now.Unix()),}// Carry the wiki page metadata onto the merged row so the dataset-level// products keep the fields the artifact API (ListArtifacts/ListWikiTopics)@@ -183,7 +243,7 @@ func (w engineWriter) DeleteDocLevelForDocs(ctx context.Context, tenant, kb stri}// StripMergedSources removes deletedDocIDs from the source_doc_ids array of-// every dataset-level (kc_merged=1) product for the dataset. The query filters+// every dataset-level (available_int=1) product for the dataset. The query filters// on source_doc_ids IN deletedDocIDs so the engine only returns rows that// actually reference a deleted doc (intersection pushed down); the survivors'// source arrays are rewritten in a single update pass driven by the shared@@ -214,12 +274,12 @@ func (w engineWriter) StripMergedSources(ctx context.Context, tenant, kb string,res, err := eng.Search(ctx, &types.SearchRequest{IndexNames: []string{baseName},KbIDs: []string{kb},- // kc_merged=1 isolates dataset-level rows; source_doc_ids IN+ // available_int=1 isolates dataset-level rows; source_doc_ids IN// deletedDocIDs pushes the intersection test into the engine so only// rows that actually reference a deleted doc are returned (Infinity// array IN means "contains at least one of").Filter: map[string]interface{}{- "kc_merged": 1,+ "available_int": 1,"source_doc_ids": deletedDocIDs,},SelectFields: []string{"id", "source_doc_ids"},@@ -309,6 +369,27 @@ func hashStr(s string) string {return hex.EncodeToString(sum[:])}+// mergedInputHash derives a canonical SHA-256 fingerprint of a WriteMerged+// batch's input evidence: the union of source_doc_ids across all products,+// sorted and deduped. Reruns of the same plan (same input documents) share the+// fingerprint, so input_hash_kwd distinguishes "same plan content" across+// executions while plan_kwd separates individual runs.+func mergedInputHash(products []kccommon.Product) string {+ seen := make(map[string]struct{}, 16)+ docs := make([]string, 0, 16)+ for _, p := range products {+ for _, d := range metaStringSlice(p.Meta, "source_doc_ids") {+ if _, ok := seen[d]; ok {+ continue+ }+ seen[d] = struct{}{}+ docs = append(docs, d)+ }+ }+ sort.Strings(docs)+ return hashStr(strings.Join(docs, "\n"))+}+// metaString extracts a string from a map value, tolerating a missing or// non-string entry.func metaString(m map[string]any, key string) string {@@ -353,3 +434,441 @@ func metaInt(m map[string]any, key string) (int64, bool) {}return 0, false}++// --- Wiki page graph materialization (wiki_entity / wiki_relation) ---++// compile_kwd values for the dataset-level products this package writes. The+// wiki variant compiles into "wiki_page" rows (per the Python writer contract+// that GetWikiAlteration / ListArtifacts / GetWikiGraph all filter on+// compile_kwd = "wiki_page"), while the page graph is materialized as the+// dedicated wiki_entity / wiki_relation buckets.+const (+ compileKwdWikiPage = "wiki_page"+ compileKwdWikiEntity = "wiki_entity"+ compileKwdWikiRelation = "wiki_relation"+ compileKwdWikiPageGraph = "wiki_page_graph" // legacy Python blob, swept on drop+)++// wikiGraphBatchSize bounds how many graph rows each parallel InsertChunks call+// carries, mirroring writeMergedBatchSize.+const wikiGraphBatchSize = 200++// wikiGraphSourceDocCap bounds the source_doc_ids array stored on a graph row so+// a heavily-shared page does not accumulate an unbounded id list.+const wikiGraphSourceDocCap = 64++// compileKwdForVariant maps a compiler Variant to the compile_kwd stamped on the+// merged chunk document. The wiki variant is special-cased because all read+// paths filter on compile_kwd = "wiki_page", not the raw variant string "wiki".+func compileKwdForVariant(v kccommon.Variant) string {+ if v == kccommon.VariantWiki {+ return compileKwdWikiPage+ }+ return string(v)+}++// wikiGraphXXHash derives a stable 16-char hex id (matches Python+// xxh64().hexdigest()) for a graph node/edge namespaced under the kb.+func wikiGraphXXHash(namespace, kb, key string) string {+ return fmt.Sprintf("%016x", xxhash.Sum64String(namespace+":"+kb+":"+key))+}++// wikiGraphBareKey reduces a full "<page_type>/<slug>" identity (or a bare+// slug) to a canonical bare key used by the graph's reverse slug index. It+// strips any "<page_type>/" prefix and normalizes underscores to hyphens so a+// bare outlink ("dong-zhuo") matches the page whose slug_kwd is+// "entity/dong_zhuo" (or "entity/dong-zhuo") regardless of the LLM's+// underscore-vs-hyphen formatting. Empty strings yield "" (never a valid key).+func wikiGraphBareKey(slug string) string {+ s := strings.TrimSpace(slug)+ if s == "" {+ return ""+ }+ if idx := strings.LastIndex(s, "/"); idx >= 0 && idx < len(s)-1 {+ s = s[idx+1:]+ }+ s = strings.ReplaceAll(s, "_", "-")+ return strings.TrimSpace(s)+}++// wikiPageProjection is the subset of a merged wiki_page row that the graph+// projection needs. It is reconstructed from the stored display columns (the+// same fields GetWikiGraph reads back), not from the JSON payload.+type wikiPageProjection struct {+ Slug string+ PageType string+ Title string+ Aliases []string+ Summary string+ // Outlinks are the other wiki pages this page links to (by full+ // "<page_type>/<slug>" identity). These inter-page links are the SOLE source+ // of wiki_relation edges: a wiki_relation row exists iff some page lists+ // another page in its Outlinks (and that target page also exists). No edge+ // comes from entity extraction, co-occurrence, or semantic similarity.+ Outlinks []string+ // SourceDocIDs are the originating document ids that produced this page.+ // For a wiki_relation the stored source_doc_ids is the UNION of both+ // endpoints' SourceDocIDs, so the relation is dropped only once neither+ // endpoint traces to a surviving document.+ SourceDocIDs []string+ SourceChunkIDs []string+}++// ProjectWikiGraph reads every merged wiki_page for the dataset and+// re-materializes the page graph. See the Writer interface doc for the+// delete-then-insert (non-atomic) contract.+func (w engineWriter) ProjectWikiGraph(ctx context.Context, tenant, kb string) error {+ pages, err := w.loadMergedWikiPages(ctx, tenant, kb)+ if err != nil {+ return err+ }+ // Telemetry: confirm how many merged wiki_page rows survived WriteMerged.+ // If this is 0 while WriteMerged reported wiki_page:N, the merged rows are+ // not queryable under (compile_kwd=wiki_page AND available_int=1) — point at the+ // stored field values, not a downstream delete.+ common.Info("knowledge_compile: ProjectWikiGraph load",+ zap.String("kb_id", kb),+ zap.Int("merged_wiki_pages_loaded", len(pages)))+ // Zero pages: the dataset has no wiki graph. Drop any stale graph rows and+ // return — a full reprojection of an empty set would only rewrite nothing.+ if len(pages) == 0 {+ return w.dropWikiGraph(ctx, tenant, kb)+ }+ rows, err := w.projectWikiGraphRows(ctx, tenant, kb, pages)+ if err != nil {+ return err+ }+ // delete-then-insert: drop the previous graph first, then write the new one.+ if err := w.dropWikiGraph(ctx, tenant, kb); err != nil {+ return err+ }+ return w.insertWikiGraphChunks(ctx, tenant, kb, rows)+}++// DropWikiGraph deletes every wiki_entity / wiki_relation row for the dataset.+func (w engineWriter) DropWikiGraph(ctx context.Context, tenant, kb string) error {+ return w.dropWikiGraph(ctx, tenant, kb)+}++// dropWikiGraph is the shared delete path for both ProjectWikiGraph (when the+// page set is empty, or before re-inserting) and DropWikiGraph. It deletes by+// kb_id + compile_kwd IN (the graph buckets), also sweeping any legacy+// wiki_page_graph blob left by an earlier Python writer so the index does not+// accumulate stale state. The dataset-wide full delete of merged rows is owned+// by DeleteDocLevelForKb elsewhere and is not repeated here.+func (w engineWriter) dropWikiGraph(ctx context.Context, tenant, kb string) error {+ eng := w.eng+ if eng == nil {+ eng = engine.Get()+ }+ if eng == nil {+ return nil+ }+ baseName := fmt.Sprintf("ragflow_%s", tenant)+ _, err := eng.DeleteChunks(ctx, map[string]interface{}{+ "kb_id": kb,+ "compile_kwd": []string{+ compileKwdWikiEntity,+ compileKwdWikiRelation,+ compileKwdWikiPageGraph,+ },+ }, baseName, kb)+ return err+}++// loadMergedWikiPages scrolls every merged wiki_page product for the dataset.+// It selects only the display columns the projection needs (slug / page_type /+// title / entity_names_kwd (aliases) / summary_with_weight / outlinks_kwd /+// source_doc_ids / source_chunk_ids), never the JSON payload.+func (w engineWriter) loadMergedWikiPages(ctx context.Context, tenant, kb string) ([]wikiPageProjection, error) {+ eng := w.eng+ if eng == nil {+ eng = engine.Get()+ }+ if eng == nil {+ return nil, nil+ }+ baseName := fmt.Sprintf("ragflow_%s", tenant)+ const batchSize = 2000+ var out []wikiPageProjection+ offset := 0+ kwdSeen := map[string]int{}+ for {+ res, err := eng.Search(ctx, &types.SearchRequest{+ IndexNames: []string{baseName},+ KbIDs: []string{kb},+ Filter: map[string]interface{}{+ "compile_kwd": compileKwdWikiPage,+ "available_int": 1,+ "kb_id": kb,+ },+ SelectFields: []string{+ "slug_kwd", "page_type_kwd", "title_kwd",+ "entity_names_kwd", "summary_with_weight", "outlinks_kwd",+ "source_doc_ids", "source_chunk_ids",+ },+ Limit: batchSize,+ Offset: offset,+ // Stable deterministic ordering so the deep-offset pages (beyond ES's+ // default result window) keep a consistent cursor and the engine can+ // switch to search_after instead of failing on deep offset.+ OrderBy: (&types.OrderByExpr{}).Asc("slug_kwd"),+ })+ if err != nil {+ return nil, err+ }+ // Telemetry: surface whether the Search filter is actually honoured. If+ // the returned rows carry compile_kwd other than wiki_page, the engine's+ // Filter map is being ignored and wiki_entity rows leak into the page+ // projection (which then has nothing under compile_kwd=wiki_page).+ for _, c := range res.Chunks {+ kwdSeen[metaString(c, "compile_kwd")]+++ }+ if len(res.Chunks) == 0 {+ break+ }+ for _, c := range res.Chunks {+ p := wikiPageProjection{+ Slug: metaString(c, "slug_kwd"),+ PageType: metaString(c, "page_type_kwd"),+ Title: metaString(c, "title_kwd"),+ Aliases: metaStringSlice(c, "entity_names_kwd"),+ Summary: metaString(c, "summary_with_weight"),+ Outlinks: metaStringSlice(c, "outlinks_kwd"),+ SourceDocIDs: metaStringSlice(c, "source_doc_ids"),+ SourceChunkIDs: metaStringSlice(c, "source_chunk_ids"),+ }+ out = append(out, p)+ }+ if len(res.Chunks) < batchSize {+ break+ }+ offset += batchSize+ }+ common.Info("knowledge_compile: loadMergedWikiPages query result",+ zap.String("kb_id", kb),+ zap.Int("rows_returned", len(out)),+ zap.Any("compile_kwd_seen", kwdSeen))+ return out, nil+}++// projectWikiGraphRows builds the wiki_entity / wiki_relation chunk documents+// from a full projection of the dataset's merged wiki pages.+//+// Graph model (mirrors the Python writer in dataset_wiki_generator.py):+// - One wiki_entity row per merged wiki page (node = page).+// - One wiki_relation row per inter-page outlink: a page lists another page+// in its Outlinks, and that target page also exists in the projection.+// Thus the EDGES of this graph are exclusively the mutual links BETWEEN+// wiki pages (page A outlinks to page B => edge A->B). No edge originates+// from entity extraction, co-occurrence, or semantic similarity.+//+// Both entities and relations are keyed on the full "<page_type>/<slug>"+// identity (matching Go wiki input), so a bare slug collision across page types+// is impossible. A relation whose target page is absent from this projection (a+// dangling edge, e.g. the target was pruned/deleted) is skipped, as is a+// self-loop (src == tgt) — the graph is rebuilt from a consistent read each+// time, so cross-batch dangling edges cannot accumulate.+func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string, pages []wikiPageProjection) ([]map[string]interface{}, error) {+ // Sort pages deterministically by full slug before building the indexes so+ // the outcome is reproducible across reprojections. The engine's row order+ // is otherwise scroll-order dependent, which made bare-slug first-match+ // resolution flip between runs.+ sort.SliceStable(pages, func(i, j int) bool {+ return pages[i].Slug < pages[j].Slug+ })+ bySlug := make(map[string]wikiPageProjection, len(pages))+ // Bare-slug index so inter-page outlinks match regardless of whether they+ // were emitted as a full "<page_type>/<slug>" identity or a bare slug. The+ // LLM writes wikitext links using bare slugs (and may mix "_" vs "-"), so+ // the wiki_page outlinks_kwd in ES is frequently bare; matching only the+ // full slug would silently drop every relation (graph has nodes, no edges).+ //+ // A bare key shared by two pages (e.g. "entity/foo" and "concept/foo" both+ // reduce to "foo") is ambiguous; such links resolve to nothing rather than+ // nondeterministically flipping between pages across runs.+ byBareSlug := make(map[string]wikiPageProjection, len(pages))+ bareAmbiguous := make(map[string]bool, len(pages))+ for _, p := range pages {+ if p.Slug == "" {+ continue+ }+ bySlug[p.Slug] = p+ if b := wikiGraphBareKey(p.Slug); b != "" {+ if prev, ok := byBareSlug[b]; ok {+ if prev.Slug != p.Slug {+ bareAmbiguous[b] = true+ }+ } else {+ byBareSlug[b] = p+ }+ }+ }++ rows := make([]map[string]interface{}, 0, len(bySlug))+ seen := make(map[string]bool, len(bySlug))+ for _, p := range bySlug {+ entityID := wikiGraphXXHash("wiki_entity", kb, p.Slug)+ if seen[entityID] {+ continue+ }+ seen[entityID] = true++ weight := len(p.Outlinks) // raw outlink count; 0 is allowed (Python parity)+ content, err := json.Marshal(map[string]any{+ "slug": p.Slug,+ "page_type": p.PageType,+ "title": p.Title,+ "aliases": p.Aliases,+ "summary": p.Summary,+ "weight": weight,+ })+ if err != nil {+ return nil, err+ }+ rows = append(rows, map[string]interface{}{+ "id": entityID,+ "doc_id": kb,+ "tenant_id": tenant,+ "kb_id": kb,+ "available_int": 1,+ "compile_kwd": compileKwdWikiEntity,+ "type_kwd": "wiki_" + p.PageType,+ "entity_type_kwd": "wiki_" + p.PageType,+ "slug_kwd": p.Slug,+ "title_kwd": p.Title,+ "aliases_kwd": p.Aliases,+ "description_with_weight": p.Summary,+ "weight_int": weight,+ "source_chunk_ids": p.SourceChunkIDs,+ "source_doc_ids": capSourceDocs(p.SourceDocIDs),+ "content_with_weight": string(content),+ })++ // Relations: one edge per outlink whose target page exists in this+ // projection. A target may be a full "<page_type>/<slug>" identity or a+ // bare slug (the latter is what the LLM emits in wikitext links); resolve+ // both so a bare outlink still produces an edge.+ for _, tgt := range p.Outlinks {+ if tgt == p.Slug {+ continue // self-loop: Python skips src == tgt+ }+ tp, ok := bySlug[tgt]+ if !ok {+ // Bare-slug outlink: normalize (strip prefix, "_"->"-") and look+ // up the reverse index. Keeps the canonical full slug for the+ // stored relation so from/to are consistent with slug_kwd. An+ // ambiguous bare key (mapped to more than one page) resolves to+ // nothing instead of nondeterministically choosing one.+ if bare := wikiGraphBareKey(tgt); bare != "" && !bareAmbiguous[bare] {+ tp, ok = byBareSlug[bare]+ if ok {+ tgt = tp.Slug+ }+ }+ }+ if !ok {+ continue // dangling edge: target page not in this projection+ }+ relID := wikiGraphXXHash("wiki_relation", kb, p.Slug+":"+tgt)+ if seen[relID] {+ continue+ }+ seen[relID] = true+ relContent, err := json.Marshal(map[string]any{+ "from": p.Slug,+ "to": tgt,+ })+ if err != nil {+ return nil, err+ }+ srcDocs := unionCap(p.SourceDocIDs, tp.SourceDocIDs)+ rows = append(rows, map[string]interface{}{+ "id": relID,+ "doc_id": kb,+ "tenant_id": tenant,+ "kb_id": kb,+ "available_int": 1,+ "compile_kwd": compileKwdWikiRelation,+ "type_kwd": compileKwdWikiRelation,+ "from_id": entityID,+ "to_id": wikiGraphXXHash("wiki_entity", kb, tgt),+ "from_kwd": p.Slug,+ "to_kwd": tgt,+ "source_doc_ids": srcDocs,+ "content_with_weight": string(relContent),+ })+ }+ }+ var relCount, entCount int+ for _, r := range rows {+ switch r["compile_kwd"] {+ case compileKwdWikiRelation:+ relCount+++ case compileKwdWikiEntity:+ entCount+++ }+ }+ common.Info("knowledge_compile: projectWikiGraphRows result",+ zap.String("kb_id", kb),+ zap.Int("pages_projected", len(bySlug)),+ zap.Int("wiki_entity_rows", entCount),+ zap.Int("wiki_relation_rows", relCount))+ return rows, nil+}++// insertWikiGraphChunks shards the graph rows and drives the inserts through the+// shared global pool, mirroring WriteMerged.+func (w engineWriter) insertWikiGraphChunks(ctx context.Context, tenant, kb string, rows []map[string]interface{}) error {+ if len(rows) == 0 {+ return nil+ }+ eng := w.eng+ if eng == nil {+ eng = engine.Get()+ }+ if eng == nil {+ return nil+ }+ baseName := fmt.Sprintf("ragflow_%s", tenant)+ jobs := make([]CompilerJob, 0, (len(rows)+wikiGraphBatchSize-1)/wikiGraphBatchSize)+ for start := 0; start < len(rows); start += wikiGraphBatchSize {+ end := start + wikiGraphBatchSize+ if end > len(rows) {+ end = len(rows)+ }+ batch := rows[start:end]+ jobs = append(jobs, func() error {+ _, err := eng.InsertChunks(ctx, batch, baseName, kb)+ return err+ })+ }+ return runCompilerJobs(ctx, jobs)+}++// capSourceDocs returns up to wikiGraphSourceDocCap source doc ids.+func capSourceDocs(ids []string) []string {+ if len(ids) <= wikiGraphSourceDocCap {+ return ids+ }+ return ids[:wikiGraphSourceDocCap]+}++// unionCap returns the union of two doc-id lists, capped at wikiGraphSourceDocCap.+func unionCap(a, b []string) []string {+ seen := make(map[string]bool, len(a)+len(b))+ out := make([]string, 0, len(a)+len(b))+ for _, id := range a {+ if !seen[id] {+ seen[id] = true+ out = append(out, id)+ }+ }+ for _, id := range b {+ if !seen[id] {+ seen[id] = true+ out = append(out, id)+ }+ }+ return capSourceDocs(out)+}diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.goindex 8f7e9bfe42..b45a04f5c1 100644--- a/internal/ingestion/knowledge_compile/writer_test.go+++ b/internal/ingestion/knowledge_compile/writer_test.go@@ -2,6 +2,7 @@ package knowledge_compileimport ("testing"+ "time"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common")@@ -28,7 +29,8 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {"source_chunk_ids": []string{"c1"},},}- m := mergedChunkMap("t1", "kb1", p)+ now := time.Now()+ m := mergedChunkMap("t1", "kb1", "run-abc", "hash-123", now, p)cases := map[string]string{"slug_kwd": "entity/alpha",@@ -37,6 +39,15 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {"page_type_kwd": "entity","topic_kwd": "Alpha","summary_with_weight": "A page about Alpha",+ "plan_kwd": "run-abc",+ "input_hash_kwd": "hash-123",+ }+ // The wall-clock audit fields must be stamped from `now`, not omitted.+ if m["create_time"] != now.Format("2006-01-02 15:04:05") {+ t.Errorf("create_time = %v, want %v", m["create_time"], now.Format("2006-01-02 15:04:05"))+ }+ if m["create_timestamp_flt"] != float64(now.Unix()) {+ t.Errorf("create_timestamp_flt = %v, want %v", m["create_timestamp_flt"], float64(now.Unix()))}for k, want := range cases {if got, _ := m[k].(string); got != want {@@ -46,8 +57,11 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {if v, _ := m["entity_names_kwd"].([]string); len(v) != 1 || v[0] != "Alpha" {t.Errorf("entity_names_kwd = %#v, want [Alpha]", m["entity_names_kwd"])}- if m["doc_id"] != "kb1" || m["kc_merged"] != 1 || m["available_int"] != 1 {- t.Errorf("merged flags wrong: doc_id=%v kc_merged=%v available_int=%v", m["doc_id"], m["kc_merged"], m["available_int"])+ if m["doc_id"] != "kb1" || m["available_int"] != 1 {+ t.Errorf("merged flags wrong: doc_id=%v available_int=%v", m["doc_id"], m["available_int"])+ }+ if _, ok := m["kc_merged"]; ok {+ t.Errorf("merged row must not persist undefined kc_merged field")}if m["q_3_vec"] == nil {t.Errorf("vector column missing")diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.goindex e106d81901..f43cf7ae50 100644--- a/internal/ingestion/pipeline/pipeline.go+++ b/internal/ingestion/pipeline/pipeline.go@@ -34,6 +34,7 @@ import ("ragflow/internal/utility""github.com/cloudwego/eino/compose"+ "go.uber.org/zap")// Pipeline is a compiled ingestion canvas plus task-scoped metadata.@@ -653,6 +654,34 @@ func (p *Pipeline) componentProgressCallback(ctx context.Context) runtime.Progremsg = ev.Component + " Error"}}+ // Surface every component lifecycle event as a structured log line so+ // a component failure (e.g. an LLM/client error) is captured in+ // ingestor_server.log even if the wrapped error never reaches the+ // higher-level "Task ... failed" branch.+ switch ev.Phase {+ case runtime.PhaseError:+ if ev.Err != nil {+ common.Error("component progress: error", ev.Err,+ zap.String("component", ev.Component),+ zap.String("task_id", p.taskID),+ zap.String("document_id", p.documentID))+ } else {+ common.Info("component progress: error",+ zap.String("component", ev.Component),+ zap.String("task_id", p.taskID),+ zap.String("document_id", p.documentID))+ }+ default:+ // Keep the message constant: msg may carry component names or+ // error-derived text, and a newline in it could forge a log record+ // (CWE-117). Pass msg and the phase as structured fields instead.+ common.Info("component progress",+ zap.String("message", msg),+ zap.Int("phase", int(ev.Phase)),+ zap.String("component", ev.Component),+ zap.String("task_id", p.taskID),+ zap.String("document_id", p.documentID))+ }p.sink.OnComponentProgress(ctx, ProgressEvent{TaskID: p.taskID,DocumentID: p.documentID,diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.goindex a67d7266d8..8a232caea2 100644--- a/internal/ingestion/task/knowledge_compiler_wiring.go+++ b/internal/ingestion/task/knowledge_compiler_wiring.go@@ -34,6 +34,7 @@ import ("ragflow/internal/service""gorm.io/gorm"+ appcommon "ragflow/internal/common")// This file is the composition-root wiring for the KnowledgeCompiler ingestion@@ -154,9 +155,22 @@ func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatRconfig.MaxTokens = req.MaxTokens}}- resp, err := c.svc.Chat(ctx, c.tenantID, llmID, msgs, config)- if err != nil {- return nil, err+ // Retry transient transport/provider failures (HTTP timeout, reset,+ // connection refused, 5xx, 429) with exponential backoff. A single+ // external-LLM hiccup must not abort the whole knowledge compile — the reply+ // is never cached, so each attempt issues a fresh request. Permanent+ // configuration/model errors (auth, unknown model) are not retried.+ var resp *models.ChatResponse+ retryErr := appcommon.RetryWithBackoff(ctx, kcChatRetryMax, kcChatRetryDelay, func() error {+ r, err := c.svc.Chat(ctx, c.tenantID, llmID, msgs, config)+ if err != nil {+ return err+ }+ resp = r+ return nil+ }, appcommon.IsTransientError)+ if retryErr != nil {+ return nil, retryErr}content := ""if resp != nil && resp.Answer != nil {@@ -165,6 +179,14 @@ func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatRreturn &kc.ChatResponse{Content: content}, nil}+// kcChatRetryMax bounds how many times a transient LLM transport failure is+// retried. Each attempt may run up to the driver's HTTP timeout, so the count+// stays small to avoid unbounded wall-clock latency inside one compile.+const kcChatRetryMax = 2++// kcChatRetryDelay is the initial exponential-backoff delay between retries.+const kcChatRetryDelay = 2 * time.Second+// kcEmbedder adapts service.ModelProviderService.GetEmbeddingModel to the// knowledge_compiler Embedder seam. Vectors are returned as []float32 to match// the component's product schema.diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex 4a77724867..6dafb7a03a 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -263,7 +263,7 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map// (available_int=1). The notification is sent only after a successful persist// and is best-effort / non-fatal — a delivery failure is logged but does not// fail the pipeline task.- if err := knowledge_compile.PublishCompleted(ctx, s.taskCtx.Tenant.ID, s.taskCtx.Doc.KbID, s.taskCtx.Doc.ID, 0); err != nil {+ if err := knowledge_compile.PublishCompleted(ctx, s.taskCtx.Tenant.ID, s.taskCtx.Doc.KbID, s.taskCtx.Doc.ID); err != nil {common.Logger.Warn(fmt.Sprintf("knowledge_compile: publish doc_completed for %s failed: %v", s.taskCtx.Doc.ID, err))}diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.goindex a094f4b2ae..7de10643de 100644--- a/internal/service/dataset_artifact_service.go+++ b/internal/service/dataset_artifact_service.go@@ -407,7 +407,7 @@ type WikiGraphRelation struct {// GetWikiGraph returns the wiki entity/relation graph for a dataset.func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, datasetID string) (*WikiGraph, error) {entityChunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,- map[string]interface{}{"compile_kwd": []string{CompileKwdWikiEntity}},+ map[string]interface{}{"compile_kwd": []string{CompileKwdWikiEntity}, "available_int": 1},[]string{"slug_kwd", "title_kwd", "aliases_kwd", "description_with_weight", "entity_type_kwd", "weight_int", "source_chunk_ids"},0, 1000, (&types.OrderByExpr{}).Desc("weight_int"))if err != nil {@@ -420,27 +420,43 @@ func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, datgraph := &WikiGraph{Entities: []WikiGraphEntity{}, Relations: []WikiGraphRelation{}}for _, c := range entityChunks {w := intValue(c["weight_int"])+ // Match the GetWikiPage / ListWikiPages contract: expose the bare slug+ // and the pure page_type (no "wiki_" prefix, no "<page_type>/" prefix)+ // so the frontend can build artifact/<page_type>/<slug> links that+ // round-trip. entity_type_kwd stores "wiki_" + page_type (e.g.+ // "wiki_topic"); strip the prefix. slug_kwd stores the full+ // "<page_type>/<slug>" form; expose the trailing bare slug.+ fullSlug := firstStringValue(c["slug_kwd"])+ bareSlug := fullSlug+ pageType := strings.TrimPrefix(firstStringValue(c["entity_type_kwd"]), "wiki_")+ if idx := strings.LastIndex(bareSlug, "/"); idx >= 0 {+ pageType = bareSlug[:idx]+ bareSlug = bareSlug[idx+1:]+ }graph.Entities = append(graph.Entities, WikiGraphEntity{- Slug: firstStringValue(c["slug_kwd"]),+ Slug: bareSlug,Name: firstStringValue(c["title_kwd"]),Aliases: toStringSlice(c["aliases_kwd"]),Description: firstStringValue(c["description_with_weight"]),- Type: firstStringValue(c["entity_type_kwd"]),+ Type: pageType,Weight: w,SourceChunkIDs: toStringSlice(c["source_chunk_ids"]),})}if len(fromKwds) > 0 {relChunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,- map[string]interface{}{"compile_kwd": []string{CompileKwdWikiRelation}, "from_kwd": fromKwds},+ map[string]interface{}{"compile_kwd": []string{CompileKwdWikiRelation}, "available_int": 1, "from_kwd": fromKwds},[]string{"from_kwd", "to_kwd"}, 0, 10000, nil)if err != nil {return nil, err}for _, c := range relChunks {+ // Relations reference endpoints by their full "<page_type>/<slug>"+ // form in ES; expose them as bare slugs so the frontend's+ // nodesBySlug (keyed on the entity's bare slug) can resolve edges.graph.Relations = append(graph.Relations, WikiGraphRelation{- From: firstStringValue(c["from_kwd"]),- To: firstStringValue(c["to_kwd"]),+ From: bareWikiSlug(firstStringValue(c["from_kwd"])),+ To: bareWikiSlug(firstStringValue(c["to_kwd"])),})}}@@ -877,6 +893,18 @@ func firstStringValue(v interface{}) string {return ""}+// bareWikiSlug strips the "<page_type>/" prefix from a full wiki slug+// ("topic/yellow-turban-rebellion" -> "yellow-turban-rebellion"), matching the+// bare-slug form the graph UI keys nodes/relations on. Slugs with no prefix are+// returned unchanged.+func bareWikiSlug(slug string) string {+ s := strings.TrimSpace(slug)+ if idx := strings.LastIndex(s, "/"); idx >= 0 && idx < len(s)-1 {+ return s[idx+1:]+ }+ return s+}+// toStringSlice normalizes an ES field value (string or list) to []string.func toStringSlice(v interface{}) []string {switch t := v.(type) {diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.goindex d40a814f5f..3121d3df16 100644--- a/internal/service/document/document_crud.go+++ b/internal/service/document/document_crud.go@@ -365,7 +365,7 @@ func (s *DocumentService) deleteDocEngineData(docID, tenantID, kbID string) {// never block the document delete, which already succeeded above.pubCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)defer cancel()- if err := knowledge_compile.PublishDeleted(pubCtx, tenantID, kbID, docID, 0); err != nil {+ if err := knowledge_compile.PublishDeleted(pubCtx, tenantID, kbID, docID); err != nil {common.Warn(fmt.Sprintf("deleteDocEngineData: publish doc_deleted for %s failed: %v", docID, err))}if s.metadataSvc != nil {diff --git a/rag/advanced_rag/knowlege_compile/wiki.py b/rag/advanced_rag/knowlege_compile/wiki.pyindex b18758dfbe..26ba5ec9dd 100644--- a/rag/advanced_rag/knowlege_compile/wiki.py+++ b/rag/advanced_rag/knowlege_compile/wiki.py@@ -1763,10 +1763,19 @@ Examples of BAD slugs (do NOT produce):- ``concept/example-name`` (just duplicate the sample)# Other rules-- Group closely related small entities onto the same page (max 3-4 per page).- BUT if a primary entity is described through several distinct thematic- sections that appear as concepts above, prefer a separate ``concept`` page- for EACH such section instead of collapsing them onto the entity page.+- Entity/concept identity is one-to-one with pages: every extracted entity and+ concept must be represented by exactly one canonical page, and each identity+ may appear in only one page's ``entity_names``. Never split an identity into+ multiple pages, page types, thematic sections, aliases, language+ transliterations, or alternate slug spellings. Put all supported sections+ for that identity on its single canonical page.+- A page may represent several closely related low-signal entities/concepts+ (max 3-4 per page), but list every represented identity in ``entity_names``+ and do not repeat any identity on another page. If the page budget is tight,+ group identities rather than omitting one or emitting a second page for it.+- Identity ownership does not limit linking: ``related_kb_pages`` should list+ every directly related canonical page supported by the input (within the+ available-page budget). Never link duplicate or non-canonical slug variants.- priority 1 = highest importance (process first).- entity_names must match the names in the entities / concepts lists above.- Target approximately {target_page_count} total pages (feel free to deviate