Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(knowledge_compile): materialize wiki page graph (wiki_entity/wiki_relation) (#17976)

Zhichang Yuf12c0ec08ac5
diff --git a/go.mod b/go.mod
index 7de8df30d6..698c7aabb1 100644
--- a/go.mod
+++ b/go.mod
@@ -20,7 +20,7 @@ require (
github.com/browserbase/stagehand-go/v3 v3.21.0
github.com/cenkalti/backoff/v5 v5.0.3
github.com/cespare/xxhash/v2 v2.3.0
- github.com/cloudwego/eino v0.9.12
+ github.com/cloudwego/eino v0.9.13
github.com/denisenkom/go-mssqldb v0.12.3
github.com/elastic/go-elasticsearch/v8 v8.19.1
github.com/eric642/e2b-go-sdk v0.1.3
diff --git a/go.sum b/go.sum
index d893f700d7..10f65eb35c 100644
--- a/go.sum
+++ b/go.sum
@@ -167,8 +167,8 @@ github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudwego/base64x v0.1.7 h1:NppS+Fgzg5ovhn4NkUXaDT3x9jldgH5ToMCqzBSi2zI=
github.com/cloudwego/base64x v0.1.7/go.mod h1:Cu1PV9zfrSf7ET2tIbWbbEy7jO7HHJ13q4X2SQ8aWYg=
-github.com/cloudwego/eino v0.9.12 h1:mHAMo5k7GdvnVD8Lc2sLyfpkxEm0S/y3PkEMhsSYt78=
-github.com/cloudwego/eino v0.9.12/go.mod h1:OBD1mrkfkt/pJa4rkg1P0VnaMeOVl7l8IAdEqY//3IQ=
+github.com/cloudwego/eino v0.9.13 h1:iD/ETS+lxnNp1VeNPqWVGPWdND6Dbf4LyINbLUlDRcM=
+github.com/cloudwego/eino v0.9.13/go.mod h1:OBD1mrkfkt/pJa4rkg1P0VnaMeOVl7l8IAdEqY//3IQ=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik=
github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4=
diff --git a/internal/agent/canvas/node_body.go b/internal/agent/canvas/node_body.go
index d01ec37b9f..7146cdf80a 100644
--- a/internal/agent/canvas/node_body.go
+++ b/internal/agent/canvas/node_body.go
@@ -40,6 +40,8 @@ import (
"time"
"ragflow/internal/agent/runtime"
+
+ "go.uber.org/zap"
)
// nodeBodyFn is the plain function shape compose.InvokableLambda accepts.
@@ -154,12 +156,10 @@ func buildNodeBodyWithOptions(ctx context.Context, cpnID, name string, params ma
if comp == nil {
return nil, fmt.Errorf("canvas: component %q (%s): factory returned nil component", cpnID, name)
}
- // Pass the class name through to the body so the per-class
- // timeout resolver (resolveTimeout) can pick the right
- // timeout without the runtime.Component interface needing
- // to expose Name(). The factory returns the class name as
- // the DSL's `component_name` field, which is also what
- // ComponentBase.Name() would have returned.
+ // Pass the class name through to the body for structured logging
+ // without the runtime.Component interface needing to expose Name().
+ // The factory returns the class name as the DSL's `component_name`
+ // field, which is also what ComponentBase.Name() would have returned.
return realComponentBodyWithOptions(cpnID, name, comp, opts), nil
}
// Fallback: no factory registered. This path is only exercised by
@@ -221,11 +221,11 @@ func componentTimeout() time.Duration {
// reach components here. Cross-cutting concerns therefore belong here,
// not inside each component's Invoke:
//
-// - per-class timeout: context.WithTimeout from resolveTimeout
-// (4-level: per-class env → per-class defaults table → uniform env
-// → 600s fallback). The lookup is per-invocation (not per-body) so
-// operators can tune COMPONENT_EXEC_TIMEOUT[_<CLASS>] at runtime
-// without rebuilding graphs.
+// - execution context: derived from the parent with cancellation only — no
+// framework-level wall-clock deadline is imposed on any component, so a
+// long knowledge-compilation step (many LLM calls) is not cut short by a
+// fixed ceiling. A cancelled task still interrupts via the parent cancel;
+// per-call model deadlines stay at the model driver.
// - progress: runtime.TrackProgress, with the callback pulled from
// ctx (nil ⇒ no observer). This makes progress a framework-level
// concern — components no longer wrap themselves.
@@ -248,8 +248,14 @@ func realComponentBody(cpnID, componentClass string, comp runtime.Component) nod
func realComponentBodyWithOptions(cpnID, componentClass string, comp runtime.Component, opts runtime.ComponentExecutionOptions) nodeBodyFn {
return func(ctx context.Context, in map[string]any) (map[string]any, error) {
- timeout := resolveTimeoutFromContext(ctx, componentClass)
- cctx, cancel := context.WithTimeout(ctx, timeout)
+ // The framework imposes no wall-clock deadline on component execution.
+ // Long-running components (notably knowledge compilation, which fans out
+ // over many model calls) exceed any fixed ceiling and would otherwise
+ // fail with "context deadline exceeded". We derive a cancellation-only
+ // context so a cancelled task (parent cancel) still interrupts the
+ // component mid-run, while no arbitrary timeout is applied. Per-call
+ // model deadlines remain enforced at the model driver.
+ cctx, cancel := context.WithCancel(ctx)
defer cancel()
var out map[string]any
@@ -261,10 +267,16 @@ func realComponentBodyWithOptions(cpnID, componentClass string, comp runtime.Com
return e
})
if invokeErr != nil {
+ // Surface the failure as a structured log line. The wrapped error
+ // already carries the full cause chain (e.g. deepseek DNS/timeout),
+ // but without this the failure only showed up as a generic
+ // "Task ... failed" line with no clear cause in the logs.
+ common.Error("canvas: component invoke failed", invokeErr,
+ zap.String("component_id", cpnID),
+ zap.String("component_class", componentClass))
switch {
case errors.Is(invokeErr, context.DeadlineExceeded):
- return nil, fmt.Errorf("canvas: component %q invoke: timeout after %s: %w",
- cpnID, timeout, invokeErr)
+ return nil, fmt.Errorf("canvas: component %q invoke: context deadline exceeded: %w", cpnID, invokeErr)
case errors.Is(invokeErr, context.Canceled):
return nil, fmt.Errorf("canvas: component %q invoke: cancelled: %w", cpnID, invokeErr)
}
diff --git a/internal/agent/canvas/node_body_per_class_timeout_integration_test.go b/internal/agent/canvas/node_body_per_class_timeout_integration_test.go
deleted file mode 100644
index 1479785096..0000000000
--- a/internal/agent/canvas/node_body_per_class_timeout_integration_test.go
+++ /dev/null
@@ -1,270 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-// node_body_per_class_timeout_integration_test.go — pins the
-// per-class timeout integration.
-//
-// Verifies that buildNodeBody's runtime.Component path actually
-// plumbs the per-class timeout (canvas/timeout.go's
-// componentDefaults table) end-to-end. The per-class table is
-// real; realComponentBody must call resolveTimeout(class) instead
-// of the uniform componentTimeout() — this test pins that
-// contract so a future refactor cannot silently regress to a
-// uniform timeout.
-
-package canvas
-
-import (
- "context"
- "errors"
- "sync/atomic"
- "testing"
- "time"
-
- "ragflow/internal/agent/runtime"
-
- "gorm.io/gorm"
-)
-
-// captureCtxComponent is a runtime.Component that records the
-// deadline of the context it was invoked with. The integration
-// test then asserts the deadline matches the per-class timeout
-// (ExeSQL=3s, TavilySearch=12s, LLM=600s, ...). The Component
-// returns immediately on success; on timeout it surfaces a
-// context.DeadlineExceeded error so the body can wrap and surface
-// it as a timeout error.
-type captureCtxComponent struct {
- name string
- deadline atomic.Int64 // unix-nano deadline observed
- timeoutOK atomic.Bool // true if ctx had a non-zero deadline
-}
-
-func (c *captureCtxComponent) Name() string { return c.name }
-func (c *captureCtxComponent) Invoke(ctx context.Context, _ *gorm.DB, _ map[string]any) (map[string]any, error) {
- if dl, ok := ctx.Deadline(); ok {
- c.deadline.Store(dl.UnixNano())
- c.timeoutOK.Store(true)
- } else {
- c.timeoutOK.Store(false)
- }
- // Block until ctx times out so the body can wrap and surface
- // the timeout error; that is the path real components hit
- // when they overrun the configured ceiling.
- <-ctx.Done()
- return nil, ctx.Err()
-}
-func (c *captureCtxComponent) Stream(_ context.Context, _ map[string]any) (<-chan map[string]any, error) {
- return nil, nil
-}
-func (c *captureCtxComponent) Inputs() map[string]string { return nil }
-func (c *captureCtxComponent) Outputs() map[string]string { return nil }
-
-var _ runtime.Component = (*captureCtxComponent)(nil)
-
-// TestBuildNodeBody_PerClassTimeout_ExeSQL_3s asserts that an
-// ExeSQL-class component body wraps the Invoke context with a
-// 3-second deadline (per the componentDefaults table in
-// timeout.go), not the uniform 600s fallback or the
-// COMPONENT_EXEC_TIMEOUT value (which we leave at the default).
-//
-// Steps:
-//
-// 1. Register a factory that returns a captureCtxComponent for
-// the class "ExeSQL".
-// 2. Call buildNodeBody("cpn", "ExeSQL", nil) — must NOT take
-// the legacyNoOp / placeholder / UserFillUp branch.
-// 3. Invoke the body with a fresh context, capture the
-// component-observed deadline via the captureCtxComponent.
-// 4. Assert the deadline is within ±500ms of 3s, AND the body
-// surfaces a wrapped context.DeadlineExceeded (the
-// realComponentBody timeout-wrap contract).
-func TestBuildNodeBody_PerClassTimeout_ExeSQL_3s(t *testing.T) {
- // Make sure uniform env is unset so the per-class table
- // is the only authority; the per-class table is the one
- // that defines "ExeSQL → 3s".
- t.Setenv("COMPONENT_EXEC_TIMEOUT", "")
- t.Setenv("COMPONENT_EXEC_TIMEOUT_EXESQL", "")
-
- cap := &captureCtxComponent{name: "ExeSQL"}
- origFactory := runtime.DefaultFactory()
- runtime.ResetDefaultFactoryForTesting()
- runtime.SetDefaultFactory(func(class string, _ map[string]any) (runtime.Component, error) {
- if class == "ExeSQL" {
- return cap, nil
- }
- return nil, errors.New("not used in this test: " + class)
- })
- t.Cleanup(func() {
- runtime.ResetDefaultFactoryForTesting()
- if origFactory != nil {
- runtime.SetDefaultFactory(origFactory)
- }
- })
-
- body, err := buildNodeBody(context.Background(), "cpn-exe", "ExeSQL", nil)
- if err != nil {
- t.Fatalf("buildNodeBody: %v", err)
- }
- start := time.Now()
- _, err = body(context.Background(), nil)
- if !errors.Is(err, context.DeadlineExceeded) {
- t.Fatalf("body err = %v, want context.DeadlineExceeded (per-class timeout should fire)", err)
- }
- if !cap.timeoutOK.Load() {
- t.Fatal("component never observed a deadline — realComponentBody did not wrap with context.WithTimeout")
- }
- // Semantic assertion: the deadline the body wired into the
- // component's ctx must be ~3s after the call started. This is
- // the load-bearing property — proving the per-class table
- // reached realComponentBody. The previous `elapsed > 5s` upper
- // bound conflated this with wall-clock scheduler jitter; if the
- // deadline is wrong (e.g. 600s fallback), sinceDeadline catches
- // it without depending on a fragile absolute threshold.
- deadline := time.Unix(0, cap.deadline.Load())
- sinceDeadline := deadline.Sub(start)
- if sinceDeadline < 2500*time.Millisecond || sinceDeadline > 3500*time.Millisecond {
- t.Errorf("ExeSQL deadline offset = %s, want ~3s", sinceDeadline)
- }
-}
-
-// TestBuildNodeBody_PerClassTimeout_TavilySearch_12s asserts the
-// 12-second web-search class. A Tavily call that hangs on the
-// upstream API should give up after 12s and let the agent
-// branch / retry, not consume 600s of wall-clock budget.
-func TestBuildNodeBody_PerClassTimeout_TavilySearch_12s(t *testing.T) {
- t.Setenv("COMPONENT_EXEC_TIMEOUT", "")
- t.Setenv("COMPONENT_EXEC_TIMEOUT_TAVILYSEARCH", "")
-
- cap := &captureCtxComponent{name: "TavilySearch"}
- origFactory := runtime.DefaultFactory()
- runtime.ResetDefaultFactoryForTesting()
- runtime.SetDefaultFactory(func(class string, _ map[string]any) (runtime.Component, error) {
- if class == "TavilySearch" {
- return cap, nil
- }
- return nil, errors.New("not used in this test: " + class)
- })
- t.Cleanup(func() {
- runtime.ResetDefaultFactoryForTesting()
- if origFactory != nil {
- runtime.SetDefaultFactory(origFactory)
- }
- })
-
- body, err := buildNodeBody(context.Background(), "cpn-tav", "TavilySearch", nil)
- if err != nil {
- t.Fatalf("buildNodeBody: %v", err)
- }
- start := time.Now()
- _, err = body(context.Background(), nil)
- if !errors.Is(err, context.DeadlineExceeded) {
- t.Fatalf("body err = %v, want context.DeadlineExceeded", err)
- }
- deadline := time.Unix(0, cap.deadline.Load())
- sinceDeadline := deadline.Sub(start)
- // Semantic: deadline the body wired is ~12s after start.
- if sinceDeadline < 11*time.Second || sinceDeadline > 13*time.Second {
- t.Errorf("TavilySearch deadline offset = %s, want ~12s", sinceDeadline)
- }
-}
-
-// TestBuildNodeBody_PerClassTimeout_UnknownClass_UniformFallback
-// asserts that a class NOT in the componentDefaults table (e.g.
-// "CustomComponent") falls through to the uniform
-// COMPONENT_EXEC_TIMEOUT env var. This is the 4-level resolution
-// contract from timeout.go: per-class env → per-class table →
-// uniform env → 600s fallback.
-//
-// We test the uniform-env fallback path (level 3) by setting
-// COMPONENT_EXEC_TIMEOUT to a non-default value and registering
-// a class that is NOT in componentDefaults.
-func TestBuildNodeBody_PerClassTimeout_UnknownClass_UniformFallback(t *testing.T) {
- t.Setenv("COMPONENT_EXEC_TIMEOUT", "5")
- t.Setenv("COMPONENT_EXEC_TIMEOUT_CUSTOMCOMPONENT", "")
-
- cap := &captureCtxComponent{name: "CustomComponent"}
- origFactory := runtime.DefaultFactory()
- runtime.ResetDefaultFactoryForTesting()
- runtime.SetDefaultFactory(func(class string, _ map[string]any) (runtime.Component, error) {
- if class == "CustomComponent" {
- return cap, nil
- }
- return nil, errors.New("not used in this test: " + class)
- })
- t.Cleanup(func() {
- runtime.ResetDefaultFactoryForTesting()
- if origFactory != nil {
- runtime.SetDefaultFactory(origFactory)
- }
- })
-
- body, err := buildNodeBody(context.Background(), "cpn-cust", "CustomComponent", nil)
- if err != nil {
- t.Fatalf("buildNodeBody: %v", err)
- }
- start := time.Now()
- _, err = body(context.Background(), nil)
- if !errors.Is(err, context.DeadlineExceeded) {
- t.Fatalf("body err = %v, want context.DeadlineExceeded", err)
- }
- deadline := time.Unix(0, cap.deadline.Load())
- sinceDeadline := deadline.Sub(start)
- // 5s uniform env should win for an unknown class. Allow a
- // 1s slack on each side.
- if sinceDeadline < 4*time.Second || sinceDeadline > 6*time.Second {
- t.Errorf("CustomComponent deadline offset = %s, want ~5s (uniform env fallback)", sinceDeadline)
- }
-}
-
-// TestBuildNodeBody_PerClassTimeout_PerClassEnvOverride asserts
-// that COMPONENT_EXEC_TIMEOUT_<CLASS> wins over the table.
-// ExeSQL's table entry is 3s; setting the per-class env to 7
-// should make the body's deadline ~7s, not 3s.
-func TestBuildNodeBody_PerClassTimeout_PerClassEnvOverride(t *testing.T) {
- t.Setenv("COMPONENT_EXEC_TIMEOUT", "")
- t.Setenv("COMPONENT_EXEC_TIMEOUT_EXESQL", "7")
-
- cap := &captureCtxComponent{name: "ExeSQL"}
- origFactory := runtime.DefaultFactory()
- runtime.ResetDefaultFactoryForTesting()
- runtime.SetDefaultFactory(func(class string, _ map[string]any) (runtime.Component, error) {
- if class == "ExeSQL" {
- return cap, nil
- }
- return nil, errors.New("not used in this test: " + class)
- })
- t.Cleanup(func() {
- runtime.ResetDefaultFactoryForTesting()
- if origFactory != nil {
- runtime.SetDefaultFactory(origFactory)
- }
- })
-
- body, err := buildNodeBody(context.Background(), "cpn-exe-ovr", "ExeSQL", nil)
- if err != nil {
- t.Fatalf("buildNodeBody: %v", err)
- }
- start := time.Now()
- _, err = body(context.Background(), nil)
- if !errors.Is(err, context.DeadlineExceeded) {
- t.Fatalf("body err = %v, want context.DeadlineExceeded", err)
- }
- deadline := time.Unix(0, cap.deadline.Load())
- sinceDeadline := deadline.Sub(start)
- if sinceDeadline < 6500*time.Millisecond || sinceDeadline > 7500*time.Millisecond {
- t.Errorf("ExeSQL deadline offset = %s, want ~7s (per-class env override)", sinceDeadline)
- }
-}
diff --git a/internal/agent/canvas/node_body_timeout_test.go b/internal/agent/canvas/node_body_timeout_test.go
index 34a74db353..03926e8cc1 100644
--- a/internal/agent/canvas/node_body_timeout_test.go
+++ b/internal/agent/canvas/node_body_timeout_test.go
@@ -14,22 +14,27 @@
// limitations under the License.
//
+// node_body_timeout_test.go — pins the cancellation-only execution context
+// contract of realComponentBody. The framework imposes no wall-clock deadline
+// on component execution (see realComponentBodyWithOptions), so a component
+// only stops when its parent context is cancelled; there is no timeout wrap to
+// test any more. These tests assert cancellation propagation and the
+// pass-through of fast components.
+
package canvas
import (
"context"
"errors"
"testing"
- "time"
"ragflow/internal/agent/runtime"
"gorm.io/gorm"
)
-// blockingComponent is a runtime.Component whose Invoke blocks until ctx
-// is cancelled. Used to test the per-component timeout wrapper in
-// realComponentBody.
+// blockingComponent is a runtime.Component whose Invoke blocks until ctx is
+// cancelled.
type blockingComponent struct{}
func (b *blockingComponent) Name() string { return "blocking" }
@@ -46,49 +51,11 @@ func (b *blockingComponent) Stream(_ context.Context, _ map[string]any) (<-chan
func (b *blockingComponent) Inputs() map[string]string { return nil }
func (b *blockingComponent) Outputs() map[string]string { return nil }
-// TestRealComponentBody_RespectsTimeout verifies that a component whose
-// Invoke blocks longer than the configured timeout causes the body to
-// return a wrapped context.DeadlineExceeded error.
-//
-// The call itself runs under context.Background(); a separate watchdog
-// goroutine fails the test if the inner timeout never fires. That keeps
-// the assertion semantic (the returned error must wrap
-// context.DeadlineExceeded) without letting an outer test context
-// manufacture the same error type and create a false positive.
-func TestRealComponentBody_RespectsTimeout(t *testing.T) {
- t.Setenv("COMPONENT_EXEC_TIMEOUT", "1")
-
- comp := &blockingComponent{}
- body := realComponentBody("test-cpn", "TestBlocking", comp)
- if body == nil {
- t.Fatalf("realComponentBody returned nil")
- }
-
- done := make(chan error, 1)
- go func() {
- _, err := body(context.Background(), map[string]any{"x": 1})
- done <- err
- }()
-
- select {
- case err := <-done:
- if err == nil {
- t.Fatalf("expected error, got nil")
- }
- if !errors.Is(err, context.DeadlineExceeded) {
- t.Errorf("expected context.DeadlineExceeded wrapped error, got: %v", err)
- }
- case <-time.After(15 * time.Second):
- t.Fatal("body did not return within 15s — timeout wrap is broken or call is hanging")
- }
-}
-
-// TestRealComponentBody_RespectsParentCancellation verifies that when
-// the parent context is already cancelled, the body surfaces a wrapped
-// context.Canceled error rather than a timeout (or a generic wrap).
+// TestRealComponentBody_RespectsParentCancellation verifies that when the
+// parent context is already cancelled, the body surfaces a wrapped
+// context.Canceled error. With no framework deadline, parent cancellation is
+// the only mechanism that stops a long-running component.
func TestRealComponentBody_RespectsParentCancellation(t *testing.T) {
- t.Setenv("COMPONENT_EXEC_TIMEOUT", "60")
-
comp := &blockingComponent{}
body := realComponentBody("test-cpn", "TestBlocking", comp)
@@ -104,13 +71,9 @@ func TestRealComponentBody_RespectsParentCancellation(t *testing.T) {
}
}
-// TestRealComponentBody_NoTimeoutWhenFast verifies that a component
-// returning immediately does not incur any timeout-induced latency or
-// error wrapping.
+// TestRealComponentBody_NoTimeoutWhenFast verifies that a component returning
+// immediately does not incur any timeout-induced latency or error wrapping.
func TestRealComponentBody_NoTimeoutWhenFast(t *testing.T) {
- t.Setenv("COMPONENT_EXEC_TIMEOUT", "60")
-
- // Stub component that returns immediately.
comp := &echoComponent{}
body := realComponentBody("test-cpn", "TestEcho", comp)
@@ -126,38 +89,8 @@ func TestRealComponentBody_NoTimeoutWhenFast(t *testing.T) {
}
}
-// TestComponentTimeout_Default verifies the default is 600s when the env
-// var is unset.
-func TestComponentTimeout_Default(t *testing.T) {
- t.Setenv("COMPONENT_EXEC_TIMEOUT", "")
- if got := componentTimeout(); got != 600*time.Second {
- t.Errorf("default timeout: got %s, want 600s", got)
- }
-}
-
-// TestComponentTimeout_HonoursEnv verifies a valid env value is parsed.
-func TestComponentTimeout_HonoursEnv(t *testing.T) {
- t.Setenv("COMPONENT_EXEC_TIMEOUT", "42")
- if got := componentTimeout(); got != 42*time.Second {
- t.Errorf("env timeout: got %s, want 42s", got)
- }
-}
-
-// TestComponentTimeout_InvalidEnvFallsBack verifies that non-numeric or
-// non-positive env values fall back to the default — invalid input must
-// never widen the timeout silently.
-func TestComponentTimeout_InvalidEnvFallsBack(t *testing.T) {
- for _, v := range []string{"abc", "0", "-5"} {
- t.Setenv("COMPONENT_EXEC_TIMEOUT", v)
- if got := componentTimeout(); got != 600*time.Second {
- t.Errorf("invalid env %q: got %s, want default 600s", v, got)
- }
- }
-}
-
-// echoComponent is a minimal runtime.Component used by the no-timeout test.
-// It returns the input map unchanged plus a __cpn_id__ tag (the body will
-// overwrite the tag, but that's fine).
+// echoComponent is a minimal runtime.Component used by the no-timeout test. It
+// returns the input map unchanged plus a __cpn_id__ tag.
type echoComponent struct{}
func (e *echoComponent) Name() string { return "echo" }
diff --git a/internal/agent/canvas/timeout.go b/internal/agent/canvas/timeout.go
deleted file mode 100644
index f283b2a7f2..0000000000
--- a/internal/agent/canvas/timeout.go
+++ /dev/null
@@ -1,157 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-// timeout.go — per-component-class Invoke timeout resolution.
-//
-// Background. The Python port decorates every component's _invoke with
-//
-// @timeout(int(os.environ.get("COMPONENT_EXEC_TIMEOUT", 10 * 60)))
-//
-// which applies a uniform 600s ceiling across all components. The Go
-// port initially did the same. Operators asked for finer granularity
-// because external HTTP lookups (Tavily, DuckDuckGo, ArXiv, ...) and
-// fast in-process helpers (ExeSQL, Invoke) blow up the wall-clock budget
-// quickly when 10 minutes is the default. The resolution here is:
-//
-// 1. COMPONENT_EXEC_TIMEOUT_<CLASS>=<seconds> wins for that class.
-// 2. Per-class default from componentDefaults.
-// 3. COMPONENT_EXEC_TIMEOUT=<seconds> uniform override (kept for
-// back-compat — operators who only set the uniform env var see no
-// change).
-// 4. Hard fallback of 600s (Python base.py default).
-//
-// All env values are parsed as seconds-suffixed durations; invalid or
-// non-positive input falls through to the next step. Invalid input must
-// never silently widen the timeout.
-package canvas
-
-import (
- "context"
- "ragflow/internal/common"
- "strconv"
- "strings"
- "time"
-)
-
-type timeoutContextKey struct{}
-
-// WithComponentTimeoutOverride forces all component invokes under ctx to use
-// the provided timeout. This lets callers with stricter execution contracts
-// reuse canvas execution without mutating the global timeout policy.
-func WithComponentTimeoutOverride(ctx context.Context, d time.Duration) context.Context {
- if d <= 0 {
- return ctx
- }
- return context.WithValue(ctx, timeoutContextKey{}, d)
-}
-
-// componentDefaults lists the per-class timeout (in seconds) used when
-// no per-class env override is set. The class keys are the PascalCase
-// names returned by Component.Name() (e.g. "LLM", "Message",
-// "TavilySearch", "Retrieval"). Names not in this table fall through to
-// the uniform env var or the hard fallback.
-//
-// Values match the Python port's per-class expectations:
-//
-// - LLM / Message / Agent — 600s (long LLM-bound work, matches base.py)
-// - Retrieval — 60s (vector + BM25 + rerank pipeline)
-// - TavilySearch / DuckDuckGo / ArXiv / GoogleScholar / SearXNG / PubMed — 12s
-// (external HTTP — fail fast and let the agent retry/branch)
-// - ExeSQL / Invoke — 3s (in-process / single HTTP hop)
-//
-// Note: the Go port registers web search as "TavilySearch" (the
-// fixture_stubs constant) rather than Python's "Tavily". We list both
-// spellings so an operator can target the Go name OR a class name that
-// matches Python's.
-var componentDefaults = map[string]time.Duration{
- "LLM": 600 * time.Second,
- "Message": 600 * time.Second,
- "Agent": 600 * time.Second,
- "Retrieval": 60 * time.Second,
- "TavilySearch": 12 * time.Second,
- "Tavily": 12 * time.Second, // Python spelling alias
- "DuckDuckGo": 12 * time.Second,
- "ArXiv": 12 * time.Second,
- "GoogleScholar": 12 * time.Second,
- "SearXNG": 12 * time.Second,
- "PubMed": 12 * time.Second,
- "ExeSQL": 3 * time.Second,
- "Invoke": 3 * time.Second,
-}
-
-// defaultComponentTimeout is the final fallback when neither the
-// per-class env, the per-class default table, nor the uniform env var
-// yields a value. Matches Python base.py: 10 * 60 = 600s.
-const defaultComponentTimeout = 600 * time.Second
-
-// resolveTimeout returns the Invoke timeout for a component of the
-// given class (the PascalCase name from Component.Name()).
-//
-// Resolution order:
-//
-// 1. COMPONENT_EXEC_TIMEOUT_<UPPERCASE_CLASS> env var, parsed as
-// "<seconds>s". Invalid or non-positive values are ignored.
-// 2. Per-class default from componentDefaults.
-// 3. COMPONENT_EXEC_TIMEOUT env var (uniform override, seconds),
-// parsed as "<seconds>s".
-// 4. defaultComponentTimeout (600s).
-//
-// Passing an empty class name still honours steps 3 and 4 — useful for
-// callers that don't know the class (e.g. a generic dispatcher).
-func resolveTimeout(componentClass string) time.Duration {
- upper := strings.ToUpper(strings.TrimSpace(componentClass))
- if upper != "" {
- if d, ok := parseSecondsEnv("COMPONENT_EXEC_TIMEOUT_" + upper); ok {
- return d
- }
- }
- if upper != "" {
- // componentDefaults uses PascalCase keys; try the original
- // spelling (an uppercased lookup would never match).
- if d, ok := componentDefaults[componentClass]; ok {
- return d
- }
- }
- if d, ok := parseSecondsEnv("COMPONENT_EXEC_TIMEOUT"); ok {
- return d
- }
- return defaultComponentTimeout
-}
-
-func resolveTimeoutFromContext(ctx context.Context, componentClass string) time.Duration {
- if ctx != nil {
- if d, ok := ctx.Value(timeoutContextKey{}).(time.Duration); ok && d > 0 {
- return d
- }
- }
- return resolveTimeout(componentClass)
-}
-
-// parseSecondsEnv reads an env var and parses it as seconds ("42" →
-// 42s). Returns (d, true) on success, (0, false) if the env var is
-// unset / empty / non-numeric / non-positive. Invalid input must
-// never widen the timeout silently.
-func parseSecondsEnv(name string) (time.Duration, bool) {
- v := common.GetEnv(name)
- if v == "" {
- return 0, false
- }
- secs, err := strconv.Atoi(strings.TrimSpace(v))
- if err != nil || secs <= 0 {
- return 0, false
- }
- return time.Duration(secs) * time.Second, true
-}
diff --git a/internal/common/logger.go b/internal/common/logger.go
index 925ddfe93a..74b322919d 100644
--- a/internal/common/logger.go
+++ b/internal/common/logger.go
@@ -157,9 +157,15 @@ func InitLogger(level string, file FileOutput, serviceName string) error {
)
if serviceName != "" {
- Logger = zap.New(core, zap.AddCallerSkip(1)).Named(serviceName)
+ Logger = zap.New(core,
+ zap.Fields(zap.Int("pid", os.Getpid())),
+ zap.AddCallerSkip(1),
+ ).Named(serviceName)
} else {
- Logger = zap.New(core, zap.AddCallerSkip(1))
+ Logger = zap.New(core,
+ zap.Fields(zap.Int("pid", os.Getpid())),
+ zap.AddCallerSkip(1),
+ )
}
Sugar = Logger.Sugar()
diff --git a/internal/common/retry.go b/internal/common/retry.go
index 757cfe37fe..a4de4be11b 100644
--- a/internal/common/retry.go
+++ b/internal/common/retry.go
@@ -17,7 +17,9 @@ package common
import (
"context"
+ "errors"
"fmt"
+ "strings"
"time"
)
@@ -74,3 +76,47 @@ func RetryWithBackoff(ctx context.Context, maxRetries int, initialDelay time.Dur
}
return fmt.Errorf("failed after %d retries: %w", maxRetries, lastErr)
}
+
+// IsTransientError reports whether err is a transient provider failure worth
+// retrying, as opposed to a permanent configuration/client error. It is the
+// shared shouldRetry predicate for provider HTTP calls.
+//
+// Classification:
+// - context.Canceled: permanent (caller cancelled; do not retry)
+// - context.DeadlineExceeded: transient (a slow provider can succeed on retry)
+// - an embedded HTTP status parsed from the error message: 5xx and 429 are
+// transient (server / rate limit); other 4xx are permanent (auth, unknown
+// model, invalid request)
+// - anything else (network reset, connection refused, timeout): transient
+//
+// Provider drivers surface HTTP statuses as "API request failed with status
+// <code>: <body>" (see BaseModel.doRequest), so parsing that prefix is enough.
+func IsTransientError(err error) bool {
+ if err == nil {
+ return false
+ }
+ if errors.Is(err, context.Canceled) {
+ return false
+ }
+ if errors.Is(err, context.DeadlineExceeded) {
+ return true
+ }
+ msg := strings.ToLower(err.Error())
+ if idx := strings.Index(msg, "status "); idx >= 0 {
+ code := 0
+ for _, r := range msg[idx+len("status "):] {
+ if r < '0' || r > '9' {
+ break
+ }
+ code = code*10 + int(r-'0')
+ }
+ if code >= 500 || code == 429 {
+ return true
+ }
+ if code >= 400 && code < 500 {
+ return false
+ }
+ }
+ // No recognizable permanent marker: treat as transient (network, timeout).
+ return true
+}
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.go
index 119da87611..c4dbe77b12 100644
--- a/internal/engine/elasticsearch/chunk.go
+++ b/internal/engine/elasticsearch/chunk.go
@@ -818,6 +818,16 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfac
"terms": map[string]interface{}{"id": ids},
})
}
+ case []string:
+ // A typed []string must be handled here; the generic loop below
+ // skips the "id" key, so without this branch a caller passing
+ // map[string]interface{}{"id": []string{...}} would build a query
+ // with no id filter and DeleteChunks could match every document.
+ if len(v) > 0 {
+ mustClauses = append(mustClauses, map[string]interface{}{
+ "terms": map[string]interface{}{"id": v},
+ })
+ }
case string:
mustClauses = append(mustClauses, map[string]interface{}{
"term": map[string]interface{}{"id": v},
@@ -856,6 +866,10 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfac
mustClauses = append(mustClauses, map[string]interface{}{
"terms": map[string]interface{}{k: listVal},
})
+ } else if listVal, ok := v.([]string); ok {
+ mustClauses = append(mustClauses, map[string]interface{}{
+ "terms": map[string]interface{}{k: listVal},
+ })
} else if _, ok := v.(string); ok {
mustClauses = append(mustClauses, map[string]interface{}{
"term": map[string]interface{}{k: v},
@@ -1275,8 +1289,6 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S
allResults = sortByScore(allResults, limit)
}
- common.Info("ES Search completed", zap.Int("returnedRows", len(allResults)), zap.Int64("totalHits", totalHits))
-
return &types.SearchResult{
Chunks: allResults,
Total: totalHits,
diff --git a/internal/engine/elasticsearch/chunk_helpers_test.go b/internal/engine/elasticsearch/chunk_helpers_test.go
index 261f4aaaf9..4959aecf23 100644
--- a/internal/engine/elasticsearch/chunk_helpers_test.go
+++ b/internal/engine/elasticsearch/chunk_helpers_test.go
@@ -2,6 +2,7 @@ package elasticsearch
import (
"context"
+ "encoding/json"
"net/http"
"net/http/httptest"
"reflect"
@@ -71,6 +72,122 @@ func TestUpdateSingleMemoryMessageWaitsForRefresh(t *testing.T) {
}
}
+func TestDeleteChunksPreservesStringSliceCondition(t *testing.T) {
+ var deleteQuery map[string]interface{}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-Elastic-Product", "Elasticsearch")
+ switch r.Method {
+ case http.MethodHead:
+ w.WriteHeader(http.StatusOK)
+ case http.MethodPost:
+ if r.URL.Path != "/ragflow_tenant/_delete_by_query" {
+ t.Errorf("path=%s, want /ragflow_tenant/_delete_by_query", r.URL.Path)
+ http.Error(w, "unexpected request path", http.StatusNotFound)
+ return
+ }
+ if err := json.NewDecoder(r.Body).Decode(&deleteQuery); err != nil {
+ t.Errorf("decode delete query: %v", err)
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"deleted":3}`))
+ default:
+ t.Errorf("method=%s", r.Method)
+ http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
+ }
+ }))
+ defer server.Close()
+
+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
+ if err != nil {
+ t.Fatalf("new elasticsearch client: %v", err)
+ }
+ engine := &Engine{client: client}
+ deleted, err := engine.DeleteChunks(context.Background(), map[string]interface{}{
+ "kb_id": "kb-1",
+ "compile_kwd": []string{"wiki_entity", "wiki_relation"},
+ }, "ragflow_tenant", "kb-1")
+ if err != nil {
+ t.Fatalf("DeleteChunks: %v", err)
+ }
+ if deleted != 3 {
+ t.Fatalf("deleted=%d, want 3", deleted)
+ }
+
+ query, ok := deleteQuery["query"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("delete query missing query: %#v", deleteQuery)
+ }
+ boolQuery, ok := query["bool"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("delete query missing bool: %#v", query)
+ }
+ must, ok := boolQuery["must"].([]interface{})
+ if !ok || len(must) != 1 {
+ t.Fatalf("delete query must=%#v, want one terms clause", boolQuery["must"])
+ }
+ terms := must[0].(map[string]interface{})["terms"].(map[string]interface{})
+ assertEqual(t, terms["compile_kwd"], []interface{}{"wiki_entity", "wiki_relation"})
+}
+
+// TestDeleteChunksIDStringSlice guards against the id filter being dropped when
+// the caller passes a typed []string. Without a dedicated []string branch in
+// the id handling, the generic loop skips the "id" key and DeleteChunks would
+// build a match_all query and delete every document in the index.
+func TestDeleteChunksIDStringSlice(t *testing.T) {
+ var deleteQuery map[string]interface{}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-Elastic-Product", "Elasticsearch")
+ switch r.Method {
+ case http.MethodHead:
+ w.WriteHeader(http.StatusOK)
+ case http.MethodPost:
+ if r.URL.Path != "/ragflow_tenant/_delete_by_query" {
+ t.Errorf("path=%s, want /ragflow_tenant/_delete_by_query", r.URL.Path)
+ http.Error(w, "unexpected request path", http.StatusNotFound)
+ return
+ }
+ if err := json.NewDecoder(r.Body).Decode(&deleteQuery); err != nil {
+ t.Errorf("decode delete query: %v", err)
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"deleted":2}`))
+ default:
+ t.Errorf("method=%s", r.Method)
+ http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
+ }
+ }))
+ defer server.Close()
+
+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
+ if err != nil {
+ t.Fatalf("new elasticsearch client: %v", err)
+ }
+ engine := &Engine{client: client}
+ if _, err = engine.DeleteChunks(context.Background(), map[string]interface{}{
+ "id": []string{"doc-a", "doc-b"},
+ }, "ragflow_tenant", "kb-1"); err != nil {
+ t.Fatalf("DeleteChunks: %v", err)
+ }
+
+ query, ok := deleteQuery["query"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("delete query missing query: %#v", deleteQuery)
+ }
+ boolQuery, ok := query["bool"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("delete query missing bool: %#v", query)
+ }
+ must, ok := boolQuery["must"].([]interface{})
+ if !ok || len(must) != 1 {
+ t.Fatalf("delete query must=%#v, want one id terms clause (match_all would be a bug)", boolQuery["must"])
+ }
+ terms, ok := must[0].(map[string]interface{})["terms"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("must[0] missing terms clause: %#v", must[0])
+ }
+ assertEqual(t, terms["id"], []interface{}{"doc-a", "doc-b"})
+}
+
func TestElasticsearchGetFieldsFiltersAndUsesIDFallback(t *testing.T) {
engine := &Engine{}
chunks := []map[string]interface{}{
diff --git a/internal/engine/types/types.go b/internal/engine/types/types.go
index e96cee1b1d..ec6c53d493 100644
--- a/internal/engine/types/types.go
+++ b/internal/engine/types/types.go
@@ -20,8 +20,6 @@ import (
"errors"
"fmt"
- "go.uber.org/zap"
-
"ragflow/internal/common"
)
@@ -137,8 +135,6 @@ type FusionExpr struct {
// LogSearchRequest logs SearchRequest in debug mode
func LogSearchRequest(engineName string, req *SearchRequest) {
- common.Info(fmt.Sprintf("Search in %s started", engineName), zap.Any("indexNames", req.IndexNames))
-
if !common.IsDebugEnabled() {
return
}
diff --git a/internal/entity/models/response_handler.go b/internal/entity/models/response_handler.go
index ee6023645b..ea537ccb1e 100644
--- a/internal/entity/models/response_handler.go
+++ b/internal/entity/models/response_handler.go
@@ -56,7 +56,7 @@ func HandleNonStreamingResponse(body []byte, modelUsage *common.ModelUsage, chat
if chatConfig != nil {
chatConfig.UsageResult = usage
model, _ := result["model"].(string)
- common.Info("StreamUsage", zap.String("model", model), zap.Int("prompt", usage.PromptTokens), zap.Int("completion", usage.CompletionTokens), zap.Int("total", usage.TotalTokens))
+ common.Info("NonStreamUsage", zap.String("model", model), zap.Int("prompt", usage.PromptTokens), zap.Int("completion", usage.CompletionTokens), zap.Int("total", usage.TotalTokens))
}
}
diff --git a/internal/entity/models/xai.go b/internal/entity/models/xai.go
index 87d9379d74..3d26b8a1f0 100644
--- a/internal/entity/models/xai.go
+++ b/internal/entity/models/xai.go
@@ -33,7 +33,7 @@ import (
)
var (
- nonStreamCallTimeout = 120 * time.Second
+ nonStreamCallTimeout = 300 * time.Second
streamCallTimeout = 10 * time.Minute
longOpCallTimeout = 10 * time.Minute
)
diff --git a/internal/ingestion/component/knowledge_compiler/common/jsonchat.go b/internal/ingestion/component/knowledge_compiler/common/jsonchat.go
index d5937d0a41..4509ef4953 100644
--- a/internal/ingestion/component/knowledge_compiler/common/jsonchat.go
+++ b/internal/ingestion/component/knowledge_compiler/common/jsonchat.go
@@ -4,11 +4,26 @@ import (
"context"
"encoding/json"
"fmt"
- "log"
+ "math/rand"
"regexp"
"strings"
+ "time"
+
+ appcommon "ragflow/internal/common"
+
+ "go.uber.org/zap"
)
+// jsonRetryMax is how many times a non-JSON (or otherwise transiently failed)
+// LLM reply is retried before GenJSON gives up. The highest-frequency LLM
+// integration must not drop a knowledge unit on a single formatting hiccup, so
+// a one-off malformed reply triggers a fresh LLM call instead of an immediate
+// failure.
+const jsonRetryMax = 5
+
+// jsonRetryDelay is the initial exponential-backoff delay between retries.
+const jsonRetryDelay = 2 * time.Second
+
// fencedJSONRE matches a ```json ... ``` or ``` ... ``` fenced block. Models
// frequently wrap JSON in such fences even when JSONMode is requested, which
// previously slipped through as an unparseable "_raw" payload and was silently
@@ -24,25 +39,68 @@ var fencedJSONRE = regexp.MustCompile("(?s)```(?:json)?\\s*(.*?)\\s*```")
// or retries rather than silently dropping the extraction — the
// highest-frequency LLM integration must not lose knowledge units on a
// formatting hiccup.
-func GenJSON(ctx context.Context, chat ChatInvoker, req ChatRequest) (map[string]any, error) {
- req.JSONMode = true
- resp, err := chat.Chat(ctx, req)
- if err != nil {
- return nil, err
+// GenJSON asks the model for a JSON reply and parses it. retryMax is an
+// optional override for jsonRetryMax: pass 0 to disable retries entirely
+// (used where the caller already budgets external calls itself, e.g. the
+// entity-merge disambiguator — retrying there would blow through the budget).
+func GenJSON(ctx context.Context, chat ChatInvoker, req ChatRequest, retryMax ...int) (map[string]any, error) {
+ maxRetries := jsonRetryMax
+ if len(retryMax) > 0 {
+ maxRetries = retryMax[0]
}
- for _, candidate := range jsonCandidates(resp.Content) {
- if m, ok := tryUnmarshalJSON(candidate); ok {
- return m, nil
+ req.JSONMode = true
+ var lastErr error
+ delay := jsonRetryDelay
+ for attempt := 0; attempt <= maxRetries; attempt++ {
+ resp, err := chat.Chat(ctx, req)
+ if err != nil {
+ // Permanent chat errors (auth, unknown model, context-length,
+ // cancelled ctx) cannot succeed on a retry; escape immediately.
+ if !appcommon.IsTransientError(err) {
+ return nil, err
+ }
+ // Transient chat failure (timeout / transport / provider); retry
+ // with a fresh LLM call.
+ lastErr = err
+ } else {
+ candidates := jsonCandidates(resp.Content)
+ for _, candidate := range candidates {
+ if m, ok := tryUnmarshalJSON(candidate); ok {
+ return m, nil
+ }
+ }
+ // A non-JSON reply must NOT be persisted/reused; discard it and
+ // immediately re-issue the call so a formatting hiccup does not
+ // abort the whole compile. Log candidate length and the unmarshal
+ // error only — the raw body may carry customer-derived content
+ // (PII), so it is deliberately excluded from the log.
+ for i, candidate := range candidates {
+ _, perr := tryUnmarshalJSONErr(candidate)
+ appcommon.Info("knowledge_compiler: GenJSON unparseable candidate",
+ zap.Int("attempt", attempt), zap.Int("candidate", i),
+ zap.Int("len", len(candidate)), zap.Error(perr))
+ }
+ lastErr = fmt.Errorf("knowledge_compiler: LLM response is not parseable JSON (%d bytes)", len(resp.Content))
+ }
+ if attempt == maxRetries {
+ break
+ }
+ appcommon.Info("knowledge_compiler: GenJSON attempt failed, retrying",
+ zap.Int("attempt", attempt), zap.Duration("delay", delay),
+ zap.Error(lastErr))
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ case <-time.After(delay + time.Duration(rand.Int63n(int64(delay/2)+1))):
+ // Jittered backoff so concurrent GenJSON jobs (parallel wiki/plan
+ // batches) do not back off in lockstep and pile up on the provider.
+ }
+ delay *= 2
+ if delay > time.Minute {
+ delay = time.Minute
}
}
- // Diagnostic: surface how far parsing got so a formatting failure is not
- // opaque. Each candidate is reported with its own unmarshal error so we can
- // tell an unfenced/truncated payload from a genuine syntax error.
- for i, candidate := range jsonCandidates(resp.Content) {
- _, err := tryUnmarshalJSONErr(candidate)
- log.Printf("knowledge_compiler: GenJSON candidate[%d] len=%d parse_err=%v body=%q", i, len(candidate), err, truncate(candidate, 300))
- }
- return nil, fmt.Errorf("knowledge_compiler: LLM response is not parseable JSON: %q", truncate(resp.Content, 200))
+ return nil, lastErr
}
// jsonCandidates yields progressively "cleaned" versions of an LLM reply that
diff --git a/internal/ingestion/component/knowledge_compiler/common/types.go b/internal/ingestion/component/knowledge_compiler/common/types.go
index 7c0635566e..04d68a4849 100644
--- a/internal/ingestion/component/knowledge_compiler/common/types.go
+++ b/internal/ingestion/component/knowledge_compiler/common/types.go
@@ -122,8 +122,8 @@ type Product struct {
ParentID string
Meta map[string]any
// Merged marks rows that already went through dataset-level dedup
- // (kc_merged=1, doc_id=kb). The consumer distinguishes these from the
- // per-document compiled rows (doc_id=<source doc>, no kc_merged) so it can
+ // (available_int=1, doc_id=kb). The consumer distinguishes these from the
+ // per-document compiled rows (doc_id=<source doc>, available_int=0) so it can
// KNN against only the merged set instead of re-deduping the whole KB.
Merged bool
}
diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.go
index e1e1f25ac9..fa44d109cb 100644
--- a/internal/ingestion/component/knowledge_compiler/component.go
+++ b/internal/ingestion/component/knowledge_compiler/component.go
@@ -13,6 +13,7 @@ import (
"strings"
"ragflow/internal/agent/runtime"
+ clog "ragflow/internal/common"
"ragflow/internal/ingestion/component/globals"
"ragflow/internal/ingestion/component/knowledge_compiler/common"
"ragflow/internal/ingestion/component/knowledge_compiler/mindmap"
@@ -23,6 +24,7 @@ import (
"ragflow/internal/service/nav"
"ragflow/internal/tokenizer"
+ "go.uber.org/zap"
"gorm.io/gorm"
)
@@ -94,7 +96,15 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in
// being lost when the upstream output map narrows it, which would otherwise
// leave the template-group lookup with an empty tenant and fail loudly.
tenantID := globals.GlobalOrInput(ctx, inputs, "tenant_id", "")
+ // The ingestion pipeline seeds kb_id (not dataset_id) into the canvas
+ // globals/inputs. In RAGFlow the knowledge base id IS the dataset id, so
+ // fall back to kb_id when dataset_id is absent. This keeps the compiler's
+ // dataset-scoped writes (merged wiki_page rows, wiki graph) keyed to the
+ // correct dataset instead of an empty string.
datasetID := globals.GlobalOrInput(ctx, inputs, "dataset_id", "")
+ if datasetID == "" {
+ datasetID = globals.GlobalOrInput(ctx, inputs, "kb_id", "")
+ }
// Resolve the compilation template spec(s). Priority:
// compilation_template_id > compilation_template_group_id. The variant is
@@ -105,6 +115,17 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in
return nil, err
}
+ // The eino wiring does not auto-merge run-level metadata into every
+ // component's input map, so doc_id may arrive only via CanvasState.Globals
+ // (seeded by the pipeline at run start) rather than inputs["doc_id"].
+ // Mirror the dataset_id/kb_id resolution above: fall back to globals so the
+ // compiler (and its wiki sub-logs) see the real doc_id instead of "unknown".
+ if d, ok := inputs["doc_id"].(string); !ok || d == "" {
+ if g := globals.GlobalOrInput(ctx, inputs, "doc_id", ""); g != "" {
+ inputs["doc_id"] = g
+ }
+ }
+
in, err := buildInputs(inputs, param)
if err != nil {
return nil, err
@@ -202,6 +223,32 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in
if err != nil {
return nil, err
}
+ // Per-doc telemetry: confirm how many compiled rows (and specifically
+ // wiki_page wiki sections vs pages) this single document produced, so a
+ // missing dataset-level wiki_page can be traced to "never generated" vs
+ // "generated then dropped".
+ var pageCount, sectionCount, entityCount, relationCount int
+ for _, p := range out.Products {
+ switch {
+ case p.Variant == common.VariantWiki && metaString(p.Meta, "kind") == "page":
+ pageCount++
+ case p.Variant == common.VariantWiki && metaString(p.Meta, "kind") == "section":
+ sectionCount++
+ case p.Variant == common.VariantWiki && metaString(p.Meta, "kind") == "entity":
+ entityCount++
+ case p.Variant == common.VariantWiki && metaString(p.Meta, "kind") == "relation":
+ relationCount++
+ }
+ }
+ clog.Info("knowledge_compiler: per-doc products generated",
+ zap.String("doc_id", in.DocID),
+ zap.Int("total_products", len(out.Products)),
+ zap.Int("wiki_page", pageCount),
+ zap.Int("wiki_section", sectionCount),
+ zap.Int("wiki_entity", entityCount),
+ zap.Int("wiki_relation", relationCount),
+ zap.Int("chunk_docs", len(compiled)),
+ )
return mergeChunks(inputs, compiled), nil
}
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/page.go b/internal/ingestion/component/knowledge_compiler/wiki/page.go
index e8cd10ea5a..d9e9df45bd 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/page.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/page.go
@@ -7,7 +7,10 @@ import (
"strings"
"unicode"
+ appcommon "ragflow/internal/common"
"ragflow/internal/ingestion/component/knowledge_compiler/common"
+
+ "go.uber.org/zap"
)
type wikiPageResult struct {
@@ -171,8 +174,15 @@ func buildPageProducts(tenantID, docID, page string, sourceChunkIDs []string) []
// per heading in its markdown body.
func buildWikiPageProducts(tenantID, docID string, pages []wikiPageResult) []common.Product {
if len(pages) == 0 {
+ appcommon.Info("wiki: buildWikiPageProducts skipped (no refined pages)",
+ zap.String("tenant_id", tenantID),
+ zap.String("doc_id", docID))
return nil
}
+ appcommon.Info("wiki: buildWikiPageProducts start",
+ zap.String("tenant_id", tenantID),
+ zap.String("doc_id", docID),
+ zap.Int("pages", len(pages)))
out := make([]common.Product, 0, len(pages)*2)
for _, page := range pages {
slug := strings.TrimSpace(page.Slug)
@@ -180,6 +190,10 @@ func buildWikiPageProducts(tenantID, docID string, pages []wikiPageResult) []com
slug = slugify(page.Title)
}
if slug == "" {
+ appcommon.Warn("wiki: page dropped (empty slug)",
+ zap.String("tenant_id", tenantID),
+ zap.String("doc_id", docID),
+ zap.String("title", page.Title))
continue
}
title := strings.TrimSpace(page.Title)
@@ -310,3 +324,24 @@ func slugify(s string) string {
}
return b.String()
}
+
+// normalizeWikiSlugHyphens canonicalizes a wiki slug to the hyphen style used
+// everywhere downstream (slug_kwd, outlinks, artifact links): the last path
+// segment's underscores become hyphens, while any "<page_type>/" prefix is
+// preserved. The LLM freely emits both "dong_zhuo" and "dong-zhuo", so this is
+// the single choke point that makes them equivalent without ad-hoc conversions
+// scattered through transformWikiLinks / the graph writer.
+func normalizeWikiSlugHyphens(slug string) string {
+ s := strings.TrimSpace(slug)
+ if s == "" {
+ return ""
+ }
+ prefix := ""
+ body := s
+ if idx := strings.LastIndex(s, "/"); idx >= 0 {
+ prefix = s[:idx+1]
+ body = s[idx+1:]
+ }
+ body = strings.ReplaceAll(body, "_", "-")
+ return prefix + body
+}
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/prompt.go b/internal/ingestion/component/knowledge_compiler/wiki/prompt.go
index 43f9cac2c9..033f745824 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/prompt.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/prompt.go
@@ -155,8 +155,19 @@ Return a JSON compilation plan with one or more page entries:
Rules:
- Return at most {max_pages} page entries for this batch.
-- Prefer one page per high-signal entity or concept when the batch supports it.
-- Merge minor or weakly-supported facts into broader topic pages instead of emitting tiny standalone pages.
+- Entity/concept identity is one-to-one with pages: every extracted entity and
+ concept must be represented by exactly one canonical page and may appear in
+ only that page's entity_names. Never split one identity across multiple
+ pages, page types, thematic sections, aliases, language transliterations, or
+ alternate slug spellings. Put all supported sections for that identity on
+ its single canonical page.
+- A page may represent several closely related low-signal entities/concepts,
+ but list every represented identity in entity_names and do not repeat any of
+ them on another page. Merge minor or weakly-supported facts into such a page
+ instead of emitting tiny standalone pages.
+- Identity ownership does not limit linking: related_kb_pages should include
+ every directly related canonical page supported by the input (up to the
+ available-page budget), and links must target canonical slugs only.
- Use page_type=entity for entity pages, page_type=concept for concept pages, and page_type=topic for cross-cutting themes.
- entity_names must name the entities and concepts that justify the page.
- related_kb_pages should list other slugs from the same plan that the page should cross-link to.
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
index 1837d390e9..e5b03fc154 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
@@ -17,8 +17,11 @@ import (
"sort"
"strings"
+ appcommon "ragflow/internal/common"
"ragflow/internal/ingestion/component/knowledge_compiler/common"
"ragflow/internal/ingestion/component/knowledge_compiler/structure"
+
+ "go.uber.org/zap"
)
// batchSubmitter fans out the MAP-stage extraction jobs on the process-wide
@@ -70,6 +73,16 @@ func wikiMapMaxTokens(modelContextLen int) int {
return max(modelContextLen-wikiMapTokenBudget, wikiMapTokenBudget)
}
+// wikiRefineMaxTokens gives the page writer (REFINE step) a generous but
+// bounded output cap so a long page body is not cut off mid-stream by a small
+// default completion limit. It reuses the map/extraction budget derivation:
+// once the input budget is consumed, the rest of the model window is handed to
+// the output. modelContextLen is the model's total context window in tokens (0
+// means unknown).
+func wikiRefineMaxTokens(modelContextLen int) int {
+ return wikiMapMaxTokens(modelContextLen)
+}
+
type wikiPipeline struct {
ctx context.Context
deps common.Deps
@@ -196,6 +209,12 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
if docID == "" {
docID = "unknown"
}
+ appcommon.Info("wiki: Run start",
+ zap.String("dataset_id", deps.DatasetID),
+ zap.String("doc_id", docID),
+ zap.Int("chunks", len(inputs.Chunks)),
+ zap.Bool("chat_ready", deps.Chat != nil),
+ zap.Bool("embed_ready", deps.Embed != nil))
p := &wikiPipeline{
ctx: ctx,
deps: deps,
@@ -207,10 +226,23 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
llmID: llmID,
}
if err := p.run(); err != nil {
+ appcommon.Error("wiki: Run pipeline failed", err,
+ zap.String("dataset_id", deps.DatasetID),
+ zap.String("doc_id", docID))
return common.Outputs{}, err
}
+ appcommon.Info("wiki: pipeline done",
+ zap.String("dataset_id", deps.DatasetID),
+ zap.String("doc_id", docID),
+ zap.Int("extracted_entities", len(p.reduced.Entities)),
+ zap.Int("plan_pages", len(p.plan.Pages)),
+ zap.Int("refined_pages", len(p.pages)))
products := buildWikiPageProducts(p.tenantID, p.docID, p.pages)
+ appcommon.Info("wiki: built page products",
+ zap.String("dataset_id", deps.DatasetID),
+ zap.String("doc_id", docID),
+ zap.Int("products", len(products)))
for i := range products {
if products[i].Meta == nil {
products[i].Meta = map[string]any{}
@@ -271,6 +303,21 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
return out, nil
}
+// runKey returns a stable identity for this wiki run's log lines. In a
+// per-document run docID is populated; in a dataset-level run (the
+// knowledge_compile aggregator) docID is empty, so fall back to a dataset
+// scope prefix. This avoids masking which dataset a MAP/REDUCE pass ran for
+// behind an uninformative "unknown".
+func (p *wikiPipeline) runKey() string {
+ if p.docID != "" {
+ return p.docID
+ }
+ if p.datasetID != "" {
+ return "dataset:" + p.datasetID
+ }
+ return "unknown"
+}
+
func (p *wikiPipeline) run() error {
if len(p.inputs.Chunks) == 0 {
return nil
@@ -278,24 +325,52 @@ func (p *wikiPipeline) run() error {
if err := p.runMap(); err != nil {
return err
}
+ appcommon.Info("wiki: MAP done",
+ zap.String("dataset_id", p.datasetID),
+ zap.String("doc_id", p.runKey()),
+ zap.Int("batches", len(p.mapExtracts)),
+ zap.Int("raw_entities", countExtractEntities(p.mapExtracts)))
p.reduced = reduceExtracts(p.mapExtracts)
// Layer embedding + LLM disambiguation onto the exact-merged entities
// (REDUCE enhancement; concepts keep exact dedup). Degrades to a no-op when
// the embedder/chat seams are unavailable.
p.reduced.Entities = p.dedupeEntities(p.reduced.Entities)
+ appcommon.Info("wiki: REDUCE done",
+ zap.String("dataset_id", p.datasetID),
+ zap.String("doc_id", p.runKey()),
+ zap.Int("entities", len(p.reduced.Entities)),
+ zap.Int("claims", len(p.reduced.Claims)))
plan, err := p.runPlan()
if err != nil {
return err
}
p.plan = plan
+ appcommon.Info("wiki: PLAN done",
+ zap.String("dataset_id", p.datasetID),
+ zap.String("doc_id", p.runKey()),
+ zap.Int("plan_pages", len(plan.Pages)),
+ zap.Int("capacity_excluded", p.planCapacityExcluded))
pages, err := p.runRefine()
if err != nil {
return err
}
p.pages = pages
+ appcommon.Info("wiki: REFINE done",
+ zap.String("dataset_id", p.datasetID),
+ zap.String("doc_id", p.runKey()),
+ zap.Int("refined_pages", len(pages)))
return nil
}
+// countExtractEntities sums the entity count across a list of map extracts.
+func countExtractEntities(extracts []wikiExtract) int {
+ n := 0
+ for _, e := range extracts {
+ n += len(e.Entities)
+ }
+ return n
+}
+
func (p *wikiPipeline) runMap() error {
// Keep each batch small enough that the LLM's entity/relation JSON output
// for the batch stays well under the model's output-token limit. 2048 input
@@ -406,7 +481,7 @@ func (p *wikiPipeline) runPlan() (wikiPlan, error) {
if err := p.ctx.Err(); err != nil {
return err
}
- plan, err := p.runPlanBatch(batch, i+1, len(batches), quota)
+ plan, err := p.runPlanBatch(batch, i+1, len(batches), quota, p.planBudget.MaxTokens)
if err != nil {
return err
}
@@ -540,10 +615,15 @@ func (p *wikiPipeline) runRefinePage(
"evidence_count": fmt.Sprintf("%d", len(evidence)),
"evidence_blocks": formatWikiEvidenceBlocks(evidence),
})
+ // wikiRefineMaxTokens gives the page writer a generous but bounded output
+ // cap so a long page is not cut off mid-stream by a small default completion
+ // limit (which would yield an unusable/cut page body).
+ rmt := wikiRefineMaxTokens(p.deps.ModelContextLen)
resp, err := p.deps.Chat.Chat(p.ctx, common.ChatRequest{
LLMID: p.llmID,
SystemPrompt: buildWikiRefineWriterSystem(""),
UserPrompt: user,
+ MaxTokens: &rmt,
})
if err != nil {
return wikiPageResult{}, err
@@ -598,7 +678,7 @@ func maxPagesForBatch(quota int) int {
return quota
}
-func (p *wikiPipeline) runPlanBatch(batch wikiExtract, batchIndex, batchTotal, quota int) (wikiPlan, error) {
+func (p *wikiPipeline) runPlanBatch(batch wikiExtract, batchIndex, batchTotal, quota, maxTokens int) (wikiPlan, error) {
user := renderWikiTemplate(wikiPlanBatchUserTemplate, map[string]string{
"doc_id": p.docID,
"batch_index": fmt.Sprintf("%d", batchIndex),
@@ -614,6 +694,7 @@ func (p *wikiPipeline) runPlanBatch(batch wikiExtract, batchIndex, batchTotal, q
LLMID: p.llmID,
SystemPrompt: wikiPlanSystem,
UserPrompt: user,
+ MaxTokens: &maxTokens,
})
if err != nil {
return wikiPlan{}, err
@@ -1239,6 +1320,11 @@ func normalizeWikiPlanPage(page wikiPlanPage) wikiPlanPage {
if page.Slug == "" {
page.Slug = slugify(page.Title)
}
+ // Canonicalize the slug to the hyphen style used by slug_kwd / outlinks /
+ // artifact links. The LLM freely mixes "dong_zhuo" and "dong-zhuo", so this
+ // single choke point makes plan slugs, slugToPageType/pageTitles keys and
+ // the writer's bare/full reverse index all agree.
+ page.Slug = normalizeWikiSlugHyphens(page.Slug)
page.Title = strings.TrimSpace(page.Title)
if page.Title == "" {
page.Title = page.Slug
@@ -1895,10 +1981,55 @@ func pageTypeOf(slug string, slugToPageType map[string]string) string {
func transformWikiLinks(content, kbID string, pageTitles, slugToPageType map[string]string) (string, []string) {
kbID = strings.TrimSpace(kbID)
+ // Resolve a wikitext slug (which may be a bare "name" or a full
+ // "<page_type>/<slug>") to the canonical full slug used as the page
+ // identifier (slug_kwd). This mirrors Python's _wiki_resolve_dead_slug:
+ // plain names / titles are reverse-mapped to the full pid, and slugs that
+ // cannot be resolved are dropped (dead links). Without this, bare slugs
+ // written by the LLM never match the full-slug page index, so wiki
+ // relations (edges) are silently lost.
+ bareToSlug := map[string]string{}
+ titleToSlug := map[string]string{}
+ // Plan slugs are canonicalized to the hyphen style upstream
+ // (normalizeWikiPlanPage -> normalizeWikiSlugHyphens), so slugToPageType /
+ // pageTitles keys are already hyphen full-slugs. LLM-authored wikitext
+ // links may still carry underscores (e.g. "dong_zhuo"); normalize both
+ // sides to hyphens so the reverse lookup matches, and always emit the
+ // resolved outlink in the canonical hyphen full-slug form.
+ for fullSlug := range slugToPageType {
+ bareToSlug[normalizeWikiSlugHyphens(lastPathSlug(fullSlug))] = fullSlug
+ }
+ for fullSlug, title := range pageTitles {
+ if t := strings.TrimSpace(title); t != "" {
+ titleToSlug[t] = fullSlug
+ }
+ }
+ resolveSlug := func(slug string) string {
+ slug = strings.TrimSpace(slug)
+ if slug == "" {
+ return ""
+ }
+ if _, ok := slugToPageType[slug]; ok {
+ return slug
+ }
+ if full, ok := bareToSlug[normalizeWikiSlugHyphens(lastPathSlug(slug))]; ok {
+ return full
+ }
+ if full, ok := titleToSlug[slug]; ok {
+ return full
+ }
+ return ""
+ }
seen := map[string]bool{}
var outlinks []string
track := func(slug string) {
- slug = strings.TrimSpace(slug)
+ slug = resolveSlug(slug)
+ // Emit the canonical hyphen full-slug so outlinks_kwd and slug_kwd
+ // always agree in format (guards against a resolved slug that still
+ // carries underscores, e.g. from an old existing page).
+ if slug != "" {
+ slug = normalizeWikiSlugHyphens(slug)
+ }
if slug == "" || seen[slug] {
return
}
@@ -1945,10 +2076,16 @@ func transformWikiLinks(content, kbID string, pageTitles, slugToPageType map[str
// <page_type>/<name> prefix, in which case that prefix is reused as the
// page_type and the name is emitted as the bare slug.
link := func(label, slug string) string {
- bareSlug := slug
- pageType := pageTypeOf(slug, slugToPageType)
- if idx := strings.Index(slug, "/"); idx >= 0 && slug[:idx] == pageType {
- bareSlug = slug[idx+1:]
+ fullSlug := resolveSlug(slug)
+ if fullSlug == "" {
+ // Unresolved (dead) slug: keep the label as plain text so the
+ // frontend does not render a broken artifact link.
+ return strings.TrimSpace(label)
+ }
+ bareSlug := fullSlug
+ pageType := pageTypeOf(fullSlug, slugToPageType)
+ if idx := strings.Index(fullSlug, "/"); idx >= 0 && fullSlug[:idx] == pageType {
+ bareSlug = fullSlug[idx+1:]
}
if pageType == "" {
pageType = "page"
@@ -1988,9 +2125,25 @@ func transformWikiLinks(content, kbID string, pageTitles, slugToPageType map[str
track(slug)
return link(displayText(slug, slug), slug)
})
+ // rawLinks counts the model-authored wikilinks in the source content. It
+ // must be counted against content, not out: by this point every [[...]] has
+ // been rewritten to markdown, so counting on out would always report zero.
+ rawLinks := wikiWikilinkSimpleRe.FindAllString(content, -1)
+ appcommon.Info("knowledge_compiler: transformWikiLinks resolved outlinks",
+ zap.Int("raw_wikilinks", len(rawLinks)),
+ zap.Int("resolved_outlinks", len(outlinks)),
+ zap.Strings("sample_resolved", firstN(outlinks, 5)),
+ zap.Strings("sample_raw", firstN(rawLinks, 5)))
return out, outlinks
}
+func firstN(s []string, n int) []string {
+ if len(s) <= n {
+ return s
+ }
+ return s[:n]
+}
+
func lastPathSlug(slug string) string {
slug = strings.TrimSpace(slug)
if idx := strings.LastIndex(slug, "/"); idx >= 0 && idx < len(slug)-1 {
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget.go
index c1807d3b4b..c9c51b1ab2 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget.go
@@ -13,10 +13,12 @@ import "sort"
// - a deterministic, mention-grounded truncation that selects the top pages
// under the global cap and reports how many were excluded.
//
-// The provider never receives an explicit output max_tokens on the wiki path
-// (see P0 in tasks/2026-08-04-wiki-go-python-gap-alignment-plan.md): output
-// scale is controlled purely through max_pages in the prompt + the in-code
-// truncation below.
+// The provider receives an explicit output max_tokens on the wiki PLAN path
+// equal to outputTokens (below): without it, small-window models default to a
+// tiny completion cap and the large plan JSON gets truncated mid-stream,
+// producing "LLM response is not parseable JSON". The REFINE (page body)
+// step uses its own cap via wikiRefineMaxTokens. capacity (max page count) and
+// outputTokens (hard completion cap) both derive from the same model window.
// Python alignment constants (wiki.py:1670-1674, 1783-1787).
const (
@@ -55,6 +57,11 @@ type wikiPlanBudget struct {
// target; that is deliberate (never ask a small-window model for more pages
// than its output can hold).
Max int
+ // MaxTokens is the explicit completion cap sent to the provider for the
+ // PLAN step. It is the same outputTokens used to derive Max and prevents
+ // the large page JSON from being truncated mid-stream by a small default
+ // completion cap.
+ MaxTokens int
}
// Cap is the page budget the planner is actually allowed to emit. It is the
@@ -104,7 +111,7 @@ func deriveWikiPlanBudget(modelContextLen, totalItems int) wikiPlanBudget {
// never be asked to emit more pages than its output capacity permits, or we
// reintroduce truncated-JSON risk. When capacity < Target, Max simply lands
// below Target and the achievable page count is capacity-bound.
- return wikiPlanBudget{Target: target, Max: maxCount}
+ return wikiPlanBudget{Target: target, Max: maxCount, MaxTokens: outputTokens}
}
// wikiExtractItemCount counts the planning items in one reduced extract. It is
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_reduce.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_reduce.go
index d506d630dd..873e71c410 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_reduce.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_reduce.go
@@ -135,6 +135,10 @@ func (p *wikiPipeline) dedupeEntities(in []wikiEntity) []wikiEntity {
// llmMergeEntityDecision asks the chat seam whether two distinct-name entities
// refer to the same real-world concept. Returns true to merge.
func (p *wikiPipeline) llmMergeEntityDecision(a, b wikiEntity) (bool, error) {
+ // Pass retryMax=0: dedupeEntities budgets llmCalls itself and counts one
+ // decision per call; letting GenJSON retry a failing chat seam would multiply
+ // each decision by (1 + jsonRetryMax) and blow through the budget (a
+ // persistent failure would issue 64 calls instead of <=16).
raw, err := common.GenJSON(p.ctx, p.deps.Chat, common.ChatRequest{
LLMID: p.llmID,
SystemPrompt: wikiReduceEntityDisambiguateSystem,
@@ -142,7 +146,7 @@ func (p *wikiPipeline) llmMergeEntityDecision(a, b wikiEntity) (bool, error) {
"entity_a": mustPrettyJSON(a),
"entity_b": mustPrettyJSON(b),
}),
- })
+ }, 0)
if err != nil {
return false, err
}
diff --git a/internal/ingestion/component/schema/chunker.go b/internal/ingestion/component/schema/chunker.go
index e920950de1..782906b6ca 100644
--- a/internal/ingestion/component/schema/chunker.go
+++ b/internal/ingestion/component/schema/chunker.go
@@ -173,9 +173,10 @@ type ChunkerOutputs struct {
type TokenChunkerParam struct {
// DelimiterMode selects the chunking strategy.
- // Allowed value: "delimiter".
- // The single-chunk "one" behavior is provided by the separate
- // OneChunker component.
+ // Allowed value: "delimiter". The legacy "token_size" value is accepted for
+ // backward compatibility and normalized to "delimiter" (it is behaviorally
+ // identical at runtime; see Validate). The single-chunk "one" behavior is
+ // provided by the separate OneChunker component.
DelimiterMode string `json:"delimiter_mode"`
// ChunkTokenSize is the target chunk size in tokens.
@@ -356,6 +357,15 @@ func (p *TokenChunkerParam) Validate() error {
// common/float_utils.py:50-58 fix for #17418.
p.OverlappedPercent = math.Round(v * 100)
}
+ // Backward-compat: Python removed the standalone "token_size" mode and
+ // treats it as identical to "delimiter" at runtime, normalizing it in
+ // check() (rag/flow/chunker/token_chunker.py). A dataset configured under
+ // the old scheme may still carry delimiter_mode="token_size"; normalize it
+ // here instead of rejecting it so those pipelines keep working (mirrors
+ // the Python behaviour exactly).
+ if p.DelimiterMode == "token_size" {
+ p.DelimiterMode = "delimiter"
+ }
switch p.DelimiterMode {
case "delimiter":
default:
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go
index 2a663c7c87..fcc6bfd931 100644
--- a/internal/ingestion/knowledge_compile/consumer.go
+++ b/internal/ingestion/knowledge_compile/consumer.go
@@ -17,6 +17,7 @@ package knowledge_compile
import (
"context"
+ "fmt"
"sort"
"sync"
"time"
@@ -47,8 +48,7 @@ type Consumer struct {
mergeThreshold float64 // KNN similarity threshold for "existing merged row is a duplicate"
mu sync.Mutex
- seqs map[string]map[string]uint64 // dataset -> docID -> last applied seq (per-doc out-of-order guard)
- tombs map[string]map[string]uint64 // dataset -> docID -> delete event seq (tombstone)
+ tombs map[string]map[string]uint64 // dataset -> docID -> delete marker (tombstone)
}
// NewConsumer constructs a Consumer driven by the given Claimer. Tests pass a
@@ -64,7 +64,6 @@ func NewConsumer(scheduler Claimer, opts ...Option) *Consumer {
pollInterval: 2 * time.Second,
sweepInterval: 30 * time.Second,
mergeThreshold: 0.99,
- seqs: map[string]map[string]uint64{},
tombs: map[string]map[string]uint64{},
}
for _, o := range opts {
@@ -215,59 +214,48 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entries
if c.tombs == nil {
c.tombs = map[string]map[string]uint64{}
}
- if c.seqs == nil {
- c.seqs = map[string]map[string]uint64{}
- }
tomb := c.tombs[kb]
- docSeqs := c.seqs[kb]
- if docSeqs == nil {
- docSeqs = map[string]uint64{}
- c.seqs[kb] = docSeqs
- }
var completed []BacklogEntry
var deleted []string
- // Per-document seq advances and tombstone clears are staged locally and
- // committed only after the write/delete paths below return without error,
- // so a failed batch leaves c.seqs/c.tombs untouched (except tombstones
- // recorded for the deletion events already present in this batch) and can
- // be retried on reclaim.
- var pendingSeq []BacklogEntry
+ // Per-document tombstone clears are staged locally and committed only after
+ // the write/delete paths below return without error, so a failed batch
+ // leaves c.tombs untouched and can be retried on reclaim.
var pendingTombClear []string
- // Reconcile each document to its highest-sequence event before classifying
- // it as deleted or completed. A completion and a deletion for the same doc
- // can land in the same batch, and completions/deletions can also arrive
- // out of order across batches. The last event (by per-doc seq) wins, so we
- // must not delete a doc that was re-ingested after its deletion, nor accept
- // a stale completion shadowed by a later deletion.
+ // Reconcile each document to its last event before classifying it as
+ // deleted or completed. A completion and a deletion for the same doc can
+ // land in the same batch, and completions/deletions can also arrive out of
+ // order across batches. The LAST event for a doc (by backlog append order)
+ // decides its fate, so we must not delete a doc that was re-ingested after
+ // its deletion, nor merge a completion shadowed by a later deletion. No
+ // sequence number is needed: the backlog is an ordered log and the last
+ // write wins, which is exactly the re-parse (completed last) vs delete
+ // (deleted last) semantics we want.
type docState struct {
winEvent EventType
- winSeq uint64
}
byDoc := make(map[string]*docState, len(entries))
for _, e := range entries {
et := EventType(e.EventType)
st := byDoc[e.DocID]
if st == nil {
- byDoc[e.DocID] = &docState{winEvent: et, winSeq: e.Seq}
+ byDoc[e.DocID] = &docState{winEvent: et}
continue
}
- if e.Seq >= st.winSeq {
- st.winEvent = et
- st.winSeq = e.Seq
- }
+ // Later entries overwrite earlier ones: last event wins.
+ st.winEvent = et
}
for docID, st := range byDoc {
switch st.winEvent {
case EventTypeDeleted:
// Record the tombstone for this batch's deletion so a stale
- // completion (seq <= delete seq) is skipped. The tombstone is
- // committed immediately (deletions are not rolled back on a
- // failed batch because re-running the delete is idempotent).
+ // completion is skipped. The tombstone is committed immediately
+ // (deletions are not rolled back on a failed batch because
+ // re-running the delete is idempotent).
if tomb == nil {
tomb = map[string]uint64{}
c.tombs[kb] = tomb
}
- tomb[docID] = st.winSeq
+ tomb[docID] = 1
deleted = append(deleted, docID)
case EventTypeCompleted:
// A re-ingest after an earlier deletion: clear the prior
@@ -275,22 +263,27 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entries
if _, hadTomb := tomb[docID]; hadTomb {
pendingTombClear = append(pendingTombClear, docID)
}
- // Seq is per-document, so the stale/duplicate check must be
- // scoped to the document, not the whole dataset (C4).
- if prev, ok := docSeqs[docID]; ok && st.winSeq <= prev {
- continue // stale / duplicate completion for this doc
- }
- // The seq advance is deferred to after the batch succeeds (see
- // below), so a transient reader/deduper/writer failure does not
- // permanently drop the completion on the next reclaim+retry.
- pendingSeq = append(pendingSeq, BacklogEntry{DocID: docID, EventType: string(EventTypeCompleted), Seq: st.winSeq})
- completed = append(completed, BacklogEntry{DocID: docID, EventType: string(EventTypeCompleted), Seq: st.winSeq})
+ completed = append(completed, BacklogEntry{DocID: docID, EventType: string(EventTypeCompleted)})
}
}
// Sort for deterministic iteration in the delete/load passes below.
sort.Strings(deleted)
c.mu.Unlock()
+ // Per-entry detail (each claim carries a frozen, closed batch of doc events;
+ // log them on a single line so a batch can be reconstructed from the ingestor
+ // log alone).
+ entryDetails := make([]string, 0, len(entries))
+ for _, e := range entries {
+ entryDetails = append(entryDetails, fmt.Sprintf("doc=%s event=%s", e.DocID, e.EventType))
+ }
+
+ common.Info("knowledge_compile: batch merge start",
+ zap.String("dataset_id", kb),
+ zap.Int("completed_docs", len(completed)),
+ zap.Int("deleted_docs", len(deleted)),
+ zap.Strings("entries", entryDetails))
+
if len(deleted) == 0 && len(completed) == 0 {
return nil
}
@@ -442,13 +435,21 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entries
return err
}
- // All merge and delete paths succeeded: commit the staged per-document seq
- // advances and tombstone clears. These are only now persisted so a failed
- // batch leaves c.seqs/c.tombs untouched and a later reclaim can retry.
- c.mu.Lock()
- for _, e := range pendingSeq {
- c.seqs[kb][e.DocID] = e.Seq
+ // Re-materialize the wiki page graph from the now-consistent merged set. This
+ // runs unconditionally after a merge: the graph is a global per-dataset
+ // projection rebuilt from a consistent read each time, so it must run even
+ // when the current batch only prunes (deletes) wiki pages, and even for a
+ // non-wiki batch where the dataset already has no wiki pages (ProjectWikiGraph
+ // then just drops any stale graph rows). The graph is reconstructible, so the
+ // cost of an occasional no-op reprojection is accepted.
+ if err := c.writer.ProjectWikiGraph(ctx, tenant, kb); err != nil {
+ return err
}
+
+ // All merge and delete paths succeeded: commit the staged tombstone
+ // clears. These are only now persisted so a failed batch leaves c.tombs
+ // untouched and a later reclaim can retry.
+ c.mu.Lock()
for _, docID := range pendingTombClear {
delete(c.tombs[kb], docID)
}
diff --git a/internal/ingestion/knowledge_compile/consumer_test.go b/internal/ingestion/knowledge_compile/consumer_test.go
index 29f0cec042..5104ad0dae 100644
--- a/internal/ingestion/knowledge_compile/consumer_test.go
+++ b/internal/ingestion/knowledge_compile/consumer_test.go
@@ -54,6 +54,8 @@ type fakeWriter struct {
written [][]kccommon.Product
deletedDocLevel []string
strippedSources []string
+ projected []string
+ dropped []string
}
func (w *fakeWriter) WriteMerged(_ context.Context, _, _ string, products []kccommon.Product) error {
@@ -82,6 +84,20 @@ func (w *fakeWriter) StripMergedSources(_ context.Context, _, _ string, docIDs [
return nil
}
+func (w *fakeWriter) ProjectWikiGraph(_ context.Context, _, kb string) error {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ w.projected = append(w.projected, kb)
+ return nil
+}
+
+func (w *fakeWriter) DropWikiGraph(_ context.Context, _, kb string) error {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ w.dropped = append(w.dropped, kb)
+ return nil
+}
+
func sampleProducts() []kccommon.Product {
return []kccommon.Product{
{ID: "p1", DocID: "d1", TenantID: "t1", Variant: kccommon.Variant("structure"),
@@ -105,7 +121,7 @@ func TestConsumerCompletedWritesMerged(t *testing.T) {
w := &fakeWriter{}
c := newTestConsumer(sch, r, w, func(string) (Deduper, error) { return NewNoopDeduper(), nil })
- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted), 1); err != nil {
+ if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted)); err != nil {
t.Fatalf("append: %v", err)
}
c.tryClaimAndProcess(context.Background())
@@ -118,6 +134,9 @@ func TestConsumerCompletedWritesMerged(t *testing.T) {
if len(w.written[0]) != 2 {
t.Fatalf("expected 2 merged products, got %d", len(w.written[0]))
}
+ if len(w.projected) != 1 || w.projected[0] != "kb1" {
+ t.Fatalf("expected 1 ProjectWikiGraph(kb1) call, got %v", w.projected)
+ }
// After ack, the claim row must be cleared (no live lease left behind).
if _, ok, _ := sch.TryClaim(context.Background()); ok {
t.Fatalf("expected no claimable row after ack")
@@ -130,22 +149,22 @@ func TestConsumerTombstoneSkipsCompletedBeforeDeleted(t *testing.T) {
w := &fakeWriter{}
c := newTestConsumer(sch, r, w, func(string) (Deduper, error) { return NewNoopDeduper(), nil })
- // completed (seq 1) before deleted (seq 2): the completion is the stale
- // original, so it must be skipped and d1's per-doc products orphaned.
- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted), 1); err != nil {
+ // completed before deleted: the deletion is the last event for d1, so it
+ // wins and d1's per-doc products are orphaned (completion skipped).
+ if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted)); err != nil {
t.Fatalf("append completed: %v", err)
}
- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeDeleted), 2); err != nil {
+ if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeDeleted)); err != nil {
t.Fatalf("append deleted: %v", err)
}
c.tryClaimAndProcess(context.Background())
w.mu.Lock()
defer w.mu.Unlock()
- // No merged write (completed skipped). The deletion is handled entirely on
- // the DocEngine: d1's per-doc products are dropped in one call and d1 is
- // stripped from every dataset-level product in one call. No products are
- // loaded into memory.
+ // No merged write (completed skipped, since deletion is the last event).
+ // The deletion is handled entirely on the DocEngine: d1's per-doc products
+ // are dropped in one call and d1 is stripped from every dataset-level
+ // product in one call. No products are loaded into memory.
if len(w.written) != 0 {
t.Fatalf("expected no merged write, got %d", len(w.written))
}
@@ -163,20 +182,20 @@ func TestConsumerReingestAfterDeletionWins(t *testing.T) {
w := &fakeWriter{}
c := newTestConsumer(sch, r, w, func(string) (Deduper, error) { return NewNoopDeduper(), nil })
- // deleted (seq 1) then completed (seq 2): the completion has the higher
- // sequence, so it wins — the doc is re-ingested, NOT deleted. The deletion
- // must not drop its per-doc products, and the completion must be merged.
- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeDeleted), 1); err != nil {
+ // deleted then completed: the completion is the last event for d1, so it
+ // wins — the doc is re-ingested, NOT deleted. The deletion must not drop
+ // its per-doc products, and the completion must be merged.
+ if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeDeleted)); err != nil {
t.Fatalf("append deleted: %v", err)
}
- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted), 2); err != nil {
+ if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted)); err != nil {
t.Fatalf("append completed: %v", err)
}
c.tryClaimAndProcess(context.Background())
w.mu.Lock()
defer w.mu.Unlock()
- // No deletion calls: the higher-seq completion overrides the deletion.
+ // No deletion calls: the last (completion) event overrides the deletion.
if len(w.deletedDocLevel) != 0 {
t.Fatalf("expected no DeleteDocLevelForDocs, got %v", w.deletedDocLevel)
}
@@ -190,13 +209,16 @@ func TestConsumerReingestAfterDeletionWins(t *testing.T) {
if len(w.written[0]) != 2 {
t.Fatalf("expected 2 merged products, got %d", len(w.written[0]))
}
+ if len(w.projected) != 1 || w.projected[0] != "kb1" {
+ t.Fatalf("expected 1 ProjectWikiGraph(kb1) call, got %v", w.projected)
+ }
}
func TestSchedulerClaimClosedBatch(t *testing.T) {
sch := NewFakeScheduler()
for i := 0; i < 40; i++ {
docID := "d" + string(rune('a'+i%26)) + string(rune('0'+i/26))
- if err := sch.Publish(context.Background(), "t1", "kb1", docID, string(EventTypeCompleted), uint64(i)); err != nil {
+ if err := sch.Publish(context.Background(), "t1", "kb1", docID, string(EventTypeCompleted)); err != nil {
t.Fatalf("append: %v", err)
}
}
@@ -229,7 +251,7 @@ func TestSchedulerClaimClosedBatch(t *testing.T) {
func TestSchedulerReclaimExpired(t *testing.T) {
sch := NewFakeScheduler()
- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted), 1); err != nil {
+ if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted)); err != nil {
t.Fatalf("append: %v", err)
}
_, ok, err := sch.Claim(context.Background(), "kb1")
@@ -278,10 +300,10 @@ func TestSchedulerStateMachineLocksStateAndCounts(t *testing.T) {
sch := NewFakeScheduler()
// Publish two docs: state=pending, backlog=2, inflight=0.
- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted), 1); err != nil {
+ if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted)); err != nil {
t.Fatalf("publish d1: %v", err)
}
- if err := sch.Publish(context.Background(), "t1", "kb1", "d2", string(EventTypeCompleted), 2); err != nil {
+ if err := sch.Publish(context.Background(), "t1", "kb1", "d2", string(EventTypeCompleted)); err != nil {
t.Fatalf("publish d2: %v", err)
}
if c, s := fakeRowCounts(sch, "kb1"); s != DatasetStatePending || c.backlog != 2 || c.inflight != 0 {
@@ -351,7 +373,7 @@ func TestSchedulerStateMachineLocksStateAndCounts(t *testing.T) {
// diagnostic, otherwise a completed state would be misread as failed.
func TestSchedulerSetErrorScopedToClaimToken(t *testing.T) {
sch := NewFakeScheduler()
- if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted), 1); err != nil {
+ if err := sch.Publish(context.Background(), "t1", "kb1", "d1", string(EventTypeCompleted)); err != nil {
t.Fatalf("publish: %v", err)
}
diff --git a/internal/ingestion/knowledge_compile/event.go b/internal/ingestion/knowledge_compile/event.go
index c0225ea752..9c4090615a 100644
--- a/internal/ingestion/knowledge_compile/event.go
+++ b/internal/ingestion/knowledge_compile/event.go
@@ -54,7 +54,6 @@ type KCCompileEvent struct {
DatasetID string `json:"dataset_id"` // the KB scope
DocID string `json:"doc_id"` // the contributing document
EventType string `json:"event_type"` // EventType value
- Seq uint64 `json:"seq"` // per-doc monotonic sequence, for out-of-order correction
Timestamp int64 `json:"ts"`
}
@@ -107,18 +106,18 @@ func DefaultClaimer() Claimer { return defaultClaimer }
// KB's durable MySQL backlog and wakes idle workers over NATS. It is a no-op
// when no Publisher has been installed (e.g. DB unavailable). A failure is
// returned so callers can log but never fail the pipeline on it.
-func PublishCompleted(ctx context.Context, tenantID, datasetID, docID string, seq uint64) error {
+func PublishCompleted(ctx context.Context, tenantID, datasetID, docID string) error {
if defaultPublisher == nil {
return nil
}
- return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeCompleted), seq)
+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeCompleted))
}
// PublishDeleted records a doc_deleted event the same way (Publish handles the
// append + notify pairing).
-func PublishDeleted(ctx context.Context, tenantID, datasetID, docID string, seq uint64) error {
+func PublishDeleted(ctx context.Context, tenantID, datasetID, docID string) error {
if defaultPublisher == nil {
return nil
}
- return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted), seq)
+ return defaultPublisher.Publish(ctx, tenantID, datasetID, docID, string(EventTypeDeleted))
}
diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.go
index a8deeee6ae..661c191d63 100644
--- a/internal/ingestion/knowledge_compile/reader.go
+++ b/internal/ingestion/knowledge_compile/reader.go
@@ -62,6 +62,7 @@ type engineReader struct {
// stored compiled chunk document.
var compiledSelectFields = []string{
"id", "doc_id", "tenant_id", "compile_kwd",
+ "available_int",
"content_with_weight", "kc_payload",
"source_chunk_ids", "source_doc_ids",
"name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd",
@@ -140,7 +141,7 @@ func productFromChunkMap(c map[string]interface{}, tenant string) (kccommon.Prod
id, _ := c["id"].(string)
docID, _ := c["doc_id"].(string)
variant, _ := c["compile_kwd"].(string)
- merged := isMerged(c["kc_merged"])
+ merged := isAvailable(c["available_int"])
meta := map[string]any{}
if v, ok := c["name_kwd"].(string); ok && v != "" {
@@ -216,7 +217,7 @@ func productFromChunkMap(c map[string]interface{}, tenant string) (kccommon.Prod
}, true
}
-// SearchSimilar runs a dense KNN over the existing merged rows (kc_merged=1,
+// SearchSimilar runs a dense KNN over the existing merged rows (available_int=1,
// compile_kwd=variant) of the KB and returns the closest hit above minScore.
func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, variant kccommon.Variant, vector []float64, topN int, minScore float64) (kccommon.Product, float64, error) {
eng := r.eng
@@ -236,11 +237,11 @@ func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, vari
Limit: topN,
SelectFields: append([]string{"id", "doc_id", "kb_id", "content_with_weight", "kc_payload",
"name_kwd", "entity_type_kwd", "from_entity_kwd", "to_entity_kwd", "slug_kwd",
- "type", "source_chunk_ids", "source_doc_ids", "kc_merged", "compile_kwd"},
+ "type", "source_chunk_ids", "source_doc_ids", "available_int", "compile_kwd"},
wikiSelectFields...),
Filter: map[string]interface{}{
- "kc_merged": 1,
- "compile_kwd": string(variant),
+ "available_int": 1,
+ "compile_kwd": string(variant),
},
MatchExprs: []interface{}{
&types.MatchDenseExpr{
@@ -267,10 +268,10 @@ func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, vari
return kccommon.Product{}, 0, nil
}
-// isMerged normalizes the boxed kc_merged field returned by the DocEngine,
+// isAvailable normalizes the boxed available_int field returned by the DocEngine,
// which may be stored as a string ("1"/"0"/"true"), a bool, or a numeric,
// depending on backend and mapping. Returns true for any positive/true form.
-func isMerged(v interface{}) bool {
+func isAvailable(v interface{}) bool {
switch t := v.(type) {
case nil:
return false
diff --git a/internal/ingestion/knowledge_compile/scheduler.go b/internal/ingestion/knowledge_compile/scheduler.go
index 03e9f15be4..479d285279 100644
--- a/internal/ingestion/knowledge_compile/scheduler.go
+++ b/internal/ingestion/knowledge_compile/scheduler.go
@@ -53,13 +53,13 @@ const (
)
// BacklogEntry is one scheduling unit appended to a KB's backlog (Option E
-// §11.4). It carries the doc id plus the original event kind/seq so the
-// consumer can re-apply the same out-of-order / tombstone guards as the
-// broker-based design without re-reading the queue.
+// §11.4). It carries the doc id plus the original event kind so the consumer
+// can re-apply the same tombstone handling as the broker-based design without
+// re-reading the queue. The backlog is an ordered log; the last event for a
+// given doc (by append order) wins, so no sequence number is needed.
type BacklogEntry struct {
DocID string `json:"doc_id"`
EventType string `json:"event_type"`
- Seq uint64 `json:"seq"`
}
// ClaimResult is returned by Scheduler.Claim: the KB's tenant plus the closed
@@ -83,7 +83,7 @@ type Publisher interface {
// Publish records one doc event in the dataset's durable backlog and wakes
// idle workers. It is transactional so concurrent publishers do not clobber
// each other's backlog, and it always pairs the append with a notify.
- Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, seq uint64) error
+ Publish(ctx context.Context, tenantID, datasetID, docID, eventType string) error
}
// Claimer is the consumer-side role (Option E §11.5). A cluster of competing
@@ -183,11 +183,11 @@ func (s *mysqlScheduler) Provision(ctx context.Context) error {
// workers. The append is transactional (concurrent publishers do not clobber
// each other's backlog), and the notify is always paired with the append so a
// producer never needs a separate Notify call.
-func (s *mysqlScheduler) Publish(ctx context.Context, tenantID, datasetID, docID, eventType string, seq uint64) error {
+func (s *mysqlScheduler) Publish(ctx context.Context, tenantID, datasetID, docID, eventType string) error {
if s.db == nil {
return nil
}
- entry := BacklogEntry{DocID: docID, EventType: eventType, Seq: seq}
+ entry := BacklogEntry{DocID: docID, EventType: eventType}
// KnowledgeCompileDataset.dataset_id is the PRIMARY KEY, so the SELECT ... FOR
// UPDATE below also takes an InnoDB gap lock for a not-yet-existing dataset;
// concurrent publishers for the same dataset serialize on that lock and the
@@ -231,8 +231,7 @@ func (s *mysqlScheduler) Publish(ctx context.Context, tenantID, datasetID, docID
zap.String("dataset_id", datasetID),
zap.String("tenant_id", tenantID),
zap.String("doc_id", docID),
- zap.String("event_type", eventType),
- zap.Uint64("seq", seq))
+ zap.String("event_type", eventType))
return s.notify(ctx, datasetID)
}
@@ -279,7 +278,8 @@ func (s *mysqlScheduler) claimRow(ctx context.Context, tx *gorm.DB, datasetID st
zap.String("tenant_id", row.TenantID),
zap.Int("batch_size", len(batch)),
zap.Int("backlog_remaining", len(backlog)-n),
- zap.String("token", row.ClaimToken))
+ zap.String("token", row.ClaimToken),
+ zap.Any("batch", batch))
return ClaimResult{DatasetID: row.DatasetID, TenantID: row.TenantID, Entries: batch, Token: row.ClaimToken}, true, nil
}
@@ -345,7 +345,8 @@ func (s *mysqlScheduler) Ack(ctx context.Context, datasetID, token string, batch
common.Info("knowledge_compile: acked dataset batch",
zap.String("dataset_id", datasetID),
zap.Int("batch_size", len(batch)),
- zap.Int("backlog_remaining", remaining))
+ zap.Int("backlog_remaining", remaining),
+ zap.Any("batch", batch))
return remaining, nil
}
@@ -528,7 +529,7 @@ func (s *mysqlScheduler) SubscribeNotify(ctx context.Context) (<-chan string, er
}
// removeEntries drops every entry in batch from the inflight slice (match by
-// doc_id + event_type + seq). The batch is the exact set the worker claimed, so
+// doc_id + event_type). The batch is the exact set the worker claimed, so
// this is a precise removal, not a "clear all" — any inflight added by a
// concurrent claim is preserved.
func removeEntries(inflight, batch []BacklogEntry) []BacklogEntry {
@@ -585,7 +586,7 @@ func NewFakeScheduler() *FakeScheduler {
func (f *FakeScheduler) Provision(_ context.Context) error { return nil }
// Publish appends one doc event and pushes a notify (same as the MySQL path).
-func (f *FakeScheduler) Publish(_ context.Context, tenantID, datasetID, docID, eventType string, seq uint64) error {
+func (f *FakeScheduler) Publish(_ context.Context, tenantID, datasetID, docID, eventType string) error {
f.mu.Lock()
defer f.mu.Unlock()
r, ok := f.rows[datasetID]
@@ -596,7 +597,7 @@ func (f *FakeScheduler) Publish(_ context.Context, tenantID, datasetID, docID, e
if r.tenant == "" {
r.tenant = tenantID
}
- r.backlog = append(r.backlog, BacklogEntry{DocID: docID, EventType: eventType, Seq: seq})
+ r.backlog = append(r.backlog, BacklogEntry{DocID: docID, EventType: eventType})
// Mirror the MySQL scheduler: surface pending unless a worker is running.
if r.state != DatasetStateRunning {
r.state = DatasetStatePending
diff --git a/internal/ingestion/knowledge_compile/wiki_graph_test.go b/internal/ingestion/knowledge_compile/wiki_graph_test.go
new file mode 100644
index 0000000000..0fc55eacaf
--- /dev/null
+++ b/internal/ingestion/knowledge_compile/wiki_graph_test.go
@@ -0,0 +1,274 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package knowledge_compile
+
+import (
+ "context"
+ "encoding/json"
+ "testing"
+)
+
+// page builds a wikiPageProjection with the full "<page_type>/<slug>" identity.
+func page(pageType, slug string, outlinks ...string) wikiPageProjection {
+ return wikiPageProjection{
+ Slug: pageType + "/" + slug,
+ PageType: pageType,
+ Title: slug,
+ Outlinks: outlinks,
+ }
+}
+
+func findEntity(rows []map[string]interface{}, slug string) map[string]interface{} {
+ for _, r := range rows {
+ if r["compile_kwd"] == compileKwdWikiEntity && r["slug_kwd"] == slug {
+ return r
+ }
+ }
+ return nil
+}
+
+func findRelation(rows []map[string]interface{}, from, to string) map[string]interface{} {
+ for _, r := range rows {
+ if r["compile_kwd"] == compileKwdWikiRelation && r["from_kwd"] == from && r["to_kwd"] == to {
+ return r
+ }
+ }
+ return nil
+}
+
+const kbForTest = "kb1"
+
+// TestProjectWikiGraphRowsCrossBatchEdgeSurvives verifies a relation whose two
+// endpoints land in different "batches" (here modeled as two pages) is still
+// materialized: the projection is global over the full page set, so edges never
+// dangle due to batching.
+func TestProjectWikiGraphRowsCrossBatchEdgeSurvives(t *testing.T) {
+ w := engineWriter{}
+ pages := []wikiPageProjection{
+ page("entity", "alpha", "entity/beta"),
+ page("entity", "beta"),
+ }
+ rows, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, pages)
+ if err != nil {
+ t.Fatalf("project: %v", err)
+ }
+ if findEntity(rows, "entity/alpha") == nil {
+ t.Fatalf("missing entity entity/alpha")
+ }
+ if findEntity(rows, "entity/beta") == nil {
+ t.Fatalf("missing entity entity/beta")
+ }
+ if findRelation(rows, "entity/alpha", "entity/beta") == nil {
+ t.Fatalf("cross-page edge entity/alpha -> entity/beta not materialized")
+ }
+}
+
+// TestProjectWikiGraphRowsDropAfterReprojection verifies a page removed from the
+// set is gone from the reprojected graph: no orphaned entity and no dangling
+// relation remain.
+func TestProjectWikiGraphRowsDropAfterReprojection(t *testing.T) {
+ w := engineWriter{}
+ // First projection has alpha -> beta and beta -> gamma.
+ first := []wikiPageProjection{
+ page("entity", "alpha", "entity/beta"),
+ page("entity", "beta", "entity/gamma"),
+ page("entity", "gamma"),
+ }
+ r1, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, first)
+ if err != nil {
+ t.Fatalf("project1: %v", err)
+ }
+ if findRelation(r1, "entity/beta", "entity/gamma") == nil {
+ t.Fatalf("expected beta->gamma before drop")
+ }
+
+ // beta is deleted: reproject from alpha + gamma only.
+ second := []wikiPageProjection{
+ page("entity", "alpha"), // alpha no longer links anywhere
+ page("entity", "gamma"),
+ }
+ r2, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, second)
+ if err != nil {
+ t.Fatalf("project2: %v", err)
+ }
+ if findEntity(r2, "entity/beta") != nil {
+ t.Fatalf("beta should be gone after drop")
+ }
+ if findRelation(r2, "entity/alpha", "entity/beta") != nil {
+ t.Fatalf("orphan alpha->beta must not survive after beta dropped")
+ }
+ if findRelation(r2, "entity/beta", "entity/gamma") != nil {
+ t.Fatalf("orphan beta->gamma must not survive after beta dropped")
+ }
+}
+
+// TestProjectWikiGraphRowsZeroOutlinkWeightIsZero verifies a page with no
+// outlinks gets weight_int == 0 (Python parity: raw outlink count, 0 allowed).
+func TestProjectWikiGraphRowsZeroOutlinkWeightIsZero(t *testing.T) {
+ w := engineWriter{}
+ pages := []wikiPageProjection{page("entity", "lonely")}
+ rows, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, pages)
+ if err != nil {
+ t.Fatalf("project: %v", err)
+ }
+ e := findEntity(rows, "entity/lonely")
+ if e == nil {
+ t.Fatalf("missing entity")
+ }
+ if wt, ok := e["weight_int"].(int); !ok || wt != 0 {
+ t.Fatalf("expected weight_int==0 for zero-outlink page, got %v", e["weight_int"])
+ }
+}
+
+// TestProjectWikiGraphRowsWeightEqualsOutlinkCount verifies weight_int equals
+// the raw outlink count (N outlinks -> weight N), not max(1, N).
+func TestProjectWikiGraphRowsWeightEqualsOutlinkCount(t *testing.T) {
+ w := engineWriter{}
+ pages := []wikiPageProjection{
+ page("entity", "hub", "entity/a", "entity/b", "entity/c"),
+ page("entity", "a"),
+ page("entity", "b"),
+ page("entity", "c"),
+ }
+ rows, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, pages)
+ if err != nil {
+ t.Fatalf("project: %v", err)
+ }
+ e := findEntity(rows, "entity/hub")
+ if e == nil {
+ t.Fatalf("missing hub entity")
+ }
+ if wt, ok := e["weight_int"].(int); !ok || wt != 3 {
+ t.Fatalf("expected weight_int==3 for 3-outlink page, got %v", e["weight_int"])
+ }
+}
+
+// TestProjectWikiGraphRowsFieldMapping verifies the reader-facing display
+// columns are written (aliases_kwd from entity_names_kwd, description_with_weight
+// from summary_with_weight), and the id is the xxhash colon-namespaced form.
+func TestProjectWikiGraphRowsFieldMapping(t *testing.T) {
+ w := engineWriter{}
+ pages := []wikiPageProjection{
+ {
+ Slug: "concept/alpha",
+ PageType: "concept",
+ Title: "Alpha",
+ Aliases: []string{"A", "Alpha Prime"},
+ Summary: "the alpha page",
+ Outlinks: nil,
+ SourceDocIDs: []string{"d1", "d2"},
+ },
+ }
+ rows, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, pages)
+ if err != nil {
+ t.Fatalf("project: %v", err)
+ }
+ // Use page_type "concept" (not "entity") so the writer's entity_type_kwd
+ // mapping ("wiki_"+PageType) is observable — asserting wiki_entity would
+ // match by coincidence with the compileKwdWikiEntity constant.
+ e := findEntity(rows, "concept/alpha")
+ if e == nil {
+ t.Fatalf("missing entity")
+ }
+ if e["aliases_kwd"] == nil {
+ t.Fatalf("expected aliases_kwd (mapped from entity_names_kwd) to be set")
+ }
+ if e["description_with_weight"] != "the alpha page" {
+ t.Fatalf("expected description_with_weight=summary, got %v", e["description_with_weight"])
+ }
+ if e["entity_type_kwd"] != "wiki_concept" {
+ t.Fatalf("expected entity_type_kwd=wiki_concept, got %v", e["entity_type_kwd"])
+ }
+ // id format: wiki_entity:{kb}:{slug} xxhashed to 16 hex chars.
+ id, _ := e["id"].(string)
+ if len(id) != 16 {
+ t.Fatalf("expected 16-char xxhash id, got %q (len %d)", id, len(id))
+ }
+ // Recompute and compare against wikiGraphXXHash.
+ wantID := wikiGraphXXHash("wiki_entity", kbForTest, "concept/alpha")
+ if id != wantID {
+ t.Fatalf("id mismatch: got %q want %q", id, wantID)
+ }
+ // content_with_weight JSON must round-trip the slug/page_type/title/aliases/summary/weight.
+ raw, _ := e["content_with_weight"].(string)
+ var decoded map[string]any
+ if err := json.Unmarshal([]byte(raw), &decoded); err != nil {
+ t.Fatalf("content_with_weight not valid JSON: %v", err)
+ }
+ if decoded["slug"] != "concept/alpha" || decoded["page_type"] != "concept" {
+ t.Fatalf("content_with_weight missing slug/page_type: %v", decoded)
+ }
+}
+
+// TestProjectWikiGraphRowsBareSlugNoCollision verifies two pages with the same
+// bare slug but different page types get distinct entities (full-slug identity).
+func TestProjectWikiGraphRowsBareSlugNoCollision(t *testing.T) {
+ w := engineWriter{}
+ pages := []wikiPageProjection{
+ page("entity", "foo"),
+ page("concept", "foo"),
+ }
+ rows, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, pages)
+ if err != nil {
+ t.Fatalf("project: %v", err)
+ }
+ if findEntity(rows, "entity/foo") == nil || findEntity(rows, "concept/foo") == nil {
+ t.Fatalf("both entity/foo and concept/foo must exist as distinct entities")
+ }
+ e1 := findEntity(rows, "entity/foo")
+ e2 := findEntity(rows, "concept/foo")
+ if e1["id"] == e2["id"] {
+ t.Fatalf("entity/foo and concept/foo must have distinct ids")
+ }
+}
+
+// TestProjectWikiGraphRowsDanglingEdgeSkipped verifies an outlink to a page not
+// present in the projection is skipped (no dangling relation).
+func TestProjectWikiGraphRowsDanglingEdgeSkipped(t *testing.T) {
+ w := engineWriter{}
+ pages := []wikiPageProjection{
+ page("entity", "alpha", "entity/missing"),
+ }
+ rows, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, pages)
+ if err != nil {
+ t.Fatalf("project: %v", err)
+ }
+ if findRelation(rows, "entity/alpha", "entity/missing") != nil {
+ t.Fatalf("dangling edge to missing target must be skipped")
+ }
+ if findEntity(rows, "entity/missing") != nil {
+ t.Fatalf("missing target must not be materialized as an entity")
+ }
+}
+
+// TestProjectWikiGraphRowsSelfLoopSkipped verifies a page whose outlinks include
+// itself does not produce a self-edge (Python parity: src == tgt is skipped).
+func TestProjectWikiGraphRowsSelfLoopSkipped(t *testing.T) {
+ w := engineWriter{}
+ pages := []wikiPageProjection{
+ page("entity", "alpha", "entity/alpha"),
+ }
+ rows, err := w.projectWikiGraphRows(context.Background(), "t1", kbForTest, pages)
+ if err != nil {
+ t.Fatalf("project: %v", err)
+ }
+ if findRelation(rows, "entity/alpha", "entity/alpha") != nil {
+ t.Fatalf("self-loop edge must be skipped (Python parity)")
+ }
+ if findEntity(rows, "entity/alpha") == nil {
+ t.Fatalf("the self-referencing entity must still exist")
+ }
+}
diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go
index 45e6fded28..6765b725de 100644
--- a/internal/ingestion/knowledge_compile/writer.go
+++ b/internal/ingestion/knowledge_compile/writer.go
@@ -19,12 +19,20 @@ import (
"context"
"crypto/sha256"
"encoding/hex"
+ "encoding/json"
"fmt"
+ "sort"
"strings"
+ "time"
+ "github.com/cespare/xxhash/v2"
+ "go.uber.org/zap"
+
+ "ragflow/internal/common"
"ragflow/internal/engine"
"ragflow/internal/engine/types"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
+ "ragflow/internal/utility"
)
// Writer persists dataset-level merged products and removes them on document
@@ -32,17 +40,26 @@ import (
type Writer interface {
// WriteMerged upserts the dataset-level merged products (available_int=1).
WriteMerged(ctx context.Context, tenant, kb string, products []kccommon.Product) error
- // DeleteDocLevelForDocs drops every per-document (doc-level, kc_merged != 1)
+ // DeleteDocLevelForDocs drops every per-document (doc-level, available_int=0)
// product of the deleted docs in a single DocEngine call. Dataset-level
// merged rows are not targeted because their doc_id equals the kb, never a
// deleted source doc id.
DeleteDocLevelForDocs(ctx context.Context, tenant, kb string, deletedDocIDs []string) error
// StripMergedSources removes deletedDocIDs from the source_doc_ids array of
- // every dataset-level (kc_merged=1) product for the dataset. It searches the
+ // every dataset-level (available_int=1) product for the dataset. It searches the
// merged set once, rewrites the source array of every non-empty survivor in a
// single update pass, and deletes (in one call) any product whose array
// became empty.
StripMergedSources(ctx context.Context, tenant, kb string, deletedDocIDs []string) error
+ // ProjectWikiGraph reads every merged wiki_page product for the dataset and
+ // re-materializes the wiki page graph (entities + relations) as
+ // wiki_entity / wiki_relation compiled rows. It is a delete-then-insert
+ // replacement: the DocEngine has no cross-delete/insert transaction boundary,
+ // so the graph is rebuilt from a consistent read and the old graph is dropped
+ // first. The graph is reconstructible, so the lack of atomicity is accepted.
+ ProjectWikiGraph(ctx context.Context, tenant, kb string) error
+ // DropWikiGraph deletes every wiki_entity / wiki_relation row for the dataset.
+ DropWikiGraph(ctx context.Context, tenant, kb string) error
}
// engineWriter persists dataset-level merged products through the global
@@ -61,6 +78,20 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, produc
if len(products) == 0 {
return nil
}
+ // Dataset-level telemetry: break the merged set down by compile_kwd so a
+ // missing wiki_page at query time can be traced to "WriteMerged never
+ // received any wiki_page products" (generation/merge bug) rather than
+ // "received then dropped" (downstream delete bug).
+ byKwd := map[string]int{}
+ for _, p := range products {
+ byKwd[compileKwdForVariant(p.Variant)]++
+ }
+ common.Info("knowledge_compile: WriteMerged dataset-level products",
+ zap.String("kb_id", kb),
+ zap.String("tenant_id", tenant),
+ zap.Int("total", len(products)),
+ zap.Any("by_compile_kwd", byKwd),
+ )
eng := w.eng
if eng == nil {
eng = engine.Get()
@@ -69,6 +100,13 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, produc
return nil
}
baseName := fmt.Sprintf("ragflow_%s", tenant)
+ // Source-tracing fields (tasks/2026-08-07-wiki-duplicates-research.md §5.1):
+ // one runID + inputHash + timestamp per WriteMerged call, shared by every
+ // shard, so rows from the same execution share plan_kwd and reruns of the
+ // same plan share input_hash_kwd.
+ now := time.Now()
+ runID := utility.GenerateUUID()
+ inputHash := mergedInputHash(products)
// Shard the rows and drive the inserts through the shared global pool
// (docengine-bounded) instead of one monolithic InsertChunks call.
jobs := make([]CompilerJob, 0, (len(products)+writeMergedBatchSize-1)/writeMergedBatchSize)
@@ -81,7 +119,19 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, produc
jobs = append(jobs, func() error {
chunks := make([]map[string]interface{}, 0, len(batch))
for _, p := range batch {
- chunks = append(chunks, mergedChunkMap(tenant, kb, p))
+ chunks = append(chunks, mergedChunkMap(tenant, kb, runID, inputHash, now, p))
+ }
+ // Telemetry: confirm the bytes actually handed to InsertChunks carry
+ // compile_kwd=wiki_page (vs the WriteMerged stats that only reflect
+ // the in-memory Variant). If this shows wiki_page but ES returns "",
+ // the engine drops the field; if this shows "" too, the map is wrong.
+ if len(chunks) > 0 {
+ common.Info("knowledge_compile: WriteMerged chunk sample",
+ zap.String("kb_id", kb),
+ zap.String("compile_kwd", metaString(chunks[0], "compile_kwd")),
+ zap.String("variant", string(batch[0].Variant)),
+ zap.String("id", metaString(chunks[0], "id")),
+ )
}
_, err := eng.InsertChunks(ctx, chunks, baseName, kb)
return err
@@ -93,21 +143,31 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, produc
// mergedChunkMap builds the chunk-index document for a dataset-level merged
// product. It uses the dataset-level idempotency key (§11.6) as `id`, never the
// per-doc key, and is always available_int=1 (searchable).
-func mergedChunkMap(tenant, kb string, p kccommon.Product) map[string]interface{} {
+//
+// runID and inputHash carry the minimal source-tracing fields (see
+// tasks/2026-08-07-wiki-duplicates-research.md §5.1): runID is the execution
+// identifier shared by every row of one WriteMerged call (plan_kwd), inputHash
+// is the canonical input fingerprint shared by reruns of the same plan
+// (input_hash_kwd), and now stamps both wall-clock audit fields. We deliberately
+// do not write the full PLAN JSON into the engine.
+func mergedChunkMap(tenant, kb, runID, inputHash string, now time.Time, p kccommon.Product) map[string]interface{} {
srcDocIDs := metaStringSlice(p.Meta, "source_doc_ids")
srcChunkIDs := metaStringSlice(p.Meta, "source_chunk_ids")
m := map[string]interface{}{
- "id": datasetLevelID(tenant, kb, p),
- "doc_id": kb,
- "tenant_id": tenant,
- "kb_id": kb,
- "available_int": 1,
- "kc_merged": 1,
- "compile_kwd": string(p.Variant),
- "content_with_weight": p.Content,
- "kc_payload": p.Content, // raw payload, for Reader reconstruction
- "source_doc_ids": srcDocIDs,
- "source_chunk_ids": srcChunkIDs,
+ "id": datasetLevelID(tenant, kb, p),
+ "doc_id": kb,
+ "tenant_id": tenant,
+ "kb_id": kb,
+ "available_int": 1,
+ "compile_kwd": compileKwdForVariant(p.Variant),
+ "content_with_weight": p.Content,
+ "kc_payload": p.Content, // raw payload, for Reader reconstruction
+ "source_doc_ids": srcDocIDs,
+ "source_chunk_ids": srcChunkIDs,
+ "plan_kwd": runID,
+ "input_hash_kwd": inputHash,
+ "create_time": now.Format("2006-01-02 15:04:05"),
+ "create_timestamp_flt": float64(now.Unix()),
}
// Carry the wiki page metadata onto the merged row so the dataset-level
// products keep the fields the artifact API (ListArtifacts/ListWikiTopics)
@@ -183,7 +243,7 @@ func (w engineWriter) DeleteDocLevelForDocs(ctx context.Context, tenant, kb stri
}
// StripMergedSources removes deletedDocIDs from the source_doc_ids array of
-// every dataset-level (kc_merged=1) product for the dataset. The query filters
+// every dataset-level (available_int=1) product for the dataset. The query filters
// on source_doc_ids IN deletedDocIDs so the engine only returns rows that
// actually reference a deleted doc (intersection pushed down); the survivors'
// source arrays are rewritten in a single update pass driven by the shared
@@ -214,12 +274,12 @@ func (w engineWriter) StripMergedSources(ctx context.Context, tenant, kb string,
res, err := eng.Search(ctx, &types.SearchRequest{
IndexNames: []string{baseName},
KbIDs: []string{kb},
- // kc_merged=1 isolates dataset-level rows; source_doc_ids IN
+ // available_int=1 isolates dataset-level rows; source_doc_ids IN
// deletedDocIDs pushes the intersection test into the engine so only
// rows that actually reference a deleted doc are returned (Infinity
// array IN means "contains at least one of").
Filter: map[string]interface{}{
- "kc_merged": 1,
+ "available_int": 1,
"source_doc_ids": deletedDocIDs,
},
SelectFields: []string{"id", "source_doc_ids"},
@@ -309,6 +369,27 @@ func hashStr(s string) string {
return hex.EncodeToString(sum[:])
}
+// mergedInputHash derives a canonical SHA-256 fingerprint of a WriteMerged
+// batch's input evidence: the union of source_doc_ids across all products,
+// sorted and deduped. Reruns of the same plan (same input documents) share the
+// fingerprint, so input_hash_kwd distinguishes "same plan content" across
+// executions while plan_kwd separates individual runs.
+func mergedInputHash(products []kccommon.Product) string {
+ seen := make(map[string]struct{}, 16)
+ docs := make([]string, 0, 16)
+ for _, p := range products {
+ for _, d := range metaStringSlice(p.Meta, "source_doc_ids") {
+ if _, ok := seen[d]; ok {
+ continue
+ }
+ seen[d] = struct{}{}
+ docs = append(docs, d)
+ }
+ }
+ sort.Strings(docs)
+ return hashStr(strings.Join(docs, "\n"))
+}
+
// metaString extracts a string from a map value, tolerating a missing or
// non-string entry.
func metaString(m map[string]any, key string) string {
@@ -353,3 +434,441 @@ func metaInt(m map[string]any, key string) (int64, bool) {
}
return 0, false
}
+
+// --- Wiki page graph materialization (wiki_entity / wiki_relation) ---
+
+// compile_kwd values for the dataset-level products this package writes. The
+// wiki variant compiles into "wiki_page" rows (per the Python writer contract
+// that GetWikiAlteration / ListArtifacts / GetWikiGraph all filter on
+// compile_kwd = "wiki_page"), while the page graph is materialized as the
+// dedicated wiki_entity / wiki_relation buckets.
+const (
+ compileKwdWikiPage = "wiki_page"
+ compileKwdWikiEntity = "wiki_entity"
+ compileKwdWikiRelation = "wiki_relation"
+ compileKwdWikiPageGraph = "wiki_page_graph" // legacy Python blob, swept on drop
+)
+
+// wikiGraphBatchSize bounds how many graph rows each parallel InsertChunks call
+// carries, mirroring writeMergedBatchSize.
+const wikiGraphBatchSize = 200
+
+// wikiGraphSourceDocCap bounds the source_doc_ids array stored on a graph row so
+// a heavily-shared page does not accumulate an unbounded id list.
+const wikiGraphSourceDocCap = 64
+
+// compileKwdForVariant maps a compiler Variant to the compile_kwd stamped on the
+// merged chunk document. The wiki variant is special-cased because all read
+// paths filter on compile_kwd = "wiki_page", not the raw variant string "wiki".
+func compileKwdForVariant(v kccommon.Variant) string {
+ if v == kccommon.VariantWiki {
+ return compileKwdWikiPage
+ }
+ return string(v)
+}
+
+// wikiGraphXXHash derives a stable 16-char hex id (matches Python
+// xxh64().hexdigest()) for a graph node/edge namespaced under the kb.
+func wikiGraphXXHash(namespace, kb, key string) string {
+ return fmt.Sprintf("%016x", xxhash.Sum64String(namespace+":"+kb+":"+key))
+}
+
+// wikiGraphBareKey reduces a full "<page_type>/<slug>" identity (or a bare
+// slug) to a canonical bare key used by the graph's reverse slug index. It
+// strips any "<page_type>/" prefix and normalizes underscores to hyphens so a
+// bare outlink ("dong-zhuo") matches the page whose slug_kwd is
+// "entity/dong_zhuo" (or "entity/dong-zhuo") regardless of the LLM's
+// underscore-vs-hyphen formatting. Empty strings yield "" (never a valid key).
+func wikiGraphBareKey(slug string) string {
+ s := strings.TrimSpace(slug)
+ if s == "" {
+ return ""
+ }
+ if idx := strings.LastIndex(s, "/"); idx >= 0 && idx < len(s)-1 {
+ s = s[idx+1:]
+ }
+ s = strings.ReplaceAll(s, "_", "-")
+ return strings.TrimSpace(s)
+}
+
+// wikiPageProjection is the subset of a merged wiki_page row that the graph
+// projection needs. It is reconstructed from the stored display columns (the
+// same fields GetWikiGraph reads back), not from the JSON payload.
+type wikiPageProjection struct {
+ Slug string
+ PageType string
+ Title string
+ Aliases []string
+ Summary string
+ // Outlinks are the other wiki pages this page links to (by full
+ // "<page_type>/<slug>" identity). These inter-page links are the SOLE source
+ // of wiki_relation edges: a wiki_relation row exists iff some page lists
+ // another page in its Outlinks (and that target page also exists). No edge
+ // comes from entity extraction, co-occurrence, or semantic similarity.
+ Outlinks []string
+ // SourceDocIDs are the originating document ids that produced this page.
+ // For a wiki_relation the stored source_doc_ids is the UNION of both
+ // endpoints' SourceDocIDs, so the relation is dropped only once neither
+ // endpoint traces to a surviving document.
+ SourceDocIDs []string
+ SourceChunkIDs []string
+}
+
+// ProjectWikiGraph reads every merged wiki_page for the dataset and
+// re-materializes the page graph. See the Writer interface doc for the
+// delete-then-insert (non-atomic) contract.
+func (w engineWriter) ProjectWikiGraph(ctx context.Context, tenant, kb string) error {
+ pages, err := w.loadMergedWikiPages(ctx, tenant, kb)
+ if err != nil {
+ return err
+ }
+ // Telemetry: confirm how many merged wiki_page rows survived WriteMerged.
+ // If this is 0 while WriteMerged reported wiki_page:N, the merged rows are
+ // not queryable under (compile_kwd=wiki_page AND available_int=1) — point at the
+ // stored field values, not a downstream delete.
+ common.Info("knowledge_compile: ProjectWikiGraph load",
+ zap.String("kb_id", kb),
+ zap.Int("merged_wiki_pages_loaded", len(pages)))
+ // Zero pages: the dataset has no wiki graph. Drop any stale graph rows and
+ // return — a full reprojection of an empty set would only rewrite nothing.
+ if len(pages) == 0 {
+ return w.dropWikiGraph(ctx, tenant, kb)
+ }
+ rows, err := w.projectWikiGraphRows(ctx, tenant, kb, pages)
+ if err != nil {
+ return err
+ }
+ // delete-then-insert: drop the previous graph first, then write the new one.
+ if err := w.dropWikiGraph(ctx, tenant, kb); err != nil {
+ return err
+ }
+ return w.insertWikiGraphChunks(ctx, tenant, kb, rows)
+}
+
+// DropWikiGraph deletes every wiki_entity / wiki_relation row for the dataset.
+func (w engineWriter) DropWikiGraph(ctx context.Context, tenant, kb string) error {
+ return w.dropWikiGraph(ctx, tenant, kb)
+}
+
+// dropWikiGraph is the shared delete path for both ProjectWikiGraph (when the
+// page set is empty, or before re-inserting) and DropWikiGraph. It deletes by
+// kb_id + compile_kwd IN (the graph buckets), also sweeping any legacy
+// wiki_page_graph blob left by an earlier Python writer so the index does not
+// accumulate stale state. The dataset-wide full delete of merged rows is owned
+// by DeleteDocLevelForKb elsewhere and is not repeated here.
+func (w engineWriter) dropWikiGraph(ctx context.Context, tenant, kb string) error {
+ eng := w.eng
+ if eng == nil {
+ eng = engine.Get()
+ }
+ if eng == nil {
+ return nil
+ }
+ baseName := fmt.Sprintf("ragflow_%s", tenant)
+ _, err := eng.DeleteChunks(ctx, map[string]interface{}{
+ "kb_id": kb,
+ "compile_kwd": []string{
+ compileKwdWikiEntity,
+ compileKwdWikiRelation,
+ compileKwdWikiPageGraph,
+ },
+ }, baseName, kb)
+ return err
+}
+
+// loadMergedWikiPages scrolls every merged wiki_page product for the dataset.
+// It selects only the display columns the projection needs (slug / page_type /
+// title / entity_names_kwd (aliases) / summary_with_weight / outlinks_kwd /
+// source_doc_ids / source_chunk_ids), never the JSON payload.
+func (w engineWriter) loadMergedWikiPages(ctx context.Context, tenant, kb string) ([]wikiPageProjection, error) {
+ eng := w.eng
+ if eng == nil {
+ eng = engine.Get()
+ }
+ if eng == nil {
+ return nil, nil
+ }
+ baseName := fmt.Sprintf("ragflow_%s", tenant)
+ const batchSize = 2000
+ var out []wikiPageProjection
+ offset := 0
+ kwdSeen := map[string]int{}
+ for {
+ res, err := eng.Search(ctx, &types.SearchRequest{
+ IndexNames: []string{baseName},
+ KbIDs: []string{kb},
+ Filter: map[string]interface{}{
+ "compile_kwd": compileKwdWikiPage,
+ "available_int": 1,
+ "kb_id": kb,
+ },
+ SelectFields: []string{
+ "slug_kwd", "page_type_kwd", "title_kwd",
+ "entity_names_kwd", "summary_with_weight", "outlinks_kwd",
+ "source_doc_ids", "source_chunk_ids",
+ },
+ Limit: batchSize,
+ Offset: offset,
+ // Stable deterministic ordering so the deep-offset pages (beyond ES's
+ // default result window) keep a consistent cursor and the engine can
+ // switch to search_after instead of failing on deep offset.
+ OrderBy: (&types.OrderByExpr{}).Asc("slug_kwd"),
+ })
+ if err != nil {
+ return nil, err
+ }
+ // Telemetry: surface whether the Search filter is actually honoured. If
+ // the returned rows carry compile_kwd other than wiki_page, the engine's
+ // Filter map is being ignored and wiki_entity rows leak into the page
+ // projection (which then has nothing under compile_kwd=wiki_page).
+ for _, c := range res.Chunks {
+ kwdSeen[metaString(c, "compile_kwd")]++
+ }
+ if len(res.Chunks) == 0 {
+ break
+ }
+ for _, c := range res.Chunks {
+ p := wikiPageProjection{
+ Slug: metaString(c, "slug_kwd"),
+ PageType: metaString(c, "page_type_kwd"),
+ Title: metaString(c, "title_kwd"),
+ Aliases: metaStringSlice(c, "entity_names_kwd"),
+ Summary: metaString(c, "summary_with_weight"),
+ Outlinks: metaStringSlice(c, "outlinks_kwd"),
+ SourceDocIDs: metaStringSlice(c, "source_doc_ids"),
+ SourceChunkIDs: metaStringSlice(c, "source_chunk_ids"),
+ }
+ out = append(out, p)
+ }
+ if len(res.Chunks) < batchSize {
+ break
+ }
+ offset += batchSize
+ }
+ common.Info("knowledge_compile: loadMergedWikiPages query result",
+ zap.String("kb_id", kb),
+ zap.Int("rows_returned", len(out)),
+ zap.Any("compile_kwd_seen", kwdSeen))
+ return out, nil
+}
+
+// projectWikiGraphRows builds the wiki_entity / wiki_relation chunk documents
+// from a full projection of the dataset's merged wiki pages.
+//
+// Graph model (mirrors the Python writer in dataset_wiki_generator.py):
+// - One wiki_entity row per merged wiki page (node = page).
+// - One wiki_relation row per inter-page outlink: a page lists another page
+// in its Outlinks, and that target page also exists in the projection.
+// Thus the EDGES of this graph are exclusively the mutual links BETWEEN
+// wiki pages (page A outlinks to page B => edge A->B). No edge originates
+// from entity extraction, co-occurrence, or semantic similarity.
+//
+// Both entities and relations are keyed on the full "<page_type>/<slug>"
+// identity (matching Go wiki input), so a bare slug collision across page types
+// is impossible. A relation whose target page is absent from this projection (a
+// dangling edge, e.g. the target was pruned/deleted) is skipped, as is a
+// self-loop (src == tgt) — the graph is rebuilt from a consistent read each
+// time, so cross-batch dangling edges cannot accumulate.
+func (w engineWriter) projectWikiGraphRows(_ context.Context, tenant, kb string, pages []wikiPageProjection) ([]map[string]interface{}, error) {
+ // Sort pages deterministically by full slug before building the indexes so
+ // the outcome is reproducible across reprojections. The engine's row order
+ // is otherwise scroll-order dependent, which made bare-slug first-match
+ // resolution flip between runs.
+ sort.SliceStable(pages, func(i, j int) bool {
+ return pages[i].Slug < pages[j].Slug
+ })
+ bySlug := make(map[string]wikiPageProjection, len(pages))
+ // Bare-slug index so inter-page outlinks match regardless of whether they
+ // were emitted as a full "<page_type>/<slug>" identity or a bare slug. The
+ // LLM writes wikitext links using bare slugs (and may mix "_" vs "-"), so
+ // the wiki_page outlinks_kwd in ES is frequently bare; matching only the
+ // full slug would silently drop every relation (graph has nodes, no edges).
+ //
+ // A bare key shared by two pages (e.g. "entity/foo" and "concept/foo" both
+ // reduce to "foo") is ambiguous; such links resolve to nothing rather than
+ // nondeterministically flipping between pages across runs.
+ byBareSlug := make(map[string]wikiPageProjection, len(pages))
+ bareAmbiguous := make(map[string]bool, len(pages))
+ for _, p := range pages {
+ if p.Slug == "" {
+ continue
+ }
+ bySlug[p.Slug] = p
+ if b := wikiGraphBareKey(p.Slug); b != "" {
+ if prev, ok := byBareSlug[b]; ok {
+ if prev.Slug != p.Slug {
+ bareAmbiguous[b] = true
+ }
+ } else {
+ byBareSlug[b] = p
+ }
+ }
+ }
+
+ rows := make([]map[string]interface{}, 0, len(bySlug))
+ seen := make(map[string]bool, len(bySlug))
+ for _, p := range bySlug {
+ entityID := wikiGraphXXHash("wiki_entity", kb, p.Slug)
+ if seen[entityID] {
+ continue
+ }
+ seen[entityID] = true
+
+ weight := len(p.Outlinks) // raw outlink count; 0 is allowed (Python parity)
+ content, err := json.Marshal(map[string]any{
+ "slug": p.Slug,
+ "page_type": p.PageType,
+ "title": p.Title,
+ "aliases": p.Aliases,
+ "summary": p.Summary,
+ "weight": weight,
+ })
+ if err != nil {
+ return nil, err
+ }
+ rows = append(rows, map[string]interface{}{
+ "id": entityID,
+ "doc_id": kb,
+ "tenant_id": tenant,
+ "kb_id": kb,
+ "available_int": 1,
+ "compile_kwd": compileKwdWikiEntity,
+ "type_kwd": "wiki_" + p.PageType,
+ "entity_type_kwd": "wiki_" + p.PageType,
+ "slug_kwd": p.Slug,
+ "title_kwd": p.Title,
+ "aliases_kwd": p.Aliases,
+ "description_with_weight": p.Summary,
+ "weight_int": weight,
+ "source_chunk_ids": p.SourceChunkIDs,
+ "source_doc_ids": capSourceDocs(p.SourceDocIDs),
+ "content_with_weight": string(content),
+ })
+
+ // Relations: one edge per outlink whose target page exists in this
+ // projection. A target may be a full "<page_type>/<slug>" identity or a
+ // bare slug (the latter is what the LLM emits in wikitext links); resolve
+ // both so a bare outlink still produces an edge.
+ for _, tgt := range p.Outlinks {
+ if tgt == p.Slug {
+ continue // self-loop: Python skips src == tgt
+ }
+ tp, ok := bySlug[tgt]
+ if !ok {
+ // Bare-slug outlink: normalize (strip prefix, "_"->"-") and look
+ // up the reverse index. Keeps the canonical full slug for the
+ // stored relation so from/to are consistent with slug_kwd. An
+ // ambiguous bare key (mapped to more than one page) resolves to
+ // nothing instead of nondeterministically choosing one.
+ if bare := wikiGraphBareKey(tgt); bare != "" && !bareAmbiguous[bare] {
+ tp, ok = byBareSlug[bare]
+ if ok {
+ tgt = tp.Slug
+ }
+ }
+ }
+ if !ok {
+ continue // dangling edge: target page not in this projection
+ }
+ relID := wikiGraphXXHash("wiki_relation", kb, p.Slug+":"+tgt)
+ if seen[relID] {
+ continue
+ }
+ seen[relID] = true
+ relContent, err := json.Marshal(map[string]any{
+ "from": p.Slug,
+ "to": tgt,
+ })
+ if err != nil {
+ return nil, err
+ }
+ srcDocs := unionCap(p.SourceDocIDs, tp.SourceDocIDs)
+ rows = append(rows, map[string]interface{}{
+ "id": relID,
+ "doc_id": kb,
+ "tenant_id": tenant,
+ "kb_id": kb,
+ "available_int": 1,
+ "compile_kwd": compileKwdWikiRelation,
+ "type_kwd": compileKwdWikiRelation,
+ "from_id": entityID,
+ "to_id": wikiGraphXXHash("wiki_entity", kb, tgt),
+ "from_kwd": p.Slug,
+ "to_kwd": tgt,
+ "source_doc_ids": srcDocs,
+ "content_with_weight": string(relContent),
+ })
+ }
+ }
+ var relCount, entCount int
+ for _, r := range rows {
+ switch r["compile_kwd"] {
+ case compileKwdWikiRelation:
+ relCount++
+ case compileKwdWikiEntity:
+ entCount++
+ }
+ }
+ common.Info("knowledge_compile: projectWikiGraphRows result",
+ zap.String("kb_id", kb),
+ zap.Int("pages_projected", len(bySlug)),
+ zap.Int("wiki_entity_rows", entCount),
+ zap.Int("wiki_relation_rows", relCount))
+ return rows, nil
+}
+
+// insertWikiGraphChunks shards the graph rows and drives the inserts through the
+// shared global pool, mirroring WriteMerged.
+func (w engineWriter) insertWikiGraphChunks(ctx context.Context, tenant, kb string, rows []map[string]interface{}) error {
+ if len(rows) == 0 {
+ return nil
+ }
+ eng := w.eng
+ if eng == nil {
+ eng = engine.Get()
+ }
+ if eng == nil {
+ return nil
+ }
+ baseName := fmt.Sprintf("ragflow_%s", tenant)
+ jobs := make([]CompilerJob, 0, (len(rows)+wikiGraphBatchSize-1)/wikiGraphBatchSize)
+ for start := 0; start < len(rows); start += wikiGraphBatchSize {
+ end := start + wikiGraphBatchSize
+ if end > len(rows) {
+ end = len(rows)
+ }
+ batch := rows[start:end]
+ jobs = append(jobs, func() error {
+ _, err := eng.InsertChunks(ctx, batch, baseName, kb)
+ return err
+ })
+ }
+ return runCompilerJobs(ctx, jobs)
+}
+
+// capSourceDocs returns up to wikiGraphSourceDocCap source doc ids.
+func capSourceDocs(ids []string) []string {
+ if len(ids) <= wikiGraphSourceDocCap {
+ return ids
+ }
+ return ids[:wikiGraphSourceDocCap]
+}
+
+// unionCap returns the union of two doc-id lists, capped at wikiGraphSourceDocCap.
+func unionCap(a, b []string) []string {
+ seen := make(map[string]bool, len(a)+len(b))
+ out := make([]string, 0, len(a)+len(b))
+ for _, id := range a {
+ if !seen[id] {
+ seen[id] = true
+ out = append(out, id)
+ }
+ }
+ for _, id := range b {
+ if !seen[id] {
+ seen[id] = true
+ out = append(out, id)
+ }
+ }
+ return capSourceDocs(out)
+}
diff --git a/internal/ingestion/knowledge_compile/writer_test.go b/internal/ingestion/knowledge_compile/writer_test.go
index 8f7e9bfe42..b45a04f5c1 100644
--- a/internal/ingestion/knowledge_compile/writer_test.go
+++ b/internal/ingestion/knowledge_compile/writer_test.go
@@ -2,6 +2,7 @@ package knowledge_compile
import (
"testing"
+ "time"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
)
@@ -28,7 +29,8 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {
"source_chunk_ids": []string{"c1"},
},
}
- m := mergedChunkMap("t1", "kb1", p)
+ now := time.Now()
+ m := mergedChunkMap("t1", "kb1", "run-abc", "hash-123", now, p)
cases := map[string]string{
"slug_kwd": "entity/alpha",
@@ -37,6 +39,15 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {
"page_type_kwd": "entity",
"topic_kwd": "Alpha",
"summary_with_weight": "A page about Alpha",
+ "plan_kwd": "run-abc",
+ "input_hash_kwd": "hash-123",
+ }
+ // The wall-clock audit fields must be stamped from `now`, not omitted.
+ if m["create_time"] != now.Format("2006-01-02 15:04:05") {
+ t.Errorf("create_time = %v, want %v", m["create_time"], now.Format("2006-01-02 15:04:05"))
+ }
+ if m["create_timestamp_flt"] != float64(now.Unix()) {
+ t.Errorf("create_timestamp_flt = %v, want %v", m["create_timestamp_flt"], float64(now.Unix()))
}
for k, want := range cases {
if got, _ := m[k].(string); got != want {
@@ -46,8 +57,11 @@ func TestMergedChunkMapKeepsWikiFields(t *testing.T) {
if v, _ := m["entity_names_kwd"].([]string); len(v) != 1 || v[0] != "Alpha" {
t.Errorf("entity_names_kwd = %#v, want [Alpha]", m["entity_names_kwd"])
}
- if m["doc_id"] != "kb1" || m["kc_merged"] != 1 || m["available_int"] != 1 {
- t.Errorf("merged flags wrong: doc_id=%v kc_merged=%v available_int=%v", m["doc_id"], m["kc_merged"], m["available_int"])
+ if m["doc_id"] != "kb1" || m["available_int"] != 1 {
+ t.Errorf("merged flags wrong: doc_id=%v available_int=%v", m["doc_id"], m["available_int"])
+ }
+ if _, ok := m["kc_merged"]; ok {
+ t.Errorf("merged row must not persist undefined kc_merged field")
}
if m["q_3_vec"] == nil {
t.Errorf("vector column missing")
diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.go
index e106d81901..f43cf7ae50 100644
--- a/internal/ingestion/pipeline/pipeline.go
+++ b/internal/ingestion/pipeline/pipeline.go
@@ -34,6 +34,7 @@ import (
"ragflow/internal/utility"
"github.com/cloudwego/eino/compose"
+ "go.uber.org/zap"
)
// Pipeline is a compiled ingestion canvas plus task-scoped metadata.
@@ -653,6 +654,34 @@ func (p *Pipeline) componentProgressCallback(ctx context.Context) runtime.Progre
msg = ev.Component + " Error"
}
}
+ // Surface every component lifecycle event as a structured log line so
+ // a component failure (e.g. an LLM/client error) is captured in
+ // ingestor_server.log even if the wrapped error never reaches the
+ // higher-level "Task ... failed" branch.
+ switch ev.Phase {
+ case runtime.PhaseError:
+ if ev.Err != nil {
+ common.Error("component progress: error", ev.Err,
+ zap.String("component", ev.Component),
+ zap.String("task_id", p.taskID),
+ zap.String("document_id", p.documentID))
+ } else {
+ common.Info("component progress: error",
+ zap.String("component", ev.Component),
+ zap.String("task_id", p.taskID),
+ zap.String("document_id", p.documentID))
+ }
+ default:
+ // Keep the message constant: msg may carry component names or
+ // error-derived text, and a newline in it could forge a log record
+ // (CWE-117). Pass msg and the phase as structured fields instead.
+ common.Info("component progress",
+ zap.String("message", msg),
+ zap.Int("phase", int(ev.Phase)),
+ zap.String("component", ev.Component),
+ zap.String("task_id", p.taskID),
+ zap.String("document_id", p.documentID))
+ }
p.sink.OnComponentProgress(ctx, ProgressEvent{
TaskID: p.taskID,
DocumentID: p.documentID,
diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.go
index a67d7266d8..8a232caea2 100644
--- a/internal/ingestion/task/knowledge_compiler_wiring.go
+++ b/internal/ingestion/task/knowledge_compiler_wiring.go
@@ -34,6 +34,7 @@ import (
"ragflow/internal/service"
"gorm.io/gorm"
+ appcommon "ragflow/internal/common"
)
// This file is the composition-root wiring for the KnowledgeCompiler ingestion
@@ -154,9 +155,22 @@ func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatR
config.MaxTokens = req.MaxTokens
}
}
- resp, err := c.svc.Chat(ctx, c.tenantID, llmID, msgs, config)
- if err != nil {
- return nil, err
+ // Retry transient transport/provider failures (HTTP timeout, reset,
+ // connection refused, 5xx, 429) with exponential backoff. A single
+ // external-LLM hiccup must not abort the whole knowledge compile — the reply
+ // is never cached, so each attempt issues a fresh request. Permanent
+ // configuration/model errors (auth, unknown model) are not retried.
+ var resp *models.ChatResponse
+ retryErr := appcommon.RetryWithBackoff(ctx, kcChatRetryMax, kcChatRetryDelay, func() error {
+ r, err := c.svc.Chat(ctx, c.tenantID, llmID, msgs, config)
+ if err != nil {
+ return err
+ }
+ resp = r
+ return nil
+ }, appcommon.IsTransientError)
+ if retryErr != nil {
+ return nil, retryErr
}
content := ""
if resp != nil && resp.Answer != nil {
@@ -165,6 +179,14 @@ func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatR
return &kc.ChatResponse{Content: content}, nil
}
+// kcChatRetryMax bounds how many times a transient LLM transport failure is
+// retried. Each attempt may run up to the driver's HTTP timeout, so the count
+// stays small to avoid unbounded wall-clock latency inside one compile.
+const kcChatRetryMax = 2
+
+// kcChatRetryDelay is the initial exponential-backoff delay between retries.
+const kcChatRetryDelay = 2 * time.Second
+
// kcEmbedder adapts service.ModelProviderService.GetEmbeddingModel to the
// knowledge_compiler Embedder seam. Vectors are returned as []float32 to match
// the component's product schema.
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index 4a77724867..6dafb7a03a 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -263,7 +263,7 @@ func (s *PipelineExecutor) processOutput(ctx context.Context, pipelineOutput map
// (available_int=1). The notification is sent only after a successful persist
// and is best-effort / non-fatal — a delivery failure is logged but does not
// fail the pipeline task.
- if err := knowledge_compile.PublishCompleted(ctx, s.taskCtx.Tenant.ID, s.taskCtx.Doc.KbID, s.taskCtx.Doc.ID, 0); err != nil {
+ if err := knowledge_compile.PublishCompleted(ctx, s.taskCtx.Tenant.ID, s.taskCtx.Doc.KbID, s.taskCtx.Doc.ID); err != nil {
common.Logger.Warn(fmt.Sprintf("knowledge_compile: publish doc_completed for %s failed: %v", s.taskCtx.Doc.ID, err))
}
diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.go
index a094f4b2ae..7de10643de 100644
--- a/internal/service/dataset_artifact_service.go
+++ b/internal/service/dataset_artifact_service.go
@@ -407,7 +407,7 @@ type WikiGraphRelation struct {
// GetWikiGraph returns the wiki entity/relation graph for a dataset.
func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, datasetID string) (*WikiGraph, error) {
entityChunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,
- map[string]interface{}{"compile_kwd": []string{CompileKwdWikiEntity}},
+ map[string]interface{}{"compile_kwd": []string{CompileKwdWikiEntity}, "available_int": 1},
[]string{"slug_kwd", "title_kwd", "aliases_kwd", "description_with_weight", "entity_type_kwd", "weight_int", "source_chunk_ids"},
0, 1000, (&types.OrderByExpr{}).Desc("weight_int"))
if err != nil {
@@ -420,27 +420,43 @@ func (s *DatasetArtifactService) GetWikiGraph(ctx context.Context, tenantID, dat
graph := &WikiGraph{Entities: []WikiGraphEntity{}, Relations: []WikiGraphRelation{}}
for _, c := range entityChunks {
w := intValue(c["weight_int"])
+ // Match the GetWikiPage / ListWikiPages contract: expose the bare slug
+ // and the pure page_type (no "wiki_" prefix, no "<page_type>/" prefix)
+ // so the frontend can build artifact/<page_type>/<slug> links that
+ // round-trip. entity_type_kwd stores "wiki_" + page_type (e.g.
+ // "wiki_topic"); strip the prefix. slug_kwd stores the full
+ // "<page_type>/<slug>" form; expose the trailing bare slug.
+ fullSlug := firstStringValue(c["slug_kwd"])
+ bareSlug := fullSlug
+ pageType := strings.TrimPrefix(firstStringValue(c["entity_type_kwd"]), "wiki_")
+ if idx := strings.LastIndex(bareSlug, "/"); idx >= 0 {
+ pageType = bareSlug[:idx]
+ bareSlug = bareSlug[idx+1:]
+ }
graph.Entities = append(graph.Entities, WikiGraphEntity{
- Slug: firstStringValue(c["slug_kwd"]),
+ Slug: bareSlug,
Name: firstStringValue(c["title_kwd"]),
Aliases: toStringSlice(c["aliases_kwd"]),
Description: firstStringValue(c["description_with_weight"]),
- Type: firstStringValue(c["entity_type_kwd"]),
+ Type: pageType,
Weight: w,
SourceChunkIDs: toStringSlice(c["source_chunk_ids"]),
})
}
if len(fromKwds) > 0 {
relChunks, _, err := s.searchCompiled(ctx, tenantID, datasetID,
- map[string]interface{}{"compile_kwd": []string{CompileKwdWikiRelation}, "from_kwd": fromKwds},
+ map[string]interface{}{"compile_kwd": []string{CompileKwdWikiRelation}, "available_int": 1, "from_kwd": fromKwds},
[]string{"from_kwd", "to_kwd"}, 0, 10000, nil)
if err != nil {
return nil, err
}
for _, c := range relChunks {
+ // Relations reference endpoints by their full "<page_type>/<slug>"
+ // form in ES; expose them as bare slugs so the frontend's
+ // nodesBySlug (keyed on the entity's bare slug) can resolve edges.
graph.Relations = append(graph.Relations, WikiGraphRelation{
- From: firstStringValue(c["from_kwd"]),
- To: firstStringValue(c["to_kwd"]),
+ From: bareWikiSlug(firstStringValue(c["from_kwd"])),
+ To: bareWikiSlug(firstStringValue(c["to_kwd"])),
})
}
}
@@ -877,6 +893,18 @@ func firstStringValue(v interface{}) string {
return ""
}
+// bareWikiSlug strips the "<page_type>/" prefix from a full wiki slug
+// ("topic/yellow-turban-rebellion" -> "yellow-turban-rebellion"), matching the
+// bare-slug form the graph UI keys nodes/relations on. Slugs with no prefix are
+// returned unchanged.
+func bareWikiSlug(slug string) string {
+ s := strings.TrimSpace(slug)
+ if idx := strings.LastIndex(s, "/"); idx >= 0 && idx < len(s)-1 {
+ return s[idx+1:]
+ }
+ return s
+}
+
// toStringSlice normalizes an ES field value (string or list) to []string.
func toStringSlice(v interface{}) []string {
switch t := v.(type) {
diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.go
index d40a814f5f..3121d3df16 100644
--- a/internal/service/document/document_crud.go
+++ b/internal/service/document/document_crud.go
@@ -365,7 +365,7 @@ func (s *DocumentService) deleteDocEngineData(docID, tenantID, kbID string) {
// never block the document delete, which already succeeded above.
pubCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
- if err := knowledge_compile.PublishDeleted(pubCtx, tenantID, kbID, docID, 0); err != nil {
+ if err := knowledge_compile.PublishDeleted(pubCtx, tenantID, kbID, docID); err != nil {
common.Warn(fmt.Sprintf("deleteDocEngineData: publish doc_deleted for %s failed: %v", docID, err))
}
if s.metadataSvc != nil {
diff --git a/rag/advanced_rag/knowlege_compile/wiki.py b/rag/advanced_rag/knowlege_compile/wiki.py
index b18758dfbe..26ba5ec9dd 100644
--- a/rag/advanced_rag/knowlege_compile/wiki.py
+++ b/rag/advanced_rag/knowlege_compile/wiki.py
@@ -1763,10 +1763,19 @@ Examples of BAD slugs (do NOT produce):
- ``concept/example-name`` (just duplicate the sample)
# Other rules
-- Group closely related small entities onto the same page (max 3-4 per page).
- BUT if a primary entity is described through several distinct thematic
- sections that appear as concepts above, prefer a separate ``concept`` page
- for EACH such section instead of collapsing them onto the entity page.
+- Entity/concept identity is one-to-one with pages: every extracted entity and
+ concept must be represented by exactly one canonical page, and each identity
+ may appear in only one page's ``entity_names``. Never split an identity into
+ multiple pages, page types, thematic sections, aliases, language
+ transliterations, or alternate slug spellings. Put all supported sections
+ for that identity on its single canonical page.
+- A page may represent several closely related low-signal entities/concepts
+ (max 3-4 per page), but list every represented identity in ``entity_names``
+ and do not repeat any identity on another page. If the page budget is tight,
+ group identities rather than omitting one or emitting a second page for it.
+- Identity ownership does not limit linking: ``related_kb_pages`` should list
+ every directly related canonical page supported by the input (within the
+ available-page budget). Never link duplicate or non-canonical slug variants.
- priority 1 = highest importance (process first).
- entity_names must match the names in the entities / concepts lists above.
- Target approximately {target_page_count} total pages (feel free to deviate