Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Refactor splitByTokens and wire wiki incremental compile (#18083)

Zhichang Yu64533e5b5e7d
diff --git a/go.mod b/go.mod
index 698c7aabb1..363f210642 100644
--- a/go.mod
+++ b/go.mod
@@ -62,6 +62,7 @@ require (
go.uber.org/zap v1.27.1
golang.org/x/crypto v0.53.0
golang.org/x/net v0.56.0
+ golang.org/x/oauth2 v0.36.0
golang.org/x/sync v0.21.0
golang.org/x/term v0.44.0
golang.org/x/text v0.38.0
@@ -211,7 +212,6 @@ require (
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/arch v0.27.0 // indirect
golang.org/x/exp v0.0.0-20231226003508-02704c960a9b // indirect
- golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/sys v0.46.0 // indirect
golang.org/x/time v0.15.0 // indirect
google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94 // indirect
diff --git a/internal/ingestion/component/knowledge_compiler/common/common_test.go b/internal/ingestion/component/knowledge_compiler/common/common_test.go
index ac9c03d365..6748216a78 100644
--- a/internal/ingestion/component/knowledge_compiler/common/common_test.go
+++ b/internal/ingestion/component/knowledge_compiler/common/common_test.go
@@ -1,9 +1,27 @@
package common
import (
+ "context"
"testing"
)
+type captureChat struct{ req ChatRequest }
+
+func (c *captureChat) Chat(_ context.Context, req ChatRequest) (*ChatResponse, error) {
+ c.req = req
+ return &ChatResponse{Content: `{"ok":true}`}, nil
+}
+
+func TestGenJSONDisablesInvokerRetry(t *testing.T) {
+ chat := &captureChat{}
+ if _, err := GenJSON(context.Background(), chat, ChatRequest{UserPrompt: "test"}, 0); err != nil {
+ t.Fatalf("GenJSON: %v", err)
+ }
+ if !chat.req.DisableRetry {
+ t.Fatal("GenJSON must disable retries in the underlying ChatInvoker")
+ }
+}
+
func vec(a, b, c float32) []float32 { return []float32{a, b, c} }
func TestMemStoreTopK(t *testing.T) {
diff --git a/internal/ingestion/component/knowledge_compiler/common/deps.go b/internal/ingestion/component/knowledge_compiler/common/deps.go
index 2c4d00fefa..43caeff9d9 100644
--- a/internal/ingestion/component/knowledge_compiler/common/deps.go
+++ b/internal/ingestion/component/knowledge_compiler/common/deps.go
@@ -28,6 +28,10 @@ type ChatRequest struct {
MaxTokens *int
APIKey string
BaseURL string
+ // DisableRetry tells the production ChatInvoker that the caller owns
+ // transient-error retries (GenJSON is such a caller). It is internal
+ // plumbing and is not part of the user-facing model request.
+ DisableRetry bool
}
// ChatResponse holds the LLM's text answer.
@@ -111,6 +115,13 @@ type Deps struct {
// deriveWikiPlanBudget, buildClusterContent) use it to size the input/output
// quotas (mirrors Python self._llm_model.max_length).
ModelContextLen int
+ // ModelMaxOutput is the chat model's generation cap (max_output), the most
+ // tokens one LLM response may emit. Cross-document merge judging packs many
+ // pairs into a single call, so the caller must cap the batch by both the
+ // input window (ModelContextLen) and this output cap; otherwise a large
+ // candidate set can overflow the model's max_output and it returns a
+ // truncated/non-JSON reply. Mirrors Python's max_tokens generation config.
+ ModelMaxOutput int
}
// DepsResolver resolves the per-run Deps from a tenant/llm/embedding triple.
diff --git a/internal/ingestion/component/knowledge_compiler/common/jsonchat.go b/internal/ingestion/component/knowledge_compiler/common/jsonchat.go
index 4509ef4953..0e7a05e959 100644
--- a/internal/ingestion/component/knowledge_compiler/common/jsonchat.go
+++ b/internal/ingestion/component/knowledge_compiler/common/jsonchat.go
@@ -49,6 +49,10 @@ func GenJSON(ctx context.Context, chat ChatInvoker, req ChatRequest, retryMax ..
maxRetries = retryMax[0]
}
req.JSONMode = true
+ // GenJSON owns retries for both transport failures and malformed JSON. Tell
+ // the production ChatInvoker to make exactly one provider request per outer
+ // GenJSON attempt, avoiding multiplicative nested retries.
+ req.DisableRetry = true
var lastErr error
delay := jsonRetryDelay
for attempt := 0; attempt <= maxRetries; attempt++ {
diff --git a/internal/ingestion/component/knowledge_compiler/structure/merge.go b/internal/ingestion/component/knowledge_compiler/structure/merge.go
index 5faa0333e8..d36f800b69 100644
--- a/internal/ingestion/component/knowledge_compiler/structure/merge.go
+++ b/internal/ingestion/component/knowledge_compiler/structure/merge.go
@@ -6,14 +6,23 @@ import (
"strings"
"sync"
+ appcommon "ragflow/internal/common"
"ragflow/internal/ingestion/component/knowledge_compiler/common"
"ragflow/internal/tokenizer"
+
+ "go.uber.org/zap"
)
-// maxBatchTokenReserve is the share of the model's token budget kept in reserve
-// for the system prompt, the batch instruction, and the model's JSON reply, so
-// a single DecideBatch sub-call never overflows max_token.
-const maxBatchTokenReserve = 0.15
+// maxWindowUsage is the fraction of the model's limits a single
+// DecideBatch sub-call may consume — applied both to the combined
+// (input+output) budget (fraction of content_length) and to the output budget
+// (fraction of max_output). The reply is an array of per-pair
+// {"duplicated","merged"} objects, and each merged object is roughly the size of
+// the longer of the two inputs, so the batch is split so the summed estimate
+// stays under this cap. 0.2 is deliberately conservative (smaller than the
+// default 0.6) so each sub-call stays well inside the window and generation cap
+// even for models with tight limits, at the cost of more, smaller sub-calls.
+const maxWindowUsage = 0.2
// Merge prompts, verbatim from Python structure.py. The merge is driven by
// the user-supplied prompts; the decision instruction lets us branch on the
@@ -99,18 +108,25 @@ type LLMMergeDecider struct {
Embed common.Embedder
Threshold float64
- // maxBatchTokens caps the estimated prompt size of a single DecideBatch
- // sub-call. When 0 (the default) the whole batch is sent in one call,
- // preserving the historic behavior. When positive the batch is split into
- // token-bounded sub-calls so a large candidate set can never overflow the
- // model's max_token window.
- maxBatchTokens int
-
- // submit runs one LLM sub-batch off the shared knowledge-compilation pool.
+ // maxBatchTotal caps the estimated combined size (prompt input + JSON reply)
+ // of a single DecideBatch sub-call. It guards the model's context window
+ // (content_length): a chunk starts a fresh one when adding a pair would push
+ // (input+output) past modelContentLength*maxWindowUsage. When 0 the
+ // whole batch is sent in one call (historic behavior).
+ maxBatchTotal int
+ // maxBatchOutput caps the estimated reply (output) size of a single
+ // DecideBatch sub-call. It guards the model's generation cap (max_output):
+ // each merged reply is roughly the size of the longer of the two inputs in a
+ // pair, so the batch is also split so the sum of per-pair output estimates
+ // stays within modelMaxOutput*maxWindowUsage. A sub-call must satisfy
+ // BOTH this cap and maxBatchTotal.
+ maxBatchOutput int
+
+ // submit runs LLM sub-batches off the shared knowledge-compilation pool.
// When nil each sub-batch runs inline (sequentially). It is injected by the
// knowledge_compile package so every stage shares one process-wide,
// vCPU-sized concurrency bound.
- submit func(ctx context.Context, fn func() error) error
+ submit func(ctx context.Context, jobs []func() error) error
mu sync.Mutex
aliases map[string]string
@@ -121,28 +137,44 @@ func NewLLMMergeDecider(chat common.ChatInvoker, llmID string, embed common.Embe
return &LLMMergeDecider{Chat: chat, LLMID: llmID, Embed: embed, Threshold: threshold, aliases: map[string]string{}}
}
-// SetMaxBatchTokens caps the estimated prompt size of a single DecideBatch
-// sub-call. Non-positive values disable per-call batching. The effective budget
-// is modelMaxTokens*(1-maxBatchTokenReserve), keeping headroom for the system
-// prompt and the JSON reply.
-func (d *LLMMergeDecider) SetMaxBatchTokens(modelMaxTokens int) {
- if modelMaxTokens <= 0 {
- d.maxBatchTokens = 0
+// SetMaxBatchTokens sets the two token budgets that bound a single DecideBatch
+// sub-call, derived from the model's two limits:
+// - modelContentLength: the model's context window (content_length); the
+// effective combined budget is modelContentLength*maxWindowUsage, i.e.
+// a sub-call may not exceed that many tokens of (prompt input + JSON reply).
+// - modelMaxOutput: the model's generation cap (max_output); the effective
+// reply budget is modelMaxOutput*maxWindowUsage.
+//
+// modelContentLength <= 0 disables both budgets (historic whole-batch behavior);
+// modelMaxOutput <= 0 falls back to combined-budget-only batching. splitByTokens
+// then keeps every sub-call inside BOTH the combined budget and the output
+// budget, so a large candidate set can never overflow the window nor the
+// generation cap.
+func (d *LLMMergeDecider) SetMaxBatchTokens(modelContentLength, modelMaxOutput int) {
+ d.maxBatchOutput = 0
+ if modelContentLength <= 0 {
+ d.maxBatchTotal = 0
return
}
- budget := int(float64(modelMaxTokens) * (1 - maxBatchTokenReserve))
- if budget <= 0 {
- d.maxBatchTokens = 0
+ totalBudget := int(float64(modelContentLength) * maxWindowUsage)
+ if totalBudget <= 0 {
+ d.maxBatchTotal = 0
return
}
- d.maxBatchTokens = budget
+ d.maxBatchTotal = totalBudget
+ if modelMaxOutput > 0 {
+ outputBudget := int(float64(modelMaxOutput) * maxWindowUsage)
+ if outputBudget > 0 {
+ d.maxBatchOutput = outputBudget
+ }
+ }
}
// SetSubmitter injects the shared knowledge-compilation pool so DecideBatch can
// run its token-bounded sub-batches concurrently. A nil submitter (the default)
-// falls back to running sub-batches sequentially. The submitter must run fn on
-// a bounded pool and return its error.
-func (d *LLMMergeDecider) SetSubmitter(submit func(ctx context.Context, fn func() error) error) {
+// falls back to running sub-batches sequentially. The submitter must enqueue
+// all jobs on a bounded pool, wait for them, and return the first error.
+func (d *LLMMergeDecider) SetSubmitter(submit func(ctx context.Context, jobs []func() error) error) {
d.submit = submit
}
@@ -229,18 +261,23 @@ type BatchMergeResult struct {
}
// DecideBatch judges every pair and returns the verdicts in input order. It is
-// the batched counterpart of Decide's single-pair judge. When a token budget is
-// configured it splits the pairs into token-bounded sub-batches (each sub-batch
-// still judged in a single LLM call, preserving pair order) so a large candidate
-// set can never overflow the model's max_token window.
+// the batched counterpart of Decide's single-pair judge. Besides deciding each
+// duplicate pair it also performs the actual merge: for a pair the LLM judges as
+// duplicated, mergePairsBatch has the model emit the combined payload (merged),
+// which the caller re-embeds and persists in place of the existing row. When a
+// token budget is configured it splits the pairs into sub-batches bounded by
+// BOTH the model's combined (input+output) budget and its output cap, each
+// sub-batch still judged in a single LLM call (preserving pair order), so a
+// large candidate set can never overflow the context window nor the generation
+// cap.
func (d *LLMMergeDecider) DecideBatch(ctx context.Context, pairs []MergePairInput) ([]BatchMergeResult, error) {
if len(pairs) == 0 {
return nil, nil
}
- if d.maxBatchTokens <= 0 {
+ if d.maxBatchTotal <= 0 && d.maxBatchOutput <= 0 {
return mergePairsBatch(ctx, d.Chat, d.LLMID, pairs)
}
- chunks := splitByTokens(pairs, d.maxBatchTokens)
+ chunks := splitByTokens(pairs, d.maxBatchTotal, d.maxBatchOutput)
// The merge decisions are LLM-bounded, not CPU-bounded: run the token-bounded
// sub-batches concurrently on the injected shared pool (vCPU-sized) when more
// than one chunk exists. Order is preserved by writing each chunk's verdicts
@@ -258,35 +295,22 @@ func (d *LLMMergeDecider) DecideBatch(ctx context.Context, pairs []MergePairInpu
return out, nil
}
results := make([][]BatchMergeResult, len(chunks))
- var (
- wg sync.WaitGroup
- errOnce sync.Once
- firstErr error
- )
+ jobs := make([]func() error, 0, len(chunks))
for i, chunk := range chunks {
i, chunk := i, chunk
- wg.Add(1)
- err := d.submit(ctx, func() error {
- defer wg.Done()
+ jobs = append(jobs, func() error {
sub, err := mergePairsBatch(ctx, d.Chat, d.LLMID, chunk)
if err != nil {
- errOnce.Do(func() { firstErr = err })
return err
}
results[i] = sub
return nil
})
- if err != nil {
- // The job was never enqueued, so the closure's wg.Done() will never
- // run — decrement here and record the failure so we don't deadlock
- // on wg.Wait() and don't silently drop the submit error.
- wg.Done()
- errOnce.Do(func() { firstErr = err })
- }
}
- wg.Wait()
- if firstErr != nil {
- return nil, firstErr
+ // Submit the complete batch in one call. The shared pool owns fan-out and
+ // bounded concurrency; the caller only waits for the pool's futures.
+ if err := d.submit(ctx, jobs); err != nil {
+ return nil, err
}
out := make([]BatchMergeResult, 0, len(pairs))
for _, sub := range results {
@@ -295,30 +319,70 @@ func (d *LLMMergeDecider) DecideBatch(ctx context.Context, pairs []MergePairInpu
return out, nil
}
-// splitByTokens groups pairs into contiguous chunks whose estimated prompt
-// size stays within budget. The original (global) Index of every pair is
-// preserved — the model echoes it back — so callers can still key verdicts by
-// the input index. A single pair that alone exceeds budget is sent on its own
-// (the model may still truncate, but we never silently drop it).
-func splitByTokens(pairs []MergePairInput, budget int) [][]MergePairInput {
+// splitByTokens groups pairs into contiguous chunks bounded by BOTH the model's
+// combined budget (prompt input + JSON reply, guarding content_length) and its
+// output budget (guarding max_output). The prompt size of a pair is roughly
+// existing+incoming; the merged reply for a pair is roughly the longer of the
+// two inputs (the merge emits the union of both objects). So a chunk starts a
+// fresh one when adding a pair would push either the combined (input+output)
+// estimate past totalBudget or the output estimate past outputBudget — keeping
+// every sub-call inside the context window AND the generation cap. The original
+// (global) Index of every pair is preserved — the model echoes it back — so
+// callers can still key verdicts by the input index. A single pair that alone
+// exceeds either budget is sent on its own (the model may still truncate, but we
+// never silently drop it).
+func splitByTokens(pairs []MergePairInput, totalBudget, outputBudget int) [][]MergePairInput {
var chunks [][]MergePairInput
cur := make([]MergePairInput, 0, len(pairs))
- used := 0
+ usedTotal := 0
+ usedOutput := 0
+ flush := func(usedTotal, usedOutput int) {
+ // Diagnostic: expose each chunk's realized budget utilization so the
+ // split can be tuned. usedTotal = Σ(input+output) and usedOutput = Σ
+ // (longer input) of the chunk's pairs; both must stay ≤ their budget for
+ // the sub-call to fit the model. "final" marks the last chunk flushed.
+ appcommon.Info("knowledge_compiler: merge batch split flush",
+ zap.Int("pairs", len(cur)),
+ zap.Int("total_budget", totalBudget),
+ zap.Int("output_budget", outputBudget),
+ zap.Int("used_total", usedTotal),
+ zap.Int("used_output", usedOutput))
+ }
+ appcommon.Info("knowledge_compiler: merge batch split start",
+ zap.Int("pairs", len(pairs)),
+ zap.Int("total_budget", totalBudget),
+ zap.Int("output_budget", outputBudget))
for _, p := range pairs {
// The pair contents dominate the prompt size; the "Pair N:" wrappers
- // are a negligible constant overhead per pair, ignored here.
- est := tokenizer.NumTokensFromString(p.Existing) + tokenizer.NumTokensFromString(p.Incoming)
- if len(cur) > 0 && used+est > budget {
+ // are a negligible constant overhead per pair, ignored here. Each input
+ // is tokenized once and reused for both the prompt and the output
+ // estimate.
+ estExisting := tokenizer.NumTokensFromString(p.Existing)
+ estIncoming := tokenizer.NumTokensFromString(p.Incoming)
+ estPrompt := estExisting + estIncoming
+ // The merged reply for a pair is at most the size of its longer input
+ // (we keep all existing fields and union in the incoming ones).
+ estOutput := estExisting
+ if estIncoming > estOutput {
+ estOutput = estIncoming
+ }
+ overOutput := outputBudget > 0 && usedOutput+estOutput > outputBudget
+ overTotal := totalBudget > 0 && usedTotal+estPrompt+estOutput > totalBudget
+ if len(cur) > 0 && (overOutput || overTotal) {
+ flush(usedTotal, usedOutput)
chunks = append(chunks, cur)
// Fresh backing array: cur[:0] shares the buffer with the chunk we
// just appended, so the next iteration would overwrite it.
cur = make([]MergePairInput, 0, len(pairs))
- used = 0
+ usedTotal = 0
+ usedOutput = 0
}
cur = append(cur, p)
- used += est
+ usedTotal += estPrompt + estOutput
+ usedOutput += estOutput
}
if len(cur) > 0 {
+ flush(usedTotal, usedOutput)
chunks = append(chunks, cur)
}
return chunks
diff --git a/internal/ingestion/component/knowledge_compiler/structure/structure_test.go b/internal/ingestion/component/knowledge_compiler/structure/structure_test.go
index 9103860ba5..4b409830fa 100644
--- a/internal/ingestion/component/knowledge_compiler/structure/structure_test.go
+++ b/internal/ingestion/component/knowledge_compiler/structure/structure_test.go
@@ -678,9 +678,23 @@ func TestLLMMergeDeciderDecideBatchEmpty(t *testing.T) {
func TestLLMMergeDeciderDecideBatchSplitsByTokenBudget(t *testing.T) {
chat := &graphChat{}
d := NewLLMMergeDecider(chat, "llm1", hashEmbedder{dim: 8}, 0.99)
+ submittedBatches := 0
+ submittedChunks := 0
+ d.SetSubmitter(func(ctx context.Context, jobs []func() error) error {
+ submittedBatches++
+ submittedChunks = len(jobs)
+ for _, job := range jobs {
+ if err := job(); err != nil {
+ return err
+ }
+ }
+ return nil
+ })
// Tiny model budget → one pair per sub-batch (exercises the split path).
- // 20 * (1-0.15) = 17 token budget, well under each ~40-token pair.
- d.SetMaxBatchTokens(20)
+ // Combined budget = 20 * 0.6 = 12 tokens, well under each ~40-token pair's
+ // (input+output) estimate. Output budget disabled (0) → combined-budget-only
+ // batching.
+ d.SetMaxBatchTokens(20, 0)
alpha := common.Product{
ID: "row-alpha",
@@ -711,6 +725,9 @@ func TestLLMMergeDeciderDecideBatchSplitsByTokenBudget(t *testing.T) {
if chat.mergeCalls != before+3 {
t.Errorf("token-split DecideBatch made %d LLM calls, want 3", chat.mergeCalls-before)
}
+ if submittedBatches != 1 || submittedChunks != 3 {
+ t.Fatalf("token-split DecideBatch submitted %d batches/%d chunks, want one batch containing all chunks", submittedBatches, submittedChunks)
+ }
if len(results) != 3 {
t.Fatalf("want 3 results, got %d", len(results))
}
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
index e5b03fc154..eeaa66779b 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go
@@ -330,11 +330,36 @@ func (p *wikiPipeline) run() error {
zap.String("doc_id", p.runKey()),
zap.Int("batches", len(p.mapExtracts)),
zap.Int("raw_entities", countExtractEntities(p.mapExtracts)))
- p.reduced = reduceExtracts(p.mapExtracts)
+ appcommon.Debug("wiki: MAP intermediate",
+ zap.String("dataset_id", p.datasetID),
+ zap.String("doc_id", p.runKey()),
+ zap.Int("batches", len(p.mapExtracts)),
+ zap.Strings("entities", extractEntitiesDebug(p.mapExtracts)),
+ zap.Strings("concepts", extractConceptDebug(p.mapExtracts)),
+ zap.Strings("claims", extractClaimDebug(p.mapExtracts)),
+ zap.Strings("relations", extractRelationDebug(p.mapExtracts)))
+ reduced := reduceExtracts(p.mapExtracts)
+ p.reduced = reduced
+ appcommon.Debug("wiki: REDUCE exact done",
+ zap.String("dataset_id", p.datasetID),
+ zap.String("doc_id", p.runKey()),
+ zap.Strings("entities", entitiesDebug(reduced.Entities)),
+ zap.Strings("concepts", conceptsDebug(reduced.Concepts)),
+ zap.Int("claims", len(reduced.Claims)),
+ zap.Strings("relations", relationsDebug(reduced.Relations)))
// Layer embedding + LLM disambiguation onto the exact-merged entities
// (REDUCE enhancement; concepts keep exact dedup). Degrades to a no-op when
// the embedder/chat seams are unavailable.
- p.reduced.Entities = p.dedupeEntities(p.reduced.Entities)
+ preDedup := reduced.Entities
+ p.reduced.Entities = p.dedupeEntities(preDedup)
+ appcommon.Debug("wiki: REDUCE embedding+LLM dedup done",
+ zap.String("dataset_id", p.datasetID),
+ zap.String("doc_id", p.runKey()),
+ zap.Int("pre_dedup_entities", len(preDedup)),
+ zap.Int("post_dedup_entities", len(p.reduced.Entities)),
+ zap.Strings("pre_dedup_names", entityNamesDebug(preDedup)),
+ zap.Strings("post_dedup_names", entityNamesDebug(p.reduced.Entities)),
+ zap.Strings("post_dedup_entities", entitiesDebug(p.reduced.Entities)))
appcommon.Info("wiki: REDUCE done",
zap.String("dataset_id", p.datasetID),
zap.String("doc_id", p.runKey()),
@@ -350,6 +375,14 @@ func (p *wikiPipeline) run() error {
zap.String("doc_id", p.runKey()),
zap.Int("plan_pages", len(plan.Pages)),
zap.Int("capacity_excluded", p.planCapacityExcluded))
+ appcommon.Debug("wiki: PLAN intermediate",
+ zap.String("dataset_id", p.datasetID),
+ zap.String("doc_id", p.runKey()),
+ zap.String("plan_title", plan.Title),
+ zap.String("plan_slug", plan.Slug),
+ zap.Int("plan_pages", len(plan.Pages)),
+ zap.Int("capacity_excluded", p.planCapacityExcluded),
+ zap.Strings("pages", planPagesDebug(plan.Pages)))
pages, err := p.runRefine()
if err != nil {
return err
@@ -359,6 +392,11 @@ func (p *wikiPipeline) run() error {
zap.String("dataset_id", p.datasetID),
zap.String("doc_id", p.runKey()),
zap.Int("refined_pages", len(pages)))
+ appcommon.Debug("wiki: REFINE intermediate",
+ zap.String("dataset_id", p.datasetID),
+ zap.String("doc_id", p.runKey()),
+ zap.Int("refined_pages", len(pages)),
+ zap.Strings("pages", pageResultsDebug(pages)))
return nil
}
@@ -371,6 +409,120 @@ func countExtractEntities(extracts []wikiExtract) int {
return n
}
+// --- Debug helpers: compact per-item descriptors for the stage-intermediate
+// logs in run(). They deliberately record only identity-level fields (name /
+// slug / type / aliases / endpoints), never full content or embedding vectors,
+// so a batch with many entities stays readable and does not blow the log line.
+
+func entityDebug(e wikiEntity) string {
+ if len(e.Aliases) == 0 {
+ return fmt.Sprintf("%s(%s)", e.Name, e.Type)
+ }
+ return fmt.Sprintf("%s(%s){aliases:%s}", e.Name, e.Type, strings.Join(e.Aliases, ","))
+}
+
+func conceptDebug(c wikiConcept) string {
+ return c.Term
+}
+
+func claimDebug(c wikiClaim) string {
+ return c.Subject + ": " + c.Statement
+}
+
+func relationDebug(r wikiRelation) string {
+ return fmt.Sprintf("%s-[%s]->%s", r.From, r.Type, r.To)
+}
+
+func extractEntitiesDebug(extracts []wikiExtract) []string {
+ var out []string
+ for bi, ex := range extracts {
+ for _, e := range ex.Entities {
+ out = append(out, fmt.Sprintf("batch%d:%s", bi, entityDebug(e)))
+ }
+ }
+ return out
+}
+
+func extractConceptDebug(extracts []wikiExtract) []string {
+ var out []string
+ for bi, ex := range extracts {
+ for _, c := range ex.Concepts {
+ out = append(out, fmt.Sprintf("batch%d:%s", bi, conceptDebug(c)))
+ }
+ }
+ return out
+}
+
+func extractClaimDebug(extracts []wikiExtract) []string {
+ var out []string
+ for bi, ex := range extracts {
+ for _, c := range ex.Claims {
+ out = append(out, fmt.Sprintf("batch%d:%s", bi, claimDebug(c)))
+ }
+ }
+ return out
+}
+
+func extractRelationDebug(extracts []wikiExtract) []string {
+ var out []string
+ for bi, ex := range extracts {
+ for _, r := range ex.Relations {
+ out = append(out, fmt.Sprintf("batch%d:%s", bi, relationDebug(r)))
+ }
+ }
+ return out
+}
+
+func entitiesDebug(es []wikiEntity) []string {
+ out := make([]string, 0, len(es))
+ for _, e := range es {
+ out = append(out, entityDebug(e))
+ }
+ return out
+}
+
+func entityNamesDebug(es []wikiEntity) []string {
+ out := make([]string, 0, len(es))
+ for _, e := range es {
+ out = append(out, e.Name)
+ }
+ return out
+}
+
+func conceptsDebug(cs []wikiConcept) []string {
+ out := make([]string, 0, len(cs))
+ for _, c := range cs {
+ out = append(out, conceptDebug(c))
+ }
+ return out
+}
+
+func relationsDebug(rs []wikiRelation) []string {
+ out := make([]string, 0, len(rs))
+ for _, r := range rs {
+ out = append(out, relationDebug(r))
+ }
+ return out
+}
+
+func planPagesDebug(pages []wikiPlanPage) []string {
+ out := make([]string, 0, len(pages))
+ for _, p := range pages {
+ out = append(out, fmt.Sprintf("%s:%s(%s) prio=%d related=%s",
+ p.Slug, p.Title, p.PageType, p.Priority, strings.Join(p.RelatedKB, ",")))
+ }
+ return out
+}
+
+func pageResultsDebug(pages []wikiPageResult) []string {
+ out := make([]string, 0, len(pages))
+ for _, p := range pages {
+ out = append(out, fmt.Sprintf("%s:%s(%s) outlinks=%s",
+ p.Slug, p.Title, p.PageType, strings.Join(p.Outlinks, ",")))
+ }
+ return out
+}
+
func (p *wikiPipeline) runMap() error {
// Keep each batch small enough that the LLM's entity/relation JSON output
// for the batch stays well under the model's output-token limit. 2048 input
@@ -1246,17 +1398,34 @@ func normalizeWikiPlan(plan wikiPlan, docID string, reduced wikiExtract) wikiPla
}
func normalizeWikiPlanPages(pages []wikiPlanPage, reduced wikiExtract) []wikiPlanPage {
- existing := map[string]wikiPlanPage{}
+ // existing maps a plan slug to its index in out (exact-slug dedup); byTitle
+ // additionally collapses pages that share the same page_type + normalized
+ // title, so a single batch never yields two "吕布" pages whose slugs only
+ // differ in transliteration (lu-bu vs lv-bu). Scheme A (§5 of the duplicates
+ // research): page identity is page_type/slug, so the key is page-type-scoped
+ // (a same-title concept and topic may legitimately coexist) and the first
+ // page seen (LLM emission order) is kept, folding the duplicate's RelatedKB
+ // into the survivor so its outlinks are not silently dropped.
+ existing := map[string]int{}
+ byTitle := map[string]int{}
out := make([]wikiPlanPage, 0, len(pages))
for _, page := range pages {
page = normalizeWikiPlanPage(page)
if page.Slug == "" {
continue
}
- if _, ok := existing[page.Slug]; ok {
+ if idx, ok := existing[page.Slug]; ok {
+ out[idx].RelatedKB = uniqueStrings(append(out[idx].RelatedKB, page.RelatedKB...))
continue
}
- existing[page.Slug] = page
+ key := wikiTitleKey(page.PageType, page.Title)
+ if idx, ok := byTitle[key]; ok {
+ out[idx].RelatedKB = uniqueStrings(append(out[idx].RelatedKB, page.RelatedKB...))
+ continue
+ }
+ idx := len(out)
+ byTitle[key] = idx
+ existing[page.Slug] = idx
out = append(out, page)
}
fallback := buildWikiFallbackPages(reduced)
@@ -1265,7 +1434,10 @@ func normalizeWikiPlanPages(pages []wikiPlanPage, reduced wikiExtract) []wikiPla
if page.Slug == "" {
continue
}
- existing[page.Slug] = page
+ page = normalizeWikiPlanPage(page)
+ idx := len(out)
+ byTitle[wikiTitleKey(page.PageType, page.Title)] = idx
+ existing[page.Slug] = idx
out = append(out, page)
}
}
@@ -1365,6 +1537,14 @@ func normalizeWikiPlanSections(sections []wikiPlanSection) []wikiPlanSection {
return out
}
+// wikiTitleKey returns the Scheme-A dedup key for a planned page: page_type
+// scoped to the normalized title. Two pages collapse only when both their type
+// and their (case/whitespace-normalized) title match — a same-title concept and
+// topic stay distinct because page identity is page_type/slug.
+func wikiTitleKey(pageType, title string) string {
+ return strings.TrimSpace(pageType) + "\x00" + normKey(title)
+}
+
func buildWikiFallbackPages(reduced wikiExtract) []wikiPlanPage {
var out []wikiPlanPage
seen := map[string]bool{}
diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go
index 2cf03c3b07..3402354d60 100644
--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go
+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go
@@ -144,6 +144,71 @@ func TestNormalizeWikiPlanPages_FallbacksToEntitiesAndConcepts(t *testing.T) {
}
}
+func TestNormalizeWikiPlanPages_DedupSameTypeSameTitle(t *testing.T) {
+ // Scheme A: two entity pages with the same title but different slug
+ // transliterations (lu-bu vs lv-bu) collapse to one, and their RelatedKB
+ // is folded into the survivor. A same-title page of another type (topic)
+ // stays distinct.
+ pages := []wikiPlanPage{
+ {
+ Slug: "entity/lu-bu",
+ Title: "吕布",
+ PageType: "entity",
+ Topic: "吕布",
+ EntityNames: []string{"吕布", "李肃"},
+ RelatedKB: []string{"entity/dong-zhuo"},
+ Priority: 1,
+ },
+ {
+ Slug: "entity/lv-bu",
+ Title: "吕布",
+ PageType: "entity",
+ Topic: "吕布",
+ EntityNames: []string{"吕布", "丁原"},
+ RelatedKB: []string{"entity/ding-yuan", "entity/dong-zhuo"},
+ Priority: 2,
+ },
+ {
+ Slug: "topic/lu-bu-legend",
+ Title: "吕布",
+ PageType: "topic",
+ Topic: "吕布生平",
+ Priority: 3,
+ },
+ }
+ got := normalizeWikiPlanPages(pages, wikiExtract{})
+ if len(got) != 2 {
+ t.Fatalf("normalizeWikiPlanPages = %d pages, want 2 (entity merged + topic)", len(got))
+ }
+ // Survivor keeps the first-emitted slug and accumulates RelatedKB.
+ var entitySlug string
+ for _, p := range got {
+ if p.PageType == "entity" {
+ entitySlug = p.Slug
+ want := []string{"entity/dong-zhuo", "entity/ding-yuan"}
+ if len(p.RelatedKB) != 2 {
+ t.Fatalf("entity RelatedKB = %v, want %v (deduped union)", p.RelatedKB, want)
+ }
+ }
+ }
+ if entitySlug != "entity/lu-bu" {
+ t.Fatalf("surviving entity slug = %q, want entity/lu-bu (first emitted)", entitySlug)
+ }
+}
+
+func TestNormalizeWikiPlanPages_DoesNotMergeAcrossTypes(t *testing.T) {
+ // Same title, different page_type: concept vs entity must stay distinct
+ // (page identity is page_type/slug).
+ pages := []wikiPlanPage{
+ {Slug: "concept/lu-bu", Title: "吕布", PageType: "concept", Priority: 1},
+ {Slug: "entity/lu-bu", Title: "吕布", PageType: "entity", Priority: 2},
+ }
+ got := normalizeWikiPlanPages(pages, wikiExtract{})
+ if len(got) != 2 {
+ t.Fatalf("normalizeWikiPlanPages = %d pages, want 2 (concept + entity)", len(got))
+ }
+}
+
func TestMergePlanCandidates_DeduplicatesWithoutLLMMerge(t *testing.T) {
p := &wikiPipeline{
docID: "doc-1",
diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.go
index fcc6bfd931..8f7a61b38f 100644
--- a/internal/ingestion/knowledge_compile/consumer.go
+++ b/internal/ingestion/knowledge_compile/consumer.go
@@ -63,7 +63,7 @@ func NewConsumer(scheduler Claimer, opts ...Option) *Consumer {
heartbeat: 20 * time.Second,
pollInterval: 2 * time.Second,
sweepInterval: 30 * time.Second,
- mergeThreshold: 0.99,
+ mergeThreshold: 0.85,
tombs: map[string]map[string]uint64{},
}
for _, o := range opts {
@@ -290,7 +290,19 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entries
deduper, err := c.factory(tenant)
if err != nil || deduper == nil {
- deduper = NewNoopDeduper()
+ // A no-op deduper would silently disable dataset-level LLM merging: every
+ // candidate (including e.g. a "吕布" wiki_page) would be written as its own
+ // merged row and duplicates would accumulate across runs. That degradation
+ // is never acceptable here, so fail loudly instead of papering over it.
+ common.Fatal("knowledge_compile: dataset-level LLM deduper unavailable, refusing to continue with a no-op merge",
+ zap.String("kb_id", kb),
+ zap.String("tenant_id", tenant),
+ zap.String("factory_err", func() string {
+ if err != nil {
+ return err.Error()
+ }
+ return "deduper factory returned nil"
+ }()))
}
deletedSet := make(map[string]bool, len(deleted))
@@ -344,6 +356,52 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entries
if err != nil {
return err
}
+ // Diagnostics: after batch dedup, verify the per-doc products still carry
+ // their embedding before the KNN/merge path consumes cand.Vector. If
+ // candidates show 0 vectors while incoming had vectors, Dedup is dropping
+ // them and both the KNN lookups and the merged rows lose embeddings.
+ {
+ incomingVec, candVec := 0, 0
+ for _, p := range incoming {
+ if len(p.Vector) > 0 {
+ incomingVec++
+ }
+ }
+ for _, p := range candidates {
+ if len(p.Vector) > 0 {
+ candVec++
+ }
+ }
+ common.Info("knowledge_compile: dedup vector audit",
+ zap.String("kb_id", kb),
+ zap.Int("incoming", len(incoming)),
+ zap.Int("incoming_with_vector", incomingVec),
+ zap.Int("candidates", len(candidates)),
+ zap.Int("candidates_with_vector", candVec))
+ }
+
+ // Diagnostics: break the KNN-input set down by variant and list the wiki_page
+ // slugs, so the reader can reconcile the number of wiki_page candidates here
+ // against the wiki_page rows WriteMerged emits (they differ when KNN merges a
+ // candidate into an existing merged row, when DecideBatch returns distinct new
+ // rows, or when candidates of the same variant collapse in-memory). This is
+ // what explains "39 wiki_page written vs 21 KNN requests".
+ {
+ byVariant := map[string]int{}
+ var wikiPageSlugs []string
+ for _, cand := range candidates {
+ v := string(cand.Variant)
+ byVariant[v]++
+ if v == "wiki_page" {
+ wikiPageSlugs = append(wikiPageSlugs, candidateIdentity(cand))
+ }
+ }
+ common.Info("knowledge_compile: candidates breakdown",
+ zap.String("kb_id", kb),
+ zap.Int("candidates", len(candidates)),
+ zap.Any("by_variant", byVariant),
+ zap.Strings("wiki_page_candidate_slugs", wikiPageSlugs))
+ }
// Then dedup each candidate against the DocEngine via KNN top1 + LLM judge,
// mirroring Python _struct_doc_storage_dedup_batch. Candidates that KNN-hit
@@ -382,6 +440,10 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entries
if hit.ID == "" {
// No sufficiently-similar merged row: insert the candidate as a new
// merged row.
+ common.Debug("knowledge_compile: KNN no hit, candidate becomes new merged row",
+ zap.String("kb_id", kb),
+ zap.String("candidate", candidateIdentity(cand)),
+ zap.String("variant", string(cand.Variant)))
cand.Merged = true
cand.DocID = kb
unmatchedMu.Lock()
@@ -389,6 +451,12 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entries
unmatchedMu.Unlock()
return nil
}
+ common.Debug("knowledge_compile: KNN hit",
+ zap.String("kb_id", kb),
+ zap.String("candidate", candidateIdentity(cand)),
+ zap.String("variant", string(cand.Variant)),
+ zap.String("hit", candidateIdentity(hit)),
+ zap.Float64("score", score))
groupsMu.Lock()
g := groupsByID[hit.ID]
if g == nil {
@@ -409,6 +477,29 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entries
// updated existing rows and the candidates judged distinct (new rows).
var newMerged []kccommon.Product
if len(groupsByID) > 0 {
+ // Diagnostics: summarize the KNN groups before the LLM merge decision so
+ // the reader can see which existing merged rows were hit and by how many
+ // candidates (e.g. whether a "吕布" candidate hit an existing 吕布 row and
+ // was then judged by the LLM).
+ {
+ type groupDump struct {
+ Existing string `json:"existing"`
+ Candidates []string `json:"candidates"`
+ Score float64 `json:"score"`
+ }
+ dumps := make([]groupDump, 0, len(groupsByID))
+ for _, g := range groupsByID {
+ cands := make([]string, 0, len(g.candidates))
+ for _, cand := range g.candidates {
+ cands = append(cands, candidateIdentity(cand))
+ }
+ dumps = append(dumps, groupDump{Existing: candidateIdentity(g.existing), Candidates: cands, Score: g.score})
+ }
+ common.Info("knowledge_compile: KNN groups for DecideBatch",
+ zap.String("kb_id", kb),
+ zap.Int("groups", len(groupsByID)),
+ zap.Any("groups_detail", dumps))
+ }
batched := make([]MergeGroup, 0, len(groupsByID))
for _, g := range groupsByID {
batched = append(batched, MergeGroup{
@@ -461,3 +552,19 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entries
zap.Int("merged_rows_written", len(mergedFinal)))
return nil
}
+
+// candidateIdentity returns a compact identity string for a product, preferring
+// the wiki slug (full "<page_type>/<slug>") when present, else the id/doc_id,
+// so KNN/dedup diagnostics can tie a candidate to the page it belongs to.
+func candidateIdentity(p kccommon.Product) string {
+ if slug, _ := p.Meta["slug"].(string); slug != "" {
+ return slug
+ }
+ if id := p.ID; id != "" {
+ return id
+ }
+ if docID := p.DocID; docID != "" {
+ return docID
+ }
+ return "<unknown>"
+}
diff --git a/internal/ingestion/knowledge_compile/dedup.go b/internal/ingestion/knowledge_compile/dedup.go
index 6cc775769c..3018b4b157 100644
--- a/internal/ingestion/knowledge_compile/dedup.go
+++ b/internal/ingestion/knowledge_compile/dedup.go
@@ -67,19 +67,19 @@ type llmDeduper struct {
}
// NewLLMDeduper builds a KB-scoped deduper from the runtime chat/embed deps.
-// llmMaxTokens is the chat model's token budget (0 disables per-batch token
-// splitting in DecideBatch).
-func NewLLMDeduper(chat kccommon.ChatInvoker, embed kccommon.Embedder, llmID string, threshold float64, llmMaxTokens int) Deduper {
+// modelContentLength is the chat model's context window (content_length) and
+// modelMaxOutput its generation cap (max_output); both bound how many pairs a
+// single DecideBatch sub-call may judge (a value <= 0 disables per-batch token
+// splitting). Together they keep every sub-call inside the input window and the
+// output cap, so a large candidate set never overflows either.
+func NewLLMDeduper(chat kccommon.ChatInvoker, embed kccommon.Embedder, llmID string, threshold float64, modelContentLength, modelMaxOutput int) Deduper {
decider := structure.NewLLMMergeDecider(chat, llmID, embed, threshold)
- decider.SetMaxBatchTokens(llmMaxTokens)
+ decider.SetMaxBatchTokens(modelContentLength, modelMaxOutput)
// Share the process-wide, vCPU-sized compiler pool so DecideBatch's
// token-bounded sub-batches run concurrently with the rest of the pipeline
- // (LLM-bounded), all under one concurrency limit. SubmitCompilerJobs enqueues
- // every sub-batch then waits on their futures on the caller goroutine — it
- // never blocks on a single job, so a stopped pool returns an error instead of
- // hanging DecideBatch.
- decider.SetSubmitter(func(ctx context.Context, fn func() error) error {
- return SubmitCompilerJobs(ctx, []CompilerJob{fn})
+ // (LLM-bounded), all under one concurrency limit.
+ decider.SetSubmitter(func(ctx context.Context, jobs []func() error) error {
+ return SubmitCompilerJobs(ctx, jobs)
})
return &llmDeduper{group: structure.NewGroupedDeduper(decider), decider: decider, embed: embed}
}
diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.go
index 661c191d63..7141fe6c57 100644
--- a/internal/ingestion/knowledge_compile/reader.go
+++ b/internal/ingestion/knowledge_compile/reader.go
@@ -21,6 +21,9 @@ import (
"fmt"
"strconv"
+ "go.uber.org/zap"
+
+ "ragflow/internal/common"
"ragflow/internal/engine"
"ragflow/internal/engine/types"
kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"
@@ -73,10 +76,18 @@ var compiledSelectFields = []string{
// compiledSelectFields) that must survive the doc→merge round-trip so the
// dataset-level merged rows keep the fields the artifact API and page renderers
// depend on (page_type_kwd/topic_kwd/title_kwd/...).
+//
+// "q_*_vec" selects the (dimension-agnostic) embedding column. Without it,
+// LoadDocProducts reconstructs products with an empty Vector, so merged rows
+// carry no embedding and the dataset-level KNN dedup (SearchSimilar on
+// available_int=1 + q_<dim>_vec) can never match an existing wiki page — the
+// graph keeps accumulating cross-run duplicates. ES accepts the wildcard in
+// both _source includes and the fields parameter.
var wikiSelectFields = []string{
"page_type_kwd", "topic_kwd", "title_kwd",
"entity_names_kwd", "summary_with_weight",
"related_kb_pages_kwd", "outlinks_kwd", "section_level_int",
+ "q_*_vec",
}
// loadDocProductsLimit is the per-page size used when scrolling a single
@@ -124,6 +135,24 @@ func (r engineReader) LoadDocProducts(ctx context.Context, tenant, kb, docID str
}
offset += loadDocProductsLimit
}
+ // Diagnostics: count how many loaded per-doc products carry an embedding
+ // and report the (deduped) vector dimensions. If this shows 0/empty while
+ // per-doc rows in ES do have q_<dim>_vec, VectorFromChunkMap is failing to
+ // restore the vector and the merged rows will end up embedding-less.
+ vecCount := 0
+ dims := map[int]int{}
+ for _, p := range out {
+ if dim := len(p.Vector); dim > 0 {
+ vecCount++
+ dims[dim]++
+ }
+ }
+ common.Info("knowledge_compile: LoadDocProducts vector audit",
+ zap.String("kb_id", kb),
+ zap.String("doc_id", docID),
+ zap.Int("products", len(out)),
+ zap.Int("with_vector", vecCount),
+ zap.Any("vector_dims", dims))
return out, nil
}
@@ -262,7 +291,11 @@ func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, vari
if !ok || !p.Merged {
continue
}
- score := toFloat64(c["score"])
+ // The DocEngine stores the dense similarity in the "_score" key (mirroring
+ // ES's _score), not "score". Reading the wrong key yields 0 and misleads
+ // downstream diagnostics (KNN groups logging). The value is informational
+ // only — KNN eligibility is enforced by the engine's min_score filter.
+ score := toFloat64(c["_score"])
return p, score, nil
}
return kccommon.Product{}, 0, nil
diff --git a/internal/ingestion/knowledge_compile/service.go b/internal/ingestion/knowledge_compile/service.go
index 8141605b9c..4acbb0975a 100644
--- a/internal/ingestion/knowledge_compile/service.go
+++ b/internal/ingestion/knowledge_compile/service.go
@@ -101,7 +101,7 @@ func defaultDeduperFactory(tenant string) (Deduper, error) {
if err != nil {
return nil, err
}
- return NewLLMDeduper(deps.Chat, deps.Embed, defaultLLMID, 0.99, deps.ModelContextLen), nil
+ return NewLLMDeduper(deps.Chat, deps.Embed, defaultLLMID, 0.99, deps.ModelContextLen, deps.ModelMaxOutput), nil
}
func generateHolder() string {
diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.go
index 6765b725de..15a006cbbe 100644
--- a/internal/ingestion/knowledge_compile/writer.go
+++ b/internal/ingestion/knowledge_compile/writer.go
@@ -107,6 +107,25 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, produc
now := time.Now()
runID := utility.GenerateUUID()
inputHash := mergedInputHash(products)
+ // Diagnostics: before building merged rows, confirm the incoming products
+ // actually carry an embedding. mergedChunkMap only writes q_<dim>_vec when
+ // len(p.Vector)>0; if this reports 0 vectors while LoadDocProducts/dedup
+ // audits reported vectors, the vector is being dropped somewhere between
+ // dedup and WriteMerged.
+ {
+ vecCount, dims := 0, map[int]int{}
+ for _, p := range products {
+ if dim := len(p.Vector); dim > 0 {
+ vecCount++
+ dims[dim]++
+ }
+ }
+ common.Info("knowledge_compile: WriteMerged vector audit",
+ zap.String("kb_id", kb),
+ zap.Int("products", len(products)),
+ zap.Int("with_vector", vecCount),
+ zap.Any("vector_dims", dims))
+ }
// Shard the rows and drive the inserts through the shared global pool
// (docengine-bounded) instead of one monolithic InsertChunks call.
jobs := make([]CompilerJob, 0, (len(products)+writeMergedBatchSize-1)/writeMergedBatchSize)
@@ -606,6 +625,10 @@ func (w engineWriter) loadMergedWikiPages(ctx context.Context, tenant, kb string
"slug_kwd", "page_type_kwd", "title_kwd",
"entity_names_kwd", "summary_with_weight", "outlinks_kwd",
"source_doc_ids", "source_chunk_ids",
+ // compile_kwd is selected purely so the query-result telemetry
+ // below can confirm the Search filter is honoured (without it the
+ // "compile_kwd_seen" audit would always read as {"":n}).
+ "compile_kwd",
},
Limit: batchSize,
Offset: offset,
diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.go
index 8a232caea2..9c9540c9ff 100644
--- a/internal/ingestion/task/knowledge_compiler_wiring.go
+++ b/internal/ingestion/task/knowledge_compiler_wiring.go
@@ -33,8 +33,9 @@ import (
"ragflow/internal/ingestion/knowledge_compile"
"ragflow/internal/service"
- "gorm.io/gorm"
appcommon "ragflow/internal/common"
+
+ "gorm.io/gorm"
)
// This file is the composition-root wiring for the KnowledgeCompiler ingestion
@@ -96,7 +97,20 @@ func newKnowledgeCompilerDepsResolver() kc.DepsResolver {
svc := service.NewModelProviderService()
return func(tenantID, llmID, embeddingModel string) (kc.Deps, error) {
if strings.TrimSpace(llmID) == "" {
- return kc.Deps{}, fmt.Errorf("knowledge_compiler: llm_id is required for production deps resolution")
+ // No explicit chat model was supplied (e.g. the dataset-level deduper
+ // is seeded with a global default that may be empty). Resolve the
+ // tenant's default chat model so cross-document LLM merging can
+ // actually run instead of failing / falling back to a no-op. Use the
+ // composite "<model>@<instance>@<provider>" reference (not the bare
+ // model name) so later Chat/ResolveModelConfig round-trips can locate
+ // the provider.
+ defaultRef, derr := svc.GetTenantDefaultModelRef(context.Background(), tenantID, entity.ModelTypeChat)
+ if derr != nil || strings.TrimSpace(defaultRef) == "" {
+ return kc.Deps{}, fmt.Errorf("knowledge_compiler: llm_id is empty and no tenant default chat model available: %w", derr)
+ }
+ llmID = defaultRef
+ // Keep the fall-through path below for ModelContextLen resolution so
+ // both explicit and default model refs share one context-window path.
}
// Resolve the chat model's context window so RAPTOR can truncate each
// cluster's texts to fit the LLM context (mirrors Python self._llm_model.max_length).
@@ -111,6 +125,16 @@ func newKnowledgeCompilerDepsResolver() kc.DepsResolver {
if ml, merr := svc.ResolveModelContextLength(ctx, tenantID, llmID); merr == nil && ml > 0 {
llmMax = ml
}
+ // Resolve the model's generation cap (max_output). Cross-document merge
+ // judging packs many pairs into one LLM call; the batch must be bounded by
+ // BOTH the input window and this output cap, so a large candidate set
+ // never overflows max_output and yields a truncated/non-JSON reply. This
+ // uses max_tokens (the generation cap), NOT content_length — see
+ // ResolveModelContextLength's comment.
+ llmMaxOutput := 0
+ if _, _, _, mo, merr := svc.ResolveModelConfig(ctx, tenantID, entity.ModelTypeChat, llmID); merr == nil && mo > 0 {
+ llmMaxOutput = mo
+ }
return kc.Deps{
Chat: &kcChatInvoker{svc: svc, tenantID: tenantID, llmID: llmID},
@@ -120,6 +144,7 @@ func newKnowledgeCompilerDepsResolver() kc.DepsResolver {
// datasetnav lock). They are wired separately when the
// surrounding pipeline supplies the backing services.
ModelContextLen: llmMax,
+ ModelMaxOutput: llmMaxOutput,
}, nil
}
}
@@ -161,15 +186,25 @@ func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatR
// is never cached, so each attempt issues a fresh request. Permanent
// configuration/model errors (auth, unknown model) are not retried.
var resp *models.ChatResponse
- retryErr := appcommon.RetryWithBackoff(ctx, kcChatRetryMax, kcChatRetryDelay, func() error {
- r, err := c.svc.Chat(ctx, c.tenantID, llmID, msgs, config)
+ call := func() error {
+ // Bound each attempt to a short deadline so a stalled LLM provider (e.g.
+ // MiniMax hanging on a large merge-judge prompt) surfaces a timeout
+ // quickly instead of blocking a compile sub-batch for minutes; the
+ // retry/backoff loop above then handles it as a transient failure.
+ attemptCtx, cancel := context.WithTimeout(ctx, kcChatAttemptTimeout)
+ defer cancel()
+ r, err := c.svc.Chat(attemptCtx, c.tenantID, llmID, msgs, config)
if err != nil {
return err
}
resp = r
return nil
- }, appcommon.IsTransientError)
- if retryErr != nil {
+ }
+ if req.DisableRetry {
+ if err := call(); err != nil {
+ return nil, err
+ }
+ } else if retryErr := appcommon.RetryWithBackoff(ctx, kcChatRetryMax, kcChatRetryDelay, call, appcommon.IsTransientError); retryErr != nil {
return nil, retryErr
}
content := ""
@@ -182,7 +217,13 @@ func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatR
// kcChatRetryMax bounds how many times a transient LLM transport failure is
// retried. Each attempt may run up to the driver's HTTP timeout, so the count
// stays small to avoid unbounded wall-clock latency inside one compile.
-const kcChatRetryMax = 2
+const kcChatRetryMax = 5
+
+// kcChatAttemptTimeout bounds a single Chat call (per retry attempt). A stalled
+// provider must surface a timeout promptly rather than hold a compile sub-batch;
+// 3 minutes is long enough for a big merge-judge prompt yet short enough that
+// several failed attempts do not stall the pipeline for many minutes.
+const kcChatAttemptTimeout = 3 * time.Minute
// kcChatRetryDelay is the initial exponential-backoff delay between retries.
const kcChatRetryDelay = 2 * time.Second
diff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.go
index 7de10643de..c43f925af5 100644
--- a/internal/service/dataset_artifact_service.go
+++ b/internal/service/dataset_artifact_service.go
@@ -20,6 +20,9 @@ import (
"sort"
"strings"
+ "golang.org/x/text/collate"
+ "golang.org/x/text/language"
+
"ragflow/internal/engine"
"ragflow/internal/engine/types"
"ragflow/internal/service/nav"
@@ -377,10 +380,23 @@ func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, d
it.PageCount = counts[t]
items = append(items, it)
}
- sort.Slice(items, func(i, j int) bool { return items[i].Topic < items[j].Topic })
+ // Sort topics by a deterministic rule. Plain UTF-8 byte order is chaotic for
+ // CJK (it sorts by Unicode code point, unrelated to pinyin/stroke). We use a
+ // CLDR-based collator (golang.org/x/text/collate) with the Chinese locale,
+ // which orders Chinese by pinyin and handles Latin/digits/other scripts
+ // correctly, case-insensitively, without assuming topics are all-Chinese.
+ sort.Slice(items, func(i, j int) bool {
+ return wikiTopicCollator.CompareString(items[i].Topic, items[j].Topic) < 0
+ })
return items, int64(len(items)), nil
}
+// wikiTopicCollator is a process-wide collator for wiki topics. language.Chinese
+// selects the CLDR zh collation (pinyin-based for Han), while Latin/digits and
+// other scripts sort naturally, case-insensitively. It is safe for concurrent
+// use after construction.
+var wikiTopicCollator = collate.New(language.Chinese)
+
// WikiGraph is the entity/relation graph for a dataset's wiki artifacts.
type WikiGraph struct {
Entities []WikiGraphEntity `json:"entities"`
diff --git a/internal/service/model_service.go b/internal/service/model_service.go
index a7eb487350..adb27d2eaf 100644
--- a/internal/service/model_service.go
+++ b/internal/service/model_service.go
@@ -3451,6 +3451,24 @@ func (m *ModelProviderService) GetTenantDefaultModelByType(ctx context.Context,
return m.ResolveModelConfig(ctx, tenantID, modelType, modelName)
}
+// GetTenantDefaultModelRef returns the composite model reference (the
+// "<model>@<instance>@<provider>" form stored on the tenant, e.g.
+// "MiniMax-M3@mm@MiniMax") that callers pass on to model config resolution.
+// Unlike GetTenantDefaultModelByType — which resolves the driver/API config and
+// returns a bare model name — this returns the reference verbatim so it can be
+// used as an llmID for a later ResolveModelConfig/Chat round-trip.
+func (m *ModelProviderService) GetTenantDefaultModelRef(ctx context.Context, tenantID string, modelType entity.ModelType) (string, error) {
+ tenant, err := m.tenantDAO.GetByID(ctx, dao.DB, tenantID)
+ if err != nil {
+ return "", fmt.Errorf("failed to get tenant: %s type %s: %w", tenantID, modelType, err)
+ }
+ modelName, _ := defaultModelRefs(tenant, modelType)
+ if strings.TrimSpace(modelName) == "" {
+ return "", fmt.Errorf("no default %s model is set", modelType)
+ }
+ return modelName, nil
+}
+
// GetModelConfigByID returns model driver and API config for a tenant_model row by its ID.
func (m *ModelProviderService) GetModelConfigByID(ctx context.Context, userID string, modelType entity.ModelType, modelID string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
common.Debug("GetModelConfigByID",