RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Refactor splitByTokens and wire wiki incremental compile (#18083)
Zhichang Yu64533e5b5e7d
diff --git a/go.mod b/go.modindex 698c7aabb1..363f210642 100644--- a/go.mod+++ b/go.mod@@ -62,6 +62,7 @@ require (go.uber.org/zap v1.27.1golang.org/x/crypto v0.53.0golang.org/x/net v0.56.0+ golang.org/x/oauth2 v0.36.0golang.org/x/sync v0.21.0golang.org/x/term v0.44.0golang.org/x/text v0.38.0@@ -211,7 +212,6 @@ require (go.yaml.in/yaml/v3 v3.0.4 // indirectgolang.org/x/arch v0.27.0 // indirectgolang.org/x/exp v0.0.0-20231226003508-02704c960a9b // indirect- golang.org/x/oauth2 v0.36.0 // indirectgolang.org/x/sys v0.46.0 // indirectgolang.org/x/time v0.15.0 // indirectgoogle.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94 // indirectdiff --git a/internal/ingestion/component/knowledge_compiler/common/common_test.go b/internal/ingestion/component/knowledge_compiler/common/common_test.goindex ac9c03d365..6748216a78 100644--- a/internal/ingestion/component/knowledge_compiler/common/common_test.go+++ b/internal/ingestion/component/knowledge_compiler/common/common_test.go@@ -1,9 +1,27 @@package commonimport (+ "context""testing")+type captureChat struct{ req ChatRequest }++func (c *captureChat) Chat(_ context.Context, req ChatRequest) (*ChatResponse, error) {+ c.req = req+ return &ChatResponse{Content: `{"ok":true}`}, nil+}++func TestGenJSONDisablesInvokerRetry(t *testing.T) {+ chat := &captureChat{}+ if _, err := GenJSON(context.Background(), chat, ChatRequest{UserPrompt: "test"}, 0); err != nil {+ t.Fatalf("GenJSON: %v", err)+ }+ if !chat.req.DisableRetry {+ t.Fatal("GenJSON must disable retries in the underlying ChatInvoker")+ }+}+func vec(a, b, c float32) []float32 { return []float32{a, b, c} }func TestMemStoreTopK(t *testing.T) {diff --git a/internal/ingestion/component/knowledge_compiler/common/deps.go b/internal/ingestion/component/knowledge_compiler/common/deps.goindex 2c4d00fefa..43caeff9d9 100644--- a/internal/ingestion/component/knowledge_compiler/common/deps.go+++ b/internal/ingestion/component/knowledge_compiler/common/deps.go@@ -28,6 +28,10 @@ type ChatRequest struct {MaxTokens *intAPIKey stringBaseURL string+ // DisableRetry tells the production ChatInvoker that the caller owns+ // transient-error retries (GenJSON is such a caller). It is internal+ // plumbing and is not part of the user-facing model request.+ DisableRetry bool}// ChatResponse holds the LLM's text answer.@@ -111,6 +115,13 @@ type Deps struct {// deriveWikiPlanBudget, buildClusterContent) use it to size the input/output// quotas (mirrors Python self._llm_model.max_length).ModelContextLen int+ // ModelMaxOutput is the chat model's generation cap (max_output), the most+ // tokens one LLM response may emit. Cross-document merge judging packs many+ // pairs into a single call, so the caller must cap the batch by both the+ // input window (ModelContextLen) and this output cap; otherwise a large+ // candidate set can overflow the model's max_output and it returns a+ // truncated/non-JSON reply. Mirrors Python's max_tokens generation config.+ ModelMaxOutput int}// DepsResolver resolves the per-run Deps from a tenant/llm/embedding triple.diff --git a/internal/ingestion/component/knowledge_compiler/common/jsonchat.go b/internal/ingestion/component/knowledge_compiler/common/jsonchat.goindex 4509ef4953..0e7a05e959 100644--- a/internal/ingestion/component/knowledge_compiler/common/jsonchat.go+++ b/internal/ingestion/component/knowledge_compiler/common/jsonchat.go@@ -49,6 +49,10 @@ func GenJSON(ctx context.Context, chat ChatInvoker, req ChatRequest, retryMax ..maxRetries = retryMax[0]}req.JSONMode = true+ // GenJSON owns retries for both transport failures and malformed JSON. Tell+ // the production ChatInvoker to make exactly one provider request per outer+ // GenJSON attempt, avoiding multiplicative nested retries.+ req.DisableRetry = truevar lastErr errordelay := jsonRetryDelayfor attempt := 0; attempt <= maxRetries; attempt++ {diff --git a/internal/ingestion/component/knowledge_compiler/structure/merge.go b/internal/ingestion/component/knowledge_compiler/structure/merge.goindex 5faa0333e8..d36f800b69 100644--- a/internal/ingestion/component/knowledge_compiler/structure/merge.go+++ b/internal/ingestion/component/knowledge_compiler/structure/merge.go@@ -6,14 +6,23 @@ import ("strings""sync"+ appcommon "ragflow/internal/common""ragflow/internal/ingestion/component/knowledge_compiler/common""ragflow/internal/tokenizer"++ "go.uber.org/zap")-// maxBatchTokenReserve is the share of the model's token budget kept in reserve-// for the system prompt, the batch instruction, and the model's JSON reply, so-// a single DecideBatch sub-call never overflows max_token.-const maxBatchTokenReserve = 0.15+// maxWindowUsage is the fraction of the model's limits a single+// DecideBatch sub-call may consume — applied both to the combined+// (input+output) budget (fraction of content_length) and to the output budget+// (fraction of max_output). The reply is an array of per-pair+// {"duplicated","merged"} objects, and each merged object is roughly the size of+// the longer of the two inputs, so the batch is split so the summed estimate+// stays under this cap. 0.2 is deliberately conservative (smaller than the+// default 0.6) so each sub-call stays well inside the window and generation cap+// even for models with tight limits, at the cost of more, smaller sub-calls.+const maxWindowUsage = 0.2// Merge prompts, verbatim from Python structure.py. The merge is driven by// the user-supplied prompts; the decision instruction lets us branch on the@@ -99,18 +108,25 @@ type LLMMergeDecider struct {Embed common.EmbedderThreshold float64- // maxBatchTokens caps the estimated prompt size of a single DecideBatch- // sub-call. When 0 (the default) the whole batch is sent in one call,- // preserving the historic behavior. When positive the batch is split into- // token-bounded sub-calls so a large candidate set can never overflow the- // model's max_token window.- maxBatchTokens int-- // submit runs one LLM sub-batch off the shared knowledge-compilation pool.+ // maxBatchTotal caps the estimated combined size (prompt input + JSON reply)+ // of a single DecideBatch sub-call. It guards the model's context window+ // (content_length): a chunk starts a fresh one when adding a pair would push+ // (input+output) past modelContentLength*maxWindowUsage. When 0 the+ // whole batch is sent in one call (historic behavior).+ maxBatchTotal int+ // maxBatchOutput caps the estimated reply (output) size of a single+ // DecideBatch sub-call. It guards the model's generation cap (max_output):+ // each merged reply is roughly the size of the longer of the two inputs in a+ // pair, so the batch is also split so the sum of per-pair output estimates+ // stays within modelMaxOutput*maxWindowUsage. A sub-call must satisfy+ // BOTH this cap and maxBatchTotal.+ maxBatchOutput int++ // submit runs LLM sub-batches off the shared knowledge-compilation pool.// When nil each sub-batch runs inline (sequentially). It is injected by the// knowledge_compile package so every stage shares one process-wide,// vCPU-sized concurrency bound.- submit func(ctx context.Context, fn func() error) error+ submit func(ctx context.Context, jobs []func() error) errormu sync.Mutexaliases map[string]string@@ -121,28 +137,44 @@ func NewLLMMergeDecider(chat common.ChatInvoker, llmID string, embed common.Embereturn &LLMMergeDecider{Chat: chat, LLMID: llmID, Embed: embed, Threshold: threshold, aliases: map[string]string{}}}-// SetMaxBatchTokens caps the estimated prompt size of a single DecideBatch-// sub-call. Non-positive values disable per-call batching. The effective budget-// is modelMaxTokens*(1-maxBatchTokenReserve), keeping headroom for the system-// prompt and the JSON reply.-func (d *LLMMergeDecider) SetMaxBatchTokens(modelMaxTokens int) {- if modelMaxTokens <= 0 {- d.maxBatchTokens = 0+// SetMaxBatchTokens sets the two token budgets that bound a single DecideBatch+// sub-call, derived from the model's two limits:+// - modelContentLength: the model's context window (content_length); the+// effective combined budget is modelContentLength*maxWindowUsage, i.e.+// a sub-call may not exceed that many tokens of (prompt input + JSON reply).+// - modelMaxOutput: the model's generation cap (max_output); the effective+// reply budget is modelMaxOutput*maxWindowUsage.+//+// modelContentLength <= 0 disables both budgets (historic whole-batch behavior);+// modelMaxOutput <= 0 falls back to combined-budget-only batching. splitByTokens+// then keeps every sub-call inside BOTH the combined budget and the output+// budget, so a large candidate set can never overflow the window nor the+// generation cap.+func (d *LLMMergeDecider) SetMaxBatchTokens(modelContentLength, modelMaxOutput int) {+ d.maxBatchOutput = 0+ if modelContentLength <= 0 {+ d.maxBatchTotal = 0return}- budget := int(float64(modelMaxTokens) * (1 - maxBatchTokenReserve))- if budget <= 0 {- d.maxBatchTokens = 0+ totalBudget := int(float64(modelContentLength) * maxWindowUsage)+ if totalBudget <= 0 {+ d.maxBatchTotal = 0return}- d.maxBatchTokens = budget+ d.maxBatchTotal = totalBudget+ if modelMaxOutput > 0 {+ outputBudget := int(float64(modelMaxOutput) * maxWindowUsage)+ if outputBudget > 0 {+ d.maxBatchOutput = outputBudget+ }+ }}// SetSubmitter injects the shared knowledge-compilation pool so DecideBatch can// run its token-bounded sub-batches concurrently. A nil submitter (the default)-// falls back to running sub-batches sequentially. The submitter must run fn on-// a bounded pool and return its error.-func (d *LLMMergeDecider) SetSubmitter(submit func(ctx context.Context, fn func() error) error) {+// falls back to running sub-batches sequentially. The submitter must enqueue+// all jobs on a bounded pool, wait for them, and return the first error.+func (d *LLMMergeDecider) SetSubmitter(submit func(ctx context.Context, jobs []func() error) error) {d.submit = submit}@@ -229,18 +261,23 @@ type BatchMergeResult struct {}// DecideBatch judges every pair and returns the verdicts in input order. It is-// the batched counterpart of Decide's single-pair judge. When a token budget is-// configured it splits the pairs into token-bounded sub-batches (each sub-batch-// still judged in a single LLM call, preserving pair order) so a large candidate-// set can never overflow the model's max_token window.+// the batched counterpart of Decide's single-pair judge. Besides deciding each+// duplicate pair it also performs the actual merge: for a pair the LLM judges as+// duplicated, mergePairsBatch has the model emit the combined payload (merged),+// which the caller re-embeds and persists in place of the existing row. When a+// token budget is configured it splits the pairs into sub-batches bounded by+// BOTH the model's combined (input+output) budget and its output cap, each+// sub-batch still judged in a single LLM call (preserving pair order), so a+// large candidate set can never overflow the context window nor the generation+// cap.func (d *LLMMergeDecider) DecideBatch(ctx context.Context, pairs []MergePairInput) ([]BatchMergeResult, error) {if len(pairs) == 0 {return nil, nil}- if d.maxBatchTokens <= 0 {+ if d.maxBatchTotal <= 0 && d.maxBatchOutput <= 0 {return mergePairsBatch(ctx, d.Chat, d.LLMID, pairs)}- chunks := splitByTokens(pairs, d.maxBatchTokens)+ chunks := splitByTokens(pairs, d.maxBatchTotal, d.maxBatchOutput)// The merge decisions are LLM-bounded, not CPU-bounded: run the token-bounded// sub-batches concurrently on the injected shared pool (vCPU-sized) when more// than one chunk exists. Order is preserved by writing each chunk's verdicts@@ -258,35 +295,22 @@ func (d *LLMMergeDecider) DecideBatch(ctx context.Context, pairs []MergePairInpureturn out, nil}results := make([][]BatchMergeResult, len(chunks))- var (- wg sync.WaitGroup- errOnce sync.Once- firstErr error- )+ jobs := make([]func() error, 0, len(chunks))for i, chunk := range chunks {i, chunk := i, chunk- wg.Add(1)- err := d.submit(ctx, func() error {- defer wg.Done()+ jobs = append(jobs, func() error {sub, err := mergePairsBatch(ctx, d.Chat, d.LLMID, chunk)if err != nil {- errOnce.Do(func() { firstErr = err })return err}results[i] = subreturn nil})- if err != nil {- // The job was never enqueued, so the closure's wg.Done() will never- // run — decrement here and record the failure so we don't deadlock- // on wg.Wait() and don't silently drop the submit error.- wg.Done()- errOnce.Do(func() { firstErr = err })- }}- wg.Wait()- if firstErr != nil {- return nil, firstErr+ // Submit the complete batch in one call. The shared pool owns fan-out and+ // bounded concurrency; the caller only waits for the pool's futures.+ if err := d.submit(ctx, jobs); err != nil {+ return nil, err}out := make([]BatchMergeResult, 0, len(pairs))for _, sub := range results {@@ -295,30 +319,70 @@ func (d *LLMMergeDecider) DecideBatch(ctx context.Context, pairs []MergePairInpureturn out, nil}-// splitByTokens groups pairs into contiguous chunks whose estimated prompt-// size stays within budget. The original (global) Index of every pair is-// preserved — the model echoes it back — so callers can still key verdicts by-// the input index. A single pair that alone exceeds budget is sent on its own-// (the model may still truncate, but we never silently drop it).-func splitByTokens(pairs []MergePairInput, budget int) [][]MergePairInput {+// splitByTokens groups pairs into contiguous chunks bounded by BOTH the model's+// combined budget (prompt input + JSON reply, guarding content_length) and its+// output budget (guarding max_output). The prompt size of a pair is roughly+// existing+incoming; the merged reply for a pair is roughly the longer of the+// two inputs (the merge emits the union of both objects). So a chunk starts a+// fresh one when adding a pair would push either the combined (input+output)+// estimate past totalBudget or the output estimate past outputBudget — keeping+// every sub-call inside the context window AND the generation cap. The original+// (global) Index of every pair is preserved — the model echoes it back — so+// callers can still key verdicts by the input index. A single pair that alone+// exceeds either budget is sent on its own (the model may still truncate, but we+// never silently drop it).+func splitByTokens(pairs []MergePairInput, totalBudget, outputBudget int) [][]MergePairInput {var chunks [][]MergePairInputcur := make([]MergePairInput, 0, len(pairs))- used := 0+ usedTotal := 0+ usedOutput := 0+ flush := func(usedTotal, usedOutput int) {+ // Diagnostic: expose each chunk's realized budget utilization so the+ // split can be tuned. usedTotal = Σ(input+output) and usedOutput = Σ+ // (longer input) of the chunk's pairs; both must stay ≤ their budget for+ // the sub-call to fit the model. "final" marks the last chunk flushed.+ appcommon.Info("knowledge_compiler: merge batch split flush",+ zap.Int("pairs", len(cur)),+ zap.Int("total_budget", totalBudget),+ zap.Int("output_budget", outputBudget),+ zap.Int("used_total", usedTotal),+ zap.Int("used_output", usedOutput))+ }+ appcommon.Info("knowledge_compiler: merge batch split start",+ zap.Int("pairs", len(pairs)),+ zap.Int("total_budget", totalBudget),+ zap.Int("output_budget", outputBudget))for _, p := range pairs {// The pair contents dominate the prompt size; the "Pair N:" wrappers- // are a negligible constant overhead per pair, ignored here.- est := tokenizer.NumTokensFromString(p.Existing) + tokenizer.NumTokensFromString(p.Incoming)- if len(cur) > 0 && used+est > budget {+ // are a negligible constant overhead per pair, ignored here. Each input+ // is tokenized once and reused for both the prompt and the output+ // estimate.+ estExisting := tokenizer.NumTokensFromString(p.Existing)+ estIncoming := tokenizer.NumTokensFromString(p.Incoming)+ estPrompt := estExisting + estIncoming+ // The merged reply for a pair is at most the size of its longer input+ // (we keep all existing fields and union in the incoming ones).+ estOutput := estExisting+ if estIncoming > estOutput {+ estOutput = estIncoming+ }+ overOutput := outputBudget > 0 && usedOutput+estOutput > outputBudget+ overTotal := totalBudget > 0 && usedTotal+estPrompt+estOutput > totalBudget+ if len(cur) > 0 && (overOutput || overTotal) {+ flush(usedTotal, usedOutput)chunks = append(chunks, cur)// Fresh backing array: cur[:0] shares the buffer with the chunk we// just appended, so the next iteration would overwrite it.cur = make([]MergePairInput, 0, len(pairs))- used = 0+ usedTotal = 0+ usedOutput = 0}cur = append(cur, p)- used += est+ usedTotal += estPrompt + estOutput+ usedOutput += estOutput}if len(cur) > 0 {+ flush(usedTotal, usedOutput)chunks = append(chunks, cur)}return chunksdiff --git a/internal/ingestion/component/knowledge_compiler/structure/structure_test.go b/internal/ingestion/component/knowledge_compiler/structure/structure_test.goindex 9103860ba5..4b409830fa 100644--- a/internal/ingestion/component/knowledge_compiler/structure/structure_test.go+++ b/internal/ingestion/component/knowledge_compiler/structure/structure_test.go@@ -678,9 +678,23 @@ func TestLLMMergeDeciderDecideBatchEmpty(t *testing.T) {func TestLLMMergeDeciderDecideBatchSplitsByTokenBudget(t *testing.T) {chat := &graphChat{}d := NewLLMMergeDecider(chat, "llm1", hashEmbedder{dim: 8}, 0.99)+ submittedBatches := 0+ submittedChunks := 0+ d.SetSubmitter(func(ctx context.Context, jobs []func() error) error {+ submittedBatches+++ submittedChunks = len(jobs)+ for _, job := range jobs {+ if err := job(); err != nil {+ return err+ }+ }+ return nil+ })// Tiny model budget → one pair per sub-batch (exercises the split path).- // 20 * (1-0.15) = 17 token budget, well under each ~40-token pair.- d.SetMaxBatchTokens(20)+ // Combined budget = 20 * 0.6 = 12 tokens, well under each ~40-token pair's+ // (input+output) estimate. Output budget disabled (0) → combined-budget-only+ // batching.+ d.SetMaxBatchTokens(20, 0)alpha := common.Product{ID: "row-alpha",@@ -711,6 +725,9 @@ func TestLLMMergeDeciderDecideBatchSplitsByTokenBudget(t *testing.T) {if chat.mergeCalls != before+3 {t.Errorf("token-split DecideBatch made %d LLM calls, want 3", chat.mergeCalls-before)}+ if submittedBatches != 1 || submittedChunks != 3 {+ t.Fatalf("token-split DecideBatch submitted %d batches/%d chunks, want one batch containing all chunks", submittedBatches, submittedChunks)+ }if len(results) != 3 {t.Fatalf("want 3 results, got %d", len(results))}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.goindex e5b03fc154..eeaa66779b 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go@@ -330,11 +330,36 @@ func (p *wikiPipeline) run() error {zap.String("doc_id", p.runKey()),zap.Int("batches", len(p.mapExtracts)),zap.Int("raw_entities", countExtractEntities(p.mapExtracts)))- p.reduced = reduceExtracts(p.mapExtracts)+ appcommon.Debug("wiki: MAP intermediate",+ zap.String("dataset_id", p.datasetID),+ zap.String("doc_id", p.runKey()),+ zap.Int("batches", len(p.mapExtracts)),+ zap.Strings("entities", extractEntitiesDebug(p.mapExtracts)),+ zap.Strings("concepts", extractConceptDebug(p.mapExtracts)),+ zap.Strings("claims", extractClaimDebug(p.mapExtracts)),+ zap.Strings("relations", extractRelationDebug(p.mapExtracts)))+ reduced := reduceExtracts(p.mapExtracts)+ p.reduced = reduced+ appcommon.Debug("wiki: REDUCE exact done",+ zap.String("dataset_id", p.datasetID),+ zap.String("doc_id", p.runKey()),+ zap.Strings("entities", entitiesDebug(reduced.Entities)),+ zap.Strings("concepts", conceptsDebug(reduced.Concepts)),+ zap.Int("claims", len(reduced.Claims)),+ zap.Strings("relations", relationsDebug(reduced.Relations)))// Layer embedding + LLM disambiguation onto the exact-merged entities// (REDUCE enhancement; concepts keep exact dedup). Degrades to a no-op when// the embedder/chat seams are unavailable.- p.reduced.Entities = p.dedupeEntities(p.reduced.Entities)+ preDedup := reduced.Entities+ p.reduced.Entities = p.dedupeEntities(preDedup)+ appcommon.Debug("wiki: REDUCE embedding+LLM dedup done",+ zap.String("dataset_id", p.datasetID),+ zap.String("doc_id", p.runKey()),+ zap.Int("pre_dedup_entities", len(preDedup)),+ zap.Int("post_dedup_entities", len(p.reduced.Entities)),+ zap.Strings("pre_dedup_names", entityNamesDebug(preDedup)),+ zap.Strings("post_dedup_names", entityNamesDebug(p.reduced.Entities)),+ zap.Strings("post_dedup_entities", entitiesDebug(p.reduced.Entities)))appcommon.Info("wiki: REDUCE done",zap.String("dataset_id", p.datasetID),zap.String("doc_id", p.runKey()),@@ -350,6 +375,14 @@ func (p *wikiPipeline) run() error {zap.String("doc_id", p.runKey()),zap.Int("plan_pages", len(plan.Pages)),zap.Int("capacity_excluded", p.planCapacityExcluded))+ appcommon.Debug("wiki: PLAN intermediate",+ zap.String("dataset_id", p.datasetID),+ zap.String("doc_id", p.runKey()),+ zap.String("plan_title", plan.Title),+ zap.String("plan_slug", plan.Slug),+ zap.Int("plan_pages", len(plan.Pages)),+ zap.Int("capacity_excluded", p.planCapacityExcluded),+ zap.Strings("pages", planPagesDebug(plan.Pages)))pages, err := p.runRefine()if err != nil {return err@@ -359,6 +392,11 @@ func (p *wikiPipeline) run() error {zap.String("dataset_id", p.datasetID),zap.String("doc_id", p.runKey()),zap.Int("refined_pages", len(pages)))+ appcommon.Debug("wiki: REFINE intermediate",+ zap.String("dataset_id", p.datasetID),+ zap.String("doc_id", p.runKey()),+ zap.Int("refined_pages", len(pages)),+ zap.Strings("pages", pageResultsDebug(pages)))return nil}@@ -371,6 +409,120 @@ func countExtractEntities(extracts []wikiExtract) int {return n}+// --- Debug helpers: compact per-item descriptors for the stage-intermediate+// logs in run(). They deliberately record only identity-level fields (name /+// slug / type / aliases / endpoints), never full content or embedding vectors,+// so a batch with many entities stays readable and does not blow the log line.++func entityDebug(e wikiEntity) string {+ if len(e.Aliases) == 0 {+ return fmt.Sprintf("%s(%s)", e.Name, e.Type)+ }+ return fmt.Sprintf("%s(%s){aliases:%s}", e.Name, e.Type, strings.Join(e.Aliases, ","))+}++func conceptDebug(c wikiConcept) string {+ return c.Term+}++func claimDebug(c wikiClaim) string {+ return c.Subject + ": " + c.Statement+}++func relationDebug(r wikiRelation) string {+ return fmt.Sprintf("%s-[%s]->%s", r.From, r.Type, r.To)+}++func extractEntitiesDebug(extracts []wikiExtract) []string {+ var out []string+ for bi, ex := range extracts {+ for _, e := range ex.Entities {+ out = append(out, fmt.Sprintf("batch%d:%s", bi, entityDebug(e)))+ }+ }+ return out+}++func extractConceptDebug(extracts []wikiExtract) []string {+ var out []string+ for bi, ex := range extracts {+ for _, c := range ex.Concepts {+ out = append(out, fmt.Sprintf("batch%d:%s", bi, conceptDebug(c)))+ }+ }+ return out+}++func extractClaimDebug(extracts []wikiExtract) []string {+ var out []string+ for bi, ex := range extracts {+ for _, c := range ex.Claims {+ out = append(out, fmt.Sprintf("batch%d:%s", bi, claimDebug(c)))+ }+ }+ return out+}++func extractRelationDebug(extracts []wikiExtract) []string {+ var out []string+ for bi, ex := range extracts {+ for _, r := range ex.Relations {+ out = append(out, fmt.Sprintf("batch%d:%s", bi, relationDebug(r)))+ }+ }+ return out+}++func entitiesDebug(es []wikiEntity) []string {+ out := make([]string, 0, len(es))+ for _, e := range es {+ out = append(out, entityDebug(e))+ }+ return out+}++func entityNamesDebug(es []wikiEntity) []string {+ out := make([]string, 0, len(es))+ for _, e := range es {+ out = append(out, e.Name)+ }+ return out+}++func conceptsDebug(cs []wikiConcept) []string {+ out := make([]string, 0, len(cs))+ for _, c := range cs {+ out = append(out, conceptDebug(c))+ }+ return out+}++func relationsDebug(rs []wikiRelation) []string {+ out := make([]string, 0, len(rs))+ for _, r := range rs {+ out = append(out, relationDebug(r))+ }+ return out+}++func planPagesDebug(pages []wikiPlanPage) []string {+ out := make([]string, 0, len(pages))+ for _, p := range pages {+ out = append(out, fmt.Sprintf("%s:%s(%s) prio=%d related=%s",+ p.Slug, p.Title, p.PageType, p.Priority, strings.Join(p.RelatedKB, ",")))+ }+ return out+}++func pageResultsDebug(pages []wikiPageResult) []string {+ out := make([]string, 0, len(pages))+ for _, p := range pages {+ out = append(out, fmt.Sprintf("%s:%s(%s) outlinks=%s",+ p.Slug, p.Title, p.PageType, strings.Join(p.Outlinks, ",")))+ }+ return out+}+func (p *wikiPipeline) runMap() error {// Keep each batch small enough that the LLM's entity/relation JSON output// for the batch stays well under the model's output-token limit. 2048 input@@ -1246,17 +1398,34 @@ func normalizeWikiPlan(plan wikiPlan, docID string, reduced wikiExtract) wikiPla}func normalizeWikiPlanPages(pages []wikiPlanPage, reduced wikiExtract) []wikiPlanPage {- existing := map[string]wikiPlanPage{}+ // existing maps a plan slug to its index in out (exact-slug dedup); byTitle+ // additionally collapses pages that share the same page_type + normalized+ // title, so a single batch never yields two "吕布" pages whose slugs only+ // differ in transliteration (lu-bu vs lv-bu). Scheme A (§5 of the duplicates+ // research): page identity is page_type/slug, so the key is page-type-scoped+ // (a same-title concept and topic may legitimately coexist) and the first+ // page seen (LLM emission order) is kept, folding the duplicate's RelatedKB+ // into the survivor so its outlinks are not silently dropped.+ existing := map[string]int{}+ byTitle := map[string]int{}out := make([]wikiPlanPage, 0, len(pages))for _, page := range pages {page = normalizeWikiPlanPage(page)if page.Slug == "" {continue}- if _, ok := existing[page.Slug]; ok {+ if idx, ok := existing[page.Slug]; ok {+ out[idx].RelatedKB = uniqueStrings(append(out[idx].RelatedKB, page.RelatedKB...))continue}- existing[page.Slug] = page+ key := wikiTitleKey(page.PageType, page.Title)+ if idx, ok := byTitle[key]; ok {+ out[idx].RelatedKB = uniqueStrings(append(out[idx].RelatedKB, page.RelatedKB...))+ continue+ }+ idx := len(out)+ byTitle[key] = idx+ existing[page.Slug] = idxout = append(out, page)}fallback := buildWikiFallbackPages(reduced)@@ -1265,7 +1434,10 @@ func normalizeWikiPlanPages(pages []wikiPlanPage, reduced wikiExtract) []wikiPlaif page.Slug == "" {continue}- existing[page.Slug] = page+ page = normalizeWikiPlanPage(page)+ idx := len(out)+ byTitle[wikiTitleKey(page.PageType, page.Title)] = idx+ existing[page.Slug] = idxout = append(out, page)}}@@ -1365,6 +1537,14 @@ func normalizeWikiPlanSections(sections []wikiPlanSection) []wikiPlanSection {return out}+// wikiTitleKey returns the Scheme-A dedup key for a planned page: page_type+// scoped to the normalized title. Two pages collapse only when both their type+// and their (case/whitespace-normalized) title match — a same-title concept and+// topic stay distinct because page identity is page_type/slug.+func wikiTitleKey(pageType, title string) string {+ return strings.TrimSpace(pageType) + "\x00" + normKey(title)+}+func buildWikiFallbackPages(reduced wikiExtract) []wikiPlanPage {var out []wikiPlanPageseen := map[string]bool{}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.goindex 2cf03c3b07..3402354d60 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go@@ -144,6 +144,71 @@ func TestNormalizeWikiPlanPages_FallbacksToEntitiesAndConcepts(t *testing.T) {}}+func TestNormalizeWikiPlanPages_DedupSameTypeSameTitle(t *testing.T) {+ // Scheme A: two entity pages with the same title but different slug+ // transliterations (lu-bu vs lv-bu) collapse to one, and their RelatedKB+ // is folded into the survivor. A same-title page of another type (topic)+ // stays distinct.+ pages := []wikiPlanPage{+ {+ Slug: "entity/lu-bu",+ Title: "吕布",+ PageType: "entity",+ Topic: "吕布",+ EntityNames: []string{"吕布", "李肃"},+ RelatedKB: []string{"entity/dong-zhuo"},+ Priority: 1,+ },+ {+ Slug: "entity/lv-bu",+ Title: "吕布",+ PageType: "entity",+ Topic: "吕布",+ EntityNames: []string{"吕布", "丁原"},+ RelatedKB: []string{"entity/ding-yuan", "entity/dong-zhuo"},+ Priority: 2,+ },+ {+ Slug: "topic/lu-bu-legend",+ Title: "吕布",+ PageType: "topic",+ Topic: "吕布生平",+ Priority: 3,+ },+ }+ got := normalizeWikiPlanPages(pages, wikiExtract{})+ if len(got) != 2 {+ t.Fatalf("normalizeWikiPlanPages = %d pages, want 2 (entity merged + topic)", len(got))+ }+ // Survivor keeps the first-emitted slug and accumulates RelatedKB.+ var entitySlug string+ for _, p := range got {+ if p.PageType == "entity" {+ entitySlug = p.Slug+ want := []string{"entity/dong-zhuo", "entity/ding-yuan"}+ if len(p.RelatedKB) != 2 {+ t.Fatalf("entity RelatedKB = %v, want %v (deduped union)", p.RelatedKB, want)+ }+ }+ }+ if entitySlug != "entity/lu-bu" {+ t.Fatalf("surviving entity slug = %q, want entity/lu-bu (first emitted)", entitySlug)+ }+}++func TestNormalizeWikiPlanPages_DoesNotMergeAcrossTypes(t *testing.T) {+ // Same title, different page_type: concept vs entity must stay distinct+ // (page identity is page_type/slug).+ pages := []wikiPlanPage{+ {Slug: "concept/lu-bu", Title: "吕布", PageType: "concept", Priority: 1},+ {Slug: "entity/lu-bu", Title: "吕布", PageType: "entity", Priority: 2},+ }+ got := normalizeWikiPlanPages(pages, wikiExtract{})+ if len(got) != 2 {+ t.Fatalf("normalizeWikiPlanPages = %d pages, want 2 (concept + entity)", len(got))+ }+}+func TestMergePlanCandidates_DeduplicatesWithoutLLMMerge(t *testing.T) {p := &wikiPipeline{docID: "doc-1",diff --git a/internal/ingestion/knowledge_compile/consumer.go b/internal/ingestion/knowledge_compile/consumer.goindex fcc6bfd931..8f7a61b38f 100644--- a/internal/ingestion/knowledge_compile/consumer.go+++ b/internal/ingestion/knowledge_compile/consumer.go@@ -63,7 +63,7 @@ func NewConsumer(scheduler Claimer, opts ...Option) *Consumer {heartbeat: 20 * time.Second,pollInterval: 2 * time.Second,sweepInterval: 30 * time.Second,- mergeThreshold: 0.99,+ mergeThreshold: 0.85,tombs: map[string]map[string]uint64{},}for _, o := range opts {@@ -290,7 +290,19 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entriesdeduper, err := c.factory(tenant)if err != nil || deduper == nil {- deduper = NewNoopDeduper()+ // A no-op deduper would silently disable dataset-level LLM merging: every+ // candidate (including e.g. a "吕布" wiki_page) would be written as its own+ // merged row and duplicates would accumulate across runs. That degradation+ // is never acceptable here, so fail loudly instead of papering over it.+ common.Fatal("knowledge_compile: dataset-level LLM deduper unavailable, refusing to continue with a no-op merge",+ zap.String("kb_id", kb),+ zap.String("tenant_id", tenant),+ zap.String("factory_err", func() string {+ if err != nil {+ return err.Error()+ }+ return "deduper factory returned nil"+ }()))}deletedSet := make(map[string]bool, len(deleted))@@ -344,6 +356,52 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entriesif err != nil {return err}+ // Diagnostics: after batch dedup, verify the per-doc products still carry+ // their embedding before the KNN/merge path consumes cand.Vector. If+ // candidates show 0 vectors while incoming had vectors, Dedup is dropping+ // them and both the KNN lookups and the merged rows lose embeddings.+ {+ incomingVec, candVec := 0, 0+ for _, p := range incoming {+ if len(p.Vector) > 0 {+ incomingVec+++ }+ }+ for _, p := range candidates {+ if len(p.Vector) > 0 {+ candVec+++ }+ }+ common.Info("knowledge_compile: dedup vector audit",+ zap.String("kb_id", kb),+ zap.Int("incoming", len(incoming)),+ zap.Int("incoming_with_vector", incomingVec),+ zap.Int("candidates", len(candidates)),+ zap.Int("candidates_with_vector", candVec))+ }++ // Diagnostics: break the KNN-input set down by variant and list the wiki_page+ // slugs, so the reader can reconcile the number of wiki_page candidates here+ // against the wiki_page rows WriteMerged emits (they differ when KNN merges a+ // candidate into an existing merged row, when DecideBatch returns distinct new+ // rows, or when candidates of the same variant collapse in-memory). This is+ // what explains "39 wiki_page written vs 21 KNN requests".+ {+ byVariant := map[string]int{}+ var wikiPageSlugs []string+ for _, cand := range candidates {+ v := string(cand.Variant)+ byVariant[v]+++ if v == "wiki_page" {+ wikiPageSlugs = append(wikiPageSlugs, candidateIdentity(cand))+ }+ }+ common.Info("knowledge_compile: candidates breakdown",+ zap.String("kb_id", kb),+ zap.Int("candidates", len(candidates)),+ zap.Any("by_variant", byVariant),+ zap.Strings("wiki_page_candidate_slugs", wikiPageSlugs))+ }// Then dedup each candidate against the DocEngine via KNN top1 + LLM judge,// mirroring Python _struct_doc_storage_dedup_batch. Candidates that KNN-hit@@ -382,6 +440,10 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entriesif hit.ID == "" {// No sufficiently-similar merged row: insert the candidate as a new// merged row.+ common.Debug("knowledge_compile: KNN no hit, candidate becomes new merged row",+ zap.String("kb_id", kb),+ zap.String("candidate", candidateIdentity(cand)),+ zap.String("variant", string(cand.Variant)))cand.Merged = truecand.DocID = kbunmatchedMu.Lock()@@ -389,6 +451,12 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entriesunmatchedMu.Unlock()return nil}+ common.Debug("knowledge_compile: KNN hit",+ zap.String("kb_id", kb),+ zap.String("candidate", candidateIdentity(cand)),+ zap.String("variant", string(cand.Variant)),+ zap.String("hit", candidateIdentity(hit)),+ zap.Float64("score", score))groupsMu.Lock()g := groupsByID[hit.ID]if g == nil {@@ -409,6 +477,29 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entries// updated existing rows and the candidates judged distinct (new rows).var newMerged []kccommon.Productif len(groupsByID) > 0 {+ // Diagnostics: summarize the KNN groups before the LLM merge decision so+ // the reader can see which existing merged rows were hit and by how many+ // candidates (e.g. whether a "吕布" candidate hit an existing 吕布 row and+ // was then judged by the LLM).+ {+ type groupDump struct {+ Existing string `json:"existing"`+ Candidates []string `json:"candidates"`+ Score float64 `json:"score"`+ }+ dumps := make([]groupDump, 0, len(groupsByID))+ for _, g := range groupsByID {+ cands := make([]string, 0, len(g.candidates))+ for _, cand := range g.candidates {+ cands = append(cands, candidateIdentity(cand))+ }+ dumps = append(dumps, groupDump{Existing: candidateIdentity(g.existing), Candidates: cands, Score: g.score})+ }+ common.Info("knowledge_compile: KNN groups for DecideBatch",+ zap.String("kb_id", kb),+ zap.Int("groups", len(groupsByID)),+ zap.Any("groups_detail", dumps))+ }batched := make([]MergeGroup, 0, len(groupsByID))for _, g := range groupsByID {batched = append(batched, MergeGroup{@@ -461,3 +552,19 @@ func (c *Consumer) processBatch(ctx context.Context, tenant, kb string, entrieszap.Int("merged_rows_written", len(mergedFinal)))return nil}++// candidateIdentity returns a compact identity string for a product, preferring+// the wiki slug (full "<page_type>/<slug>") when present, else the id/doc_id,+// so KNN/dedup diagnostics can tie a candidate to the page it belongs to.+func candidateIdentity(p kccommon.Product) string {+ if slug, _ := p.Meta["slug"].(string); slug != "" {+ return slug+ }+ if id := p.ID; id != "" {+ return id+ }+ if docID := p.DocID; docID != "" {+ return docID+ }+ return "<unknown>"+}diff --git a/internal/ingestion/knowledge_compile/dedup.go b/internal/ingestion/knowledge_compile/dedup.goindex 6cc775769c..3018b4b157 100644--- a/internal/ingestion/knowledge_compile/dedup.go+++ b/internal/ingestion/knowledge_compile/dedup.go@@ -67,19 +67,19 @@ type llmDeduper struct {}// NewLLMDeduper builds a KB-scoped deduper from the runtime chat/embed deps.-// llmMaxTokens is the chat model's token budget (0 disables per-batch token-// splitting in DecideBatch).-func NewLLMDeduper(chat kccommon.ChatInvoker, embed kccommon.Embedder, llmID string, threshold float64, llmMaxTokens int) Deduper {+// modelContentLength is the chat model's context window (content_length) and+// modelMaxOutput its generation cap (max_output); both bound how many pairs a+// single DecideBatch sub-call may judge (a value <= 0 disables per-batch token+// splitting). Together they keep every sub-call inside the input window and the+// output cap, so a large candidate set never overflows either.+func NewLLMDeduper(chat kccommon.ChatInvoker, embed kccommon.Embedder, llmID string, threshold float64, modelContentLength, modelMaxOutput int) Deduper {decider := structure.NewLLMMergeDecider(chat, llmID, embed, threshold)- decider.SetMaxBatchTokens(llmMaxTokens)+ decider.SetMaxBatchTokens(modelContentLength, modelMaxOutput)// Share the process-wide, vCPU-sized compiler pool so DecideBatch's// token-bounded sub-batches run concurrently with the rest of the pipeline- // (LLM-bounded), all under one concurrency limit. SubmitCompilerJobs enqueues- // every sub-batch then waits on their futures on the caller goroutine — it- // never blocks on a single job, so a stopped pool returns an error instead of- // hanging DecideBatch.- decider.SetSubmitter(func(ctx context.Context, fn func() error) error {- return SubmitCompilerJobs(ctx, []CompilerJob{fn})+ // (LLM-bounded), all under one concurrency limit.+ decider.SetSubmitter(func(ctx context.Context, jobs []func() error) error {+ return SubmitCompilerJobs(ctx, jobs)})return &llmDeduper{group: structure.NewGroupedDeduper(decider), decider: decider, embed: embed}}diff --git a/internal/ingestion/knowledge_compile/reader.go b/internal/ingestion/knowledge_compile/reader.goindex 661c191d63..7141fe6c57 100644--- a/internal/ingestion/knowledge_compile/reader.go+++ b/internal/ingestion/knowledge_compile/reader.go@@ -21,6 +21,9 @@ import ("fmt""strconv"+ "go.uber.org/zap"++ "ragflow/internal/common""ragflow/internal/engine""ragflow/internal/engine/types"kccommon "ragflow/internal/ingestion/component/knowledge_compiler/common"@@ -73,10 +76,18 @@ var compiledSelectFields = []string{// compiledSelectFields) that must survive the doc→merge round-trip so the// dataset-level merged rows keep the fields the artifact API and page renderers// depend on (page_type_kwd/topic_kwd/title_kwd/...).+//+// "q_*_vec" selects the (dimension-agnostic) embedding column. Without it,+// LoadDocProducts reconstructs products with an empty Vector, so merged rows+// carry no embedding and the dataset-level KNN dedup (SearchSimilar on+// available_int=1 + q_<dim>_vec) can never match an existing wiki page — the+// graph keeps accumulating cross-run duplicates. ES accepts the wildcard in+// both _source includes and the fields parameter.var wikiSelectFields = []string{"page_type_kwd", "topic_kwd", "title_kwd","entity_names_kwd", "summary_with_weight","related_kb_pages_kwd", "outlinks_kwd", "section_level_int",+ "q_*_vec",}// loadDocProductsLimit is the per-page size used when scrolling a single@@ -124,6 +135,24 @@ func (r engineReader) LoadDocProducts(ctx context.Context, tenant, kb, docID str}offset += loadDocProductsLimit}+ // Diagnostics: count how many loaded per-doc products carry an embedding+ // and report the (deduped) vector dimensions. If this shows 0/empty while+ // per-doc rows in ES do have q_<dim>_vec, VectorFromChunkMap is failing to+ // restore the vector and the merged rows will end up embedding-less.+ vecCount := 0+ dims := map[int]int{}+ for _, p := range out {+ if dim := len(p.Vector); dim > 0 {+ vecCount+++ dims[dim]+++ }+ }+ common.Info("knowledge_compile: LoadDocProducts vector audit",+ zap.String("kb_id", kb),+ zap.String("doc_id", docID),+ zap.Int("products", len(out)),+ zap.Int("with_vector", vecCount),+ zap.Any("vector_dims", dims))return out, nil}@@ -262,7 +291,11 @@ func (r engineReader) SearchSimilar(ctx context.Context, tenant, kb string, variif !ok || !p.Merged {continue}- score := toFloat64(c["score"])+ // The DocEngine stores the dense similarity in the "_score" key (mirroring+ // ES's _score), not "score". Reading the wrong key yields 0 and misleads+ // downstream diagnostics (KNN groups logging). The value is informational+ // only — KNN eligibility is enforced by the engine's min_score filter.+ score := toFloat64(c["_score"])return p, score, nil}return kccommon.Product{}, 0, nildiff --git a/internal/ingestion/knowledge_compile/service.go b/internal/ingestion/knowledge_compile/service.goindex 8141605b9c..4acbb0975a 100644--- a/internal/ingestion/knowledge_compile/service.go+++ b/internal/ingestion/knowledge_compile/service.go@@ -101,7 +101,7 @@ func defaultDeduperFactory(tenant string) (Deduper, error) {if err != nil {return nil, err}- return NewLLMDeduper(deps.Chat, deps.Embed, defaultLLMID, 0.99, deps.ModelContextLen), nil+ return NewLLMDeduper(deps.Chat, deps.Embed, defaultLLMID, 0.99, deps.ModelContextLen, deps.ModelMaxOutput), nil}func generateHolder() string {diff --git a/internal/ingestion/knowledge_compile/writer.go b/internal/ingestion/knowledge_compile/writer.goindex 6765b725de..15a006cbbe 100644--- a/internal/ingestion/knowledge_compile/writer.go+++ b/internal/ingestion/knowledge_compile/writer.go@@ -107,6 +107,25 @@ func (w engineWriter) WriteMerged(ctx context.Context, tenant, kb string, producnow := time.Now()runID := utility.GenerateUUID()inputHash := mergedInputHash(products)+ // Diagnostics: before building merged rows, confirm the incoming products+ // actually carry an embedding. mergedChunkMap only writes q_<dim>_vec when+ // len(p.Vector)>0; if this reports 0 vectors while LoadDocProducts/dedup+ // audits reported vectors, the vector is being dropped somewhere between+ // dedup and WriteMerged.+ {+ vecCount, dims := 0, map[int]int{}+ for _, p := range products {+ if dim := len(p.Vector); dim > 0 {+ vecCount+++ dims[dim]+++ }+ }+ common.Info("knowledge_compile: WriteMerged vector audit",+ zap.String("kb_id", kb),+ zap.Int("products", len(products)),+ zap.Int("with_vector", vecCount),+ zap.Any("vector_dims", dims))+ }// Shard the rows and drive the inserts through the shared global pool// (docengine-bounded) instead of one monolithic InsertChunks call.jobs := make([]CompilerJob, 0, (len(products)+writeMergedBatchSize-1)/writeMergedBatchSize)@@ -606,6 +625,10 @@ func (w engineWriter) loadMergedWikiPages(ctx context.Context, tenant, kb string"slug_kwd", "page_type_kwd", "title_kwd","entity_names_kwd", "summary_with_weight", "outlinks_kwd","source_doc_ids", "source_chunk_ids",+ // compile_kwd is selected purely so the query-result telemetry+ // below can confirm the Search filter is honoured (without it the+ // "compile_kwd_seen" audit would always read as {"":n}).+ "compile_kwd",},Limit: batchSize,Offset: offset,diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.goindex 8a232caea2..9c9540c9ff 100644--- a/internal/ingestion/task/knowledge_compiler_wiring.go+++ b/internal/ingestion/task/knowledge_compiler_wiring.go@@ -33,8 +33,9 @@ import ("ragflow/internal/ingestion/knowledge_compile""ragflow/internal/service"- "gorm.io/gorm"appcommon "ragflow/internal/common"++ "gorm.io/gorm")// This file is the composition-root wiring for the KnowledgeCompiler ingestion@@ -96,7 +97,20 @@ func newKnowledgeCompilerDepsResolver() kc.DepsResolver {svc := service.NewModelProviderService()return func(tenantID, llmID, embeddingModel string) (kc.Deps, error) {if strings.TrimSpace(llmID) == "" {- return kc.Deps{}, fmt.Errorf("knowledge_compiler: llm_id is required for production deps resolution")+ // No explicit chat model was supplied (e.g. the dataset-level deduper+ // is seeded with a global default that may be empty). Resolve the+ // tenant's default chat model so cross-document LLM merging can+ // actually run instead of failing / falling back to a no-op. Use the+ // composite "<model>@<instance>@<provider>" reference (not the bare+ // model name) so later Chat/ResolveModelConfig round-trips can locate+ // the provider.+ defaultRef, derr := svc.GetTenantDefaultModelRef(context.Background(), tenantID, entity.ModelTypeChat)+ if derr != nil || strings.TrimSpace(defaultRef) == "" {+ return kc.Deps{}, fmt.Errorf("knowledge_compiler: llm_id is empty and no tenant default chat model available: %w", derr)+ }+ llmID = defaultRef+ // Keep the fall-through path below for ModelContextLen resolution so+ // both explicit and default model refs share one context-window path.}// Resolve the chat model's context window so RAPTOR can truncate each// cluster's texts to fit the LLM context (mirrors Python self._llm_model.max_length).@@ -111,6 +125,16 @@ func newKnowledgeCompilerDepsResolver() kc.DepsResolver {if ml, merr := svc.ResolveModelContextLength(ctx, tenantID, llmID); merr == nil && ml > 0 {llmMax = ml}+ // Resolve the model's generation cap (max_output). Cross-document merge+ // judging packs many pairs into one LLM call; the batch must be bounded by+ // BOTH the input window and this output cap, so a large candidate set+ // never overflows max_output and yields a truncated/non-JSON reply. This+ // uses max_tokens (the generation cap), NOT content_length — see+ // ResolveModelContextLength's comment.+ llmMaxOutput := 0+ if _, _, _, mo, merr := svc.ResolveModelConfig(ctx, tenantID, entity.ModelTypeChat, llmID); merr == nil && mo > 0 {+ llmMaxOutput = mo+ }return kc.Deps{Chat: &kcChatInvoker{svc: svc, tenantID: tenantID, llmID: llmID},@@ -120,6 +144,7 @@ func newKnowledgeCompilerDepsResolver() kc.DepsResolver {// datasetnav lock). They are wired separately when the// surrounding pipeline supplies the backing services.ModelContextLen: llmMax,+ ModelMaxOutput: llmMaxOutput,}, nil}}@@ -161,15 +186,25 @@ func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatR// is never cached, so each attempt issues a fresh request. Permanent// configuration/model errors (auth, unknown model) are not retried.var resp *models.ChatResponse- retryErr := appcommon.RetryWithBackoff(ctx, kcChatRetryMax, kcChatRetryDelay, func() error {- r, err := c.svc.Chat(ctx, c.tenantID, llmID, msgs, config)+ call := func() error {+ // Bound each attempt to a short deadline so a stalled LLM provider (e.g.+ // MiniMax hanging on a large merge-judge prompt) surfaces a timeout+ // quickly instead of blocking a compile sub-batch for minutes; the+ // retry/backoff loop above then handles it as a transient failure.+ attemptCtx, cancel := context.WithTimeout(ctx, kcChatAttemptTimeout)+ defer cancel()+ r, err := c.svc.Chat(attemptCtx, c.tenantID, llmID, msgs, config)if err != nil {return err}resp = rreturn nil- }, appcommon.IsTransientError)- if retryErr != nil {+ }+ if req.DisableRetry {+ if err := call(); err != nil {+ return nil, err+ }+ } else if retryErr := appcommon.RetryWithBackoff(ctx, kcChatRetryMax, kcChatRetryDelay, call, appcommon.IsTransientError); retryErr != nil {return nil, retryErr}content := ""@@ -182,7 +217,13 @@ func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatR// kcChatRetryMax bounds how many times a transient LLM transport failure is// retried. Each attempt may run up to the driver's HTTP timeout, so the count// stays small to avoid unbounded wall-clock latency inside one compile.-const kcChatRetryMax = 2+const kcChatRetryMax = 5++// kcChatAttemptTimeout bounds a single Chat call (per retry attempt). A stalled+// provider must surface a timeout promptly rather than hold a compile sub-batch;+// 3 minutes is long enough for a big merge-judge prompt yet short enough that+// several failed attempts do not stall the pipeline for many minutes.+const kcChatAttemptTimeout = 3 * time.Minute// kcChatRetryDelay is the initial exponential-backoff delay between retries.const kcChatRetryDelay = 2 * time.Seconddiff --git a/internal/service/dataset_artifact_service.go b/internal/service/dataset_artifact_service.goindex 7de10643de..c43f925af5 100644--- a/internal/service/dataset_artifact_service.go+++ b/internal/service/dataset_artifact_service.go@@ -20,6 +20,9 @@ import ("sort""strings"+ "golang.org/x/text/collate"+ "golang.org/x/text/language"+"ragflow/internal/engine""ragflow/internal/engine/types""ragflow/internal/service/nav"@@ -377,10 +380,23 @@ func (s *DatasetArtifactService) ListWikiTopics(ctx context.Context, tenantID, dit.PageCount = counts[t]items = append(items, it)}- sort.Slice(items, func(i, j int) bool { return items[i].Topic < items[j].Topic })+ // Sort topics by a deterministic rule. Plain UTF-8 byte order is chaotic for+ // CJK (it sorts by Unicode code point, unrelated to pinyin/stroke). We use a+ // CLDR-based collator (golang.org/x/text/collate) with the Chinese locale,+ // which orders Chinese by pinyin and handles Latin/digits/other scripts+ // correctly, case-insensitively, without assuming topics are all-Chinese.+ sort.Slice(items, func(i, j int) bool {+ return wikiTopicCollator.CompareString(items[i].Topic, items[j].Topic) < 0+ })return items, int64(len(items)), nil}+// wikiTopicCollator is a process-wide collator for wiki topics. language.Chinese+// selects the CLDR zh collation (pinyin-based for Han), while Latin/digits and+// other scripts sort naturally, case-insensitively. It is safe for concurrent+// use after construction.+var wikiTopicCollator = collate.New(language.Chinese)+// WikiGraph is the entity/relation graph for a dataset's wiki artifacts.type WikiGraph struct {Entities []WikiGraphEntity `json:"entities"`diff --git a/internal/service/model_service.go b/internal/service/model_service.goindex a7eb487350..adb27d2eaf 100644--- a/internal/service/model_service.go+++ b/internal/service/model_service.go@@ -3451,6 +3451,24 @@ func (m *ModelProviderService) GetTenantDefaultModelByType(ctx context.Context,return m.ResolveModelConfig(ctx, tenantID, modelType, modelName)}+// GetTenantDefaultModelRef returns the composite model reference (the+// "<model>@<instance>@<provider>" form stored on the tenant, e.g.+// "MiniMax-M3@mm@MiniMax") that callers pass on to model config resolution.+// Unlike GetTenantDefaultModelByType — which resolves the driver/API config and+// returns a bare model name — this returns the reference verbatim so it can be+// used as an llmID for a later ResolveModelConfig/Chat round-trip.+func (m *ModelProviderService) GetTenantDefaultModelRef(ctx context.Context, tenantID string, modelType entity.ModelType) (string, error) {+ tenant, err := m.tenantDAO.GetByID(ctx, dao.DB, tenantID)+ if err != nil {+ return "", fmt.Errorf("failed to get tenant: %s type %s: %w", tenantID, modelType, err)+ }+ modelName, _ := defaultModelRefs(tenant, modelType)+ if strings.TrimSpace(modelName) == "" {+ return "", fmt.Errorf("no default %s model is set", modelType)+ }+ return modelName, nil+}+// GetModelConfigByID returns model driver and API config for a tenant_model row by its ID.func (m *ModelProviderService) GetModelConfigByID(ctx context.Context, userID string, modelType entity.ModelType, modelID string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {common.Debug("GetModelConfigByID",