RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: report knowledge compilation LLM failures (#19313)
buua43696fcfd9905ee
diff --git a/internal/ingestion/component/knowledge_compiler/common/common_test.go b/internal/ingestion/component/knowledge_compiler/common/common_test.goindex 6748216a78..f9679f4b46 100644--- a/internal/ingestion/component/knowledge_compiler/common/common_test.go+++ b/internal/ingestion/component/knowledge_compiler/common/common_test.go@@ -2,7 +2,11 @@ package commonimport ("context"+ "errors"+ "strings""testing"++ "ragflow/internal/agent/runtime")type captureChat struct{ req ChatRequest }@@ -22,6 +26,42 @@ func TestGenJSONDisablesInvokerRetry(t *testing.T) {}}+type failingChat struct{ err error }++func (c failingChat) Chat(context.Context, ChatRequest) (*ChatResponse, error) {+ return nil, c.err+}++func TestGenJSONReportsLLMFailure(t *testing.T) {+ var messages []string+ ctx := runtime.WithProgressMessageCallback(t.Context(), func(component, message string) {+ if component != "Compiler" {+ t.Fatalf("component = %q, want Compiler", component)+ }+ messages = append(messages, message)+ })+ err := errors.New("API request failed with status 401:\ninvalid key")+ if _, gotErr := GenJSON(ctx, failingChat{err: err}, ChatRequest{}, 0); !errors.Is(gotErr, err) {+ t.Fatalf("GenJSON error = %v, want %v", gotErr, err)+ }+ if len(messages) != 1 {+ t.Fatalf("messages = %v, want one error message", messages)+ }+ if !strings.Contains(messages[0], "[ERROR] LLM call failed (attempt 1/1)") || strings.Contains(messages[0], "\n") {+ t.Fatalf("unexpected progress message: %q", messages[0])+ }+}++func TestCompactErrorRedactsCredentials(t *testing.T) {+ got := CompactError(errors.New(`status=401 api_key="sk-secret-value" password=topsecret`))+ if strings.Contains(got, "sk-secret-value") || strings.Contains(got, "topsecret") {+ t.Fatalf("CompactError leaked credentials: %q", got)+ }+ if !strings.Contains(got, "api_key=[REDACTED]") || !strings.Contains(got, "password=[REDACTED]") {+ t.Fatalf("CompactError did not redact credential fields: %q", got)+ }+}+func vec(a, b, c float32) []float32 { return []float32{a, b, c} }func TestMemStoreTopK(t *testing.T) {diff --git a/internal/ingestion/component/knowledge_compiler/common/deps.go b/internal/ingestion/component/knowledge_compiler/common/deps.goindex 1d3855a6b0..56f396cd92 100644--- a/internal/ingestion/component/knowledge_compiler/common/deps.go+++ b/internal/ingestion/component/knowledge_compiler/common/deps.go@@ -23,8 +23,8 @@ type ChatRequest struct {// knowledge compilation pins extraction at 0.1 and merge judging at 0.0,// so variants set it per call site.Temperature *float64- // MaxTokens caps the generated summary length (mirrors Python's- // {"max_tokens": max(self._max_token, 512)} config, issue #10235).+ // MaxTokens is an optional per-call override. Normal knowledge-compilation+ // calls use the selected model's configured max_output value.MaxTokens *intAPIKey stringBaseURL string@@ -172,9 +172,7 @@ type Deps struct {TenantID stringDatasetID string// ModelContextLen is the chat model's context window in tokens- // (content_length). The prompt-budget helpers (wikiMapMaxTokens,- // deriveWikiPlanBudget, buildClusterContent) use it to size the input/output- // quotas (mirrors Python self._llm_model.max_length).+ // (content_length). Prompt-packing helpers use it to size input quotas.ModelContextLen int// ModelMaxOutput is the chat model's generation cap (max_output), the most// tokens one LLM response may emit. Cross-document merge judging packs manydiff --git a/internal/ingestion/component/knowledge_compiler/common/jsonchat.go b/internal/ingestion/component/knowledge_compiler/common/jsonchat.goindex 0e7a05e959..e2917aafac 100644--- a/internal/ingestion/component/knowledge_compiler/common/jsonchat.go+++ b/internal/ingestion/component/knowledge_compiler/common/jsonchat.go@@ -9,6 +9,7 @@ import ("strings""time"+ "ragflow/internal/agent/runtime"appcommon "ragflow/internal/common""go.uber.org/zap"@@ -61,6 +62,7 @@ func GenJSON(ctx context.Context, chat ChatInvoker, req ChatRequest, retryMax ..// Permanent chat errors (auth, unknown model, context-length,// cancelled ctx) cannot succeed on a retry; escape immediately.if !appcommon.IsTransientError(err) {+ reportLLMFailure(ctx, attempt, maxRetries, 0, err)return nil, err}// Transient chat failure (timeout / transport / provider); retry@@ -87,8 +89,10 @@ func GenJSON(ctx context.Context, chat ChatInvoker, req ChatRequest, retryMax ..lastErr = fmt.Errorf("knowledge_compiler: LLM response is not parseable JSON (%d bytes)", len(resp.Content))}if attempt == maxRetries {+ reportLLMFailure(ctx, attempt, maxRetries, 0, lastErr)break}+ reportLLMFailure(ctx, attempt, maxRetries, delay, lastErr)appcommon.Info("knowledge_compiler: GenJSON attempt failed, retrying",zap.Int("attempt", attempt), zap.Duration("delay", delay),zap.Error(lastErr))@@ -107,6 +111,34 @@ func GenJSON(ctx context.Context, chat ChatInvoker, req ChatRequest, retryMax ..return nil, lastErr}+func reportLLMFailure(ctx context.Context, attempt, maxRetries int, delay time.Duration, err error) {+ message := fmt.Sprintf("[ERROR] LLM call failed (attempt %d/%d): %s", attempt+1, maxRetries+1, CompactError(err))+ if delay > 0 {+ message += fmt.Sprintf("; retrying in %s", delay)+ }+ runtime.ReportProgressMessage(ctx, "Compiler", message)+}++// CompactError produces a bounded, single-line error suitable for progress+// messages. Provider errors may contain credentials, so redact common secret+// fields and API-key-shaped values before exposing the result to users.+func CompactError(err error) string {+ if err == nil {+ return "unknown error"+ }+ const maxLength = 1000+ message := strings.Join(strings.Fields(err.Error()), " ")+ message = errorCredentialRE.ReplaceAllString(message, "$1=[REDACTED]")+ message = errorAPIKeyRE.ReplaceAllString(message, "[REDACTED]")+ if len(message) > maxLength {+ return message[:maxLength] + "..."+ }+ return message+}++var errorCredentialRE = regexp.MustCompile(`(?i)(api[-_ ]?key|access[-_ ]?token|authorization|password|secret)\s*["']?\s*[:=]\s*["']?[^,\s}"']+`)+var errorAPIKeyRE = regexp.MustCompile(`\bsk-[A-Za-z0-9_-]+`)+// jsonCandidates yields progressively "cleaned" versions of an LLM reply that// may contain JSON: the raw text, a fenced ```json ... ``` block, and the// outermost {...} span.diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.goindex 3feb4cf622..8b53367891 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go@@ -65,29 +65,6 @@ const wikiMapTokenBudget = 2048const wikiRefineProgressStep = 5-// wikiMapMaxTokens derives the extraction output budget from the model's-// context length and the per-batch input budget: once the batch has consumed-// wikiMapTokenBudget input tokens, the rest of the window is handed to the-// output — but never below the input budget itself, so a small-input batch can-// still get a proportionally large extraction payload. modelContextLen is the-// model's total context window in tokens (0 means unknown).-func wikiMapMaxTokens(modelContextLen int) int {- if modelContextLen <= 0 {- modelContextLen = common.DefaultLLMContextLength- }- return max(modelContextLen-wikiMapTokenBudget, wikiMapTokenBudget)-}--// wikiRefineMaxTokens gives the page writer (REFINE step) a generous but-// bounded output cap so a long page body is not cut off mid-stream by a small-// default completion limit. It reuses the map/extraction budget derivation:-// once the input budget is consumed, the rest of the model window is handed to-// the output. modelContextLen is the model's total context window in tokens (0-// means unknown).-func wikiRefineMaxTokens(modelContextLen int) int {- return wikiMapMaxTokens(modelContextLen)-}-type wikiPipeline struct {ctx context.Contextdeps common.Deps@@ -300,17 +277,22 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commofor i, p := range products {texts[i] = p.Content}+ runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf("Wiki EMBEDDING Started: products=%d", len(texts)))vectors, err := deps.Embed.Encode(ctx, texts)if err != nil {+ runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf("[ERROR] Wiki EMBEDDING Failed: products=%d error=%s", len(texts), common.CompactError(err)))return common.Outputs{}, err}- for i := range products {- if i < len(vectors) {- products[i].Vector = vectors[i]- }+ if len(vectors) != len(texts) {+ err = fmt.Errorf("wiki: embedder returned %d vectors for %d products", len(vectors), len(texts))+ runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf("[ERROR] Wiki EMBEDDING Failed: products=%d error=%s", len(texts), common.CompactError(err)))+ return common.Outputs{}, err+ }+ runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf("Wiki EMBEDDING Done: products=%d vectors=%d", len(texts), len(vectors)))+ if err := assignWikiProductVectors(products, vectors); err != nil {+ return common.Outputs{}, err}}-store := common.NewMemStore()decider := structure.CosineDecider{Threshold: param.SimilarityThreshold}stats := structure.MergeStats{}@@ -349,6 +331,16 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commoreturn out, nil}+func assignWikiProductVectors(products []common.Product, vectors [][]float32) error {+ if len(products) != len(vectors) {+ return fmt.Errorf("wiki: embedder returned %d vectors for %d products", len(vectors), len(products))+ }+ for i := range products {+ products[i].Vector = vectors[i]+ }+ return nil+}+// runKey returns a stable identity for this wiki run's log lines. In a// per-document run docID is populated; in a dataset-level run (the// knowledge_compile aggregator) docID is empty, so fall back to a dataset@@ -708,18 +700,10 @@ func runMapBatches(func (p *wikiPipeline) mapBatch(batch []common.Chunk) (wikiExtract, error) {parserConfig, _ := p.inputs.VariantSpecific["parser_config"].(map[string]any)user, _ := buildWikiMapPrompt(p.docID, batch, parserConfig, p.param.Language)- // Give the extraction step a generous output budget so the entity/relation- // JSON is not silently truncated by the model's default output cap (that- // produced "unexpected end of JSON input" from GenJSON). The output budget is- // tied to the per-batch input budget: once the batch consumes- // wikiMapTokenBudget tokens of the model's context, the remainder is left- // for the extraction payload (and never less than the input budget itself).- mt := wikiMapMaxTokens(p.deps.ModelContextLen)raw, err := common.GenJSON(p.ctx, p.deps.Chat, common.ChatRequest{LLMID: p.llmID,SystemPrompt: wikiMapSystem,UserPrompt: user,- MaxTokens: &mt,})if err != nil {return wikiExtract{}, err@@ -745,7 +729,7 @@ func (p *wikiPipeline) runLegacyPlan() (wikiPlan, error) {for _, b := range batches {totalItems += wikiExtractItemCount(b)}- p.planBudget = deriveWikiPlanBudget(p.deps.ModelContextLen, totalItems)+ p.planBudget = deriveWikiPlanBudget(p.deps.ModelMaxOutput, totalItems)// Quota allocation must use the achievable cap (min(Target, Max)): when the// model's output capacity is smaller than the item-count-derived target, the// planner must be asked for at most Max pages so the sum of per-batch@@ -778,7 +762,7 @@ func (p *wikiPipeline) runLegacyPlan() (wikiPlan, error) {if err := p.ctx.Err(); err != nil {return err}- plan, err := p.runPlanBatch(batch, i+1, len(batches), quota, p.planBudget.MaxTokens)+ plan, err := p.runPlanBatch(batch, i+1, len(batches), quota)if err != nil {return err}@@ -934,15 +918,10 @@ func (p *wikiPipeline) runRefinePage("evidence_count": fmt.Sprintf("%d", len(evidence)),"evidence_blocks": formatWikiEvidenceBlocks(evidence),})- // wikiRefineMaxTokens gives the page writer a generous but bounded output- // cap so a long page is not cut off mid-stream by a small default completion- // limit (which would yield an unusable/cut page body).- rmt := wikiRefineMaxTokens(p.deps.ModelContextLen)resp, err := p.deps.Chat.Chat(p.ctx, common.ChatRequest{LLMID: p.llmID,SystemPrompt: buildWikiRefineWriterSystem(""),UserPrompt: user,- MaxTokens: &rmt,})if err != nil {return wikiPageResult{}, err@@ -1006,7 +985,7 @@ func maxPagesForBatch(quota int) int {return quota}-func (p *wikiPipeline) runPlanBatch(batch wikiExtract, batchIndex, batchTotal, quota, maxTokens int) (wikiPlan, error) {+func (p *wikiPipeline) runPlanBatch(batch wikiExtract, batchIndex, batchTotal, quota int) (wikiPlan, error) {planningModeRules := "Entity mode: create exactly one page for each extracted entity or concept. Do not merge multiple identities into one page, and each page's entity_names must contain only its own identity."if p.wikiMode() == "topic" {planningModeRules = "Topic mode: closely related extracted entities or concepts may be combined into one page when the source evidence supports it. Include every combined identity in entity_names."@@ -1027,7 +1006,6 @@ func (p *wikiPipeline) runPlanBatch(batch wikiExtract, batchIndex, batchTotal, qLLMID: p.llmID,SystemPrompt: wikiPlanSystem,UserPrompt: user,- MaxTokens: &maxTokens,})if err != nil {return wikiPlan{}, errdiff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget.goindex c9c51b1ab2..56d317f4d3 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget.go@@ -6,27 +6,24 @@ import "sort"// wiki variant with Python's wiki.py://// - a global target_page_count derived from item count (clamp(8, total//3, 60));-// - a dynamic max_page_count derived from the model's context window that acts+// - a dynamic max_page_count derived from the model's configured output cap that acts// as an unbreakable hard cap (output-token capacity vs page-token estimate);// - per-batch page quotas distributed by largest-remainder so the quota sum// equals the global target and no batch silently multiplies the page count;// - a deterministic, mention-grounded truncation that selects the top pages// under the global cap and reports how many were excluded.//-// The provider receives an explicit output max_tokens on the wiki PLAN path-// equal to outputTokens (below): without it, small-window models default to a-// tiny completion cap and the large plan JSON gets truncated mid-stream,-// producing "LLM response is not parseable JSON". The REFINE (page body)-// step uses its own cap via wikiRefineMaxTokens. capacity (max page count) and-// outputTokens (hard completion cap) both derive from the same model window.+// The LLM request uses the selected model's max_output configuration. This+// budget only uses that same value to keep the requested page count within the+// configured output capacity.// Python alignment constants (wiki.py:1670-1674, 1783-1787).const (- wikiPlanMaxOutputTokens = 4096- wikiPlanOutputSafetyTokens = 256- wikiPlanPageTokenEstimate = 48- wikiPlanTargetPageCountMin = 8- wikiPlanTargetPageCountMax = 60+ wikiPlanDefaultOutputTokens = 4096+ wikiPlanOutputSafetyTokens = 256+ wikiPlanPageTokenEstimate = 48+ wikiPlanTargetPageCountMin = 8+ wikiPlanTargetPageCountMax = 60)// wikiTargetPageCount mirrors Python _wiki_target_page_count:@@ -57,11 +54,6 @@ type wikiPlanBudget struct {// target; that is deliberate (never ask a small-window model for more pages// than its output can hold).Max int- // MaxTokens is the explicit completion cap sent to the provider for the- // PLAN step. It is the same outputTokens used to derive Max and prevents- // the large page JSON from being truncated mid-stream by a small default- // completion cap.- MaxTokens int}// Cap is the page budget the planner is actually allowed to emit. It is the@@ -75,24 +67,15 @@ func (b wikiPlanBudget) Cap() int {return b.Target}-// deriveWikiPlanBudget computes the global page budget from the model's context-// window and the reduced item count, mirroring Python's-// output_tokens / output_page_capacity / max_page_count derivation-// (wiki.py:2066-2078). modelContextLen is the chat model's context window in-// tokens (0 means unknown).-func deriveWikiPlanBudget(modelContextLen, totalItems int) wikiPlanBudget {+// deriveWikiPlanBudget computes the global page budget from the model's+// configured generation cap and the reduced item count. A missing generation+// cap uses a conservative internal estimate without changing the LLM request.+func deriveWikiPlanBudget(modelMaxOutput, totalItems int) wikiPlanBudget {target := wikiTargetPageCount(totalItems)- if modelContextLen <= 0 {- modelContextLen = 8192- }- // output_tokens = min(4096, max(1024, int(model_context * 0.4))).- outputTokens := modelContextLen * 2 / 5 // 0.4- if outputTokens < 1024 {- outputTokens = 1024- }- if outputTokens > wikiPlanMaxOutputTokens {- outputTokens = wikiPlanMaxOutputTokens+ outputTokens := modelMaxOutput+ if outputTokens <= 0 {+ outputTokens = wikiPlanDefaultOutputTokens}capacity := (outputTokens - wikiPlanOutputSafetyTokens) / wikiPlanPageTokenEstimate@@ -111,7 +94,7 @@ func deriveWikiPlanBudget(modelContextLen, totalItems int) wikiPlanBudget {// never be asked to emit more pages than its output capacity permits, or we// reintroduce truncated-JSON risk. When capacity < Target, Max simply lands// below Target and the achievable page count is capacity-bound.- return wikiPlanBudget{Target: target, Max: maxCount, MaxTokens: outputTokens}+ return wikiPlanBudget{Target: target, Max: maxCount}}// wikiExtractItemCount counts the planning items in one reduced extract. It isdiff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget_test.goindex 2a26cb2c36..f67d2423ca 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget_test.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_budget_test.go@@ -32,11 +32,11 @@ func TestWikiTargetPageCount_Clamp(t *testing.T) {// TestDeriveWikiPlanBudget_MaxReflectsOutputCapacity locks the corrected P0// contract: Max is the unbreakable output-capacity bound and is NOT raised back-// up to Target. A small-window model must never be asked for more pages than its-// output capacity permits.+// up to Target. A model with a small configured output limit must never be+// asked for more pages than that limit permits.func TestDeriveWikiPlanBudget_MaxReflectsOutputCapacity(t *testing.T) {- // Tiny window (modelLen=1024): output_tokens = max(1024, 1024*0.4=409) =- // 1024; capacity = (1024-256)//48 = 16. For a large item count+ // A 1024-token configured output limit gives capacity+ // (1024-256)//48 = 16. For a large item count// (target=60), Max must stay at 16 (capacity-bound), NOT be raised to 60.b := deriveWikiPlanBudget(1024, 1000)if b.Target != 60 {@@ -45,34 +45,42 @@ func TestDeriveWikiPlanBudget_MaxReflectsOutputCapacity(t *testing.T) {if b.Max != 16 {t.Fatalf("Max = %d, want 16 (capacity-bound, must not re-raise to Target 60)", b.Max)}- // A tiny item count with the same window: target = 8, max = min(16, 16, 16)+ // A tiny item count with the same output limit: target = 8,+ // max = min(16, 16, 16)// = 16.b = deriveWikiPlanBudget(1024, 1)if b.Max != 16 {t.Fatalf("Max = %d, want 16", b.Max)}- // A roomy window: Max = min(capacity, target+8, target*2). For total=1000- // (target 60) and window 8192: output=3276, capacity=62 -> max=min(62,68,120)=62.+ // A roomy output limit: Max = min(capacity, target+8, target*2). For+ // total=1000 (target 60) and max_output=8192: capacity=165, so+ // max=min(165,68,120)=68.b = deriveWikiPlanBudget(8192, 1000)- if b.Max != 62 {- t.Fatalf("Max = %d, want 62", b.Max)+ if b.Max != 68 {+ t.Fatalf("Max = %d, want 68", b.Max)}}func TestDeriveWikiPlanBudget_OutputCapacityBounds(t *testing.T) {- // With a 8192 model: output_tokens = min(4096, max(1024, 8192*0.4=3276))- // = 3276; capacity = (3276-256)//48 = 62. For total=1000 target=60,- // max = min(62, max(68, 120)) = 62. Max must equal 62 and be >= target 60.+ // With max_output=8192, capacity = (8192-256)//48 = 165. For+ // total=1000 target=60, max = min(165, 68, 120) = 68.b := deriveWikiPlanBudget(8192, 1000)if b.Target != 60 {t.Fatalf("Target = %d, want 60", b.Target)}- want := 62+ want := 68if b.Max != want {t.Fatalf("Max = %d, want %d (output-token capacity)", b.Max, want)}}+func TestDeriveWikiPlanBudget_UsesDefaultWhenModelOutputUnknown(t *testing.T) {+ b := deriveWikiPlanBudget(0, 1000)+ if b.Target != 60 || b.Max != 68 {+ t.Fatalf("budget = %#v, want target=60 max=68", b)+ }+}+func TestAllocatePlanQuotas_SumsToTarget(t *testing.T) {batches := []wikiExtract{{Entities: make([]wikiEntity, 5)},diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.goindex 6bfacd93d1..1430c5abc5 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go@@ -101,28 +101,6 @@ func TestPackWikiPlanBatches_SplitsLargeInput(t *testing.T) {}}-// TestWikiMapMaxTokens_OutputBudgetTracksInputBudget locks the input/output-// budget coupling: the extraction MaxTokens must leave at least the whole-// wikiMapTokenBudget input budget of headroom and, with a roomy model, give the-// output the rest of the context window after the batch's input is reserved.-func TestWikiMapMaxTokens_OutputBudgetTracksInputBudget(t *testing.T) {- // Unknown model context -> default window (DefaultLLMContextLength). Output- // gets the whole window minus the input budget.- got := wikiMapMaxTokens(0)- if want := common.DefaultLLMContextLength - wikiMapTokenBudget; got != want {- t.Fatalf("wikiMapMaxTokens(0) = %d, want %d", got, want)- }- // A model window that barely fits one batch must still grant at least the- // input budget of output space (never starve the output).- if got := wikiMapMaxTokens(2048); got != wikiMapTokenBudget {- t.Fatalf("wikiMapMaxTokens(2048) = %d, want %d (floor at input budget)", got, wikiMapTokenBudget)- }- // A roomy model: output = window - input budget.- if got := wikiMapMaxTokens(16384); got != 16384-wikiMapTokenBudget {- t.Fatalf("wikiMapMaxTokens(16384) = %d, want %d", got, 16384-wikiMapTokenBudget)- }-}-func TestRunMapBatches_PreservesBatchOrderWithSubmitter(t *testing.T) {previous := batchSubmitterdefer SetBatchSubmitter(previous)@@ -231,6 +209,17 @@ func TestNormalizeWikiPlanPagesDoesNotUseEntityTitleAsTopic(t *testing.T) {type topicPathEmbedStub struct{}+func TestAssignWikiProductVectorsRejectsMismatch(t *testing.T) {+ products := []common.Product{{Content: "first"}, {Content: "second"}}+ err := assignWikiProductVectors(products, [][]float32{{1, 0}})+ if err == nil {+ t.Fatal("expected an error when the embedding count does not match products")+ }+ if products[0].Vector != nil || products[1].Vector != nil {+ t.Fatalf("products were partially assigned after mismatch: %#v", products)+ }+}+func (topicPathEmbedStub) Encode(_ context.Context, texts []string) ([][]float32, error) {out := make([][]float32, len(texts))for i, text := range texts {diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_topic_plan.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_topic_plan.goindex 46b59e5cd7..5c3102ef57 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_topic_plan.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_topic_plan.go@@ -201,7 +201,7 @@ func (p *wikiPipeline) runTopicPlan() (wikiPlan, error) {for _, community := range communities {totalItems += wikiExtractItemCount(community)}- p.planBudget = deriveWikiPlanBudget(p.deps.ModelContextLen, totalItems)+ p.planBudget = deriveWikiPlanBudget(p.deps.ModelMaxOutput, totalItems)quotas := allocatePlanQuotas(communities, p.planBudget.Cap())approved := wikiExtract{}plans := make([]wikiPlan, len(communities))@@ -221,7 +221,7 @@ func (p *wikiPipeline) runTopicPlan() (wikiPlan, error) {if err := p.ctx.Err(); err != nil {return err}- plan, err := p.runPlanBatch(community, i+1, len(communities), quota, p.planBudget.MaxTokens)+ plan, err := p.runPlanBatch(community, i+1, len(communities), quota)if err != nil {return err}diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.goindex 04cb1f2f21..a73a081073 100644--- a/internal/ingestion/pipeline/pipeline.go+++ b/internal/ingestion/pipeline/pipeline.go@@ -742,7 +742,9 @@ func (p *Pipeline) componentProgressCallback(ctx context.Context) runtime.Progrezap.String("task_id", p.taskID),zap.String("document_id", p.documentID))}- p.sink.OnComponentProgress(ctx, ProgressEvent{+ sinkCtx, cancel := progressSinkContext(ctx)+ defer cancel()+ p.sink.OnComponentProgress(sinkCtx, ProgressEvent{TaskID: p.taskID,DocumentID: p.documentID,Component: ev.Component,@@ -767,6 +769,15 @@ func (p *Pipeline) componentProgressMessageCallback(ctx context.Context) runtimezap.String("task_id", p.taskID),zap.String("document_id", p.documentID),zap.String("message", message))- sink.OnComponentMessage(ctx, p.taskID, p.documentID, component, message)+ sinkCtx, cancel := progressSinkContext(ctx)+ defer cancel()+ sink.OnComponentMessage(sinkCtx, p.taskID, p.documentID, component, message)}}++func progressSinkContext(ctx context.Context) (context.Context, context.CancelFunc) {+ if ctx.Err() == nil {+ return context.WithCancel(ctx)+ }+ return context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)+}diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.goindex c506276754..9a60b13269 100644--- a/internal/ingestion/task/knowledge_compiler_wiring.go+++ b/internal/ingestion/task/knowledge_compiler_wiring.go@@ -23,6 +23,7 @@ import ("sync/atomic""time"+ "ragflow/internal/agent/runtime""ragflow/internal/dao""ragflow/internal/engine"enginetypes "ragflow/internal/engine/types"@@ -522,9 +523,10 @@ func newKnowledgeCompilerDepsResolver() kc.DepsResolver {llmMax := kc.DefaultLLMContextLength// Bound the model-config lookup so a stalled provider/instance DB read// cannot block document ingestion indefinitely.- ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)- defer cancel()- if ml, merr := svc.ResolveModelContextLength(ctx, tenantID, llmID); merr == nil && ml > 0 {+ contextLengthCtx, cancelContextLength := context.WithTimeout(context.Background(), 30*time.Second)+ ml, contextLengthErr := svc.ResolveModelContextLength(contextLengthCtx, tenantID, llmID)+ cancelContextLength()+ if contextLengthErr == nil && ml > 0 {llmMax = ml}// Resolve the model's generation cap (max_output). Cross-document merge@@ -534,12 +536,15 @@ func newKnowledgeCompilerDepsResolver() kc.DepsResolver {// uses max_tokens (the generation cap), NOT content_length — see// ResolveModelContextLength's comment.llmMaxOutput := 0- if _, _, _, mo, merr := svc.ResolveModelConfig(ctx, tenantID, entity.ModelTypeChat, llmID); merr == nil && mo > 0 {+ modelConfigCtx, cancelModelConfig := context.WithTimeout(context.Background(), 30*time.Second)+ _, _, _, mo, modelConfigErr := svc.ResolveModelConfig(modelConfigCtx, tenantID, entity.ModelTypeChat, llmID)+ cancelModelConfig()+ if modelConfigErr == nil && mo > 0 {llmMaxOutput = mo}return kc.Deps{- Chat: &kcChatInvoker{svc: svc, tenantID: tenantID, llmID: llmID},+ Chat: &kcChatInvoker{svc: svc, tenantID: tenantID, llmID: llmID, maxTokens: llmMaxOutput},Embed: &kcEmbedder{svc: svc, tenantID: tenantID, embdID: embeddingModel},WikiPages: &kcWikiPageStore{docEngine: engine.Get()},WikiMapVersions: knowledge_compile.NewWikiMapVersionStore(engine.Get()),@@ -555,9 +560,10 @@ func newKnowledgeCompilerDepsResolver() kc.DepsResolver {// kcChatInvoker adapts service.ModelProviderService.Chat to the// knowledge_compiler ChatInvoker seam.type kcChatInvoker struct {- svc *service.ModelProviderService- tenantID string- llmID string+ svc *service.ModelProviderService+ tenantID string+ llmID string+ maxTokens int}func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatResponse, error) {@@ -572,15 +578,18 @@ func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatR// Python's knowledge compilation pins per-call-site temperatures// (extraction 0.1, merge judging 0.0); nil leaves the driver default.var config *models.ChatConfig- if req.Temperature != nil || req.MaxTokens != nil {+ if req.Temperature != nil || req.MaxTokens != nil || c.maxTokens > 0 {config = &models.ChatConfig{}if req.Temperature != nil {config.Temperature = req.Temperature}- // MaxTokens caps the generated summary length (mirrors Python's- // {"max_tokens": max(self._max_token, 512)}, issue #10235).+ // Normal knowledge-compilation calls use the generation cap resolved+ // from the selected model's max_output configuration. Specialized+ // variants may still provide a smaller per-call cap.if req.MaxTokens != nil {config.MaxTokens = req.MaxTokens+ } else if c.maxTokens > 0 {+ config.MaxTokens = &c.maxTokens}}// Retry transient transport/provider failures (HTTP timeout, reset,@@ -589,15 +598,23 @@ func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatR// is never cached, so each attempt issues a fresh request. Permanent// configuration/model errors (auth, unknown model) are not retried.var resp *models.ChatResponse+ attempt := 0call := func() error {- // Bound each attempt to a short deadline so a stalled LLM provider (e.g.- // MiniMax hanging on a large merge-judge prompt) surfaces a timeout- // quickly instead of blocking a compile sub-batch for minutes; the- // retry/backoff loop above then handles it as a transient failure.+ attempt+++ // Bound each attempt so a stalled LLM provider eventually releases its+ // compile sub-batch. Knowledge compilation prompts can be large and some+ // providers legitimately need several minutes to return a response.attemptCtx, cancel := context.WithTimeout(ctx, kcChatAttemptTimeout)defer cancel()r, err := c.svc.Chat(attemptCtx, c.tenantID, llmID, msgs, config)if err != nil {+ if !req.DisableRetry {+ message := fmt.Sprintf("[ERROR] LLM call failed (attempt %d/%d): %s", attempt, kcChatRetryMax+1, kc.CompactError(err))+ if appcommon.IsTransientError(err) && attempt <= kcChatRetryMax {+ message += "; retrying with exponential backoff"+ }+ runtime.ReportProgressMessage(ctx, "Compiler", message)+ }return err}resp = r@@ -622,10 +639,9 @@ func (c *kcChatInvoker) Chat(ctx context.Context, req kc.ChatRequest) (*kc.ChatR// stays small to avoid unbounded wall-clock latency inside one compile.const kcChatRetryMax = 5-// kcChatAttemptTimeout bounds a single Chat call (per retry attempt). A stalled-// provider must surface a timeout promptly rather than hold a compile sub-batch;-// 3 minutes is long enough for a big merge-judge prompt yet short enough that-// several failed attempts do not stall the pipeline for many minutes.+// kcChatAttemptTimeout bounds a single Chat call per retry attempt. It matches+// the non-streaming provider deadline so the knowledge-compiler adapter does+// not cancel a valid long-running response before the provider does.const kcChatAttemptTimeout = 3 * time.Minute// kcChatRetryDelay is the initial exponential-backoff delay between retries.@@ -669,13 +685,13 @@ func (e *kcEmbedder) Encode(ctx context.Context, texts []string) ([][]float32, e}batchTexts := texts[start:end]jobs = append(jobs, func() error {- if err = ctx.Err(); err != nil {- return err+ if jobErr := ctx.Err(); jobErr != nil {+ return jobErr}var embeds []models.EmbeddingData- embeds, err = mdl.ModelDriver.Embed(ctx, mdl.ModelName, models.EmbedRequest{Texts: batchTexts}, mdl.APIConfig, config, nil)- if err != nil {- return fmt.Errorf("knowledge_compiler: embed: %w", err)+ embeds, jobErr := mdl.ModelDriver.Embed(ctx, mdl.ModelName, models.EmbedRequest{Texts: batchTexts}, mdl.APIConfig, config, nil)+ if jobErr != nil {+ return fmt.Errorf("knowledge_compiler: embed: %w", jobErr)}vecs := make([][]float32, len(embeds))for i, v := range embeds {