RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestion): report knowledge compiler progress fractions (#20902)
jay77721a93f41655886
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.goindex 9635f5a6d3..9f6196fbc2 100644--- a/internal/ingestion/component/extractor.go+++ b/internal/ingestion/component/extractor.go@@ -54,6 +54,7 @@ import ("ragflow/internal/entity""ragflow/internal/entity/models""ragflow/internal/ingestion/chunkcache"+ "ragflow/internal/ingestion/component/progress""ragflow/internal/ingestion/component/schema""ragflow/internal/tokenizer""ragflow/internal/utility"@@ -677,17 +678,17 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs mapkeywordsOn := c.Param.Keywords.EnabledtagsOn := c.Param.Tags.EnabledremainingOn := c.Param.Questions.Enabled || c.Param.Summary.Enabled || c.Param.Metadata.Enabled- phaseWindows := newFractionSplit(keywordsOn, tagsOn, remainingOn)+ phaseWindows := progress.NewSplit(keywordsOn, tagsOn, remainingOn)if err := runtime.WithTimeout(ctx, extractorTimeout, func(timeoutCtx context.Context) error {// Phase 1: Keywords extraction (if enabled), running across chunks via extractorPool.- if err := c.runAutoKeywordsPool(timeoutCtx, db, in, phaseWindows.take(keywordsOn)); err != nil {+ if err := c.runAutoKeywordsPool(timeoutCtx, db, in, phaseWindows.Take(keywordsOn)); err != nil {return err}// Phase 2: Tag phase (if enabled), benefiting from title and freshly extracted keywords.if tagsOn {- tagged, tagErr := c.runAutoTags(timeoutCtx, db, in, phaseWindows.take(tagsOn))+ tagged, tagErr := c.runAutoTags(timeoutCtx, db, in, phaseWindows.Take(tagsOn))if tagErr != nil {return tagErr}@@ -695,7 +696,7 @@ func (c *ExtractorComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map}// Phase 3: Remaining extractions (Questions, Summary, Metadata) via extractorPool.- return c.runRemainingExtractions(timeoutCtx, db, in, phaseWindows.take(remainingOn))+ return c.runRemainingExtractions(timeoutCtx, db, in, phaseWindows.Take(remainingOn))}); err != nil {return nil, fmt.Errorf("extractor: %w", err)}@@ -901,7 +902,7 @@ func (c *ExtractorComponent) runAutoSummary(ctx context.Context, db *gorm.DB, in// runAutoKeywordsPool dispatches keyword extraction across all chunks concurrently// using extractorPool before the tagging stage.-func (c *ExtractorComponent) runAutoKeywordsPool(ctx context.Context, db *gorm.DB, in extractorInputs, w fractionWindow) error {+func (c *ExtractorComponent) runAutoKeywordsPool(ctx context.Context, db *gorm.DB, in extractorInputs, w progress.Window) error {if !c.Param.Keywords.Enabled || len(in.chunks) == 0 {return nil}@@ -929,7 +930,7 @@ func (c *ExtractorComponent) runAutoKeywordsPool(ctx context.Context, db *gorm.D// runRemainingExtractions dispatches auto questions / summary / metadata// extractions across all chunks concurrently using extractorPool.-func (c *ExtractorComponent) runRemainingExtractions(ctx context.Context, db *gorm.DB, in extractorInputs, w fractionWindow) error {+func (c *ExtractorComponent) runRemainingExtractions(ctx context.Context, db *gorm.DB, in extractorInputs, w progress.Window) error {if !c.Param.Questions.Enabled && !c.Param.Summary.Enabled && !c.Param.Metadata.Enabled {return nil}@@ -975,7 +976,7 @@ func (c *ExtractorComponent) remainingExtractionJob(ctx context.Context, db *gor// Each settled future advances the phase's fraction window, so a chunk-heavy// LLM extraction reports in-flight progress instead of going quiet until the// whole phase is done.-func awaitFutures(ctx context.Context, futs []utility.WorkerPoolFuture[extractorJob, struct{}], w fractionWindow) error {+func awaitFutures(ctx context.Context, futs []utility.WorkerPoolFuture[extractorJob, struct{}], w progress.Window) error {if len(futs) == 0 {return nil}@@ -988,7 +989,7 @@ func awaitFutures(ctx context.Context, futs []utility.WorkerPoolFuture[extractorwg.Add(1)go func(f utility.WorkerPoolFuture[extractorJob, struct{}]) {defer wg.Done()- defer func() { w.report(ctx, float64(done.Add(1))/float64(total)) }()+ defer func() { w.Report(ctx, float64(done.Add(1))/float64(total)) }()res, werr := f.Wait(ctx)if werr != nil {emu.Lock()diff --git a/internal/ingestion/component/extractor_tag.go b/internal/ingestion/component/extractor_tag.goindex 71d663517d..807ac2dfaf 100644--- a/internal/ingestion/component/extractor_tag.go+++ b/internal/ingestion/component/extractor_tag.go@@ -24,6 +24,7 @@ import ("ragflow/internal/common""ragflow/internal/ingestion/chunkcache""ragflow/internal/ingestion/component/globals"+ "ragflow/internal/ingestion/component/progress""ragflow/internal/ingestion/component/schema""ragflow/internal/tokenizer")@@ -366,10 +367,10 @@ func isHighConfidenceMatch(matched *schema.TaggedChunk) bool {return false}-func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in extractorInputs, w fractionWindow) ([]map[string]any, error) {+func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in extractorInputs, w progress.Window) ([]map[string]any, error) {// Hand the whole window to the next phase on every exit path, including the// skip paths below that do no measurable work.- defer w.report(ctx, 1)+ defer w.Report(ctx, 1)lang := in.langif lang == "" {lang = detectTextLanguage(in.chunks)@@ -420,7 +421,7 @@ func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in exwg.Add(1)go func(idx int) {defer wg.Done()- defer func() { w.report(ctx, float64(tagged.Add(1))/float64(total)) }()+ defer func() { w.Report(ctx, float64(tagged.Add(1))/float64(total)) }()select {case sem <- struct{}{}:defer func() { <-sem }()diff --git a/internal/ingestion/component/knowledge_compiler/common/deps.go b/internal/ingestion/component/knowledge_compiler/common/deps.goindex d2f3652951..7428669f95 100644--- a/internal/ingestion/component/knowledge_compiler/common/deps.go+++ b/internal/ingestion/component/knowledge_compiler/common/deps.go@@ -6,6 +6,8 @@ import ("sync""gorm.io/gorm"++ "ragflow/internal/ingestion/component/progress")// DefaultLLMContextLength is the fallback chat-model context window (tokens)@@ -165,6 +167,8 @@ type RedisClient interface {// interfaces so tests inject stubs; production wiring lives in// internal/ingestion/task (see PORT_PLAN.md §4 dependency injection seam).type Deps struct {+ // Progress is the fraction budget allocated to this compilation template.+ Progress progress.WindowChat ChatInvokerEmbed EmbedderTokenizer Tokenizerdiff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.goindex a4fe62d74e..712c682b33 100644--- a/internal/ingestion/component/knowledge_compiler/component.go+++ b/internal/ingestion/component/knowledge_compiler/component.go@@ -20,6 +20,7 @@ import ("ragflow/internal/ingestion/component/knowledge_compiler/structure""ragflow/internal/ingestion/component/knowledge_compiler/tree""ragflow/internal/ingestion/component/knowledge_compiler/wiki"+ "ragflow/internal/ingestion/component/progress""ragflow/internal/ingestion/component/schema""ragflow/internal/tokenizer"@@ -88,6 +89,9 @@ func (c *KnowledgeCompilerComponent) Outputs() map[string]string {// {output_format:"chunks", chunks:[...]}.func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {_ = db+ if err := ctx.Err(); err != nil {+ return nil, err+ }param := c.Param// Resolve the run-level tenant scope from the shared CanvasState.Globals// bag first (seeded by the pipeline at run start), falling back to the@@ -134,7 +138,7 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in// template's id and kind. All products are buffered (no streaming sink), so// the post-run loop below covers every row (M1).var out common.Outputs- for _, spec := range specs {+ for specIndex, spec := range specs {variant, err := common.KindToVariant(spec.Kind)if err != nil {return nil, err@@ -157,6 +161,8 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, in}deps.TenantID = tenantIDdeps.DatasetID = datasetID+ // Reserve 1% for converting and merging all products.+ deps.Progress = (progress.Window{Span: 0.99}).Sub(specIndex, len(specs))// Per-spec Inputs copy: each spec must get its own VariantSpecific map,// otherwise specIn.VariantSpecific below would mutate the shared map and@@ -201,6 +207,10 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, inreturn nil, err}+ if err := ctx.Err(); err != nil {+ return nil, err+ }+ deps.Progress.Report(ctx, 1)for i := range o.Products {if o.Products[i].Meta == nil {o.Products[i].Meta = map[string]any{}@@ -260,6 +270,10 @@ func (c *KnowledgeCompilerComponent) Invoke(ctx context.Context, db *gorm.DB, inif len(out.WikiActiveStates) > 0 {result["wiki_active_map_states"] = wikiActiveStateValues(out.WikiActiveStates)}+ if err := ctx.Err(); err != nil {+ return nil, err+ }+ (progress.Window{Span: 1}).Report(ctx, 1)return result, nil}diff --git a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.goindex 97d527342f..f9ef23bbed 100644--- a/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go+++ b/internal/ingestion/component/knowledge_compiler/mindmap/mindmap.go@@ -70,6 +70,7 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commobatches := packMindmapBatches(inputs.Chunks, deps.Tokenizer)results := make([]mindmapBatchResult, len(batches))jobs := make([]func() error, 0, len(batches))+ generated := deps.Progress.Sub(0, 3).Counter(ctx, len(batches))for i, batch := range batches {i, batch := i, batchjobs = append(jobs, func() error {@@ -91,6 +92,7 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo// Distinct slice index per batch → no cross-goroutine contention.if tree, parseErr := parseJSONTreeWithError(resp.Content, batch.ids); parseErr == nil {results[i].tree = tree+ generated()return nil} else {lastParseErr = parseErr@@ -99,6 +101,7 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commoruntime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf("[ERROR] Mindmap batch %d/%d JSON parsing failed after %d attempts: %s",i+1, len(batches), mindmapJSONRetryMax, common.CompactError(lastParseErr)))+ generated()return nil})}@@ -109,12 +112,14 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commoreturn common.Outputs{}, err}+ deps.Progress.Sub(0, 3).Report(ctx, 1)// Merge every successfully parsed JSON tree. A failed batch is skipped after// its user-facing error is reported and must not erase other batches.root := mergeMindmapBatchResults(results)products := treeToProducts(tenantID, docID, root, mindmapSourceChunkIDs(inputs.Chunks))+ deps.Progress.Sub(1, 3).Report(ctx, 1)// Batched embedding of each node's content for downstream vector search.if len(products) > 0 && deps.Embed != nil {texts := make([]string, len(products))diff --git a/internal/ingestion/component/knowledge_compiler/progress_test.go b/internal/ingestion/component/knowledge_compiler/progress_test.gonew file mode 100644index 0000000000..255d2f919b--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/progress_test.go@@ -0,0 +1,200 @@+package knowledge_compiler++import (+ "context"+ "errors"+ "math"+ "sync"+ "testing"++ "gorm.io/gorm"++ "ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/component/knowledge_compiler/common"+)++type compilerFractions struct {+ mu sync.Mutex+ values []float64+}++func (r *compilerFractions) context(ctx context.Context, t *testing.T) context.Context {+ return runtime.BindComponentFraction(runtime.WithProgressFractionCallback(ctx, func(id string, f float64) {+ if id != "Compiler:test" {+ t.Errorf("fraction attributed to %q", id)+ }+ r.mu.Lock()+ defer r.mu.Unlock()+ r.values = append(r.values, f)+ }), "Compiler:test")+}++func (r *compilerFractions) snapshot() []float64 {+ r.mu.Lock()+ defer r.mu.Unlock()+ return append([]float64(nil), r.values...)+}++func compilerProgressInputs() map[string]any {+ return map[string]any{+ "tenant_id": "t1", "dataset_id": "ds1", "doc_id": "d1",+ "parser_config": graphParserConfig(),+ "chunks": []any{+ map[string]any{"id": "c1", "text": "Alpha is linked to Beta"},+ map[string]any{"id": "c2", "text": "Beta is linked to Gamma"},+ },+ }+}++func TestCompilerProgressAdvancesWithinTemplates(t *testing.T) {+ for _, variant := range []string{"structure", "wiki", "mindmap", "tree"} {+ t.Run(variant, func(t *testing.T) {+ if variant == "structure" {+ installMockDeps(t)+ } else {+ installProseDeps(t)+ }+ installVariantTemplateResolver(t, variant)+ c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{"compilation_template_id": "tpl-" + variant})+ if err != nil {+ t.Fatal(err)+ }+ r := &compilerFractions{}+ _, err = c.Invoke(r.context(t.Context(), t), nil, compilerProgressInputs())+ if err != nil {+ t.Fatal(err)+ }+ values := r.snapshot()+ intermediate := false+ for _, f := range values {+ // One template owns [0, 0.99]; its completion boundary alone+ // must not satisfy the assertion for internal phase progress.+ if f > 0 && f < 0.99 {+ intermediate = true+ }+ }+ if !intermediate {+ t.Fatalf("no intermediate progress: %v", values)+ }+ if len(values) == 0 || values[len(values)-1] != 1 {+ t.Fatalf("successful compiler did not finish progress: %v", values)+ }+ })+ }+}++func TestCompilerProgressReservesBudgetForLaterTemplates(t *testing.T) {+ r := &compilerFractions{}+ calls := 0+ common.SetDepsResolver(func(_ context.Context, tenant, _, _ string) (common.Deps, error) {+ calls+++ if calls == 2 {+ values := r.snapshot()+ if len(values) == 0 {+ t.Error("first template did not advance progress")+ } else if math.Abs(values[len(values)-1]-0.495) > 1e-9 {+ t.Errorf("first template ended at %v, want 0.495 with budget reserved for the second template and merging", values[len(values)-1])+ }+ for _, f := range values {+ if f <= 0 || f >= 0.5 {+ t.Errorf("first template consumed later budget: %v", values)+ break+ }+ }+ }+ return common.Deps{Chat: proseChat{}, Embed: mockEmbedder{dim: 8}, TenantID: tenant}, nil+ })+ t.Cleanup(func() { common.SetDepsResolver(nil) })+ common.SetGroupResolver(func(context.Context, *gorm.DB, string, []string) ([]string, error) {+ return []string{"tpl-mindmap", "tpl-mindmap"}, nil+ })+ t.Cleanup(func() { common.SetGroupResolver(testGroupResolver) })+ installVariantTemplateResolver(t, "mindmap")+ c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{"compilation_template_group_id": "group"})+ if err != nil {+ t.Fatal(err)+ }+ if _, err := c.Invoke(r.context(t.Context(), t), nil, compilerProgressInputs()); err != nil {+ t.Fatal(err)+ }+ if calls != 2 {+ t.Fatalf("ran %d templates", calls)+ }+}++type failedProgressEmbedder struct{ cancel context.CancelFunc }++func (failedProgressEmbedder) Dimensions() int { return 8 }+func (e failedProgressEmbedder) Encode(context.Context, []string) ([][]float32, error) {+ if e.cancel != nil {+ e.cancel()+ return nil, context.Canceled+ }+ return nil, errors.New("embedding failed")+}++func TestCompilerProgressDoesNotFinishOnFailureOrCancellation(t *testing.T) {+ for _, canceled := range []bool{false, true} {+ t.Run(map[bool]string{false: "failure", true: "cancellation"}[canceled], func(t *testing.T) {+ ctx, cancel := context.WithCancel(t.Context())+ defer cancel()+ embed := failedProgressEmbedder{}+ if canceled {+ embed.cancel = cancel+ }+ common.SetDepsResolver(func(context.Context, string, string, string) (common.Deps, error) {+ return common.Deps{Chat: proseChat{}, Embed: embed}, nil+ })+ t.Cleanup(func() { common.SetDepsResolver(nil) })+ installVariantTemplateResolver(t, "mindmap")+ c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{"compilation_template_id": "tpl-mindmap"})+ if err != nil {+ t.Fatal(err)+ }+ r := &compilerFractions{}+ if _, err := c.Invoke(r.context(ctx, t), nil, compilerProgressInputs()); err == nil {+ t.Fatal("expected embedding error")+ }+ values := r.snapshot()+ if len(values) == 0 {+ t.Fatal("completed generation did not advance progress")+ }+ for _, f := range values {+ if f >= 0.99 {+ t.Fatalf("unfinished compiler reported full progress: %v", values)+ }+ }+ })+ }+}++func TestCompilerProgressStructureExtractionUsesMainBudget(t *testing.T) {+ t.Setenv("STRUCT_MAX_CHUNKS_PER_BATCH", "1")+ installMockDeps(t)+ installVariantTemplateResolver(t, "structure")+ c, err := NewKnowledgeCompilerComponent("Compiler", map[string]any{"compilation_template_id": "tpl-structure"})+ if err != nil {+ t.Fatal(err)+ }+ inputs := compilerProgressInputs()+ inputs["chunks"] = append(inputs["chunks"].([]any), map[string]any{"id": "c3", "text": "Alpha is linked to Gamma"})+ r := &compilerFractions{}+ if _, err := c.Invoke(r.context(t.Context(), t), nil, inputs); err != nil {+ t.Fatal(err)+ }+ values := r.snapshot()+ // Three extraction batches should advance a five-node run from 60%+ // through 65.28%, 70.56%, and 75.84%, before the faster follow-up stages.+ for _, want := range []float64{0.264, 0.528, 0.792} {+ found := false+ for _, f := range values {+ if math.Abs(f-want) < 1e-9 {+ found = true+ break+ }+ }+ if !found {+ t.Errorf("missing extraction progress %v: %v", want, values)+ }+ }+}diff --git a/internal/ingestion/component/knowledge_compiler/structure/structure.go b/internal/ingestion/component/knowledge_compiler/structure/structure.goindex 32e56e6f37..52eff20572 100644--- a/internal/ingestion/component/knowledge_compiler/structure/structure.go+++ b/internal/ingestion/component/knowledge_compiler/structure/structure.go@@ -143,6 +143,13 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo// The progress callback is supplied by the caller and is not required to be// goroutine-safe; pool workers report out of order, so serialise it.var progressMu sync.Mutex+ // LLM extraction owns 80%; the four subsequent phases share the remainder.+ mapWindow := deps.Progress+ mapWindow.Span *= 0.8+ remainingWindow := deps.Progress+ remainingWindow.Base += mapWindow.Span+ remainingWindow.Span -= mapWindow.Span+ mapDone := mapWindow.Counter(ctx, len(batches))for i, batch := range batches {i, batch := i, batchjobs = append(jobs, func() error {@@ -150,6 +157,7 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo"%s-template: extracting batch %d/%d", compileType, i+1, len(batches)))packed, batchIDs := PackBatch(batch)if len(batchIDs) == 0 {+ mapDone()return nil}nodes, edges, err := extractHypergraph(ctx, deps, cfg, nodePrompt, edgePromptTmpl, packed)@@ -178,6 +186,7 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo"%s-template: batch %d/%d done: %d entities, %d relations",compileType, i+1, len(batches), len(nodes), len(edges)))progressMu.Unlock()+ mapDone()return nil})}@@ -187,6 +196,7 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commoif err := runBatches(ctx, jobs); err != nil {return common.Outputs{}, err}+ mapWindow.Report(ctx, 1)// Embed each extracted batch serially after all MAP jobs have returned.// This avoids nesting Embed.Encode (and its batch jobs) inside a worker// already occupied by the shared compiler pool.@@ -201,27 +211,33 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo}perBatch[i] = rowsrowCount += len(rows)+ remainingWindow.Sub(0, 4).Report(ctx, float64(i+1)/float64(len(extracted)))runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf("%s-template: embedded batch %d/%d (%d rows so far)", compileType, i+1, len(batches), rowCount))}runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf("%s-template: deduplicating %d row(s)", compileType, rowCount))+ remainingWindow.Sub(0, 4).Report(ctx, 1)// ---- DEDUP ----// Sequential in batch order so merge outcomes are deterministic and match// Python's _struct_local_dedup (which folds docs in list order).decider := NewLLMMergeDecider(deps.Chat, llmID, deps.Embed, param.SimilarityThreshold)deduper := NewGroupedDeduper(decider)+ dedupDone := remainingWindow.Sub(1, 4).Counter(ctx, rowCount)for _, rows := range perBatch {for _, row := range rows {if err := deduper.Add(ctx, row); err != nil {return common.Outputs{}, err}+ dedupDone()}}+ remainingWindow.Sub(1, 4).Report(ctx, 1)if err := deduper.RewriteRelations(ctx, decider.Aliases(), deps.Embed); err != nil {return common.Outputs{}, err}+ remainingWindow.Sub(2, 4).Report(ctx, 1)stats := deduper.Stats()prods := filterSelfLoopRelations(deduper.Rows())runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf(diff --git a/internal/ingestion/component/knowledge_compiler/tree/claims.go b/internal/ingestion/component/knowledge_compiler/tree/claims.goindex 5706c93c59..185c5bc9ad 100644--- a/internal/ingestion/component/knowledge_compiler/tree/claims.go+++ b/internal/ingestion/component/knowledge_compiler/tree/claims.go@@ -256,7 +256,7 @@ func ExtractClaimsForChunks(ctx context.Context, deps common.Deps, llmID string,// which advances the counter only for finished batches; workers// finish out of order, so the figure jumps by batch size).completed += len(batch)- runtime.ReportComponentFraction(ctx, float64(completed)/float64(total))+ deps.Progress.Sub(1, 5).Report(ctx, float64(completed)/float64(total))if reporter.shouldReport(completed) {runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf("tree-template: extracting claims for chunk %d/%d", completed, total))}diff --git a/internal/ingestion/component/knowledge_compiler/tree/raptor.go b/internal/ingestion/component/knowledge_compiler/tree/raptor.goindex 07aeddefce..ad25e506ed 100644--- a/internal/ingestion/component/knowledge_compiler/tree/raptor.go+++ b/internal/ingestion/component/knowledge_compiler/tree/raptor.go@@ -56,12 +56,14 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commoreturn common.Outputs{}, err}+ deps.Progress.Sub(2, 5).Report(ctx, 1)// Rename duplicate summary titles whose descriptions differ (Python// rewrite_duplicate_tree_names), BEFORE the graph projection: titles are// the entity names and the relation endpoints, so two nodes sharing a// title would render as one ambiguous node. Best-effort — a rewrite// failure must not cost us the tree.rewriteDuplicateTreeNames(ctx, deps, llmID, products)+ deps.Progress.Sub(3, 5).Report(ctx, 1)// Project the RAPTOR tree onto the {entities, relations} structure-graph// shape (Python raptor_tree_to_graph) and persist it as entity/relation rows@@ -182,6 +184,7 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID str}embeddings = toFloat64Matrix(vectors)}+ deps.Progress.Sub(0, 5).Report(ctx, 1)// Extract claims before clustering so each cluster can be summarised from// its members' claims instead of their truncated raw text. Claims are far// more compact than the source, so the per-chunk truncation in@@ -200,6 +203,7 @@ func buildTree(ctx context.Context, deps common.Deps, llmID, tenantID, docID strrfcommon.Info("tree: extracted claims", zap.Int("chunks_with_claims", len(claimsByChunk)), zap.Int("chunks", len(chunks)))}}+ deps.Progress.Sub(1, 5).Report(ctx, 1)if claimOut != nil {*claimOut = claimsByChunk}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/progress_test.go b/internal/ingestion/component/knowledge_compiler/wiki/progress_test.gonew file mode 100644index 0000000000..1b5c639b50--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/wiki/progress_test.go@@ -0,0 +1,72 @@+package wiki++import (+ "context"+ "fmt"+ "math"+ "testing"++ "ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/component/knowledge_compiler/common"+ "ragflow/internal/ingestion/component/progress"+)++func serialProgressBatches(t *testing.T) {+ t.Helper()+ previous := batchSubmitter+ batchSubmitter = func(_ context.Context, jobs []func() error) error {+ for _, job := range jobs {+ if err := job(); err != nil {+ return err+ }+ }+ return nil+ }+ t.Cleanup(func() { batchSubmitter = previous })+}++func wikiProgressContext(t *testing.T, values *[]float64) context.Context {+ t.Helper()+ return runtime.BindComponentFraction(runtime.WithProgressFractionCallback(t.Context(), func(_ string, f float64) {+ *values = append(*values, f)+ }), "Compiler:wiki")+}++func TestWikiProgressMapReportsBeforeAllBatchesFinish(t *testing.T) {+ serialProgressBatches(t)+ var values []float64+ ctx := wikiProgressContext(t, &values)+ _, err := runMapBatches(ctx, [][]common.Chunk{{{ID: "first"}}, {{ID: "second"}}}, progress.Window{Base: 0.2, Span: 0.4}, func(batch []common.Chunk) (wikiExtract, error) {+ if batch[0].ID == "second" && (len(values) != 1 || math.Abs(values[0]-0.4) > 1e-9) {+ t.Errorf("first batch did not advance progress before second started: %v", values)+ }+ return wikiExtract{}, nil+ })+ if err != nil {+ t.Fatal(err)+ }+ if len(values) != 2 || math.Abs(values[1]-0.6) > 1e-9 {+ t.Fatalf("MAP progress = %v, want [0.4 0.6]", values)+ }+}++func TestWikiProgressTopicPlanCountsScheduledJobs(t *testing.T) {+ serialProgressBatches(t)+ var values []float64+ ctx := wikiProgressContext(t, &values)+ p := &wikiPipeline{+ ctx: ctx, llmID: "llm", docID: "doc",+ deps: common.Deps{Chat: batchPlanChatStub{}, Embed: reconcileEmbedStub{}, ModelMaxOutput: 304, Progress: progress.Window{Span: 1}},+ }+ for i := 0; i < 30; i++ {+ p.reduced.Entities = append(p.reduced.Entities, wikiEntity{Name: fmt.Sprintf("entity-%d", i), Type: "project"})+ }+ if _, err := p.runTopicPlan(); err != nil {+ t.Fatal(err)+ }+ // The output budget schedules one planner job; quota-zero communities+ // must not keep the PLAN window below its end (3/6).+ if len(values) != 1 || math.Abs(values[0]-0.5) > 1e-9 {+ t.Fatalf("PLAN progress = %v, want [0.5]", values)+ }+}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki.goindex 8d40926172..f7fb750a89 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki.go@@ -23,6 +23,7 @@ import (appcommon "ragflow/internal/common""ragflow/internal/ingestion/component/knowledge_compiler/common""ragflow/internal/ingestion/component/knowledge_compiler/structure"+ "ragflow/internal/ingestion/component/progress""go.uber.org/zap")@@ -293,6 +294,7 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commoreturn common.Outputs{}, err}}+ deps.Progress.Sub(4, 6).Report(ctx, 1)store := common.NewMemStore()decider := structure.CosineDecider{Threshold: param.SimilarityThreshold}stats := structure.MergeStats{}@@ -380,6 +382,7 @@ func (p *wikiPipeline) run() error {zap.Strings("claims", extractClaimDebug(p.mapExtracts)),zap.Strings("relations", extractRelationDebug(p.mapExtracts)))runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki REDUCE Started: input_entities=%d input_concepts=%d input_claims=%d input_relations=%d", countExtractEntities(p.mapExtracts), countExtractConcepts(p.mapExtracts), countExtractClaims(p.mapExtracts), countExtractRelations(p.mapExtracts)))+ p.deps.Progress.Sub(0, 6).Report(p.ctx, 1)reduced := reduceExtracts(p.mapExtracts)p.reduced = reducedappcommon.Debug("wiki: REDUCE exact done",@@ -395,6 +398,7 @@ func (p *wikiPipeline) run() error {zap.Int("entities", len(p.reduced.Entities)),zap.Int("claims", len(p.reduced.Claims)))runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki REDUCE Done: output_entities=%d output_concepts=%d output_claims=%d output_relations=%d", len(p.reduced.Entities), len(p.reduced.Concepts), len(p.reduced.Claims), len(p.reduced.Relations)))+ p.deps.Progress.Sub(1, 6).Report(p.ctx, 1)// PLAN uses the same topic-assignment path in both modes. Entity mode keeps// one identity per page, while topic mode may merge related identities in// the planner before assigning the resulting pages to MAP topics.@@ -406,6 +410,7 @@ func (p *wikiPipeline) run() error {return err}appcommon.Info("wiki: PLAN done", zap.String("dataset_id", p.datasetID), zap.String("doc_id", p.runKey()), zap.String("mode", p.wikiMode()), zap.Int("plan_pages", len(plan.Pages)))+ p.deps.Progress.Sub(2, 6).Report(p.ctx, 1)p.plan = planp.selectAffectedPages(plan.Pages)appcommon.Info("wiki: PLAN done",@@ -427,6 +432,7 @@ func (p *wikiPipeline) run() error {if err != nil {return err}+ p.deps.Progress.Sub(3, 6).Report(p.ctx, 1)p.pages = pagesactiveState, err := p.buildActiveMapState(plan.Pages)if err != nil {@@ -648,7 +654,7 @@ func (p *wikiPipeline) runMap() error {// extraction payload; oversize batches caused truncated, unparseable JSON// (unexpected end of JSON input) on the real pipeline.batches := common.PackBatches(p.inputs.Chunks, wikiMapTokenBudget, p.deps.Tokenizer)- extracts, err := runMapBatches(p.ctx, batches, p.mapBatch)+ extracts, err := runMapBatches(p.ctx, batches, p.deps.Progress.Sub(0, 6), p.mapBatch)if err != nil {return err}@@ -669,12 +675,14 @@ func (p *wikiPipeline) wikiMode() string {func runMapBatches(ctx context.Context,batches [][]common.Chunk,+ window progress.Window,mapBatch func([]common.Chunk) (wikiExtract, error),) ([]wikiExtract, error) {if len(batches) == 0 {return nil, nil}extracts := make([]wikiExtract, len(batches))+ completed := window.Counter(ctx, len(batches))jobs := make([]func() error, 0, len(batches))for i, batch := range batches {i, batch := i, batch@@ -688,6 +696,7 @@ func runMapBatches(}// Distinct slice index per batch keeps results stable without locks.extracts[i] = extract+ completed()return nil})}@@ -839,6 +848,7 @@ func (p *wikiPipeline) runLegacyPlan() (wikiPlan, error) {// can never leak back into the plan via buildWikiFallbackPages.approved := wikiExtract{}plans := make([]wikiPlan, len(batches))+ var planDone func()jobs := make([]func() error, 0, len(batches))for i, batch := range batches {i, batch := i, batch@@ -864,9 +874,11 @@ func (p *wikiPipeline) runLegacyPlan() (wikiPlan, error) {}// Distinct slice index keeps results stable without locks.plans[i] = plan+ planDone()return nil})}+ planDone = p.deps.Progress.Sub(2, 6).Counter(p.ctx, len(jobs))if err := runBatches(p.ctx, jobs); err != nil {return wikiPlan{}, err}@@ -943,6 +955,7 @@ func (p *wikiPipeline) runRefine() ([]wikiPageResult, error) {results[i] = resprogressMu.Lock()completed+++ p.deps.Progress.Sub(3, 6).Report(p.ctx, float64(completed)/float64(len(pages)))if shouldReportRefineProgress(completed, len(pages)) {runtime.ReportProgressMessage(p.ctx, "Compiler", fmt.Sprintf("Wiki REFINE Progress: completed_pages=%d total_pages=%d", completed, len(pages)))}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.goindex ae61daa703..aeeadff05b 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_map_cache.go@@ -111,7 +111,7 @@ func (p *wikiPipeline) runVersionedMap() error {}batches := common.PackBatches(misses, wikiMapTokenBudget, p.deps.Tokenizer)- batchExtracts, err := runMapBatches(p.ctx, batches, p.mapBatch)+ batchExtracts, err := runMapBatches(p.ctx, batches, p.deps.Progress.Sub(0, 6).Sub(0, 2), p.mapBatch)if err != nil {return err}@@ -159,7 +159,7 @@ func (p *wikiPipeline) runVersionedMap() error {uncacheable = append(uncacheable, chunk)}uncacheableBatches := common.PackBatches(uncacheable, wikiMapTokenBudget, p.deps.Tokenizer)- uncachedExtracts, err := runMapBatches(p.ctx, uncacheableBatches, p.mapBatch)+ uncachedExtracts, err := runMapBatches(p.ctx, uncacheableBatches, p.deps.Progress.Sub(0, 6).Sub(1, 2), p.mapBatch)if err != nil {return err}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.goindex d1a0ea5c27..e1fdf74f1d 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_test.go@@ -10,6 +10,7 @@ import ("time""ragflow/internal/ingestion/component/knowledge_compiler/common"+ "ragflow/internal/ingestion/component/progress")func TestReduceExtracts_MergesProvenance(t *testing.T) {@@ -194,7 +195,7 @@ func TestRunMapBatches_PreservesBatchOrderWithSubmitter(t *testing.T) {{{ID: "fast-1", Text: "fast-1"}},{{ID: "fast-2", Text: "fast-2"}},}- got, err := runMapBatches(context.Background(), batches, func(batch []common.Chunk) (wikiExtract, error) {+ got, err := runMapBatches(context.Background(), batches, progress.Window{}, func(batch []common.Chunk) (wikiExtract, error) {if batch[0].ID == "slow" {time.Sleep(25 * time.Millisecond)}diff --git a/internal/ingestion/component/knowledge_compiler/wiki/wiki_topic_plan.go b/internal/ingestion/component/knowledge_compiler/wiki/wiki_topic_plan.goindex 5c3102ef57..09a037cb80 100644--- a/internal/ingestion/component/knowledge_compiler/wiki/wiki_topic_plan.go+++ b/internal/ingestion/component/knowledge_compiler/wiki/wiki_topic_plan.go@@ -205,6 +205,7 @@ func (p *wikiPipeline) runTopicPlan() (wikiPlan, error) {quotas := allocatePlanQuotas(communities, p.planBudget.Cap())approved := wikiExtract{}plans := make([]wikiPlan, len(communities))+ var planDone func()jobs := make([]func() error, 0, len(communities))for i, community := range communities {i, community := i, community@@ -230,9 +231,11 @@ func (p *wikiPipeline) runTopicPlan() (wikiPlan, error) {plan.Pages[pageIndex].PlanGroup = group}plans[i] = plan+ planDone()return nil})}+ planDone = p.deps.Progress.Sub(2, 6).Counter(p.ctx, len(jobs))if err := runBatches(p.ctx, jobs); err != nil {return wikiPlan{}, err}diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.goindex 424aa67a25..d5c0cd2afe 100644--- a/internal/ingestion/component/media_dispatch.go+++ b/internal/ingestion/component/media_dispatch.go@@ -35,6 +35,7 @@ import (deepdocpdf "ragflow/internal/deepdoc/parser/pdf""ragflow/internal/entity"modelModule "ragflow/internal/entity/models"+ "ragflow/internal/ingestion/component/progress""ragflow/internal/ingestion/component/schema""ragflow/internal/parser/parser""ragflow/internal/utility"@@ -116,7 +117,7 @@ func maybeDispatchImage(text, err = extractImageText(ctx, img)}release()- parsed := dispatchParse(ctx, fileType, filename, binary, setups, fractionWindow{})+ parsed := dispatchParse(ctx, fileType, filename, binary, setups, progress.Window{})if parsed.Err != nil {return parsed, true, parsed.Err}diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.goindex 1234c34116..6d45b28cc9 100644--- a/internal/ingestion/component/parser.go+++ b/internal/ingestion/component/parser.go@@ -88,6 +88,7 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/common""ragflow/internal/ingestion/component/globals"+ "ragflow/internal/ingestion/component/progress""ragflow/internal/ingestion/component/schema""ragflow/internal/parser/parser""ragflow/internal/utility"@@ -544,13 +545,13 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[stif !handledVision && !handledMedia && !handledImage && !handledAudio {parsePagesOn := pdfPageProgressSupported(fileTypeExt, setups)enhanceOn := c.enableVisionEnhancement- split := newFractionSplit(parsePagesOn, enhanceOn)- pageWin := split.take(parsePagesOn)- visionWin := split.take(enhanceOn)+ split := progress.NewSplit(parsePagesOn, enhanceOn)+ pageWin := split.Take(parsePagesOn)+ visionWin := split.Take(enhanceOn)dispatched = dispatchParse(ctx, fileTypeExt, filename, binary, setups, pageWin)if dispatched.Err == nil && ctx.Err() == nil {- pageWin.report(ctx, 1.0)+ pageWin.Report(ctx, 1.0)}if c.enableVisionEnhancement {diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.goindex 460bdae1df..3cc586e942 100644--- a/internal/ingestion/component/parser_dispatch.go+++ b/internal/ingestion/component/parser_dispatch.go@@ -29,6 +29,7 @@ import ("strings""ragflow/internal/common"+ "ragflow/internal/ingestion/component/progress""ragflow/internal/ingestion/component/schema""ragflow/internal/parser/parser""ragflow/internal/utility"@@ -76,7 +77,7 @@ func configureParserFromSetups(p any, fileType utility.FileType, setups map[stri// re-reading setups. lib_type is no longer threaded through: the// Python dispatcher picks a single backend per family and the Go// constructors mirror that.-func dispatchParse(ctx context.Context, fileType utility.FileType, filename string, data []byte, setups map[string]schema.ParserSetup, window fractionWindow) parser.ParseResult {+func dispatchParse(ctx context.Context, fileType utility.FileType, filename string, data []byte, setups map[string]schema.ParserSetup, window progress.Window) parser.ParseResult {if fileType == utility.FileTypeOTHER {// Unknown / unset family. The component treats the bytes// as text pages; splitIntoPages handles it. We return no@@ -139,16 +140,16 @@ func pdfPageProgressSupported(fileType utility.FileType, setups map[string]schem// the component exits. Only the local DeepDOC backend fires it; remote// engines (MinerU, PaddleOCR, Docling, …) never call the callback and// the fraction stays at zero.-func attachPDFPageProgress(ctx context.Context, p parser.ParseResultProducer, window fractionWindow) {+func attachPDFPageProgress(ctx context.Context, p parser.ParseResultProducer, window progress.Window) {pdfParser, ok := p.(*parser.PDFParser)- if !ok || window.span <= 0 {+ if !ok || window.Span <= 0 {return}pdfParser.OnPageDone = func(done, total int) {if total <= 0 {return}- window.report(ctx, float64(done)/float64(total))+ window.Report(ctx, float64(done)/float64(total))}}diff --git a/internal/ingestion/component/parser_dispatch_progress_test.go b/internal/ingestion/component/parser_dispatch_progress_test.goindex 56bb6667a7..710bb53f90 100644--- a/internal/ingestion/component/parser_dispatch_progress_test.go+++ b/internal/ingestion/component/parser_dispatch_progress_test.go@@ -9,6 +9,7 @@ import ("testing""ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/component/progress""ragflow/internal/ingestion/component/schema""ragflow/internal/parser/parser""ragflow/internal/utility"@@ -34,7 +35,7 @@ func TestAttachPDFPageProgress_ReportsFraction(t *testing.T) {ctx := boundFractionContext(t, "Parser:pdf", &seen)pdfParser := parser.NewPDFParser()- attachPDFPageProgress(ctx, pdfParser, fractionWindow{base: 0, span: 1})+ attachPDFPageProgress(ctx, pdfParser, progress.Window{Base: 0, Span: 1})if pdfParser.OnPageDone == nil {t.Fatal("expected OnPageDone to be wired")}@@ -58,7 +59,7 @@ func TestAttachPDFPageProgress_IgnoresUnknownTotal(t *testing.T) {ctx := boundFractionContext(t, "Parser:pdf", &seen)pdfParser := parser.NewPDFParser()- attachPDFPageProgress(ctx, pdfParser, fractionWindow{base: 0, span: 1})+ attachPDFPageProgress(ctx, pdfParser, progress.Window{Base: 0, Span: 1})pdfParser.OnPageDone(1, 0)if len(seen) != 0 {@@ -70,7 +71,7 @@ func TestAttachPDFPageProgress_LeavesOtherParsersAlone(t *testing.T) {var seen []fractionObservationctx := boundFractionContext(t, "Parser:txt", &seen)- attachPDFPageProgress(ctx, parser.NewTextParser(), fractionWindow{base: 0, span: 1})+ attachPDFPageProgress(ctx, parser.NewTextParser(), progress.Window{Base: 0, Span: 1})if len(seen) != 0 {t.Errorf("expected no fraction from a non-PDF parser, got %v", seen)@@ -82,7 +83,7 @@ func TestAttachPDFPageProgress_ZeroWindowSilent(t *testing.T) {ctx := boundFractionContext(t, "Parser:pdf", &seen)pdfParser := parser.NewPDFParser()- attachPDFPageProgress(ctx, pdfParser, fractionWindow{})+ attachPDFPageProgress(ctx, pdfParser, progress.Window{})if pdfParser.OnPageDone != nil {t.Fatal("expected OnPageDone to remain nil for zero window")}@@ -96,7 +97,7 @@ func TestAttachPDFPageProgress_ReportsFractionInWindow(t *testing.T) {ctx := boundFractionContext(t, "Parser:pdf", &seen)pdfParser := parser.NewPDFParser()- attachPDFPageProgress(ctx, pdfParser, fractionWindow{base: 0.1, span: 0.4})+ attachPDFPageProgress(ctx, pdfParser, progress.Window{Base: 0.1, Span: 0.4})if pdfParser.OnPageDone == nil {t.Fatal("expected OnPageDone to be wired")}diff --git a/internal/ingestion/component/progress/fraction.go b/internal/ingestion/component/progress/fraction.gonew file mode 100644index 0000000000..126c2ec0fd--- /dev/null+++ b/internal/ingestion/component/progress/fraction.go@@ -0,0 +1,66 @@+// Package progress maps ingestion subphases into a component's fraction budget.+package progress++import (+ "context"+ "sync/atomic"++ "ragflow/internal/agent/runtime"+)++// Window is the part of a component's 0..1 budget owned by one phase.+// Its zero value disables reporting.+type Window struct{ Base, Span float64 }++// Report maps a phase ratio to the component's framework-bound fraction.+func (w Window) Report(ctx context.Context, ratio float64) {+ if w.Span <= 0 || ctx.Err() != nil {+ return+ }+ ratio = min(max(ratio, 0), 1)+ runtime.ReportComponentFraction(ctx, w.Base+w.Span*ratio)+}++// Sub divides a window into total equal parts and returns part index.+func (w Window) Sub(index, total int) Window {+ if total <= 0 {+ return Window{}+ }+ span := w.Span / float64(total)+ return Window{Base: w.Base + float64(index)*span, Span: span}+}++// Counter returns a concurrency-safe reporter to call after each completed job.+func (w Window) Counter(ctx context.Context, total int) func() {+ var done atomic.Int64+ return func() {+ if total > 0 {+ w.Report(ctx, float64(done.Add(1))/float64(total))+ }+ }+}++// split hands out one equal window per enabled sequential phase.+type split struct {+ phases int+ next int+}++func NewSplit(enabled ...bool) *split {+ s := &split{}+ for _, on := range enabled {+ if on {+ s.phases+++ }+ }+ return s+}++func (s *split) Take(enabled bool) Window {+ if !enabled {+ return Window{}+ }+ w := (Window{Span: 1}).Sub(s.next, s.phases)+ s.next+++ return w+}diff --git a/internal/ingestion/component/progress/fraction_test.go b/internal/ingestion/component/progress/fraction_test.gonew file mode 100644index 0000000000..7b830196f1--- /dev/null+++ b/internal/ingestion/component/progress/fraction_test.go@@ -0,0 +1,54 @@+package progress++import (+ "context"+ "math"+ "sort"+ "sync"+ "testing"++ "ragflow/internal/agent/runtime"+)++func TestNestedWindowCountsConcurrentCompletions(t *testing.T) {+ var mu sync.Mutex+ var values []float64+ ctx := runtime.BindComponentFraction(runtime.WithProgressFractionCallback(t.Context(), func(id string, f float64) {+ mu.Lock()+ defer mu.Unlock()+ if id != "Compiler:test" {+ t.Errorf("component = %q", id)+ }+ values = append(values, f)+ }), "Compiler:test")+ // Second of two templates, first of two phases: component budget [0.5,0.75].+ completed := (Window{Span: 1}).Sub(1, 2).Sub(0, 2).Counter(ctx, 4)+ var wg sync.WaitGroup+ for range 4 {+ wg.Go(completed)+ }+ wg.Wait()+ sort.Float64s(values)+ want := []float64{0.5625, 0.625, 0.6875, 0.75}+ if len(values) != len(want) {+ t.Fatalf("fractions = %v, want %v", values, want)+ }+ for i, f := range values {+ if math.Abs(f-want[i]) > 1e-9 {+ t.Errorf("fraction[%d] = %v, want %v", i, f, want[i])+ }+ }+}++func TestCancelledAndEmptyWindowsDoNotAdvanceProgress(t *testing.T) {+ ctx := runtime.BindComponentFraction(runtime.WithProgressFractionCallback(t.Context(), func(_ string, f float64) {+ t.Errorf("unexpected fraction: %v", f)+ }), "Compiler:test")+ Window{}.Report(ctx, 1)+ (Window{Span: 1}).Counter(ctx, 0)()+ NewSplit(false, false).Take(true).Report(ctx, 1)+ canceled, cancel := context.WithCancel(ctx)+ cancel()+ (Window{Span: 1}).Report(canceled, 1)+ (Window{Span: 1}).Counter(canceled, 1)()+}diff --git a/internal/ingestion/component/progress_fraction.go b/internal/ingestion/component/progress_fraction.godeleted file mode 100644index 2f7c10d656..0000000000--- a/internal/ingestion/component/progress_fraction.go+++ /dev/null@@ -1,51 +0,0 @@-package component--import (- "context"-- "ragflow/internal/agent/runtime"-)--// fractionWindow is the slice of a component's 0..1 fraction budget owned by one-// sequential phase. Splitting the budget keeps the reported fraction climbing-// across phases: the run-level progress tracker keeps a high-water mark, so a-// second phase restarting from zero would freeze the bar until the component-// exits.-type fractionWindow struct{ base, span float64 }--// report maps a phase completion ratio into the window and forwards it to the-// run-level fraction channel. The zero window reports nothing, which keeps-// disabled phases free of budget arithmetic.-func (w fractionWindow) report(ctx context.Context, ratio float64) {- if w.span <= 0 {- return- }- ratio = min(max(ratio, 0), 1)- runtime.ReportComponentFraction(ctx, w.base+w.span*ratio)-}--// fractionSplit hands out one equal window per enabled phase, in phase order.-type fractionSplit struct {- phases int- next int-}--func newFractionSplit(enabled ...bool) *fractionSplit {- s := &fractionSplit{}- for _, on := range enabled {- if on {- s.phases++- }- }- return s-}--func (s *fractionSplit) take(enabled bool) fractionWindow {- if !enabled {- return fractionWindow{}- }- span := 1 / float64(s.phases)- w := fractionWindow{base: float64(s.next) * span, span: span}- s.next++- return w-}diff --git a/internal/ingestion/component/progress_fraction_test.go b/internal/ingestion/component/progress_fraction_test.goindex fb2805af0c..d2ffa0ec68 100644--- a/internal/ingestion/component/progress_fraction_test.go+++ b/internal/ingestion/component/progress_fraction_test.go@@ -11,6 +11,7 @@ import ("testing""ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/component/progress""ragflow/internal/ingestion/component/schema""ragflow/internal/utility")@@ -43,27 +44,27 @@ func (r *fractionRecorder) fractions() []float64 {}func TestFractionSplit_HandsOutMonotonicWindows(t *testing.T) {- split := newFractionSplit(true, false, true)+ split := progress.NewSplit(true, false, true)- first := split.take(true)- skipped := split.take(false)- second := split.take(true)+ first := split.Take(true)+ skipped := split.Take(false)+ second := split.Take(true)- if first.base != 0 || first.span != 0.5 {+ if first.Base != 0 || first.Span != 0.5 {t.Errorf("first window = %+v, want base 0 span 0.5", first)}- if skipped.base != 0 || skipped.span != 0 {+ if skipped.Base != 0 || skipped.Span != 0 {t.Errorf("disabled window = %+v, want the zero window", skipped)}- if second.base != 0.5 || second.span != 0.5 {+ if second.Base != 0.5 || second.Span != 0.5 {t.Errorf("second window = %+v, want base 0.5 span 0.5", second)}rec := &fractionRecorder{}ctx := rec.context(t, "Extractor:abc")- first.report(ctx, 0.5)- skipped.report(ctx, 1)- second.report(ctx, 1)+ first.Report(ctx, 0.5)+ skipped.Report(ctx, 1)+ second.Report(ctx, 1)got := rec.fractions()want := []float64{0.25, 1}@@ -80,10 +81,10 @@ func TestFractionSplit_HandsOutMonotonicWindows(t *testing.T) {func TestFractionWindow_ClampsRatio(t *testing.T) {rec := &fractionRecorder{}ctx := rec.context(t, "Extractor:abc")- w := fractionWindow{base: 0.5, span: 0.5}+ w := progress.Window{Base: 0.5, Span: 0.5}- w.report(ctx, -1)- w.report(ctx, 2)+ w.Report(ctx, -1)+ w.Report(ctx, 2)got := rec.fractions()if len(got) != 2 || got[0] != 0.5 || got[1] != 1 {@@ -104,7 +105,7 @@ func TestAwaitFutures_ReportsEverySettledJob(t *testing.T) {futs = append(futs, f)}- w := fractionWindow{base: 0.5, span: 0.5}+ w := progress.Window{Base: 0.5, Span: 0.5}if err := awaitFutures(ctx, futs, w); err != nil {t.Fatalf("awaitFutures: %v", err)}@@ -115,15 +116,15 @@ func TestAwaitFutures_ReportsEverySettledJob(t *testing.T) {}highest := got[0]for i, f := range got {- if f < w.base || f > w.base+w.span {- t.Errorf("report %d = %v, outside window [%v %v]", i, f, w.base, w.base+w.span)+ if f < w.Base || f > w.Base+w.Span {+ t.Errorf("report %d = %v, outside window [%v %v]", i, f, w.Base, w.Base+w.Span)}highest = max(highest, f)}// Goroutines report concurrently, so only the highest value proves the// phase reached the end of its window.- if highest != w.base+w.span {- t.Errorf("highest report = %v, want %v", highest, w.base+w.span)+ if highest != w.Base+w.Span {+ t.Errorf("highest report = %v, want %v", highest, w.Base+w.Span)}}@@ -131,7 +132,7 @@ func TestAwaitFutures_NoJobsNoReports(t *testing.T) {rec := &fractionRecorder{}ctx := rec.context(t, "Extractor:abc")- if err := awaitFutures(ctx, nil, fractionWindow{span: 1}); err != nil {+ if err := awaitFutures(ctx, nil, progress.Window{Span: 1}); err != nil {t.Fatalf("awaitFutures: %v", err)}if got := rec.fractions(); len(got) != 0 {diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.goindex f97cb7b95b..1d2c0602bb 100644--- a/internal/ingestion/component/vision_enhancement.go+++ b/internal/ingestion/component/vision_enhancement.go@@ -38,6 +38,7 @@ import (pdflayout "ragflow/internal/deepdoc/parser/pdf/layout""ragflow/internal/entity"modelModule "ragflow/internal/entity/models"+ "ragflow/internal/ingestion/component/progress""ragflow/internal/parser/parser""ragflow/internal/rag/prompts""ragflow/internal/utility"@@ -133,7 +134,7 @@ func maybeDispatchVisionEnhancement(dispatched parser.ParseResult,inputs map[string]any,vision visionSettings,- window fractionWindow,+ window progress.Window,) (parser.ParseResult, bool, error) {w := window@@ -307,7 +308,7 @@ func maybeDispatchVisionEnhancement(if resource == nil {releaseSlot()n := settled.Add(1)- w.report(ctx, float64(n)/float64(total))+ w.Report(ctx, float64(n)/float64(total))continue}if !resource.VLMDataValidated && !isUsableVisionImage(resource.VLMData) {@@ -316,7 +317,7 @@ func maybeDispatchVisionEnhancement(}releaseSlot()n := settled.Add(1)- w.report(ctx, float64(n)/float64(total))+ w.Report(ctx, float64(n)/float64(total))continue}wg.Add(1)@@ -326,7 +327,7 @@ func maybeDispatchVisionEnhancement(defer func() {if ctx.Err() == nil {n := settled.Add(1)- w.report(ctx, float64(n)/float64(total))+ w.Report(ctx, float64(n)/float64(total))}}()@@ -354,7 +355,7 @@ func maybeDispatchVisionEnhancement(if err := ctx.Err(); err != nil {return dispatched, modified, err}- w.report(ctx, 1.0)+ w.Report(ctx, 1.0)failedItems := 0firstFailure := ""for _, failure := range failures {diff --git a/internal/ingestion/component/vision_enhancement_progress_test.go b/internal/ingestion/component/vision_enhancement_progress_test.goindex 74626dec21..124623b267 100644--- a/internal/ingestion/component/vision_enhancement_progress_test.go+++ b/internal/ingestion/component/vision_enhancement_progress_test.go@@ -9,6 +9,7 @@ import ("ragflow/internal/dao"modelModule "ragflow/internal/entity/models"+ "ragflow/internal/ingestion/component/progress""ragflow/internal/parser/parser""ragflow/internal/utility")@@ -31,7 +32,7 @@ func TestVisionEnhancement_ReportsPerFigureFractions(t *testing.T) {},}- win := fractionWindow{base: 0.5, span: 0.5}+ win := progress.Window{Base: 0.5, Span: 0.5}_, handled, err := maybeDispatchVisionEnhancement(ctx, dao.DB, utility.FileTypeXLSX, dispatched,map[string]any{"tenant_id": "t1"},@@ -84,7 +85,7 @@ func TestVisionEnhancement_SettledFractionsCountFailedFigures(t *testing.T) {},}- win := fractionWindow{base: 0.2, span: 0.8}+ win := progress.Window{Base: 0.2, Span: 0.8}_, _, err := maybeDispatchVisionEnhancement(ctx, dao.DB, utility.FileTypeXLSX, dispatched,map[string]any{"tenant_id": "t1"},@@ -135,7 +136,7 @@ func TestVisionEnhancement_ZeroWindowReportsNothing(t *testing.T) {ctx, dao.DB, utility.FileTypeXLSX, dispatched,map[string]any{"tenant_id": "t1"},visionSettings{},- fractionWindow{},+ progress.Window{},)if err != nil {t.Fatalf("maybeDispatchVisionEnhancement: %v", err)@@ -166,7 +167,7 @@ func TestVisionEnhancement_CancelledContextReportsNothing(t *testing.T) {},}- win := fractionWindow{base: 0.5, span: 0.5}+ win := progress.Window{Base: 0.5, Span: 0.5}_, _, _ = maybeDispatchVisionEnhancement(ctx, dao.DB, utility.FileTypeXLSX, dispatched,map[string]any{"tenant_id": "t1"},diff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.goindex 9b0e04490e..191b48b5f8 100644--- a/internal/ingestion/component/vision_enhancement_test.go+++ b/internal/ingestion/component/vision_enhancement_test.go@@ -35,6 +35,7 @@ import (deepdoctype "ragflow/internal/deepdoc/parser/type""ragflow/internal/entity"modelModule "ragflow/internal/entity/models"+ "ragflow/internal/ingestion/component/progress""ragflow/internal/ingestion/component/schema""ragflow/internal/parser/parser""ragflow/internal/utility"@@ -227,7 +228,7 @@ func TestVisionEnhancement_AppendsVLMToExistingText(t *testing.T) {res, handled, err := maybeDispatchVisionEnhancement(t.Context(), dao.DB, utility.FileTypeXLSX, dispatched,map[string]any{"tenant_id": "t1"},- visionSettings{}, fractionWindow{})+ visionSettings{}, progress.Window{})if err != nil {t.Fatalf("maybeDispatchVisionEnhancement: %v", err)}@@ -263,7 +264,7 @@ func TestVisionEnhancementPreparesPDFSourceBeforeImageAdmission(t *testing.T) {}if _, _, err := maybeDispatchVisionEnhancement(t.Context(), dao.DB, utility.FileTypePDF, dispatched,- map[string]any{"tenant_id": "t1"}, visionSettings{}, fractionWindow{},+ map[string]any{"tenant_id": "t1"}, visionSettings{}, progress.Window{},); err != nil {t.Fatalf("maybeDispatchVisionEnhancement: %v", err)}@@ -286,7 +287,7 @@ func TestVisionEnhancement_ReportsVLMInvocationFailure(t *testing.T) {}result, _, err := maybeDispatchVisionEnhancement(t.Context(), dao.DB, utility.FileTypeDOCX, dispatched,map[string]any{"tenant_id": "t1"},- visionSettings{}, fractionWindow{})+ visionSettings{}, progress.Window{})if err != nil {t.Fatalf("maybeDispatchVisionEnhancement: %v", err)}@@ -314,7 +315,7 @@ func TestVisionEnhancement_CropFailureFallsBackToInlineImage(t *testing.T) {}result, handled, err := maybeDispatchVisionEnhancement(t.Context(), dao.DB, utility.FileTypePDF, dispatched,- map[string]any{"tenant_id": "t1"}, visionSettings{}, fractionWindow{})+ map[string]any{"tenant_id": "t1"}, visionSettings{}, progress.Window{})if err != nil {t.Fatalf("maybeDispatchVisionEnhancement: %v", err)}@@ -349,7 +350,7 @@ func TestVisionEnhancement_EnhancesPDFTablesRegardlessOfOCRProvider(t *testing.Tres, handled, err := maybeDispatchVisionEnhancement(t.Context(), dao.DB, utility.FileTypePDF, dispatched,map[string]any{"tenant_id": "t1"},- visionSettings{}, fractionWindow{})+ visionSettings{}, progress.Window{})if err != nil {t.Fatalf("maybeDispatchVisionEnhancement: %v", err)}@@ -385,7 +386,7 @@ func TestVisionEnhancement_EnhancesTableAndCellImages(t *testing.T) {result, handled, err := maybeDispatchVisionEnhancement(t.Context(), dao.DB, utility.FileTypePDF, dispatched,- map[string]any{"tenant_id": "t1"}, visionSettings{}, fractionWindow{})+ map[string]any{"tenant_id": "t1"}, visionSettings{}, progress.Window{})if err != nil {t.Fatalf("maybeDispatchVisionEnhancement: %v", err)}@@ -414,7 +415,7 @@ func TestVisionEnhancement_EnhancesExternalPDFImage(t *testing.T) {result, handled, err := maybeDispatchVisionEnhancement(t.Context(), dao.DB, utility.FileTypePDF, dispatched,- map[string]any{"tenant_id": "t1"}, visionSettings{}, fractionWindow{})+ map[string]any{"tenant_id": "t1"}, visionSettings{}, progress.Window{})if err != nil {t.Fatalf("maybeDispatchVisionEnhancement: %v", err)}@@ -465,7 +466,7 @@ func TestVisionEnhancement_EnhancesJSONImagesAndTables(t *testing.T) {tc.fileType,dispatched,map[string]any{"tenant_id": "t1", "lang": "Japanese"},- visionSettings{}, fractionWindow{})+ visionSettings{}, progress.Window{})if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -545,7 +546,7 @@ func TestVisionEnhancement_LanguagePriority(t *testing.T) {_, handled, err := maybeDispatchVisionEnhancement(t.Context(), dao.DB, utility.FileTypePDF, dispatched, tc.inputs,- visionSettings{}, fractionWindow{})+ visionSettings{}, progress.Window{})if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -586,7 +587,7 @@ func TestVisionEnhancement_MarkdownOutputUntouched(t *testing.T) {utility.FileTypeDOCX,dispatched,map[string]any{"tenant_id": "t1"},- visionSettings{}, fractionWindow{})+ visionSettings{}, progress.Window{})if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -617,7 +618,7 @@ func TestVisionEnhancement_UsesVisualPayloadRegardlessOfFileType(t *testing.T) {utility.FileTypeOTHER,dispatched,map[string]any{"tenant_id": "t1"},- visionSettings{}, fractionWindow{})+ visionSettings{}, progress.Window{})if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -647,7 +648,7 @@ func TestVisionEnhancement_EmptyOrNoTenantSkipped(t *testing.T) {utility.FileTypeDOCX,dispatched,map[string]any{},- visionSettings{}, fractionWindow{})+ visionSettings{}, progress.Window{})if err != nil || handled {t.Errorf("handled=%v, err=%v, want false, nil for missing tenant_id", handled, err)}@@ -674,7 +675,7 @@ func TestVisionEnhancement_DispatchedErrSkipped(t *testing.T) {utility.FileTypePDF,dispatched,map[string]any{"tenant_id": "t1"},- visionSettings{}, fractionWindow{})+ visionSettings{}, progress.Window{})if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -712,7 +713,7 @@ func TestVisionEnhancement_ContextCancellation(t *testing.T) {utility.FileTypePDF,dispatched,map[string]any{"tenant_id": "t1"},- visionSettings{}, fractionWindow{})+ visionSettings{}, progress.Window{})if !errors.Is(err, context.Canceled) {t.Fatalf("err = %v, want context.Canceled", err)}@@ -754,7 +755,7 @@ func TestVisionEnhancement_NonStringImageFieldFiltered(t *testing.T) {utility.FileTypePDF,dispatched,map[string]any{"tenant_id": "t1"},- visionSettings{}, fractionWindow{})+ visionSettings{}, progress.Window{})if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -797,7 +798,7 @@ func TestVisionEnhancement_MoreThanConcurrencyItems(t *testing.T) {utility.FileTypePDF,dispatched,map[string]any{"tenant_id": "t1"},- visionSettings{}, fractionWindow{})+ visionSettings{}, progress.Window{})if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -838,7 +839,7 @@ func TestVisionEnhancement_PlainTextResponseNotTruncated(t *testing.T) {utility.FileTypePDF,dispatched,map[string]any{"tenant_id": "t1"},- visionSettings{}, fractionWindow{})+ visionSettings{}, progress.Window{})if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -877,7 +878,7 @@ func TestVisionEnhancement_PromptBuilderErrorSkipped(t *testing.T) {utility.FileTypePDF,dispatched,map[string]any{"tenant_id": "t1"},- visionSettings{}, fractionWindow{})+ visionSettings{}, progress.Window{})if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -916,7 +917,7 @@ func TestVisionEnhancement_ModelResolveFailureSkipped(t *testing.T) {utility.FileTypePDF,dispatched,map[string]any{"tenant_id": "t1"},- visionSettings{}, fractionWindow{})+ visionSettings{}, progress.Window{})if err != nil {t.Fatalf("unexpected error: %v", err)}@@ -968,7 +969,7 @@ func TestVisionEnhancement_CancellationStopsSchedulingWithManyItems(t *testing.Tutility.FileTypePDF,dispatched,map[string]any{"tenant_id": "t1"},- visionSettings{}, fractionWindow{})+ visionSettings{}, progress.Window{})if !errors.Is(err, context.Canceled) {t.Fatalf("err = %v, want context.Canceled", err)}@@ -1191,7 +1192,7 @@ func TestVisionEnhancement_GlobalModelPreferredOverPDFParser(t *testing.T) {utility.FileTypePDF,dispatched,map[string]any{"tenant_id": "t1"},- visionSettings{modelID: "custom-vlm@provider"}, fractionWindow{})+ visionSettings{modelID: "custom-vlm@provider"}, progress.Window{})if err != nil {t.Fatalf("maybeDispatchVisionEnhancement: %v", err)}@@ -1233,7 +1234,7 @@ func TestVisionEnhancement_InvalidImageDataSkipped(t *testing.T) {utility.FileTypePDF,dispatched,map[string]any{"tenant_id": "t1"},- visionSettings{}, fractionWindow{})+ visionSettings{}, progress.Window{})if err != nil {t.Fatalf("maybeDispatchVisionEnhancement: %v", err)}diff --git a/internal/ingestion/component/vision_source_admission_test.go b/internal/ingestion/component/vision_source_admission_test.goindex 992a5bec3d..874e9d86b2 100644--- a/internal/ingestion/component/vision_source_admission_test.go+++ b/internal/ingestion/component/vision_source_admission_test.go@@ -25,6 +25,7 @@ import ("time""ragflow/internal/dao"+ "ragflow/internal/ingestion/component/progress""ragflow/internal/parser/parser""ragflow/internal/utility")@@ -78,7 +79,7 @@ func TestVisionSourceFetchDoesNotBlockImageAdmission(t *testing.T) {dispatched := parser.ParseResult{OutputFormat: "json", JSON: []map[string]any{newItem()}}res, _, err := maybeDispatchVisionEnhancement(context.Background(), dao.DB, utility.FileTypePDF, dispatched,- inputs, visionSettings{}, fractionWindow{},+ inputs, visionSettings{}, progress.Window{},)if err != nil {t.Errorf("maybeDispatchVisionEnhancement: %v", err)