RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestion): share extractor concurrency across tag extraction (#21891)
jay777218d44dca39ba5
diff --git a/docker/.env b/docker/.envindex e6bab62333..992a3d5144 100644--- a/docker/.env+++ b/docker/.env@@ -377,11 +377,6 @@ DOC_BULK_SIZE=${DOC_BULK_SIZE:-4}# Defaults to 16 if EMBEDDING_BATCH_SIZE is not set in the environment.EMBEDDING_BATCH_SIZE=${EMBEDDING_BATCH_SIZE:-16}-# Extractor LLM call timeout in seconds, including retries and backoff.-# Applies to keyword, question, summary, and metadata extraction.-# Default: 180. Invalid or non-positive values use the default.-# EXTRACTOR_CHAT_TIMEOUT=180-# Connector ingest: documents (e.g. one MySQL row = one file) written per batch.# Defaults to 2. Raise this for large RDBMS syncs so fewer write round-trips hit MySQL.# INDEX_BATCH_SIZE=32@@ -550,10 +545,22 @@ LLM_POOL_RATE_LIMIT_RETRY_MAX_DELAY=30.0# RECYCLE_SHUTDOWN_TIMEOUT=300# ------------------------------------------------------------------------------# Ingestor worker concurrency (overrides ingestor.concurrent_workers in service_conf.yaml)+# Ingestor# -----------------------------------------------------------------------------+# Worker concurrency (overrides ingestor.concurrent_workers in service_conf.yaml).# RAGFLOW_INGESTOR_CONCURRENT_WORKERS=1+# Maximum concurrent Extractor chunk jobs, shared across all ingestion workers+# in one process. This bounds concurrency, not requests or tokens per minute.+# Default: 10. Lower it for providers with tighter rate limits.+# Restart the ingestor after changing this value.+# EXTRACTOR_CONCURRENCY=10++# Extractor LLM call timeout in seconds, including retries and backoff.+# Applies to keyword, question, summary, and metadata extraction.+# Default: 180. Invalid or non-positive values use the default.+# EXTRACTOR_CHAT_TIMEOUT=180+# -----------------------------------------------------------------------------# Media cache limits (Go ingestion)# -----------------------------------------------------------------------------diff --git a/internal/common/environments.go b/internal/common/environments.goindex d3f907e61c..6f84cec141 100644--- a/internal/common/environments.go+++ b/internal/common/environments.go@@ -288,6 +288,10 @@ const (EnvGoogleDriveWebOAuthRedirectURI = "GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI"EnvSpacyModelDir = "SPACY_MODEL_DIR"+ // EnvExtractorConcurrency bounds concurrent chunk extraction jobs across+ // all Extractor invocations in one process.+ EnvExtractorConcurrency = "EXTRACTOR_CONCURRENCY"+// EnvExtractorChatTimeout sets the Extractor LLM call timeout in seconds,// including retries and backoff. It does not apply to tag extraction.EnvExtractorChatTimeout = "EXTRACTOR_CHAT_TIMEOUT"diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.goindex 8e895bda59..c45a59e0d0 100644--- a/internal/ingestion/component/extractor.go+++ b/internal/ingestion/component/extractor.go@@ -99,35 +99,26 @@ var (// and question extraction calls (generator.py:230,245).const extractorTemperature = 0.2-// defaultExtractorConcurrency bounds concurrent LLM extraction calls-// process-wide. It mirrors Python's chat_limiter bound-// (agent/component/base.py:353 asyncio.Semaphore(MAX_CONCURRENT_CHATS,-// default 10)) so the Go port rate-limits auto-keywords / auto-questions /-// auto-metadata extraction the same way Python does.+// defaultExtractorConcurrency bounds concurrent chunk extraction jobs+// process-wide to reduce provider rate-limit pressure.const defaultExtractorConcurrency = 10// extractorJob is one chunk's auto-extraction unit of work.type extractorJob func() error-// extractorPool is the process-wide bounded worker pool that drives-// cross-chunk concurrency for auto-keywords / auto-questions /-// auto-metadata extraction. It mirrors the parser/structure/mindmap-// WorkerPool usage, but is held globally so every Extractor-// invocation shares one rate limiter instead of spinning up one pool-// per batch. The pool only bounds concurrency (it is never StopWait'd),-// so per-invocation completion is tracked by runAutoExtractions with its-// own sync.WaitGroup + first-error collection — concurrent Invoke calls-// therefore do not disturb each other.+// extractorPool bounds chunk extraction concurrency across all Extractor+// invocations in this process: keywords, LLM tags, questions, summary, and+// metadata share the same workers. Each invocation waits for its own futures.var extractorPool = utility.NewWorkerPool[extractorJob, struct{}](extractorConcurrency(),extractorConcurrency()*4,func(_ context.Context, j extractorJob) (struct{}, error) { return struct{}{}, j() },)-// extractorConcurrency resolves the pool size from MAX_CONCURRENT_CHATS-// (Python parity) with a safe default of defaultExtractorConcurrency.+// extractorConcurrency resolves the pool size from EXTRACTOR_CONCURRENCY+// with a safe default of defaultExtractorConcurrency.func extractorConcurrency() int {- if v := os.Getenv("MAX_CONCURRENT_CHATS"); v != "" {+ if v := os.Getenv(common.EnvExtractorConcurrency); v != "" {if n, err := strconv.Atoi(v); err == nil && n > 0 {return n}@@ -135,15 +126,6 @@ func extractorConcurrency() int {return defaultExtractorConcurrency}-// SetExtractorConcurrency overrides the global extractor pool size at-// runtime (e.g. from service init or tests). Mirrors tuning Python's-// chat_limiter bound.-func SetExtractorConcurrency(n int) {- if n > 0 {- extractorPool.Resize(n)- }-}-// extractorTopNPattern matches the {{ topn }} placeholder accepted in// keyword/question system prompts.// It is replaced with the configured top_n so the count slider staysdiff --git a/internal/ingestion/component/extractor_tag.go b/internal/ingestion/component/extractor_tag.goindex f2a0c8d75b..2e06aaed69 100644--- a/internal/ingestion/component/extractor_tag.go+++ b/internal/ingestion/component/extractor_tag.go@@ -11,8 +11,6 @@ import ("sort""strconv""strings"- "sync"- "sync/atomic""time""github.com/cespare/xxhash/v2"@@ -27,6 +25,7 @@ import ("ragflow/internal/ingestion/component/progress""ragflow/internal/ingestion/component/schema""ragflow/internal/tokenizer"+ "ragflow/internal/utility")const (@@ -58,8 +57,6 @@ const (rankDecayWeightPower = 0.4)-const taggerLLMConcurrency = 8-var taggerTimeout = 60 * time.Secondconst taggerPromptTmpl = `## Role@@ -405,34 +402,8 @@ func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in ex}}- if len(docsToTag) > 0 {- driver, model, apiKey, baseURL, err := resolveExtractorChatTarget(ctx, db, in.llmID)- if err != nil {- common.Warn("extractor tag: resolve model failed, skipping LLM tagging", zap.Error(err))- }- if driver != "" && model != "" {- inv := getExtractorChatInvoker()- sem := make(chan struct{}, taggerLLMConcurrency)- var wg sync.WaitGroup- var tagged atomic.Int64- total := int64(len(docsToTag))-- for i := range docsToTag {- wg.Add(1)- go func(idx int) {- defer wg.Done()- defer func() { w.Report(ctx, float64(tagged.Add(1))/float64(total)) }()- select {- case sem <- struct{}{}:- defer func() { <-sem }()- case <-ctx.Done():- return- }- llmTagChunk(ctx, db, inv, docsToTag[idx], indexed.allTags, examples, in.cache, in.llmID, in.modelID, driver, model, apiKey, baseURL, topN, indexed)- }(i)- }- wg.Wait()- }+ if err := c.runLLMTags(ctx, db, in, docsToTag, indexed, examples, w); err != nil {+ return nil, err}taggedCount := 0@@ -454,6 +425,34 @@ func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in exreturn in.chunks, nil}+// runLLMTags submits unmatched chunks to the shared Extractor pool.+func (c *ExtractorComponent) runLLMTags(ctx context.Context, db *gorm.DB, in extractorInputs, chunks []map[string]any, indexed *MemoryTagIndex, examples []schema.TaggedChunk, w progress.Window) error {+ if len(chunks) == 0 {+ return nil+ }+ driver, model, apiKey, baseURL, err := resolveExtractorChatTarget(ctx, db, in.llmID)+ if err != nil {+ common.Warn("extractor tag: resolve model failed, skipping LLM tagging", zap.Error(err))+ }+ if driver == "" || model == "" {+ return nil+ }+ inv := getExtractorChatInvoker()+ topN := c.Param.Tags.TopN+ futs := make([]utility.WorkerPoolFuture[extractorJob, struct{}], 0, len(chunks))+ for _, chunk := range chunks {+ f, err := extractorPool.Submit(ctx, func() error {+ llmTagChunk(ctx, db, inv, chunk, indexed.allTags, examples, in.cache, in.llmID, in.modelID, driver, model, apiKey, baseURL, topN, indexed)+ return nil+ })+ if err != nil {+ return err+ }+ futs = append(futs, f)+ }+ return awaitFutures(ctx, futs, w)+}+func (c *ExtractorComponent) resolveTagSource(ctx context.Context, db *gorm.DB, lang string) (*MemoryTagIndex, bool) {if c.Param.Tags.TagFileID == "" {return nil, falsediff --git a/internal/ingestion/component/extractor_tag_test.go b/internal/ingestion/component/extractor_tag_test.goindex e7b604c509..3851e4a502 100644--- a/internal/ingestion/component/extractor_tag_test.go+++ b/internal/ingestion/component/extractor_tag_test.go@@ -15,10 +15,70 @@ import ("ragflow/internal/common""ragflow/internal/dao""ragflow/internal/entity"+ "ragflow/internal/ingestion/component/progress""ragflow/internal/ingestion/component/schema""ragflow/internal/tokenizer"+ "ragflow/internal/utility")+func TestExtractorTagsShareExtractionConcurrency(t *testing.T) {+ pushExtractorTagChatStub(t, map[string]string{"__static__": `{"finance": 8}`})+ pushExtractorTagTargetResolverStub(t)++ previous := extractorPool+ pool := utility.NewWorkerPool[extractorJob, struct{}](1, 4,+ func(_ context.Context, job extractorJob) (struct{}, error) { return struct{}{}, job() })+ extractorPool = pool+ t.Cleanup(func() { pool.StopWait(); extractorPool = previous })+ release := make(chan struct{})+ defer close(release)+ started := make(chan struct{})+ if _, err := pool.Submit(t.Context(), func() error { close(started); <-release; return nil }); err != nil {+ t.Fatal(err)+ }+ <-started++ c := &ExtractorComponent{Param: schema.ExtractorParam{}.Defaults()}+ c.Param.Tags.TopN = 1+ chunks := []map[string]any{{"text": "unrelated astronomy observations"}, {"text": "unrelated mountain geology"}}+ indexed := &MemoryTagIndex{allTags: map[string]float64{"finance": 1}}+ ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)+ defer cancel()+ done := make(chan error, 1)+ go func() {+ done <- c.runLLMTags(ctx, nil, extractorInputs{llmID: "test@test", modelID: "test@test"}, chunks, indexed, nil, progress.Window{})+ }()++ // The occupied shared slot must hold up tags, rather than letting their+ // own pool bypass extraction work already in flight.+ ticker := time.NewTicker(time.Millisecond)+ defer ticker.Stop()+ for pool.Stats().SubmittedTotal < 3 {+ select {+ case err := <-done:+ t.Fatalf("tagging bypassed the occupied extraction pool: %v", err)+ case <-ctx.Done():+ t.Fatal("tagging did not queue in the shared extraction pool")+ case <-ticker.C:+ }+ }+ release <- struct{}{}+ select {+ case err := <-done:+ if err != nil {+ t.Fatal(err)+ }+ case <-ctx.Done():+ t.Fatal("tagging did not finish after the extraction slot was released")+ }+ for i, chunk := range chunks {+ weights, ok := chunk[common.TAG_FLD].(OrderedTagWeights)+ if !ok || weights["finance"] != 8 {+ t.Fatalf("chunk %d tags = %v, want finance=8", i, chunk[common.TAG_FLD])+ }+ }+}+type stubExtractorTagChat struct {responses map[string]string}diff --git a/internal/ingestion/knowledge_compile/pool.go b/internal/ingestion/knowledge_compile/pool.goindex 6b527dffd3..294678cfd3 100644--- a/internal/ingestion/knowledge_compile/pool.go+++ b/internal/ingestion/knowledge_compile/pool.go@@ -52,7 +52,7 @@ var compilerPool = utility.NewWorkerPool[CompilerJob, struct{}](// compilerConcurrency resolves the global pool size. It defaults to the host// vCPU count, overridable via KC_COMPILE_CONCURRENCY (mirroring the extractor-// pool's MAX_CONCURRENT_CHATS tuning knob).+// pool's EXTRACTOR_CONCURRENCY tuning knob).func compilerConcurrency() int {if v := os.Getenv("KC_COMPILE_CONCURRENCY"); v != "" {if n, err := strconv.Atoi(v); err == nil && n > 0 {@@ -67,7 +67,7 @@ func compilerConcurrency() int {}// SetCompilerConcurrency overrides the global pool size at runtime (e.g. from-// service init or tests). Mirrors SetExtractorConcurrency.+// service init or tests).func SetCompilerConcurrency(n int) {if n > 0 {compilerPool.Resize(n)