Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): share extractor concurrency across tag extraction (#21891)

jay777218d44dca39ba5
diff --git a/docker/.env b/docker/.env
index e6bab62333..992a3d5144 100644
--- a/docker/.env
+++ b/docker/.env
@@ -377,11 +377,6 @@ DOC_BULK_SIZE=${DOC_BULK_SIZE:-4}
# Defaults to 16 if EMBEDDING_BATCH_SIZE is not set in the environment.
EMBEDDING_BATCH_SIZE=${EMBEDDING_BATCH_SIZE:-16}
-# Extractor LLM call timeout in seconds, including retries and backoff.
-# Applies to keyword, question, summary, and metadata extraction.
-# Default: 180. Invalid or non-positive values use the default.
-# EXTRACTOR_CHAT_TIMEOUT=180
-
# Connector ingest: documents (e.g. one MySQL row = one file) written per batch.
# Defaults to 2. Raise this for large RDBMS syncs so fewer write round-trips hit MySQL.
# INDEX_BATCH_SIZE=32
@@ -550,10 +545,22 @@ LLM_POOL_RATE_LIMIT_RETRY_MAX_DELAY=30.0
# RECYCLE_SHUTDOWN_TIMEOUT=300
# -----------------------------------------------------------------------------
-# Ingestor worker concurrency (overrides ingestor.concurrent_workers in service_conf.yaml)
+# Ingestor
# -----------------------------------------------------------------------------
+# Worker concurrency (overrides ingestor.concurrent_workers in service_conf.yaml).
# RAGFLOW_INGESTOR_CONCURRENT_WORKERS=1
+# Maximum concurrent Extractor chunk jobs, shared across all ingestion workers
+# in one process. This bounds concurrency, not requests or tokens per minute.
+# Default: 10. Lower it for providers with tighter rate limits.
+# Restart the ingestor after changing this value.
+# EXTRACTOR_CONCURRENCY=10
+
+# Extractor LLM call timeout in seconds, including retries and backoff.
+# Applies to keyword, question, summary, and metadata extraction.
+# Default: 180. Invalid or non-positive values use the default.
+# EXTRACTOR_CHAT_TIMEOUT=180
+
# -----------------------------------------------------------------------------
# Media cache limits (Go ingestion)
# -----------------------------------------------------------------------------
diff --git a/internal/common/environments.go b/internal/common/environments.go
index d3f907e61c..6f84cec141 100644
--- a/internal/common/environments.go
+++ b/internal/common/environments.go
@@ -288,6 +288,10 @@ const (
EnvGoogleDriveWebOAuthRedirectURI = "GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI"
EnvSpacyModelDir = "SPACY_MODEL_DIR"
+ // EnvExtractorConcurrency bounds concurrent chunk extraction jobs across
+ // all Extractor invocations in one process.
+ EnvExtractorConcurrency = "EXTRACTOR_CONCURRENCY"
+
// EnvExtractorChatTimeout sets the Extractor LLM call timeout in seconds,
// including retries and backoff. It does not apply to tag extraction.
EnvExtractorChatTimeout = "EXTRACTOR_CHAT_TIMEOUT"
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go
index 8e895bda59..c45a59e0d0 100644
--- a/internal/ingestion/component/extractor.go
+++ b/internal/ingestion/component/extractor.go
@@ -99,35 +99,26 @@ var (
// and question extraction calls (generator.py:230,245).
const extractorTemperature = 0.2
-// defaultExtractorConcurrency bounds concurrent LLM extraction calls
-// process-wide. It mirrors Python's chat_limiter bound
-// (agent/component/base.py:353 asyncio.Semaphore(MAX_CONCURRENT_CHATS,
-// default 10)) so the Go port rate-limits auto-keywords / auto-questions /
-// auto-metadata extraction the same way Python does.
+// defaultExtractorConcurrency bounds concurrent chunk extraction jobs
+// process-wide to reduce provider rate-limit pressure.
const defaultExtractorConcurrency = 10
// extractorJob is one chunk's auto-extraction unit of work.
type extractorJob func() error
-// extractorPool is the process-wide bounded worker pool that drives
-// cross-chunk concurrency for auto-keywords / auto-questions /
-// auto-metadata extraction. It mirrors the parser/structure/mindmap
-// WorkerPool usage, but is held globally so every Extractor
-// invocation shares one rate limiter instead of spinning up one pool
-// per batch. The pool only bounds concurrency (it is never StopWait'd),
-// so per-invocation completion is tracked by runAutoExtractions with its
-// own sync.WaitGroup + first-error collection — concurrent Invoke calls
-// therefore do not disturb each other.
+// extractorPool bounds chunk extraction concurrency across all Extractor
+// invocations in this process: keywords, LLM tags, questions, summary, and
+// metadata share the same workers. Each invocation waits for its own futures.
var extractorPool = utility.NewWorkerPool[extractorJob, struct{}](
extractorConcurrency(),
extractorConcurrency()*4,
func(_ context.Context, j extractorJob) (struct{}, error) { return struct{}{}, j() },
)
-// extractorConcurrency resolves the pool size from MAX_CONCURRENT_CHATS
-// (Python parity) with a safe default of defaultExtractorConcurrency.
+// extractorConcurrency resolves the pool size from EXTRACTOR_CONCURRENCY
+// with a safe default of defaultExtractorConcurrency.
func extractorConcurrency() int {
- if v := os.Getenv("MAX_CONCURRENT_CHATS"); v != "" {
+ if v := os.Getenv(common.EnvExtractorConcurrency); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
return n
}
@@ -135,15 +126,6 @@ func extractorConcurrency() int {
return defaultExtractorConcurrency
}
-// SetExtractorConcurrency overrides the global extractor pool size at
-// runtime (e.g. from service init or tests). Mirrors tuning Python's
-// chat_limiter bound.
-func SetExtractorConcurrency(n int) {
- if n > 0 {
- extractorPool.Resize(n)
- }
-}
-
// extractorTopNPattern matches the {{ topn }} placeholder accepted in
// keyword/question system prompts.
// It is replaced with the configured top_n so the count slider stays
diff --git a/internal/ingestion/component/extractor_tag.go b/internal/ingestion/component/extractor_tag.go
index f2a0c8d75b..2e06aaed69 100644
--- a/internal/ingestion/component/extractor_tag.go
+++ b/internal/ingestion/component/extractor_tag.go
@@ -11,8 +11,6 @@ import (
"sort"
"strconv"
"strings"
- "sync"
- "sync/atomic"
"time"
"github.com/cespare/xxhash/v2"
@@ -27,6 +25,7 @@ import (
"ragflow/internal/ingestion/component/progress"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/tokenizer"
+ "ragflow/internal/utility"
)
const (
@@ -58,8 +57,6 @@ const (
rankDecayWeightPower = 0.4
)
-const taggerLLMConcurrency = 8
-
var taggerTimeout = 60 * time.Second
const taggerPromptTmpl = `## Role
@@ -405,34 +402,8 @@ func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in ex
}
}
- if len(docsToTag) > 0 {
- driver, model, apiKey, baseURL, err := resolveExtractorChatTarget(ctx, db, in.llmID)
- if err != nil {
- common.Warn("extractor tag: resolve model failed, skipping LLM tagging", zap.Error(err))
- }
- if driver != "" && model != "" {
- inv := getExtractorChatInvoker()
- sem := make(chan struct{}, taggerLLMConcurrency)
- var wg sync.WaitGroup
- var tagged atomic.Int64
- total := int64(len(docsToTag))
-
- for i := range docsToTag {
- wg.Add(1)
- go func(idx int) {
- defer wg.Done()
- defer func() { w.Report(ctx, float64(tagged.Add(1))/float64(total)) }()
- select {
- case sem <- struct{}{}:
- defer func() { <-sem }()
- case <-ctx.Done():
- return
- }
- llmTagChunk(ctx, db, inv, docsToTag[idx], indexed.allTags, examples, in.cache, in.llmID, in.modelID, driver, model, apiKey, baseURL, topN, indexed)
- }(i)
- }
- wg.Wait()
- }
+ if err := c.runLLMTags(ctx, db, in, docsToTag, indexed, examples, w); err != nil {
+ return nil, err
}
taggedCount := 0
@@ -454,6 +425,34 @@ func (c *ExtractorComponent) runAutoTags(ctx context.Context, db *gorm.DB, in ex
return in.chunks, nil
}
+// runLLMTags submits unmatched chunks to the shared Extractor pool.
+func (c *ExtractorComponent) runLLMTags(ctx context.Context, db *gorm.DB, in extractorInputs, chunks []map[string]any, indexed *MemoryTagIndex, examples []schema.TaggedChunk, w progress.Window) error {
+ if len(chunks) == 0 {
+ return nil
+ }
+ driver, model, apiKey, baseURL, err := resolveExtractorChatTarget(ctx, db, in.llmID)
+ if err != nil {
+ common.Warn("extractor tag: resolve model failed, skipping LLM tagging", zap.Error(err))
+ }
+ if driver == "" || model == "" {
+ return nil
+ }
+ inv := getExtractorChatInvoker()
+ topN := c.Param.Tags.TopN
+ futs := make([]utility.WorkerPoolFuture[extractorJob, struct{}], 0, len(chunks))
+ for _, chunk := range chunks {
+ f, err := extractorPool.Submit(ctx, func() error {
+ llmTagChunk(ctx, db, inv, chunk, indexed.allTags, examples, in.cache, in.llmID, in.modelID, driver, model, apiKey, baseURL, topN, indexed)
+ return nil
+ })
+ if err != nil {
+ return err
+ }
+ futs = append(futs, f)
+ }
+ return awaitFutures(ctx, futs, w)
+}
+
func (c *ExtractorComponent) resolveTagSource(ctx context.Context, db *gorm.DB, lang string) (*MemoryTagIndex, bool) {
if c.Param.Tags.TagFileID == "" {
return nil, false
diff --git a/internal/ingestion/component/extractor_tag_test.go b/internal/ingestion/component/extractor_tag_test.go
index e7b604c509..3851e4a502 100644
--- a/internal/ingestion/component/extractor_tag_test.go
+++ b/internal/ingestion/component/extractor_tag_test.go
@@ -15,10 +15,70 @@ import (
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
+ "ragflow/internal/ingestion/component/progress"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/tokenizer"
+ "ragflow/internal/utility"
)
+func TestExtractorTagsShareExtractionConcurrency(t *testing.T) {
+ pushExtractorTagChatStub(t, map[string]string{"__static__": `{"finance": 8}`})
+ pushExtractorTagTargetResolverStub(t)
+
+ previous := extractorPool
+ pool := utility.NewWorkerPool[extractorJob, struct{}](1, 4,
+ func(_ context.Context, job extractorJob) (struct{}, error) { return struct{}{}, job() })
+ extractorPool = pool
+ t.Cleanup(func() { pool.StopWait(); extractorPool = previous })
+ release := make(chan struct{})
+ defer close(release)
+ started := make(chan struct{})
+ if _, err := pool.Submit(t.Context(), func() error { close(started); <-release; return nil }); err != nil {
+ t.Fatal(err)
+ }
+ <-started
+
+ c := &ExtractorComponent{Param: schema.ExtractorParam{}.Defaults()}
+ c.Param.Tags.TopN = 1
+ chunks := []map[string]any{{"text": "unrelated astronomy observations"}, {"text": "unrelated mountain geology"}}
+ indexed := &MemoryTagIndex{allTags: map[string]float64{"finance": 1}}
+ ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+ defer cancel()
+ done := make(chan error, 1)
+ go func() {
+ done <- c.runLLMTags(ctx, nil, extractorInputs{llmID: "test@test", modelID: "test@test"}, chunks, indexed, nil, progress.Window{})
+ }()
+
+ // The occupied shared slot must hold up tags, rather than letting their
+ // own pool bypass extraction work already in flight.
+ ticker := time.NewTicker(time.Millisecond)
+ defer ticker.Stop()
+ for pool.Stats().SubmittedTotal < 3 {
+ select {
+ case err := <-done:
+ t.Fatalf("tagging bypassed the occupied extraction pool: %v", err)
+ case <-ctx.Done():
+ t.Fatal("tagging did not queue in the shared extraction pool")
+ case <-ticker.C:
+ }
+ }
+ release <- struct{}{}
+ select {
+ case err := <-done:
+ if err != nil {
+ t.Fatal(err)
+ }
+ case <-ctx.Done():
+ t.Fatal("tagging did not finish after the extraction slot was released")
+ }
+ for i, chunk := range chunks {
+ weights, ok := chunk[common.TAG_FLD].(OrderedTagWeights)
+ if !ok || weights["finance"] != 8 {
+ t.Fatalf("chunk %d tags = %v, want finance=8", i, chunk[common.TAG_FLD])
+ }
+ }
+}
+
type stubExtractorTagChat struct {
responses map[string]string
}
diff --git a/internal/ingestion/knowledge_compile/pool.go b/internal/ingestion/knowledge_compile/pool.go
index 6b527dffd3..294678cfd3 100644
--- a/internal/ingestion/knowledge_compile/pool.go
+++ b/internal/ingestion/knowledge_compile/pool.go
@@ -52,7 +52,7 @@ var compilerPool = utility.NewWorkerPool[CompilerJob, struct{}](
// compilerConcurrency resolves the global pool size. It defaults to the host
// vCPU count, overridable via KC_COMPILE_CONCURRENCY (mirroring the extractor
-// pool's MAX_CONCURRENT_CHATS tuning knob).
+// pool's EXTRACTOR_CONCURRENCY tuning knob).
func compilerConcurrency() int {
if v := os.Getenv("KC_COMPILE_CONCURRENCY"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
@@ -67,7 +67,7 @@ func compilerConcurrency() int {
}
// SetCompilerConcurrency overrides the global pool size at runtime (e.g. from
-// service init or tests). Mirrors SetExtractorConcurrency.
+// service init or tests).
func SetCompilerConcurrency(n int) {
if n > 0 {
compilerPool.Resize(n)