Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(server): collapse duplicate ingestor concurrency resolver and drop redundant Max prefix (#21588)

xugangqiang9742be8b5bd9
diff --git a/cmd/args_flag_test.go b/cmd/args_flag_test.go
index 9c83b80f60..539395cb05 100644
--- a/cmd/args_flag_test.go
+++ b/cmd/args_flag_test.go
@@ -39,7 +39,7 @@ func TestParseArgsStandardSyntaxMatrix(t *testing.T) {
{"--port", "8080"},
{"--admin-host", "1.2.3.4:5678"},
{"--deepdoc-inference-concurrency", "4"},
- {"--ingestor-max-concurrent-workers", "8"},
+ {"--ingestor-concurrent-workers", "8"},
{"--ingestor-page-concurrency", "4"},
{"--deepdoc-inference-cpu-cores", "2"},
}
@@ -333,7 +333,7 @@ func TestParseArgsValueEdgeCases(t *testing.T) {
func TestParseArgsMissingValue(t *testing.T) {
valueFlags := []string{
"--log-level", "--config", "--name", "--port", "--admin-host",
- "--deepdoc-inference-concurrency", "--ingestor-max-concurrent-workers",
+ "--deepdoc-inference-concurrency", "--ingestor-concurrent-workers",
"--ingestor-page-concurrency", "--deepdoc-inference-cpu-cores",
}
for _, flag := range valueFlags {
@@ -356,7 +356,7 @@ func TestParseArgsMissingValue(t *testing.T) {
// integer flags, preserving the pre-migration validation rules.
func TestParseArgsNumericValidation(t *testing.T) {
positiveInvalid := []string{"0", "-1", "abc", "1.5"}
- for _, flag := range []string{"--deepdoc-inference-concurrency", "--ingestor-max-concurrent-workers", "--ingestor-page-concurrency"} {
+ for _, flag := range []string{"--deepdoc-inference-concurrency", "--ingestor-concurrent-workers", "--ingestor-page-concurrency"} {
for _, v := range positiveInvalid {
if _, err := parseArgsFrom([]string{"ragflow_server", "--api", flag + "=" + v}); err == nil {
t.Errorf("%s=%s error = nil, want rejection", flag, v)
@@ -420,15 +420,15 @@ func TestParseArgsPresenceSemantics(t *testing.T) {
t.Fatalf("parseArgsFrom error = %v", err)
}
for name, got := range map[string]bool{
- "logLevel": args.LogLevel != nil,
- "configPath": args.ConfigPath != nil,
- "port": args.Port != nil,
- "adminHost": args.AdminHost != nil,
- "name": args.Name != nil,
- "deepdocInferenceConcurrency": args.DeepdocInferenceConcurrency != nil,
- "ingestorMaxConcurrentWorkers": args.IngestorMaxConcurrentWorkers != nil,
- "ingestorPageConcurrency": args.IngestorPageConcurrency != nil,
- "deepdocInferenceCPUCores": args.DeepdocInferenceCPUCores != nil,
+ "logLevel": args.LogLevel != nil,
+ "configPath": args.ConfigPath != nil,
+ "port": args.Port != nil,
+ "adminHost": args.AdminHost != nil,
+ "name": args.Name != nil,
+ "deepdocInferenceConcurrency": args.DeepdocInferenceConcurrency != nil,
+ "ingestorConcurrentWorkers": args.IngestorConcurrentWorkers != nil,
+ "ingestorPageConcurrency": args.IngestorPageConcurrency != nil,
+ "deepdocInferenceCPUCores": args.DeepdocInferenceCPUCores != nil,
} {
if got {
t.Errorf("%s set with no flag on the command line", name)
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go
index ace822d810..1d6533b120 100644
--- a/cmd/ragflow_server.go
+++ b/cmd/ragflow_server.go
@@ -133,7 +133,7 @@ func parseArgsFrom(argv []string) (*bootstrap.ServerArgs, error) {
fs.IntVar(&port, "port", 0, "")
fs.IntVar(&port, "p", 0, "")
fs.IntVar(&deepdocConcurrency, "deepdoc-inference-concurrency", 0, "")
- fs.IntVar(&ingestorWorkers, "ingestor-max-concurrent-workers", 0, "")
+ fs.IntVar(&ingestorWorkers, "ingestor-concurrent-workers", 0, "")
fs.IntVar(&pageConcurrency, "ingestor-page-concurrency", 0, "")
fs.IntVar(&cpuCores, "deepdoc-inference-cpu-cores", 0, "")
fs.BoolVar(&initSuperUser, "init-superuser", false, "")
@@ -174,8 +174,8 @@ func parseArgsFrom(argv []string) (*bootstrap.ServerArgs, error) {
args.Port = &port
case "deepdoc-inference-concurrency":
args.DeepdocInferenceConcurrency = &deepdocConcurrency
- case "ingestor-max-concurrent-workers":
- args.IngestorMaxConcurrentWorkers = &ingestorWorkers
+ case "ingestor-concurrent-workers":
+ args.IngestorConcurrentWorkers = &ingestorWorkers
case "ingestor-page-concurrency":
args.IngestorPageConcurrency = &pageConcurrency
case "deepdoc-inference-cpu-cores":
@@ -221,8 +221,8 @@ func parseArgsFrom(argv []string) (*bootstrap.ServerArgs, error) {
if args.DeepdocInferenceConcurrency != nil && *args.DeepdocInferenceConcurrency <= 0 {
return nil, fmt.Errorf("invalid --deepdoc-inference-concurrency %d: must be positive", *args.DeepdocInferenceConcurrency)
}
- if args.IngestorMaxConcurrentWorkers != nil && *args.IngestorMaxConcurrentWorkers <= 0 {
- return nil, fmt.Errorf("invalid --ingestor-max-concurrent-workers %d: must be positive", *args.IngestorMaxConcurrentWorkers)
+ if args.IngestorConcurrentWorkers != nil && *args.IngestorConcurrentWorkers <= 0 {
+ return nil, fmt.Errorf("invalid --ingestor-concurrent-workers %d: must be positive", *args.IngestorConcurrentWorkers)
}
if args.IngestorPageConcurrency != nil && *args.IngestorPageConcurrency <= 0 {
return nil, fmt.Errorf("invalid --ingestor-page-concurrency %d: must be positive", *args.IngestorPageConcurrency)
@@ -288,7 +288,7 @@ func printHelp(args *bootstrap.ServerArgs) {
fmt.Fprintf(os.Stderr, " -p, --port int \tServer port (overrides config file)\n")
fmt.Fprintf(os.Stderr, " --admin-host string\tAdmin server host:port (ingestor, syncer, deepdoc)\n")
fmt.Fprintf(os.Stderr, " --name string \tServer name (ingestor, syncer, deepdoc)\n")
- fmt.Fprintf(os.Stderr, " --ingestor-max-concurrent-workers int\tIngestor NATS worker count K, range [1, 256] (default: 1)\n")
+ fmt.Fprintf(os.Stderr, " --ingestor-concurrent-workers int\tIngestor NATS worker count K, range [1, 256] (default: 1)\n")
fmt.Fprintf(os.Stderr, " --ingestor-page-concurrency int\tProcess-wide page concurrency N, range [1, 16] (default: 2)\n")
fmt.Fprintf(os.Stderr, " --init-superuser\tInitialize superuser account (admin)\n")
fmt.Fprintf(os.Stderr, " -v, --version \tPrint version information and exit\n")
@@ -329,7 +329,7 @@ func printHelp(args *bootstrap.ServerArgs) {
fmt.Fprintf(os.Stderr, "Options:\n")
fmt.Fprintf(os.Stderr, " -f --config string\tPath to config file\n")
fmt.Fprintf(os.Stderr, " --name string\t\t\tIngestion server name (default: \"default_ingestion\")\n")
- fmt.Fprintf(os.Stderr, " --ingestor-max-concurrent-workers int\tIngestor NATS worker count K, range [1, 256] (default: 1)\n")
+ fmt.Fprintf(os.Stderr, " --ingestor-concurrent-workers int\tIngestor NATS worker count K, range [1, 256] (default: 1)\n")
fmt.Fprintf(os.Stderr, " --ingestor-page-concurrency int\tProcess-wide page concurrency N, range [1, 16] (default: 2)\n")
fmt.Fprintf(os.Stderr, " --admin-host string\tAdmin server host:port (overrides config file)\n")
fmt.Fprintf(os.Stderr, " -v, --version \t\tPrint version information and exit\n")
diff --git a/cmd/ragflow_server_deepdoc_test.go b/cmd/ragflow_server_deepdoc_test.go
index 5b96b1ebac..afbfdeba9c 100644
--- a/cmd/ragflow_server_deepdoc_test.go
+++ b/cmd/ragflow_server_deepdoc_test.go
@@ -69,25 +69,25 @@ func TestParseArgsDeepDocInferenceConcurrency(t *testing.T) {
}
}
-// TestParseArgsIngestorMaxConcurrentWorkers pins the contract that the CLI
-// parser rejects a non-positive or non-integer --ingestor-max-concurrent-workers
+// TestParseArgsIngestorConcurrentWorkers pins the contract that the CLI
+// parser rejects a non-positive or non-integer --ingestor-concurrent-workers
// up front (both the "--flag=value" and "--flag value" forms), and accepts a
// positive value.
-func TestParseArgsIngestorMaxConcurrentWorkers(t *testing.T) {
+func TestParseArgsIngestorConcurrentWorkers(t *testing.T) {
cases := []struct {
name string
args []string
wantErr bool
want int
}{
- {"equals zero", []string{"prog", "--ingestor-max-concurrent-workers=0"}, true, 0},
- {"equals negative", []string{"prog", "--ingestor-max-concurrent-workers=-3"}, true, 0},
- {"equals nonint", []string{"prog", "--ingestor-max-concurrent-workers=abc"}, true, 0},
- {"space zero", []string{"prog", "--ingestor-max-concurrent-workers", "0"}, true, 0},
- {"space negative", []string{"prog", "--ingestor-max-concurrent-workers", "-3"}, true, 0},
- {"space nonint", []string{"prog", "--ingestor-max-concurrent-workers", "abc"}, true, 0},
- {"equals positive", []string{"prog", "--ingestor-max-concurrent-workers=12"}, false, 12},
- {"space positive", []string{"prog", "--ingestor-max-concurrent-workers", "12"}, false, 12},
+ {"equals zero", []string{"prog", "--ingestor-concurrent-workers=0"}, true, 0},
+ {"equals negative", []string{"prog", "--ingestor-concurrent-workers=-3"}, true, 0},
+ {"equals nonint", []string{"prog", "--ingestor-concurrent-workers=abc"}, true, 0},
+ {"space zero", []string{"prog", "--ingestor-concurrent-workers", "0"}, true, 0},
+ {"space negative", []string{"prog", "--ingestor-concurrent-workers", "-3"}, true, 0},
+ {"space nonint", []string{"prog", "--ingestor-concurrent-workers", "abc"}, true, 0},
+ {"equals positive", []string{"prog", "--ingestor-concurrent-workers=12"}, false, 12},
+ {"space positive", []string{"prog", "--ingestor-concurrent-workers", "12"}, false, 12},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
@@ -105,8 +105,8 @@ func TestParseArgsIngestorMaxConcurrentWorkers(t *testing.T) {
if err != nil {
t.Fatalf("parseArgs() error = %v, want nil for %v", err, tc.args)
}
- if got.IngestorMaxConcurrentWorkers == nil || *got.IngestorMaxConcurrentWorkers != tc.want {
- t.Fatalf("ingestorMaxConcurrentWorkers = %v, want %d for %v", got.IngestorMaxConcurrentWorkers, tc.want, tc.args)
+ if got.IngestorConcurrentWorkers == nil || *got.IngestorConcurrentWorkers != tc.want {
+ t.Fatalf("ingestorConcurrentWorkers = %v, want %d for %v", got.IngestorConcurrentWorkers, tc.want, tc.args)
}
})
}
diff --git a/conf/service_conf.yaml b/conf/service_conf.yaml
index 4d40e16eea..7801da4c7d 100644
--- a/conf/service_conf.yaml
+++ b/conf/service_conf.yaml
@@ -98,10 +98,10 @@ otel:
ingestor:
mq_type: 'nats'
# NATS consumer worker count (K). How many ingestion tasks run in parallel.
- # Override with the env var RAGFLOW_INGESTOR_MAX_CONCURRENT_WORKERS, or the
- # CLI flag --ingestor-max-concurrent-workers (highest precedence).
+ # Override with the env var RAGFLOW_INGESTOR_CONCURRENT_WORKERS, or the
+ # CLI flag --ingestor-concurrent-workers (highest precedence).
# Valid range [1, 256]. Default 1.
- max_concurrent_workers: 1
+ concurrent_workers: 1
# Process-wide page concurrency (N). Total PDF pages parsed concurrently across
# the whole process (shared by all ingestor workers). Override with the env var
# RAGFLOW_INGESTOR_PAGE_CONCURRENCY, or the CLI flag --ingestor-page-concurrency
diff --git a/docker/.env b/docker/.env
index 168a86fe2f..5bd9c1e544 100644
--- a/docker/.env
+++ b/docker/.env
@@ -540,6 +540,11 @@ LLM_POOL_RATE_LIMIT_RETRY_MAX_DELAY=30.0
# before cancelling them. Only has an effect when MAX_TASKS_PER_WORKER > 0.
# RECYCLE_SHUTDOWN_TIMEOUT=300
+# -----------------------------------------------------------------------------
+# Ingestor worker concurrency (overrides ingestor.concurrent_workers in service_conf.yaml)
+# -----------------------------------------------------------------------------
+# RAGFLOW_INGESTOR_CONCURRENT_WORKERS=1
+
# -----------------------------------------------------------------------------
# DeepDoc (in-process)
# -----------------------------------------------------------------------------
diff --git a/docker/service_conf.yaml.template b/docker/service_conf.yaml.template
index 54f1277e34..0714b1ab74 100644
--- a/docker/service_conf.yaml.template
+++ b/docker/service_conf.yaml.template
@@ -126,10 +126,10 @@ otel:
ingestor:
mq_type: 'nats'
# NATS consumer worker count (K). How many ingestion tasks run in parallel.
- # Override with the env var RAGFLOW_INGESTOR_MAX_CONCURRENT_WORKERS, or the
- # CLI flag --ingestor-max-concurrent-workers (highest precedence).
+ # Override with the env var RAGFLOW_INGESTOR_CONCURRENT_WORKERS, or the
+ # CLI flag --ingestor-concurrent-workers (highest precedence).
# Valid range [1, 256]. Default 1.
- max_concurrent_workers: 1
+ concurrent_workers: 1
# Process-wide page concurrency (N). Total PDF pages parsed concurrently across
# the whole process (shared by all ingestor workers). Override with the env var
# RAGFLOW_INGESTOR_PAGE_CONCURRENCY, or the CLI flag --ingestor-page-concurrency
diff --git a/internal/common/environments.go b/internal/common/environments.go
index f1fab34ff9..d6323efc67 100644
--- a/internal/common/environments.go
+++ b/internal/common/environments.go
@@ -310,11 +310,11 @@ const (
// ingestor.inference_concurrency config key and is itself overridden by the
// --deepdoc-inference-concurrency CLI flag.
EnvDeepDocInferenceConcurrency = "RAGFLOW_DEEPDOC_INFERENCE_CONCURRENCY"
- // EnvIngestorMaxConcurrentWorkers bounds how many ingestion tasks the
+ // EnvIngestorConcurrentWorkers bounds how many ingestion tasks the
// ingestor runs in parallel (the NATS consumer worker count). It overrides
- // the ingestor.max_concurrent_workers config key and is itself overridden
- // by the --ingestor-max-concurrent-workers CLI flag.
- EnvIngestorMaxConcurrentWorkers = "RAGFLOW_INGESTOR_MAX_CONCURRENT_WORKERS"
+ // the ingestor.concurrent_workers config key and is itself overridden
+ // by the --ingestor-concurrent-workers CLI flag.
+ EnvIngestorConcurrentWorkers = "RAGFLOW_INGESTOR_CONCURRENT_WORKERS"
// EnvIngestorPageConcurrency bounds how many pages of a single document are
// parsed concurrently inside one ingestor worker. It overrides the
// ingestor.page_concurrency config key and is itself overridden by the
diff --git a/internal/ingestion/component/vision_source_admission_test.go b/internal/ingestion/component/vision_source_admission_test.go
index 20de2598de..e097a0d123 100644
--- a/internal/ingestion/component/vision_source_admission_test.go
+++ b/internal/ingestion/component/vision_source_admission_test.go
@@ -42,7 +42,7 @@ import (
// of capacity can fix — the waiter is outlasted, not outnumbered.
//
// Two concurrent calls model two documents parsed in the same process, which
-// requires ingestor.max_concurrent_workers >= 2.
+// requires ingestor.concurrent_workers >= 2.
func TestVisionSourceFetchDoesNotBlockImageAdmission(t *testing.T) {
if testing.Short() {
t.Skip("needs the stub fetch to outlast visionMediaItemBudget")
diff --git a/internal/ingestion/service/execute_memory_task_panic_test.go b/internal/ingestion/service/execute_memory_task_panic_test.go
index fcf8410221..1a076a00e8 100644
--- a/internal/ingestion/service/execute_memory_task_panic_test.go
+++ b/internal/ingestion/service/execute_memory_task_panic_test.go
@@ -17,7 +17,7 @@ import (
// the memory-extraction path (HandleSaveToMemoryTask) is recovered by
// executeMemoryTask. Before the fix, executeMemoryTask had no recover guard
// (unlike the ingestion path's settleMessage), so a panic crashed the worker
-// goroutine. With max_concurrent_workers defaults that can be as low as 1, a
+// goroutine. With concurrent_workers defaults that can be as low as 1, a
// single panicking memory task permanently removed the only worker, after which
// every normal document parse (PDF etc.) stalled in the queue — exactly the
// "parsing works at first, then breaks after some memory tasks" symptom.
diff --git a/internal/server/bootstrap/args.go b/internal/server/bootstrap/args.go
index 869515772f..07619adf7d 100644
--- a/internal/server/bootstrap/args.go
+++ b/internal/server/bootstrap/args.go
@@ -36,9 +36,9 @@ type ServerArgs struct {
// DeepdocInferenceConcurrency, when set, overrides the DeepDoc inference
// concurrency from env/config. nil means "unspecified".
DeepdocInferenceConcurrency *int
- // IngestorMaxConcurrentWorkers, when set, overrides the ingestor worker
+ // IngestorConcurrentWorkers, when set, overrides the ingestor worker
// count (NATS consumer count K) from env/config. nil means "unspecified".
- IngestorMaxConcurrentWorkers *int
+ IngestorConcurrentWorkers *int
// IngestorPageConcurrency, when set, overrides the per-document page
// concurrency (N) from env/config. nil means "unspecified".
IngestorPageConcurrency *int
diff --git a/internal/server/config/ingestor_config.go b/internal/server/config/ingestor_config.go
index fcb540326d..255134f0b4 100644
--- a/internal/server/config/ingestor_config.go
+++ b/internal/server/config/ingestor_config.go
@@ -29,17 +29,26 @@ import (
)
type IngestorConfig struct {
- // MaxConcurrentWorkers bounds how many ingestion tasks the ingestor runs in
+ // ConcurrentWorkers bounds how many ingestion tasks the ingestor runs in
// parallel — it is the NATS consumer worker count. Valid range [1, 256],
// enforced at startup by the CLI/env/config resolver (GetIngestor*); a
// value outside that range is a fatal startup error. Default 1.
- MaxConcurrentWorkers int `mapstructure:"max_concurrent_workers"`
+ ConcurrentWorkers int `mapstructure:"concurrent_workers"`
+ // ConcurrentWorkersSet is true when concurrent_workers was explicitly
+ // present in the config file. It lets the server distinguish "unset" (fall
+ // back to the default of 1) from an explicit value such as 0 or a negative
+ // number, which is invalid for the worker count and must be rejected rather
+ // than silently defaulted.
+ ConcurrentWorkersSet bool
// PageConcurrency bounds the total number of PDF pages parsed concurrently
// across the whole process (it sizes the single shared page worker pool that
// all ingestor workers submit to), not a per-document or per-worker limit.
// Valid range [1, 16], enforced at startup by the CLI/env/config resolver; a
// value outside that range is a fatal startup error. Default 2.
PageConcurrency int `mapstructure:"page_concurrency"`
+ // PageConcurrencySet is true when page_concurrency was explicitly present in
+ // the config file, mirroring ConcurrentWorkersSet.
+ PageConcurrencySet bool
// CompilerPoolSize bounds the process-wide knowledge-compilation worker
// pool that drives the cross-doc KNN / LLM-merge / write stages. 0/negative
// falls back to runtime.NumCPU() (or KC_COMPILE_CONCURRENCY if set).
@@ -74,7 +83,7 @@ type IngestorConfig struct {
// configured values can be validated against a single source of truth.
const (
// MinIngestorWorkers / MaxIngestorWorkers are the inclusive bounds for
- // ingestor.max_concurrent_workers (the NATS consumer count K).
+ // ingestor.concurrent_workers (the NATS consumer count K).
MinIngestorWorkers = 1
MaxIngestorWorkers = 256
// MinPageConcurrency / MaxPageConcurrency are the inclusive bounds for
@@ -85,7 +94,7 @@ const (
func (c *Config) ParseIngestorConfig(v *viper.Viper) error {
// Default Ingestor config.
- c.ingestor.MaxConcurrentWorkers = 1
+ c.ingestor.ConcurrentWorkers = 1
c.ingestor.PageConcurrency = 2
c.ingestor.CompilerPoolSize = 0
c.ingestor.deepDocInferenceConcurrency = 1
@@ -101,11 +110,13 @@ func (c *Config) ParseIngestorConfig(v *viper.Viper) error {
return nil
}
- if sub.IsSet("max_concurrent_workers") {
- c.ingestor.MaxConcurrentWorkers = sub.GetInt("max_concurrent_workers")
+ if sub.IsSet("concurrent_workers") {
+ c.ingestor.ConcurrentWorkers = sub.GetInt("concurrent_workers")
+ c.ingestor.ConcurrentWorkersSet = true
}
if sub.IsSet("page_concurrency") {
c.ingestor.PageConcurrency = sub.GetInt("page_concurrency")
+ c.ingestor.PageConcurrencySet = true
}
if sub.IsSet("compiler_pool_size") {
c.ingestor.CompilerPoolSize = sub.GetInt("compiler_pool_size")
@@ -212,28 +223,32 @@ func (c *Config) GetDeepDocInferenceCPUCores(argValue *int) (int, bool, error) {
return val, explicit, nil
}
-// GetIngestorMaxConcurrentWorkers applies the precedence
+// GetIngestorConcurrentWorkers applies the precedence
// CLI flag > environment variable > config file > default(1) and returns
// the resolved ingestor worker count (K), validating it against the
// inclusive range [MinIngestorWorkers, MaxIngestorWorkers]. An out-of-range
-// or noninteger value at any layer is a fatal startup error.
-func (c *Config) GetIngestorMaxConcurrentWorkers(argValue *int) (int, error) {
- val := MinIngestorWorkers
- if c.ingestor.MaxConcurrentWorkers > 0 {
- val = c.ingestor.MaxConcurrentWorkers
- }
- if v := strings.TrimSpace(os.Getenv(common.EnvIngestorMaxConcurrentWorkers)); v != "" {
+// or noninteger value at any layer is a fatal startup error. An explicitly
+// configured zero or negative value is rejected (not silently defaulted):
+// the config-layer presence is tracked by ConcurrentWorkersSet so an absent
+// key falls back to the default while an explicit 0/-N fails fast.
+func (c *Config) GetIngestorConcurrentWorkers(argValue *int) (int, error) {
+ val := c.ingestor.ConcurrentWorkers
+ set := c.ingestor.ConcurrentWorkersSet
+ if v := strings.TrimSpace(os.Getenv(common.EnvIngestorConcurrentWorkers)); v != "" {
n, err := strconv.Atoi(v)
if err != nil {
- return 0, fmt.Errorf("invalid %s %q: %w", common.EnvIngestorMaxConcurrentWorkers, v, err)
+ return 0, fmt.Errorf("invalid %s %q: %w", common.EnvIngestorConcurrentWorkers, v, err)
}
- val = n
+ val, set = n, true
}
if argValue != nil {
- val = *argValue
+ val, set = *argValue, true
+ }
+ if !set {
+ val = MinIngestorWorkers
}
if val < MinIngestorWorkers || val > MaxIngestorWorkers {
- return 0, fmt.Errorf("ingestor max_concurrent_workers %d out of range [%d, %d]",
+ return 0, fmt.Errorf("ingestor concurrent_workers %d out of range [%d, %d]",
val, MinIngestorWorkers, MaxIngestorWorkers)
}
return val, nil
@@ -244,20 +259,23 @@ func (c *Config) GetIngestorMaxConcurrentWorkers(argValue *int) (int, error) {
// the resolved per-document page concurrency (N), validating it against
// the inclusive range [MinPageConcurrency, MaxPageConcurrency]. An
// out-of-range or noninteger value at any layer is a fatal startup error.
+// As with GetIngestorConcurrentWorkers, an explicitly configured zero or
+// negative page_concurrency is rejected rather than silently defaulted.
func (c *Config) GetIngestorPageConcurrency(argValue *int) (int, error) {
- val := 2 // default page concurrency
- if c.ingestor.PageConcurrency > 0 {
- val = c.ingestor.PageConcurrency
- }
+ val := c.ingestor.PageConcurrency
+ set := c.ingestor.PageConcurrencySet
if v := strings.TrimSpace(os.Getenv(common.EnvIngestorPageConcurrency)); v != "" {
n, err := strconv.Atoi(v)
if err != nil {
return 0, fmt.Errorf("invalid %s %q: %w", common.EnvIngestorPageConcurrency, v, err)
}
- val = n
+ val, set = n, true
}
if argValue != nil {
- val = *argValue
+ val, set = *argValue, true
+ }
+ if !set {
+ val = 2 // default page concurrency
}
if val < MinPageConcurrency || val > MaxPageConcurrency {
return 0, fmt.Errorf("ingestor page_concurrency %d out of range [%d, %d]",
diff --git a/internal/server/config/ingestor_config_test.go b/internal/server/config/ingestor_config_test.go
index 4da4bc074b..3660bbcdab 100644
--- a/internal/server/config/ingestor_config_test.go
+++ b/internal/server/config/ingestor_config_test.go
@@ -27,15 +27,15 @@ import (
func TestParseIngestorConfigReadsWorkerAndCompilerSettings(t *testing.T) {
v := viper.New()
v.Set("ingestor", map[string]any{
- "max_concurrent_workers": 4,
- "compiler_pool_size": 2,
+ "concurrent_workers": 4,
+ "compiler_pool_size": 2,
})
config := &Config{}
if err := config.ParseIngestorConfig(v); err != nil {
t.Fatalf("ParseIngestorConfig: %v", err)
}
- if got := config.GetIngestorConfig().MaxConcurrentWorkers; got != 4 {
+ if got := config.GetIngestorConfig().ConcurrentWorkers; got != 4 {
t.Fatalf("max concurrent workers = %d, want 4", got)
}
if got := config.GetIngestorConfig().CompilerPoolSize; got != 2 {
@@ -51,7 +51,7 @@ func TestParseIngestorConfigDefaultsToK1AndN2(t *testing.T) {
if err := config.ParseIngestorConfig(viper.New()); err != nil {
t.Fatalf("ParseIngestorConfig: %v", err)
}
- if got := config.GetIngestorConfig().MaxConcurrentWorkers; got != 1 {
+ if got := config.GetIngestorConfig().ConcurrentWorkers; got != 1 {
t.Fatalf("max concurrent workers = %d, want 1", got)
}
if got := config.GetIngestorConfig().PageConcurrency; got != 2 {
@@ -64,16 +64,16 @@ func TestParseIngestorConfigDefaultsToK1AndN2(t *testing.T) {
func TestParseIngestorConfigReadsPageConcurrency(t *testing.T) {
v := viper.New()
v.Set("ingestor", map[string]any{
- "max_concurrent_workers": 3,
- "page_concurrency": 10,
- "compiler_pool_size": 2,
+ "concurrent_workers": 3,
+ "page_concurrency": 10,
+ "compiler_pool_size": 2,
})
config := &Config{}
if err := config.ParseIngestorConfig(v); err != nil {
t.Fatalf("ParseIngestorConfig: %v", err)
}
- if got := config.GetIngestorConfig().MaxConcurrentWorkers; got != 3 {
+ if got := config.GetIngestorConfig().ConcurrentWorkers; got != 3 {
t.Fatalf("max concurrent workers = %d, want 3", got)
}
if got := config.GetIngestorConfig().PageConcurrency; got != 10 {
@@ -83,7 +83,7 @@ func TestParseIngestorConfigReadsPageConcurrency(t *testing.T) {
// TestParseIngestorConfigToleratesOutOfRangeWorkers pins the precedence
// contract: ParseIngestorConfig must NOT fail-fast on an out-of-range
-// ingestor.max_concurrent_workers. Range validation belongs to the
+// ingestor.concurrent_workers. Range validation belongs to the
// CLI/env/config resolver (config.MinIngestorWorkers / MaxIngestorWorkers),
// which runs later during startup, so a config value that a higher-priority
// source (CLI flag or env var) overrides must survive config load. The resolver
@@ -100,10 +100,10 @@ func TestParseIngestorConfigToleratesOutOfRangeWorkers(t *testing.T) {
} {
t.Run(tc.name, func(t *testing.T) {
v := viper.New()
- v.Set("ingestor", map[string]any{"max_concurrent_workers": tc.value})
+ v.Set("ingestor", map[string]any{"concurrent_workers": tc.value})
config := &Config{}
if err := config.ParseIngestorConfig(v); err != nil {
- t.Fatalf("ParseIngestorConfig rejected max_concurrent_workers=%d: %v (validation moved to resolver)", tc.value, err)
+ t.Fatalf("ParseIngestorConfig rejected concurrent_workers=%d: %v (validation moved to resolver)", tc.value, err)
}
})
}
@@ -132,30 +132,39 @@ func TestParseIngestorConfigToleratesOutOfRangePageConcurrency(t *testing.T) {
}
}
-// TestGetIngestorMaxConcurrentWorkers pins the precedence
+// TestGetIngestorConcurrentWorkers pins the precedence
// CLI > environment > config file > default(1) and the inclusive [1, 256]
// range guard on the FINAL value. An out-of-range or non-integer value at
-// any layer is a fatal startup error.
-func TestGetIngestorMaxConcurrentWorkers(t *testing.T) {
- const envKey = common.EnvIngestorMaxConcurrentWorkers
+// any layer is a fatal startup error. An explicit zero or negative value at
+// the config layer is rejected (not silently defaulted), so an operator typo
+// such as concurrent_workers: 0 or -1 fails fast instead of silently running
+// at the default of 1.
+func TestGetIngestorConcurrentWorkers(t *testing.T) {
+ const envKey = common.EnvIngestorConcurrentWorkers
cases := []struct {
name string
configured int
+ set bool // true iff concurrent_workers was explicitly present in the config file
env string // "" means leave unset
argValue *int
want int
wantErr bool
}{
- {"default", 0, "", nil, 1, false},
- {"config only", 5, "", nil, 5, false},
- {"env overrides config", 5, "8", nil, 8, false},
- {"cli overrides env and config", 5, "8", intPtr(12), 12, false},
- {"env invalid errors", 5, "notanint", nil, 0, true},
- {"env only", 0, "9", nil, 9, false},
- {"config above range errors", 300, "", nil, 0, true},
- {"cli below range errors", 0, "", intPtr(0), 0, true},
- {"cli above range errors", 0, "", intPtr(1000), 0, true},
+ {"default", 0, false, "", nil, 1, false},
+ {"config only", 5, true, "", nil, 5, false},
+ {"env overrides config", 5, true, "8", nil, 8, false},
+ {"cli overrides env and config", 5, true, "8", intPtr(12), 12, false},
+ {"env invalid errors", 5, true, "notanint", nil, 0, true},
+ {"env only", 0, false, "9", nil, 9, false},
+ {"config above range errors", 300, true, "", nil, 0, true},
+ {"cli below range errors", 0, false, "", intPtr(0), 0, true},
+ {"cli above range errors", 0, false, "", intPtr(1000), 0, true},
+ // Adversarial: an explicitly present zero/negative config value must be
+ // rejected, distinguishing "explicit 0" (set=true) from "absent" (set=false
+ // -> default 1). The old >0 guard silently defaulted both to 1.
+ {"config explicit zero errors", 0, true, "", nil, 0, true},
+ {"config negative errors", -5, true, "", nil, 0, true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
@@ -165,8 +174,9 @@ func TestGetIngestorMaxConcurrentWorkers(t *testing.T) {
if err := c.ParseIngestorConfig(viper.New()); err != nil {
t.Fatalf("ParseIngestorConfig: %v", err)
}
- c.ingestor.MaxConcurrentWorkers = tc.configured
- got, err := c.GetIngestorMaxConcurrentWorkers(tc.argValue)
+ c.ingestor.ConcurrentWorkers = tc.configured
+ c.ingestor.ConcurrentWorkersSet = tc.set
+ got, err := c.GetIngestorConcurrentWorkers(tc.argValue)
if tc.wantErr {
if err == nil {
t.Fatalf("got nil error, want rejection (configured=%d env=%q argValue=%v)",
@@ -188,26 +198,34 @@ func TestGetIngestorMaxConcurrentWorkers(t *testing.T) {
// TestGetIngestorPageConcurrency pins the precedence
// CLI > environment > config file > default(2) and the inclusive [1, 16]
// range guard on the FINAL value. An out-of-range or non-integer value at
-// any layer is a fatal startup error.
+// any layer is a fatal startup error. An explicit zero or negative value at
+// the config layer is rejected (not silently defaulted to 2), matching the
+// env/CLI behavior.
func TestGetIngestorPageConcurrency(t *testing.T) {
const envKey = common.EnvIngestorPageConcurrency
cases := []struct {
name string
configured int
+ set bool // true iff page_concurrency was explicitly present in the config file
env string // "" means leave unset
argValue *int
want int
wantErr bool
}{
- {"default", 0, "", nil, 2, false},
- {"config only", 6, "", nil, 6, false},
- {"env overrides config", 6, "8", nil, 8, false},
- {"cli overrides env and config", 6, "8", intPtr(12), 12, false},
- {"env invalid errors", 6, "notanint", nil, 0, true},
- {"env only", 0, "9", nil, 9, false},
- {"config above range errors", 20, "", nil, 0, true},
- {"cli below range errors", 0, "", intPtr(0), 0, true},
+ {"default", 0, false, "", nil, 2, false},
+ {"config only", 6, true, "", nil, 6, false},
+ {"env overrides config", 6, true, "8", nil, 8, false},
+ {"cli overrides env and config", 6, true, "8", intPtr(12), 12, false},
+ {"env invalid errors", 6, true, "notanint", nil, 0, true},
+ {"env only", 0, false, "9", nil, 9, false},
+ {"config above range errors", 20, true, "", nil, 0, true},
+ {"cli below range errors", 0, false, "", intPtr(0), 0, true},
+ // Adversarial: an explicitly present zero/negative config value must be
+ // rejected, distinguishing "explicit 0" (set=true) from "absent" (set=false
+ // -> default 2).
+ {"config explicit zero errors", 0, true, "", nil, 0, true},
+ {"config negative errors", -5, true, "", nil, 0, true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
@@ -217,6 +235,7 @@ func TestGetIngestorPageConcurrency(t *testing.T) {
t.Fatalf("ParseIngestorConfig: %v", err)
}
c.ingestor.PageConcurrency = tc.configured
+ c.ingestor.PageConcurrencySet = tc.set
got, err := c.GetIngestorPageConcurrency(tc.argValue)
if tc.wantErr {
if err == nil {
diff --git a/internal/server/deepdoc/server.go b/internal/server/deepdoc/server.go
index 22aa309885..7e75bcbfff 100644
--- a/internal/server/deepdoc/server.go
+++ b/internal/server/deepdoc/server.go
@@ -34,7 +34,6 @@ import (
infnative "ragflow/internal/deepdoc/parser/pdf/inference/native_analyzer"
"ragflow/internal/server"
"ragflow/internal/server/bootstrap"
- "ragflow/internal/server/ingestor"
)
// inferenceTotalCores returns the CPU budget the DeepDoc inference config is
@@ -134,7 +133,7 @@ func RegisterNativeDeepDoc(arguments *bootstrap.ServerArgs) error {
// Resolve the per-document page concurrency (N) from CLI > env > config >
// default(2) and register it with the shared page worker pool. Fail fast on
// an out-of-range value.
- pageConcurrency, err := ingestor.ResolveIngestorPageConcurrency(arguments, server.GetConfig().GetIngestorConfig().PageConcurrency)
+ pageConcurrency, err := server.GetConfig().GetIngestorPageConcurrency(arguments.IngestorPageConcurrency)
if err != nil {
return err
}
diff --git a/internal/server/ingestor/server.go b/internal/server/ingestor/server.go
index 0053a18af2..1e1e0cbad1 100644
--- a/internal/server/ingestor/server.go
+++ b/internal/server/ingestor/server.go
@@ -21,10 +21,7 @@ package ingestor
import (
"context"
"fmt"
- "os"
"ragflow/internal/server/local"
- "strconv"
- "strings"
"time"
"go.uber.org/zap"
@@ -39,67 +36,12 @@ import (
"ragflow/internal/ingestion/task"
"ragflow/internal/server"
"ragflow/internal/server/bootstrap"
- "ragflow/internal/server/config"
"ragflow/internal/service"
"ragflow/internal/service/nav"
"ragflow/internal/service/nlp"
"ragflow/internal/tokenizer"
)
-// resolveIngestorMaxConcurrentWorkers applies the precedence
-// CLI flag > environment variable > config file > default(1) and returns the
-// resolved ingestor worker count (K), validating it against the inclusive
-// range [MinIngestorWorkers, MaxIngestorWorkers]. An out-of-range or
-// noninteger value at any layer is a fatal startup error.
-func resolveIngestorMaxConcurrentWorkers(args *bootstrap.ServerArgs, configured int) (int, error) {
- val := config.MinIngestorWorkers
- if configured > 0 {
- val = configured
- }
- if v := strings.TrimSpace(os.Getenv(common.EnvIngestorMaxConcurrentWorkers)); v != "" {
- n, err := strconv.Atoi(v)
- if err != nil {
- return 0, fmt.Errorf("invalid %s %q: %w", common.EnvIngestorMaxConcurrentWorkers, v, err)
- }
- val = n
- }
- if args.IngestorMaxConcurrentWorkers != nil {
- val = *args.IngestorMaxConcurrentWorkers
- }
- if val < config.MinIngestorWorkers || val > config.MaxIngestorWorkers {
- return 0, fmt.Errorf("ingestor max_concurrent_workers %d out of range [%d, %d]",
- val, config.MinIngestorWorkers, config.MaxIngestorWorkers)
- }
- return val, nil
-}
-
-// resolveIngestorPageConcurrency applies the precedence
-// CLI flag > environment variable > config file > default(2) and returns the
-// resolved per-document page concurrency (N), validating it against the
-// inclusive range [MinPageConcurrency, MaxPageConcurrency]. An out-of-range or
-// noninteger value at any layer is a fatal startup error.
-func ResolveIngestorPageConcurrency(args *bootstrap.ServerArgs, configured int) (int, error) {
- val := 2 // default page concurrency
- if configured > 0 {
- val = configured
- }
- if v := strings.TrimSpace(os.Getenv(common.EnvIngestorPageConcurrency)); v != "" {
- n, err := strconv.Atoi(v)
- if err != nil {
- return 0, fmt.Errorf("invalid %s %q: %w", common.EnvIngestorPageConcurrency, v, err)
- }
- val = n
- }
- if args.IngestorPageConcurrency != nil {
- val = *args.IngestorPageConcurrency
- }
- if val < config.MinPageConcurrency || val > config.MaxPageConcurrency {
- return 0, fmt.Errorf("ingestor page_concurrency %d out of range [%d, %d]",
- val, config.MinPageConcurrency, config.MaxPageConcurrency)
- }
- return val, nil
-}
-
func logTokenizerCounters() {
var available, unavailable []string
for _, status := range tokenizer.CounterStatuses() {
@@ -148,7 +90,7 @@ func Run(ctx context.Context, cancel context.CancelFunc, serverName string, args
ingestorCfg := globalConfig.GetIngestorConfig()
// Resolve the ingestor worker count (K) from CLI > env > config > default(1)
// and fail fast on an out-of-range value.
- workers, err := resolveIngestorMaxConcurrentWorkers(args, ingestorCfg.MaxConcurrentWorkers)
+ workers, err := globalConfig.GetIngestorConcurrentWorkers(args.IngestorConcurrentWorkers)
if err != nil {
return err
}
diff --git a/internal/server/ingestor/server_test.go b/internal/server/ingestor/server_test.go
deleted file mode 100644
index 15a79a3675..0000000000
--- a/internal/server/ingestor/server_test.go
+++ /dev/null
@@ -1,118 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package ingestor
-
-import (
- "testing"
-
- "ragflow/internal/common"
- "ragflow/internal/server/bootstrap"
-)
-
-func intPtr(n int) *int { return &n }
-
-func TestResolveIngestorMaxConcurrentWorkers(t *testing.T) {
- const envKey = common.EnvIngestorMaxConcurrentWorkers
-
- cases := []struct {
- name string
- configured int
- env string // "" means leave unset
- cli *int
- want int
- wantErr bool
- }{
- {"default", 0, "", nil, 1, false},
- {"config only", 5, "", nil, 5, false},
- {"env overrides config", 5, "8", nil, 8, false},
- {"cli overrides env and config", 5, "8", intPtr(12), 12, false},
- {"env invalid errors", 5, "notanint", nil, 0, true},
- {"env only", 0, "9", nil, 9, false},
- {"config above range errors", 300, "", nil, 300, true},
- {"cli below range errors", 0, "", intPtr(0), 0, true},
- {"cli above range errors", 0, "", intPtr(1000), 0, true},
- }
- for _, tc := range cases {
- t.Run(tc.name, func(t *testing.T) {
- // t.Setenv restores the previous value after the subtest.
- t.Setenv(envKey, tc.env)
- args := &bootstrap.ServerArgs{IngestorMaxConcurrentWorkers: tc.cli}
- got, err := resolveIngestorMaxConcurrentWorkers(args, tc.configured)
- if tc.wantErr {
- if err == nil {
- t.Fatalf("got nil error, want rejection (configured=%d env=%q cli=%v)",
- tc.configured, tc.env, tc.cli)
- }
- return
- }
- if err != nil {
- t.Fatalf("unexpected error %v", err)
- }
- if got != tc.want {
- t.Fatalf("got %d, want %d (configured=%d env=%q cli=%v)",
- got, tc.want, tc.configured, tc.env, tc.cli)
- }
- })
- }
-}
-
-// TestResolveIngestorPageConcurrency pins the precedence
-// CLI > environment > config file > default(2) and the inclusive [1, 16]
-// range guard. An out-of-range or non-integer value at any layer is a fatal
-// startup error.
-func TestResolveIngestorPageConcurrency(t *testing.T) {
- const envKey = common.EnvIngestorPageConcurrency
-
- cases := []struct {
- name string
- configured int
- env string // "" means leave unset
- cli *int
- want int
- wantErr bool
- }{
- {"default", 0, "", nil, 2, false},
- {"config only", 6, "", nil, 6, false},
- {"env overrides config", 6, "8", nil, 8, false},
- {"cli overrides env and config", 6, "8", intPtr(12), 12, false},
- {"env invalid errors", 6, "notanint", nil, 0, true},
- {"env only", 0, "9", nil, 9, false},
- {"config above range errors", 20, "", nil, 20, true},
- {"cli below range errors", 0, "", intPtr(0), 0, true},
- }
- for _, tc := range cases {
- t.Run(tc.name, func(t *testing.T) {
- t.Setenv(envKey, tc.env)
- args := &bootstrap.ServerArgs{IngestorPageConcurrency: tc.cli}
- got, err := ResolveIngestorPageConcurrency(args, tc.configured)
- if tc.wantErr {
- if err == nil {
- t.Fatalf("got nil error, want rejection (configured=%d env=%q cli=%v)",
- tc.configured, tc.env, tc.cli)
- }
- return
- }
- if err != nil {
- t.Fatalf("unexpected error %v", err)
- }
- if got != tc.want {
- t.Fatalf("got %d, want %d (configured=%d env=%q cli=%v)",
- got, tc.want, tc.configured, tc.env, tc.cli)
- }
- })
- }
-}