Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(structure): read STRUCT_MAX_CHUNKS_PER_BATCH via common.GetEnvInt (#21984)

xugangqiangf1a132b89d58
diff --git a/internal/ingestion/component/knowledge_compiler/structure/structure.go b/internal/ingestion/component/knowledge_compiler/structure/structure.go
index 52eff20572..45d4bc777f 100644
--- a/internal/ingestion/component/knowledge_compiler/structure/structure.go
+++ b/internal/ingestion/component/knowledge_compiler/structure/structure.go
@@ -9,11 +9,10 @@ package structure
import (
"context"
"fmt"
- "os"
- "strconv"
"sync"
"ragflow/internal/agent/runtime"
+ appcommon "ragflow/internal/common"
"ragflow/internal/ingestion/component/knowledge_compiler/common"
)
@@ -70,6 +69,17 @@ func structureInputBudget(modelContextLen, promptOverhead int) int {
return budget
}
+// structureBatchCap returns the optional STRUCT_MAX_CHUNKS_PER_BATCH override
+// (0 means off). appcommon.GetEnvInt falls back to 0 on an unset or unparseable
+// value, so the caller only needs to check cap > 0.
+func structureBatchCap() int {
+ v := appcommon.GetEnvInt("STRUCT_MAX_CHUNKS_PER_BATCH", 0)
+ if v <= 0 {
+ return 0
+ }
+ return v
+}
+
// Run executes the structure variant:
// 1. MAP — per-batch two-stage (node → edge) extraction, parallel across
// batches, results kept in batch order (mirrors _run_chunked_pipeline).
@@ -125,8 +135,8 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
// per extraction batch (0 = window-packed only — a heading has to see its
// whole section to own it; the 4-per-batch rule belongs to tree's claim
// harvesting). Overridable for benchmarking, mirrored verbatim.
- if v, err := strconv.Atoi(os.Getenv("STRUCT_MAX_CHUNKS_PER_BATCH")); err == nil && v > 0 {
- batches = capBatchChunkCount(batches, v)
+ if cap := structureBatchCap(); cap > 0 {
+ batches = capBatchChunkCount(batches, cap)
}
runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf(
"%s-template: %d chunk(s) -> %d batch(es)", compileType, len(inputs.Chunks), len(batches)))
diff --git a/internal/ingestion/component/knowledge_compiler/structure/structure_test.go b/internal/ingestion/component/knowledge_compiler/structure/structure_test.go
index 237e1d34ca..8d92f82127 100644
--- a/internal/ingestion/component/knowledge_compiler/structure/structure_test.go
+++ b/internal/ingestion/component/knowledge_compiler/structure/structure_test.go
@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"math"
+ "os"
"strings"
"testing"
@@ -912,3 +913,41 @@ func TestGroupedDeduperTracksNameAfterSemanticMerge(t *testing.T) {
t.Fatalf("semantic and exact-name merges lost provenance: %v", ids)
}
}
+
+// TestStructureBatchCap locks the STRUCT_MAX_CHUNKS_PER_BATCH env override:
+// invalid or non-positive values disable the cap (return 0), and a positive
+// value sets the cap.
+func TestStructureBatchCap(t *testing.T) {
+ const key = "STRUCT_MAX_CHUNKS_PER_BATCH"
+ cases := []struct {
+ name string
+ set string // "" means unset
+ want int
+ }{
+ {"unset", "", 0},
+ {"zero", "0", 0},
+ {"negative", "-3", 0},
+ {"non-numeric", "abc", 0},
+ {"positive", "5", 5},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ orig, had := os.LookupEnv(key)
+ if c.set == "" {
+ os.Unsetenv(key)
+ } else {
+ os.Setenv(key, c.set)
+ }
+ t.Cleanup(func() {
+ if had {
+ os.Setenv(key, orig)
+ } else {
+ os.Unsetenv(key)
+ }
+ })
+ if got := structureBatchCap(); got != c.want {
+ t.Fatalf("structureBatchCap() = %d, want %d (env=%q)", got, c.want, c.set)
+ }
+ })
+ }
+}
diff --git a/internal/ingestion/knowledge_compile/pool.go b/internal/ingestion/knowledge_compile/pool.go
index 294678cfd3..36587c2cea 100644
--- a/internal/ingestion/knowledge_compile/pool.go
+++ b/internal/ingestion/knowledge_compile/pool.go
@@ -17,10 +17,9 @@ package knowledge_compile
import (
"context"
- "os"
"runtime"
- "strconv"
+ "ragflow/internal/common"
"ragflow/internal/utility"
)
@@ -50,14 +49,23 @@ var compilerPool = utility.NewWorkerPool[CompilerJob, struct{}](
func(_ context.Context, j CompilerJob) (struct{}, error) { return struct{}{}, j() },
)
+// kcCompileConcurrencyOverride returns the optional KC_COMPILE_CONCURRENCY
+// override (0 means off). common.GetEnvInt falls back to 0 on an unset or
+// unparseable value, so the caller only needs to check v > 0.
+func kcCompileConcurrencyOverride() int {
+ v := common.GetEnvInt("KC_COMPILE_CONCURRENCY", 0)
+ if v <= 0 {
+ return 0
+ }
+ return v
+}
+
// compilerConcurrency resolves the global pool size. It defaults to the host
// vCPU count, overridable via KC_COMPILE_CONCURRENCY (mirroring the extractor
// pool's EXTRACTOR_CONCURRENCY tuning knob).
func compilerConcurrency() int {
- if v := os.Getenv("KC_COMPILE_CONCURRENCY"); v != "" {
- if n, err := strconv.Atoi(v); err == nil && n > 0 {
- return n
- }
+ if v := kcCompileConcurrencyOverride(); v > 0 {
+ return v
}
n := runtime.NumCPU()
if n <= 0 {
diff --git a/internal/ingestion/knowledge_compile/pool_test.go b/internal/ingestion/knowledge_compile/pool_test.go
new file mode 100644
index 0000000000..6e8c70fb9c
--- /dev/null
+++ b/internal/ingestion/knowledge_compile/pool_test.go
@@ -0,0 +1,44 @@
+package knowledge_compile
+
+import (
+ "os"
+ "testing"
+)
+
+// TestKCCompileConcurrencyOverride locks the KC_COMPILE_CONCURRENCY env
+// override: unset / zero / negative / non-numeric all mean "no override" (0),
+// and a positive value is returned as-is. Mirrors TestStructureBatchCap.
+func TestKCCompileConcurrencyOverride(t *testing.T) {
+ const key = "KC_COMPILE_CONCURRENCY"
+ cases := []struct {
+ name string
+ set string // "" means unset
+ want int
+ }{
+ {"unset", "", 0},
+ {"zero", "0", 0},
+ {"negative", "-2", 0},
+ {"non-numeric", "xyz", 0},
+ {"positive", "8", 8},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ orig, had := os.LookupEnv(key)
+ if c.set == "" {
+ os.Unsetenv(key)
+ } else {
+ os.Setenv(key, c.set)
+ }
+ t.Cleanup(func() {
+ if had {
+ os.Setenv(key, orig)
+ } else {
+ os.Unsetenv(key)
+ }
+ })
+ if got := kcCompileConcurrencyOverride(); got != c.want {
+ t.Fatalf("kcCompileConcurrencyOverride() = %d, want %d (env=%q)", got, c.want, c.set)
+ }
+ })
+ }
+}