RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(structure): read STRUCT_MAX_CHUNKS_PER_BATCH via common.GetEnvInt (#21984)
xugangqiangf1a132b89d58
diff --git a/internal/ingestion/component/knowledge_compiler/structure/structure.go b/internal/ingestion/component/knowledge_compiler/structure/structure.goindex 52eff20572..45d4bc777f 100644--- a/internal/ingestion/component/knowledge_compiler/structure/structure.go+++ b/internal/ingestion/component/knowledge_compiler/structure/structure.go@@ -9,11 +9,10 @@ package structureimport ("context""fmt"- "os"- "strconv""sync""ragflow/internal/agent/runtime"+ appcommon "ragflow/internal/common""ragflow/internal/ingestion/component/knowledge_compiler/common")@@ -70,6 +69,17 @@ func structureInputBudget(modelContextLen, promptOverhead int) int {return budget}+// structureBatchCap returns the optional STRUCT_MAX_CHUNKS_PER_BATCH override+// (0 means off). appcommon.GetEnvInt falls back to 0 on an unset or unparseable+// value, so the caller only needs to check cap > 0.+func structureBatchCap() int {+ v := appcommon.GetEnvInt("STRUCT_MAX_CHUNKS_PER_BATCH", 0)+ if v <= 0 {+ return 0+ }+ return v+}+// Run executes the structure variant:// 1. MAP — per-batch two-stage (node → edge) extraction, parallel across// batches, results kept in batch order (mirrors _run_chunked_pipeline).@@ -125,8 +135,8 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo// per extraction batch (0 = window-packed only — a heading has to see its// whole section to own it; the 4-per-batch rule belongs to tree's claim// harvesting). Overridable for benchmarking, mirrored verbatim.- if v, err := strconv.Atoi(os.Getenv("STRUCT_MAX_CHUNKS_PER_BATCH")); err == nil && v > 0 {- batches = capBatchChunkCount(batches, v)+ if cap := structureBatchCap(); cap > 0 {+ batches = capBatchChunkCount(batches, cap)}runtime.ReportProgressMessage(ctx, "Compiler", fmt.Sprintf("%s-template: %d chunk(s) -> %d batch(es)", compileType, len(inputs.Chunks), len(batches)))diff --git a/internal/ingestion/component/knowledge_compiler/structure/structure_test.go b/internal/ingestion/component/knowledge_compiler/structure/structure_test.goindex 237e1d34ca..8d92f82127 100644--- a/internal/ingestion/component/knowledge_compiler/structure/structure_test.go+++ b/internal/ingestion/component/knowledge_compiler/structure/structure_test.go@@ -5,6 +5,7 @@ import ("encoding/json""fmt""math"+ "os""strings""testing"@@ -912,3 +913,41 @@ func TestGroupedDeduperTracksNameAfterSemanticMerge(t *testing.T) {t.Fatalf("semantic and exact-name merges lost provenance: %v", ids)}}++// TestStructureBatchCap locks the STRUCT_MAX_CHUNKS_PER_BATCH env override:+// invalid or non-positive values disable the cap (return 0), and a positive+// value sets the cap.+func TestStructureBatchCap(t *testing.T) {+ const key = "STRUCT_MAX_CHUNKS_PER_BATCH"+ cases := []struct {+ name string+ set string // "" means unset+ want int+ }{+ {"unset", "", 0},+ {"zero", "0", 0},+ {"negative", "-3", 0},+ {"non-numeric", "abc", 0},+ {"positive", "5", 5},+ }+ for _, c := range cases {+ t.Run(c.name, func(t *testing.T) {+ orig, had := os.LookupEnv(key)+ if c.set == "" {+ os.Unsetenv(key)+ } else {+ os.Setenv(key, c.set)+ }+ t.Cleanup(func() {+ if had {+ os.Setenv(key, orig)+ } else {+ os.Unsetenv(key)+ }+ })+ if got := structureBatchCap(); got != c.want {+ t.Fatalf("structureBatchCap() = %d, want %d (env=%q)", got, c.want, c.set)+ }+ })+ }+}diff --git a/internal/ingestion/knowledge_compile/pool.go b/internal/ingestion/knowledge_compile/pool.goindex 294678cfd3..36587c2cea 100644--- a/internal/ingestion/knowledge_compile/pool.go+++ b/internal/ingestion/knowledge_compile/pool.go@@ -17,10 +17,9 @@ package knowledge_compileimport ("context"- "os""runtime"- "strconv"+ "ragflow/internal/common""ragflow/internal/utility")@@ -50,14 +49,23 @@ var compilerPool = utility.NewWorkerPool[CompilerJob, struct{}](func(_ context.Context, j CompilerJob) (struct{}, error) { return struct{}{}, j() },)+// kcCompileConcurrencyOverride returns the optional KC_COMPILE_CONCURRENCY+// override (0 means off). common.GetEnvInt falls back to 0 on an unset or+// unparseable value, so the caller only needs to check v > 0.+func kcCompileConcurrencyOverride() int {+ v := common.GetEnvInt("KC_COMPILE_CONCURRENCY", 0)+ if v <= 0 {+ return 0+ }+ return v+}+// compilerConcurrency resolves the global pool size. It defaults to the host// vCPU count, overridable via KC_COMPILE_CONCURRENCY (mirroring the extractor// pool's EXTRACTOR_CONCURRENCY tuning knob).func compilerConcurrency() int {- if v := os.Getenv("KC_COMPILE_CONCURRENCY"); v != "" {- if n, err := strconv.Atoi(v); err == nil && n > 0 {- return n- }+ if v := kcCompileConcurrencyOverride(); v > 0 {+ return v}n := runtime.NumCPU()if n <= 0 {diff --git a/internal/ingestion/knowledge_compile/pool_test.go b/internal/ingestion/knowledge_compile/pool_test.gonew file mode 100644index 0000000000..6e8c70fb9c--- /dev/null+++ b/internal/ingestion/knowledge_compile/pool_test.go@@ -0,0 +1,44 @@+package knowledge_compile++import (+ "os"+ "testing"+)++// TestKCCompileConcurrencyOverride locks the KC_COMPILE_CONCURRENCY env+// override: unset / zero / negative / non-numeric all mean "no override" (0),+// and a positive value is returned as-is. Mirrors TestStructureBatchCap.+func TestKCCompileConcurrencyOverride(t *testing.T) {+ const key = "KC_COMPILE_CONCURRENCY"+ cases := []struct {+ name string+ set string // "" means unset+ want int+ }{+ {"unset", "", 0},+ {"zero", "0", 0},+ {"negative", "-2", 0},+ {"non-numeric", "xyz", 0},+ {"positive", "8", 8},+ }+ for _, c := range cases {+ t.Run(c.name, func(t *testing.T) {+ orig, had := os.LookupEnv(key)+ if c.set == "" {+ os.Unsetenv(key)+ } else {+ os.Setenv(key, c.set)+ }+ t.Cleanup(func() {+ if had {+ os.Setenv(key, orig)+ } else {+ os.Unsetenv(key)+ }+ })+ if got := kcCompileConcurrencyOverride(); got != c.want {+ t.Fatalf("kcCompileConcurrencyOverride() = %d, want %d (env=%q)", got, c.want, c.set)+ }+ })+ }+}