Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: avoid nested compiler pool embedding deadlock (#19486)

buua436289100188d83
diff --git a/internal/entity/models/base_model.go b/internal/entity/models/base_model.go
index a642d097ed..992da05fe1 100644
--- a/internal/entity/models/base_model.go
+++ b/internal/entity/models/base_model.go
@@ -466,7 +466,7 @@ func NewDriverHTTPClient(allowPrivate bool) *http.Client {
t.MaxIdleConnsPerHost = 10
t.IdleConnTimeout = 90 * time.Second
t.DisableCompression = false
- t.ResponseHeaderTimeout = 5 * 60 * time.Second
+ t.ResponseHeaderTimeout = 20 * time.Minute
t.TLSHandshakeTimeout = 30 * time.Second
var rt http.RoundTripper = t
diff --git a/internal/entity/models/xai.go b/internal/entity/models/xai.go
index 50d3d33804..00ae4b2b20 100644
--- a/internal/entity/models/xai.go
+++ b/internal/entity/models/xai.go
@@ -33,9 +33,9 @@ import (
)
var (
- nonStreamCallTimeout = 300 * time.Second
- streamCallTimeout = 10 * time.Minute
- longOpCallTimeout = 10 * time.Minute
+ nonStreamCallTimeout = 20 * time.Minute
+ streamCallTimeout = 20 * time.Minute
+ longOpCallTimeout = 20 * time.Minute
)
// XAIModel implements ModelDriver for xAI (Grok models)
diff --git a/internal/ingestion/component/knowledge_compiler/structure/structure.go b/internal/ingestion/component/knowledge_compiler/structure/structure.go
index 60f207564d..d3af89ad92 100644
--- a/internal/ingestion/component/knowledge_compiler/structure/structure.go
+++ b/internal/ingestion/component/knowledge_compiler/structure/structure.go
@@ -84,9 +84,15 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
// ---- MAP ----
batches := common.PackBatches(inputs.Chunks, structureBatchTokenBudget, deps.Tokenizer)
+ type extractedBatch struct {
+ nodes, edges []map[string]any
+ batchIDs []string
+ }
+ extracted := make([]extractedBatch, len(batches))
perBatch := make([][]common.Product, len(batches))
jobs := make([]func() error, 0, len(batches))
for i, batch := range batches {
+ i, batch := i, batch
jobs = append(jobs, func() error {
packed, batchIDs := PackBatch(batch)
if len(batchIDs) == 0 {
@@ -96,12 +102,9 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
if err != nil {
return err
}
- rows, err := buildRows(ctx, deps, cfg, nodes, edges, batchIDs)
- if err != nil {
- return err
- }
- // Distinct slice index per batch → no cross-goroutine contention.
- perBatch[i] = rows
+ // Keep embedding out of the compiler-pool worker. buildRows calls
+ // Embed.Encode, which may submit its own batch jobs to that pool.
+ extracted[i] = extractedBatch{nodes: nodes, edges: edges, batchIDs: batchIDs}
return nil
})
}
@@ -111,6 +114,19 @@ func Run(ctx context.Context, deps common.Deps, param common.Param, inputs commo
if err := runBatches(ctx, jobs); err != nil {
return common.Outputs{}, err
}
+ // Embed each extracted batch serially after all MAP jobs have returned.
+ // This avoids nesting Embed.Encode (and its batch jobs) inside a worker
+ // already occupied by the shared compiler pool.
+ for i, result := range extracted {
+ if len(result.batchIDs) == 0 {
+ continue
+ }
+ rows, err := buildRows(ctx, deps, cfg, result.nodes, result.edges, result.batchIDs)
+ if err != nil {
+ return common.Outputs{}, err
+ }
+ perBatch[i] = rows
+ }
// ---- DEDUP ----
// Sequential in batch order so merge outcomes are deterministic and match
diff --git a/internal/ingestion/task/knowledge_compiler_wiring.go b/internal/ingestion/task/knowledge_compiler_wiring.go
index 9a60b13269..9a0dcaca1c 100644
--- a/internal/ingestion/task/knowledge_compiler_wiring.go
+++ b/internal/ingestion/task/knowledge_compiler_wiring.go
@@ -642,7 +642,7 @@ const kcChatRetryMax = 5
// kcChatAttemptTimeout bounds a single Chat call per retry attempt. It matches
// the non-streaming provider deadline so the knowledge-compiler adapter does
// not cancel a valid long-running response before the provider does.
-const kcChatAttemptTimeout = 3 * time.Minute
+const kcChatAttemptTimeout = 20 * time.Minute
// kcChatRetryDelay is the initial exponential-backoff delay between retries.
const kcChatRetryDelay = 2 * time.Second