Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): document and validate extractor timeout env (#21119)

jay777211d2d0497024b
diff --git a/docker/.env b/docker/.env
index f378323ab7..a98ff61fe9 100644
--- a/docker/.env
+++ b/docker/.env
@@ -367,6 +367,11 @@ DOC_BULK_SIZE=${DOC_BULK_SIZE:-4}
# Defaults to 16 if EMBEDDING_BATCH_SIZE is not set in the environment.
EMBEDDING_BATCH_SIZE=${EMBEDDING_BATCH_SIZE:-16}
+# Extractor LLM call timeout in seconds, including retries and backoff.
+# Applies to keyword, question, summary, and metadata extraction.
+# Default: 180. Invalid or non-positive values use the default.
+# EXTRACTOR_CHAT_TIMEOUT=180
+
# Connector ingest: documents (e.g. one MySQL row = one file) written per batch.
# Defaults to 2. Raise this for large RDBMS syncs so fewer write round-trips hit MySQL.
# INDEX_BATCH_SIZE=32
diff --git a/internal/common/environments.go b/internal/common/environments.go
index 11f87739d9..4f17013ed8 100644
--- a/internal/common/environments.go
+++ b/internal/common/environments.go
@@ -255,6 +255,10 @@ const (
EnvGoogleDriveWebOAuthRedirectURI = "GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI"
EnvSpacyModelDir = "SPACY_MODEL_DIR"
+ // EnvExtractorChatTimeout sets the Extractor LLM call timeout in seconds,
+ // including retries and backoff. It does not apply to tag extraction.
+ EnvExtractorChatTimeout = "EXTRACTOR_CHAT_TIMEOUT"
+
// EnvDeepDocModelDir points the in-process (Go) DeepDoc backend at the
// model snapshot (see common.DeepDocModelFiles); mirrors
// the RAGFlow default model dir (internal/rag/res/deepdoc).
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go
index 296c7e3236..39614bc609 100644
--- a/internal/ingestion/component/extractor.go
+++ b/internal/ingestion/component/extractor.go
@@ -39,6 +39,7 @@ import (
"encoding/json"
"errors"
"fmt"
+ "math"
"os"
"regexp"
"sort"
@@ -66,20 +67,16 @@ import (
const componentNameExtractor = "Extractor"
-// defaultExtractorChatTimeout bounds one LLM chat call. It is a hung-call
-// ceiling, not a pacing target: a fast model answers in seconds and never
-// approaches it, so per-model speed variance needs no tuning — only a hung
-// provider burns the full budget. Larger than the tag path's 60s because a
-// thinking model legitimately needs longer on open-ended prompts, and the
-// driver-level 20m stays as the absolute backstop.
+// defaultExtractorChatTimeout bounds one Extractor LLM call, including
+// retries and backoff. Tag extraction has its own timeout.
const defaultExtractorChatTimeout = 180 * time.Second
// extractorChatTimeout resolves the per-call LLM budget from
// EXTRACTOR_CHAT_TIMEOUT (seconds), falling back to
// defaultExtractorChatTimeout on unset/invalid values.
func extractorChatTimeout() time.Duration {
- if v := os.Getenv("EXTRACTOR_CHAT_TIMEOUT"); v != "" {
- if n, err := strconv.Atoi(strings.TrimSpace(v)); err == nil && n > 0 {
+ if v := os.Getenv(common.EnvExtractorChatTimeout); v != "" {
+ if n, err := strconv.ParseInt(strings.TrimSpace(v), 10, 64); err == nil && n > 0 && n <= math.MaxInt64/int64(time.Second) {
return time.Duration(n) * time.Second
}
}
diff --git a/internal/ingestion/component/extractor_call_timeout_test.go b/internal/ingestion/component/extractor_call_timeout_test.go
index 2be55bd7bf..2f2a2852e0 100644
--- a/internal/ingestion/component/extractor_call_timeout_test.go
+++ b/internal/ingestion/component/extractor_call_timeout_test.go
@@ -24,6 +24,7 @@ import (
"testing"
"time"
+ "ragflow/internal/common"
"ragflow/internal/ingestion/component/schema"
)
@@ -71,7 +72,7 @@ func keywordsOnlyExtractor() *ExtractorComponent {
// TestExtractorChatTimeoutSkipsSlowChunk: one hung chunk must not fail the
// document. The healthy chunk keeps its extraction, the hung one is skipped.
func TestExtractorChatTimeoutSkipsSlowChunk(t *testing.T) {
- t.Setenv("EXTRACTOR_CHAT_TIMEOUT", "1")
+ t.Setenv(common.EnvExtractorChatTimeout, "1")
stub := withRouteStubChatInvoker(t)
c := keywordsOnlyExtractor()
@@ -102,7 +103,7 @@ func TestExtractorChatTimeoutSkipsSlowChunk(t *testing.T) {
// TestExtractorChatTimeoutAllHangFails: every chunk hung means the model is
// down — the run must fail loudly, not silently produce empty extractions.
func TestExtractorChatTimeoutAllHangFails(t *testing.T) {
- t.Setenv("EXTRACTOR_CHAT_TIMEOUT", "1")
+ t.Setenv(common.EnvExtractorChatTimeout, "1")
withRouteStubChatInvoker(t)
c := keywordsOnlyExtractor()
@@ -123,7 +124,7 @@ func TestExtractorChatTimeoutAllHangFails(t *testing.T) {
// TestExtractorParentCancelPropagates: task stop must still fail fast, never
// degrade into per-chunk skips.
func TestExtractorParentCancelPropagates(t *testing.T) {
- t.Setenv("EXTRACTOR_CHAT_TIMEOUT", "1")
+ t.Setenv(common.EnvExtractorChatTimeout, "1")
withRouteStubChatInvoker(t)
ctx, cancel := context.WithCancel(t.Context())
@@ -150,12 +151,19 @@ func TestExtractorChatTimeoutEnv(t *testing.T) {
}{
{name: "unset falls back to default", value: "", want: defaultExtractorChatTimeout},
{name: "override wins", value: "1", want: time.Second},
+ {name: "whitespace falls back", value: " \t", want: defaultExtractorChatTimeout},
+ {name: "trimmed override wins", value: " 240 ", want: 240 * time.Second},
+ {name: "duration suffix falls back", value: "180s", want: defaultExtractorChatTimeout},
+ {name: "fractional seconds fall back", value: "1.5", want: defaultExtractorChatTimeout},
+ {name: "maximum duration seconds", value: "9223372036", want: 9223372036 * time.Second},
+ {name: "duration overflow falls back", value: "9223372037", want: defaultExtractorChatTimeout},
+ {name: "integer overflow falls back", value: "9223372036854775808", want: defaultExtractorChatTimeout},
{name: "zero falls back", value: "0", want: defaultExtractorChatTimeout},
{name: "negative falls back", value: "-5", want: defaultExtractorChatTimeout},
{name: "non-numeric falls back", value: "abc", want: defaultExtractorChatTimeout},
} {
t.Run(tt.name, func(t *testing.T) {
- t.Setenv("EXTRACTOR_CHAT_TIMEOUT", tt.value)
+ t.Setenv(common.EnvExtractorChatTimeout, tt.value)
if got := extractorChatTimeout(); got != tt.want {
t.Errorf("extractorChatTimeout() = %v, want %v", got, tt.want)
}
@@ -197,7 +205,7 @@ func withSummaryHangStub(t *testing.T) {
// summaries hang for every chunk. The phase must succeed and keep the
// questions instead of failing as all-timed-out.
func TestExtractorPartialTimeoutKeepsCompleted(t *testing.T) {
- t.Setenv("EXTRACTOR_CHAT_TIMEOUT", "1")
+ t.Setenv(common.EnvExtractorChatTimeout, "1")
withSummaryHangStub(t)
c := &ExtractorComponent{Param: schema.ExtractorParam{