Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Go: handle llama-server over-limit rejections in embed and rerank (#20784)

Zhichang Yu9b5d73bd7c9f
diff --git a/conf/models/openai-api-compatible.json b/conf/models/openai-api-compatible.json
index 99486b1528..dd9002a7b1 100644
--- a/conf/models/openai-api-compatible.json
+++ b/conf/models/openai-api-compatible.json
@@ -5,6 +5,7 @@
"chat": "chat/completions",
"models": "models",
"embedding": "embeddings",
+ "rerank": "rerank",
"asr": "audio/transcriptions",
"tts": "audio/speech"
},
diff --git a/internal/entity/models/model_test.go b/internal/entity/models/model_test.go
index e48924ee0b..8ebab3172a 100644
--- a/internal/entity/models/model_test.go
+++ b/internal/entity/models/model_test.go
@@ -61,6 +61,21 @@ func readProviderConfig(t *testing.T, fileName string) []byte {
return nil
}
+// A provider that supports rerank must declare its rerank url_suffix. Without it
+// the request is posted to the base URL (".../") instead of "/rerank", which the
+// server answers with a generic 500 — the failure reported in #16033.
+func TestOpenAIAPICompatibleDeclaresRerankSuffix(t *testing.T) {
+ var cfg struct {
+ URLSuffix URLSuffix `json:"url_suffix"`
+ }
+ if err := json.Unmarshal(readProviderConfig(t, "openai-api-compatible.json"), &cfg); err != nil {
+ t.Fatalf("parse openai-api-compatible.json: %v", err)
+ }
+ if strings.TrimSpace(cfg.URLSuffix.Rerank) == "" {
+ t.Fatal("openai-api-compatible.json declares no rerank url_suffix")
+ }
+}
+
// setupProviderTestDir creates a temporary directory populated with provider
// config files and conf/all_models.json, then changes the working directory to
// it. InitProviderManager hardcodes a read of conf/all_models.json relative to
diff --git a/internal/entity/models/non_chat_usage_test.go b/internal/entity/models/non_chat_usage_test.go
index a677af4f43..7bd8f3f25b 100644
--- a/internal/entity/models/non_chat_usage_test.go
+++ b/internal/entity/models/non_chat_usage_test.go
@@ -133,6 +133,18 @@ func TestProviderEmbeddingAndRerankUsage(t *testing.T) {
embedInput: 7,
embedTotal: 7,
},
+ {
+ // The OpenAI-API-Compatible provider reuses the vLLM driver, which is
+ // how a locally served embedding model (llama-server, vLLM) is added.
+ // It must report usage so the ingest path can calibrate its tokenizer.
+ name: "openai-api-compatible (vLLM)",
+ newModel: func(baseURL string) ModelDriver {
+ return NewOpenAIAPICompatibleModel(map[string]string{"default": baseURL}, URLSuffix{Embedding: "embedding"})
+ },
+ embeddingBody: `{"id":"embed-vllm","data":[{"embedding":[0.1,0.2],"index":0}],"usage":{"prompt_tokens":7,"total_tokens":7}}`,
+ embedInput: 7,
+ embedTotal: 7,
+ },
}
for _, tc := range cases {
diff --git a/internal/entity/models/rerank_model_test.go b/internal/entity/models/rerank_model_test.go
index 2dc12d1700..7319925696 100644
--- a/internal/entity/models/rerank_model_test.go
+++ b/internal/entity/models/rerank_model_test.go
@@ -89,3 +89,158 @@ func TestNewRerankModelDefaultsMaxTokens(t *testing.T) {
t.Fatalf("MaxTokens = %d, want %d", model.MaxTokens, defaultMaxRerankTokens)
}
}
+
+// scriptedRerankDriver returns whatever fail produces for each call, and records
+// every request's documents so a test can see whether they shrank.
+type scriptedRerankDriver struct {
+ ModelDriver
+ fail func(request RerankRequest, call int) error
+ calls int
+ docs [][]string
+}
+
+func (d *scriptedRerankDriver) Rerank(_ context.Context, _ *string, request RerankRequest, _ *APIConfig, _ *RerankConfig, _ *common.ModelUsage) (*RerankResponse, error) {
+ d.calls++
+ d.docs = append(d.docs, request.Documents)
+ if d.fail != nil {
+ if err := d.fail(request, d.calls); err != nil {
+ return nil, err
+ }
+ }
+ return &RerankResponse{Data: []RerankResult{{Index: 0, RelevanceScore: 1}}}, nil
+}
+
+func llamaRerankOverLimit() error {
+ return errors.New(`vLLM rerank API error: 500 Internal Server Error, body: {"error":{"message":"input is too large to process. increase the physical batch size"}}`)
+}
+
+// A provider that rejects a window we computed from its own declared limit must
+// not fail the call: the documents are re-cut with a smaller budget and the
+// calibration is told, exactly as the embedding path does.
+func TestRerankModelShrinksOnOverLimit(t *testing.T) {
+ t.Setenv(common.EnvRerankTokenLimitMode, "truncate")
+ document := strings.Repeat("token ", 2000)
+ driver := &scriptedRerankDriver{
+ fail: func(request RerankRequest, _ int) error {
+ if len(request.Documents[0]) > 1024 {
+ return llamaRerankOverLimit()
+ }
+ return nil
+ },
+ }
+ apiKey := t.Name()
+ model := NewRerankModel(driver, nil, &APIConfig{ApiKey: &apiKey}, 512)
+ if _, err := model.Rerank(t.Context(), RerankRequest{Query: "q", Documents: []string{document}}, nil, nil); err != nil {
+ t.Fatalf("Rerank() error = %v, want success after shrinking", err)
+ }
+ if driver.calls < 2 {
+ t.Fatalf("driver calls = %d, want a retry with a smaller budget", driver.calls)
+ }
+ first, last := driver.docs[0][0], driver.docs[len(driver.docs)-1][0]
+ if len(last) >= len(first) {
+ t.Fatalf("documents did not shrink across attempts: first=%d bytes last=%d bytes", len(first), len(last))
+ }
+ // The rejection must teach the calibration from the attempt actually sent: the
+ // count of the trimmed documents, not the longer original, which would
+ // understate the ratio and leave it at the 1.05 default.
+ ratio, _, rejects, ok := tokenizer.DefaultCalibration().Stats(model.QuotaKey())
+ if !ok || rejects == 0 {
+ t.Fatalf("calibration saw no over-limit observation: ok=%t rejects=%d", ok, rejects)
+ }
+ if ratio <= tokenizer.DefaultUncountedRatioUpper {
+ t.Fatalf("calibration ratio = %.3f, want above the %.2f default", ratio, tokenizer.DefaultUncountedRatioUpper)
+ }
+}
+
+// An exact tokenizer has no calibration: Limiter.Calibration() returns nil. A
+// provider can still reject such an input (its physical batch may be smaller than
+// the declared window), and the retry must keep working - ObserveOverLimit is
+// nil-safe, so there is nothing to guard.
+func TestRerankModelRetriesWithExactCounter(t *testing.T) {
+ t.Setenv(common.EnvRerankTokenLimitMode, "truncate")
+ t.Setenv("TOKENIZER_EMBEDDING_TOKENIZER", tokenizer.CounterCL100K)
+ document := strings.Repeat("token ", 2000)
+ driver := &scriptedRerankDriver{
+ fail: func(request RerankRequest, _ int) error {
+ if len(request.Documents[0]) > 1024 {
+ return llamaRerankOverLimit()
+ }
+ return nil
+ },
+ }
+ name := "cl100k-model"
+ model := NewRerankModel(driver, &name, nil, 512)
+ if cal, _ := model.limiter.Calibration(); cal != nil {
+ t.Fatalf("expected an exact limiter (nil calibration), got %v", cal)
+ }
+ if _, err := model.Rerank(t.Context(), RerankRequest{Query: "q", Documents: []string{document}}, nil, nil); err != nil {
+ t.Fatalf("Rerank() error = %v, want success after shrinking", err)
+ }
+ if driver.calls < 2 {
+ t.Fatalf("driver calls = %d, want a retry with a smaller budget", driver.calls)
+ }
+}
+
+// Only over-limit rejections are retried; a genuine failure is returned at once.
+func TestRerankModelDoesNotRetryOtherErrors(t *testing.T) {
+ t.Setenv(common.EnvRerankTokenLimitMode, "truncate")
+ driver := &scriptedRerankDriver{
+ fail: func(RerankRequest, int) error { return errors.New("connection refused") },
+ }
+ model := NewRerankModel(driver, nil, nil, 512)
+ _, err := model.Rerank(t.Context(), RerankRequest{Query: "q", Documents: []string{"doc"}}, nil, nil)
+ if err == nil || !strings.Contains(err.Error(), "connection refused") {
+ t.Fatalf("Rerank() error = %v, want the driver error", err)
+ }
+ if driver.calls != 1 {
+ t.Fatalf("driver calls = %d, want 1 (no retry for a non-size error)", driver.calls)
+ }
+}
+
+func TestRerankModelQuotaKeyNamesDeployment(t *testing.T) {
+ baseA, baseB, apiKey, name := "http://a/v1", "http://b/v1", "k", "bge-reranker-v2-m3"
+ keyA := NewRerankModel(&captureRerankDriver{}, &name, &APIConfig{BaseURL: &baseA, ApiKey: &apiKey}, 10).QuotaKey()
+ keyB := NewRerankModel(&captureRerankDriver{}, &name, &APIConfig{BaseURL: &baseB, ApiKey: &apiKey}, 10).QuotaKey()
+ if keyA == "" {
+ t.Fatal("QuotaKey() is empty")
+ }
+ if keyA == keyB {
+ t.Fatalf("QuotaKey() = %q for two endpoints, want them distinct", keyA)
+ }
+ unnamed := NewRerankModel(&captureRerankDriver{}, nil, &APIConfig{BaseURL: &baseA, ApiKey: &apiKey}, 10).QuotaKey()
+ if unnamed == keyA {
+ t.Fatalf("QuotaKey() ignored the model name: %q", unnamed)
+ }
+ // With no BaseURL the driver falls back to its provider's default endpoint, so
+ // the provider name is what keeps two deployments apart.
+ noURL := &APIConfig{ApiKey: &apiKey}
+ provA := NewRerankModelWithInfo(&captureRerankDriver{}, &name, noURL, 10, &ModelInfo{ProviderName: "VLLM"}).QuotaKey()
+ provB := NewRerankModelWithInfo(&captureRerankDriver{}, &name, noURL, 10, &ModelInfo{ProviderName: "OpenAI-API-Compatible"}).QuotaKey()
+ if provA == provB {
+ t.Fatalf("QuotaKey() ignored the provider when BaseURL is empty: %q", provA)
+ }
+}
+
+// passthrough sends the input without measuring it, so it must not be blocked by a
+// missing tokenizer asset; the modes that do count still refuse.
+func TestRerankModelPassthroughIgnoresUnavailableTokenizer(t *testing.T) {
+ document := strings.Repeat("token ", 40)
+
+ t.Setenv(common.EnvRerankTokenLimitMode, "passthrough")
+ driver := &captureRerankDriver{}
+ model := NewRerankModel(driver, nil, nil, 10)
+ model.limiterErr = errors.New("model tokenizer is declared but its asset is unavailable")
+ if _, err := model.Rerank(t.Context(), RerankRequest{Query: "q", Documents: []string{document}}, nil, nil); err != nil {
+ t.Fatalf("passthrough must not require the tokenizer counter: %v", err)
+ }
+ if driver.calls != 1 {
+ t.Fatalf("driver calls = %d, want 1", driver.calls)
+ }
+
+ t.Setenv(common.EnvRerankTokenLimitMode, "truncate")
+ refusing := NewRerankModel(&captureRerankDriver{}, nil, nil, 10)
+ refusing.limiterErr = errors.New("model tokenizer is declared but its asset is unavailable")
+ if _, err := refusing.Rerank(t.Context(), RerankRequest{Query: "q", Documents: []string{document}}, nil, nil); err == nil {
+ t.Fatal("truncate must refuse when the declared tokenizer asset is unavailable")
+ }
+}
diff --git a/internal/entity/models/types.go b/internal/entity/models/types.go
index 111e47dd15..a7df795561 100644
--- a/internal/entity/models/types.go
+++ b/internal/entity/models/types.go
@@ -354,34 +354,58 @@ func (m *EmbeddingModel) ResolveTokenizerID() string {
return GetModelTokenizer(name)
}
-// QuotaKey names the deployment this embedding model counts against: endpoint,
-// region, model name and an API-key prefix. The tokenizer belongs to the model, but
-// what a provider accepts is per deployment - the same model behind two endpoints
-// can have different windows - so everything that learns a real/own token ratio
-// (the ingest embedder, the dataset-nav embedder, the knowledge-compiler embedder)
-// has to key that ratio the same way; otherwise each path re-learns the same
-// rejection and none of them tightens for the others.
+// QuotaKey names the deployment this embedding model counts against: provider,
+// endpoint, region, model name and an API-key prefix. The tokenizer belongs to the
+// model, but what a provider accepts is per deployment - the same model behind two
+// endpoints can have different windows - so everything that learns a real/own token
+// ratio (the ingest embedder, the dataset-nav embedder, the knowledge-compiler
+// embedder) has to key that ratio the same way; otherwise each path re-learns the
+// same rejection and none of them tightens for the others.
func (m *EmbeddingModel) QuotaKey() string {
if m == nil {
return ""
}
- var baseURL, region, apiKey, modelName string
- if cfg := m.APIConfig; cfg != nil {
- if cfg.BaseURL != nil {
- baseURL = *cfg.BaseURL
+ return quotaKeyFor(providerNameOf(m.info), m.APIConfig, m.ModelName)
+}
+
+// providerNameOf is the provider a model was resolved from, or "" when it was built
+// without catalog metadata (tests, direct construction).
+func providerNameOf(info *ModelInfo) string {
+ if info == nil {
+ return ""
+ }
+ return info.ProviderName
+}
+
+// quotaKeyFor is the shared key used by every model that calibrates a real/own
+// token ratio against a provider deployment: provider, resolved endpoint, region,
+// model name and a short API-key digest. The tokenizer belongs to the model, but the
+// accepted window belongs to the deployment, and the digest keeps two tenants of the
+// same endpoint from sharing one learned ratio.
+//
+// providerName matters when BaseURL is empty: the driver then falls back to the
+// endpoint its provider config supplies for that region, so provider + region is what
+// identifies the deployment. Without it two providers that resolve to different
+// default endpoints but agree on region, model and key would share one learned ratio,
+// and the tighter window would over-truncate the other's requests.
+func quotaKeyFor(providerName string, apiConfig *APIConfig, modelName *string) string {
+ var baseURL, region, apiKey, name string
+ if apiConfig != nil {
+ if apiConfig.BaseURL != nil {
+ baseURL = *apiConfig.BaseURL
}
- if cfg.Region != nil {
- region = *cfg.Region
+ if apiConfig.Region != nil {
+ region = *apiConfig.Region
}
- if cfg.ApiKey != nil {
- apiKey = *cfg.ApiKey
+ if apiConfig.ApiKey != nil {
+ apiKey = *apiConfig.ApiKey
}
}
- if m.ModelName != nil {
- modelName = *m.ModelName
+ if modelName != nil {
+ name = *modelName
}
sum := sha256.Sum256([]byte(apiKey))
- return fmt.Sprintf("%s|%s|%s|%x", baseURL, region, modelName, sum[:8])
+ return fmt.Sprintf("%s|%s|%s|%s|%x", providerName, baseURL, region, name, sum[:8])
}
// RerankModel wraps a ModelDriver with rerank-specific configuration
@@ -401,27 +425,43 @@ func NewRerankModel(driver ModelDriver, modelName *string, apiConfig *APIConfig,
if maxTokens <= 0 {
maxTokens = defaultMaxRerankTokens
}
- tokenizerID := ""
- if modelName != nil {
- tokenizerID = GetModelTokenizer(*modelName)
- }
- return &RerankModel{
+ model := &RerankModel{
ModelDriver: driver,
ModelName: modelName,
APIConfig: apiConfig,
MaxTokens: maxTokens,
- limiter: tokenizer.LimiterFor(tokenizerID, "", tokenizer.DefaultCalibration()),
- limiterErr: tokenizer.RefuseUnavailableCounter(tokenizerID, "rerank"),
}
+ model.initLimiter()
+ return model
}
// NewRerankModelWithInfo creates a rerank model with resolved metadata.
func NewRerankModelWithInfo(driver ModelDriver, modelName *string, apiConfig *APIConfig, maxTokens int, info *ModelInfo) *RerankModel {
model := NewRerankModel(driver, modelName, apiConfig, maxTokens)
model.info = cloneModelInfo(info)
+ // Rebuild the limiter now that the provider is known: it is part of the
+ // calibration key (see quotaKeyFor), and it was built without it above.
+ model.initLimiter()
return model
}
+// initLimiter binds the model's declared tokenizer and its deployment calibration
+// key. Safe to call more than once; the constructors call it after the model's
+// metadata is set so the key can include the provider.
+func (r *RerankModel) initLimiter() {
+ tokenizerID := ""
+ if r.ModelName != nil {
+ tokenizerID = GetModelTokenizer(*r.ModelName)
+ }
+ // The calibration key is the deployment, not the model: rerank and embedding
+ // that share an endpoint still learn separate ratios, but every rerank caller
+ // of one deployment keys the same ratio and tightens it together (see
+ // RerankModel.Rerank, which records over-limit rejections against it).
+ quotaKey := r.QuotaKey()
+ r.limiter = tokenizer.LimiterFor(tokenizerID, quotaKey, tokenizer.DefaultCalibration())
+ r.limiterErr = tokenizer.RefuseUnavailableCounter(tokenizerID, quotaKey)
+}
+
// Info returns metadata for this selected rerank model.
func (r *RerankModel) Info() *ModelInfo {
if r == nil {
@@ -430,8 +470,26 @@ func (r *RerankModel) Info() *ModelInfo {
return cloneModelInfo(r.info)
}
+// QuotaKey names the deployment this rerank model counts against, so its
+// calibration is keyed the same way as an embedding model on the same endpoint.
+func (r *RerankModel) QuotaKey() string {
+ if r == nil {
+ return ""
+ }
+ return quotaKeyFor(providerNameOf(r.info), r.APIConfig, r.ModelName)
+}
+
// Rerank calculates similarity between query and texts. Rerank input is measured
// and truncated with the model's declared tokenizer, just like embedding input.
+// An over-limit rejection is recovered like the embedding path: the documents are
+// re-cut with a smaller budget and the calibration is told, as an attempt - if every
+// budget is rejected the provider's error is still returned.
+//
+// The window bounds the query PLUS each document, not a single text, so only the
+// documents can shrink - the query is fixed. RERANK_TOKEN_LIMIT_MODE decides
+// whether that is wanted at all: "passthrough" sends the input untouched, and
+// "raise_error" reports an oversized document instead of trimming it, so neither
+// retries; only "truncate" (the default) shrinks-and-retries.
func (r *RerankModel) Rerank(ctx context.Context, request RerankRequest, rerankConfig *RerankConfig, modelUsage *common.ModelUsage) (*RerankResponse, error) {
if r == nil || r.ModelDriver == nil {
return nil, errors.New("rerank model: driver is nil")
@@ -440,12 +498,6 @@ func (r *RerankModel) Rerank(ctx context.Context, request RerankRequest, rerankC
if maxTokens <= 0 {
maxTokens = defaultMaxRerankTokens
}
- if r.limiterErr != nil {
- return nil, r.limiterErr
- }
- limiter := r.limiter
- counter := limiter.Counter()
- effectiveMaxTokens := limiter.Limit(maxTokens)
mode := strings.ToLower(strings.TrimSpace(common.GetEnv(common.EnvRerankTokenLimitMode)))
if mode == "" {
mode = "truncate"
@@ -453,26 +505,62 @@ func (r *RerankModel) Rerank(ctx context.Context, request RerankRequest, rerankC
if mode != "truncate" && mode != "passthrough" && mode != "raise_error" {
return nil, fmt.Errorf("%w: invalid %s %q; expected %q, %q, or %q", ErrRerankTokenLimitPolicy, common.EnvRerankTokenLimitMode, mode, "truncate", "passthrough", "raise_error")
}
- if mode != "passthrough" && len(request.Documents) > 0 {
- queryTokens := counter.Count(request.Query)
- if mode == "truncate" {
- documentBudget := max(effectiveMaxTokens-queryTokens, 0)
- documentTokens := tokenizer.OverLimitLadder(documentBudget)[0]
- documents := make([]string, len(request.Documents))
- for i, document := range request.Documents {
- documents[i] = counter.TrimToLimit(document, documentTokens)
- }
- request.Documents = documents
- } else {
- for i, document := range request.Documents {
- inputTokens := queryTokens + counter.Count(document)
- if inputTokens > effectiveMaxTokens {
- return nil, fmt.Errorf("%w: rerank input at document index %d has %d tokens, exceeding the configured maximum of %d", ErrRerankTokenLimitPolicy, i, inputTokens, effectiveMaxTokens)
- }
+ // passthrough opted out of measuring the input; an empty document list has
+ // nothing to cut. Neither needs the tokenizer counter, so the "declared
+ // tokenizer asset is missing" refusal does not apply to them.
+ if mode == "passthrough" || len(request.Documents) == 0 {
+ return r.ModelDriver.Rerank(ctx, r.ModelName, request, r.APIConfig, rerankConfig, modelUsage)
+ }
+ // Both remaining modes count tokens, so an unavailable declared tokenizer is a
+ // refusal rather than a silent fallback to the calibrated estimate.
+ if r.limiterErr != nil {
+ return nil, r.limiterErr
+ }
+ limiter := r.limiter
+ counter := limiter.Counter()
+ effectiveMaxTokens := limiter.Limit(maxTokens)
+
+ queryTokens := counter.Count(request.Query)
+ documentBudget := max(effectiveMaxTokens-queryTokens, 0)
+
+ if mode == "raise_error" {
+ for i, document := range request.Documents {
+ inputTokens := queryTokens + counter.Count(document)
+ if inputTokens > effectiveMaxTokens {
+ return nil, fmt.Errorf("%w: rerank input at document index %d has %d tokens, exceeding the configured maximum of %d", ErrRerankTokenLimitPolicy, i, inputTokens, effectiveMaxTokens)
}
}
+ return r.ModelDriver.Rerank(ctx, r.ModelName, request, r.APIConfig, rerankConfig, modelUsage)
+ }
+
+ // mode == "truncate": pre-cut each document to the window, then shrink and
+ // retry if the provider still rejects an input as over its window. A provider
+ // that rejects a window computed from its own declared limit is telling us the
+ // declared limit was too optimistic (an estimate, or a smaller real window);
+ // re-cutting keeps the call alive, and the observation tightens the next one.
+ original := request.Documents
+ cal, calKey := limiter.Calibration()
+ var lastErr error
+ for _, budget := range tokenizer.OverLimitLadder(documentBudget) {
+ attempt := request
+ attempt.Documents = make([]string, len(original))
+ for i, document := range original {
+ attempt.Documents[i] = counter.TrimToLimit(document, budget)
+ }
+ resp, err := r.ModelDriver.Rerank(ctx, r.ModelName, attempt, r.APIConfig, rerankConfig, modelUsage)
+ if err == nil {
+ return resp, nil
+ }
+ if !tokenizer.IsOverLimitError(err) {
+ return nil, err
+ }
+ // The provider rejected the documents we actually sent, so the true ratio is
+ // at least maxTokens / our count of that attempt's largest pair - not the
+ // original's, which is longer and would understate the ratio.
+ cal.ObserveOverLimit(calKey, queryTokens+tokenizer.OwnTokenMax(attempt.Documents, counter), maxTokens)
+ lastErr = err
}
- return r.ModelDriver.Rerank(ctx, r.ModelName, request, r.APIConfig, rerankConfig, modelUsage)
+ return nil, lastErr
}
// OCRModel wraps a ModelDriver with optical-character-recognition configuration.
diff --git a/internal/entity/models/vllm.go b/internal/entity/models/vllm.go
index 0d40701005..b0d91d89da 100644
--- a/internal/entity/models/vllm.go
+++ b/internal/entity/models/vllm.go
@@ -153,10 +153,16 @@ func (v *VllmModel) ChatStreamlyWithSender(ctx context.Context, modelName string
// Encode encodes a list of texts into embeddings
type vllmEmbeddingResponse struct {
+ ID string `json:"id"`
Data []struct {
Index int `json:"index"`
Embedding []float64 `json:"embedding"`
} `json:"data"`
+ // llama-server and vLLM report the request's token usage alongside the vectors.
+ Usage struct {
+ PromptTokens int `json:"prompt_tokens"`
+ TotalTokens int `json:"total_tokens"`
+ } `json:"usage"`
}
// Embed embeds a list of texts into embeddings
@@ -241,6 +247,15 @@ func (v *VllmModel) Embed(ctx context.Context, modelName *string, request EmbedR
embeddings = append(embeddings, embeddingData)
}
+ // The provider's own token count is the oracle the ingest path calibrates its
+ // tokenizer against (see internal/tokenizer: ObserveUsage). Without it a
+ // llama-server deployment that counts with the calibrated estimate never leaves
+ // its default ratio. This mirrors OpenAIModel.Embed.
+ recordResponseUsage(modelUsage, parsed.ID, &TokenUsage{
+ PromptTokens: parsed.Usage.PromptTokens,
+ TotalTokens: parsed.Usage.TotalTokens,
+ }, "embedding")
+
return embeddings, nil
}
@@ -335,10 +350,16 @@ type vllmRerankRequest struct {
// field is intentionally ignored — callers reconstruct text from the
// original input via Index.
type vllmRerankResponse struct {
+ ID string `json:"id"`
Results []struct {
Index int `json:"index"`
RelevanceScore float64 `json:"relevance_score"`
} `json:"results"`
+ // vLLM reports the request's token usage alongside the scores.
+ Usage struct {
+ PromptTokens int `json:"prompt_tokens"`
+ TotalTokens int `json:"total_tokens"`
+ } `json:"usage"`
}
// Rerank scores documents against the query using a vLLM rerank model
@@ -439,6 +460,13 @@ func (v *VllmModel) Rerank(ctx context.Context, modelName *string, request Reran
})
}
+ // Report the provider's token usage, mirroring OpenAIModel.Embed and this
+ // driver's Embed, so rerank calls are counted instead of always reporting zero.
+ recordResponseUsage(modelUsage, parsed.ID, &TokenUsage{
+ PromptTokens: parsed.Usage.PromptTokens,
+ TotalTokens: parsed.Usage.TotalTokens,
+ }, "rerank")
+
return &rerankResponse, nil
}
diff --git a/internal/entity/models/vllm_rerank_test.go b/internal/entity/models/vllm_rerank_test.go
index 03be2f9f44..b63f51fe08 100644
--- a/internal/entity/models/vllm_rerank_test.go
+++ b/internal/entity/models/vllm_rerank_test.go
@@ -7,6 +7,8 @@ import (
"net/http/httptest"
"strings"
"testing"
+
+ "ragflow/internal/common"
)
func newVllmRerankServer(t *testing.T, expectAuth string, handler func(t *testing.T, body map[string]interface{}, w http.ResponseWriter)) *httptest.Server {
@@ -224,3 +226,32 @@ func TestVllmRerankRejectsOutOfRangeIndex(t *testing.T) {
t.Errorf("expected out-of-range error, got %v", err)
}
}
+
+// The rerank response's usage is the oracle a local reranker's tokenizer
+// calibration has; without it a llama-server/vLLM deployment reports 0 tokens.
+func TestVllmRerankReportsUsage(t *testing.T) {
+ withSSRFBypass(t)
+ ctx := t.Context()
+ srv := newVllmRerankServer(t, "Bearer test-key", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {
+ _ = json.NewEncoder(w).Encode(map[string]interface{}{
+ "id": "rerank-1",
+ "results": []map[string]interface{}{{"index": 0, "relevance_score": 0.5}},
+ "usage": map[string]interface{}{"prompt_tokens": 7, "total_tokens": 7},
+ })
+ })
+ defer srv.Close()
+
+ model := newVllmModelForTest(srv.URL)
+ apiKey := "test-key"
+ modelName := "BAAI/bge-reranker-v2-m3"
+ usage := &common.ModelUsage{}
+ if _, err := model.Rerank(ctx, &modelName, RerankRequest{Query: "q", Documents: []string{"a"}}, &APIConfig{ApiKey: &apiKey}, &RerankConfig{}, usage); err != nil {
+ t.Fatalf("Rerank failed: %v", err)
+ }
+ if usage.InputTokens != 7 || usage.TotalTokens != 7 {
+ t.Fatalf("usage=(%d,%d), want (7,7)", usage.InputTokens, usage.TotalTokens)
+ }
+ if usage.Type != "rerank" {
+ t.Fatalf("usage.Type=%q, want rerank", usage.Type)
+ }
+}
diff --git a/internal/tokenizer/embedlimit.go b/internal/tokenizer/embedlimit.go
index 18b56075d0..45d753d639 100644
--- a/internal/tokenizer/embedlimit.go
+++ b/internal/tokenizer/embedlimit.go
@@ -555,6 +555,11 @@ var OverLimitMarkers = []string{
"token limit",
"reduce the length",
"maximum allowed",
+ // llama-server's other embedding rejection: a 400 with type
+ // "exceed_context_size_error" when the input fits the physical batch but not the
+ // slot's context. Its wording ("larger than the max context size") matches none of
+ // the phrases above, so the 4xx status alone would not classify it.
+ "max context size",
}
// OverLimitCodes are the provider error codes that mean the same thing. 20015 is
@@ -566,9 +571,46 @@ var OverLimitCodes = []string{"20015"}
// Request" matches, "1400" and "4000" do not.
var overLimitStatus = regexp.MustCompile(`\b(?:400|413|422)\b`)
+// serverOverLimitStatus matches an HTTP 500 as a delimited number. It must be
+// applied to statusField(msg), not the whole message: a token count inside the
+// response body ("input (500 tokens) is too large to process") is not a status.
+var serverOverLimitStatus = regexp.MustCompile(`\b500\b`)
+
+// statusField returns the part of a driver error that carries the HTTP status:
+// everything before the response body. Drivers format the error as
+// "<provider> API error: <status>, body: <body>" (or "... API error: status <N>,
+// body: <body>"), so the status is always in the head. Matching the whole message
+// would read a token count in <body> as the status — e.g. a 503 whose body says
+// "input (500 tokens) is too large to process" would be treated as a 500.
+// A message without a body marker is returned unchanged.
+func statusField(msg string) string {
+ if i := strings.Index(msg, "body:"); i >= 0 {
+ return msg[:i]
+ }
+ return msg
+}
+
+// serverOverLimitMarkers are the wordings that make a 5xx an over-limit rejection
+// after all. They are matched ONLY together with serverOverLimitStatus, so a genuine
+// server failure (500 without this wording, or any other 5xx) still returns at once.
+//
+// llama-server reports an embedding input over its physical batch (-ub) as HTTP 500
+// (llama.cpp maps ERROR_TYPE_SERVER to a server error) even though a shorter input
+// succeeds. Its message carries one of these phrases. Treating it as over-limit is
+// what lets the calibration learn from it and the input be re-cut; the generic "a
+// 5xx is the provider's problem" rule would fail the whole document instead.
+var serverOverLimitMarkers = []string{
+ "is too large to process",
+}
+
// IsOverLimitError reports whether err is an over-limit rejection rather than a
-// rate limit or a genuine failure. Only 4xx rejections qualify: a 5xx is the
-// provider's problem and a shorter input would not fix it.
+// rate limit or a genuine failure.
+//
+// A 4xx (400/413/422) qualifies when its message carries a known phrase or provider
+// code. A 5xx does NOT, with one narrow exception: llama-server answers an input over
+// its physical batch with 500 plus a specific phrase, and a shorter input does fix it
+// (see serverOverLimitMarkers). The 500 is matched only in the status field, and any
+// other 5xx is still the provider's problem.
//
// Status codes and provider codes are matched as DELIMITED numbers. A substring test
// would read provider code 120015 as SiliconFlow's 20015 (and 1400 as 400), and the
@@ -580,14 +622,25 @@ func IsOverLimitError(err error) bool {
}
msg := strings.ToLower(err.Error())
if !overLimitStatus.MatchString(msg) {
- return false
+ if !serverOverLimitStatus.MatchString(statusField(msg)) {
+ return false
+ }
+ return containsAnyMarker(msg, serverOverLimitMarkers)
}
- for _, marker := range OverLimitMarkers {
+ return containsAnyMarker(msg, OverLimitMarkers) || containsAnyCode(msg, OverLimitCodes)
+}
+
+func containsAnyMarker(msg string, markers []string) bool {
+ for _, marker := range markers {
if strings.Contains(msg, marker) {
return true
}
}
- for _, code := range OverLimitCodes {
+ return false
+}
+
+func containsAnyCode(msg string, codes []string) bool {
+ for _, code := range codes {
if hasDelimitedNumber(msg, code) {
return true
}
diff --git a/internal/tokenizer/embedlimit_test.go b/internal/tokenizer/embedlimit_test.go
index 8bc24eb489..d36fe6892d 100644
--- a/internal/tokenizer/embedlimit_test.go
+++ b/internal/tokenizer/embedlimit_test.go
@@ -304,6 +304,31 @@ func TestIsOverLimitErrorMatchesDelimitedNumbers(t *testing.T) {
{"code 120015", `SILICONFLOW API error: 400 Bad Request, body: {"code":120015}`, false},
{"code 200150", `SILICONFLOW API error: 400 Bad Request, body: {"code":200150}`, false},
{"status 1400", `API error: 1400 Bad Request, body: too long`, false},
+ // llama-server's two embedding rejections. The first is wrongly reported as
+ // HTTP 500 but is fixed by a shorter input; the second is a 400 whose wording
+ // matches none of the generic phrases, so both need explicit handling.
+ {
+ "llama-server over physical batch (500)",
+ `vLLM embeddings API error: 500 Internal Server Error, body: {"error":{"code":500,"message":"input (2068 tokens) is too large to process. increase the physical batch size (current batch size: 2048)","type":"server_error"}}`,
+ true,
+ },
+ {
+ "llama-server wording under another 5xx",
+ `vLLM embeddings API error: 503 Service Unavailable, body: {"error":{"code":503,"message":"input (2068 tokens) is too large to process. increase the physical batch size (current batch size: 2048)","type":"server_error"}}`,
+ false,
+ },
+ {
+ // A 500 in the body (a token count) must not be read as the HTTP status:
+ // only the status field counts.
+ "llama-server wording under 503 with 500 in the body",
+ `vLLM embeddings API error: 503 Service Unavailable, body: {"error":{"code":503,"message":"input (500 tokens) is too large to process. increase the physical batch size","type":"server_error"}}`,
+ false,
+ },
+ {
+ "llama-server over slot context (400)",
+ `vLLM embeddings API error: 400 Bad Request, body: {"error":{"code":400,"message":"input (2068 tokens) is larger than the max context size (2048 tokens). skipping","type":"exceed_context_size_error"}}`,
+ true,
+ },
// Rate limits and provider failures must not be mistaken for size.
{"rate limit", `SILICONFLOW API error: 429 Too Many Requests, body: {"message":"Request was rejected due to rate limiting. Details: TPM limit reached."}`, false},
{"server error", `SILICONFLOW API error: 500 Internal Server Error, body: too long`, false},