RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Go: handle llama-server over-limit rejections in embed and rerank (#20784)
Zhichang Yu9b5d73bd7c9f
diff --git a/conf/models/openai-api-compatible.json b/conf/models/openai-api-compatible.jsonindex 99486b1528..dd9002a7b1 100644--- a/conf/models/openai-api-compatible.json+++ b/conf/models/openai-api-compatible.json@@ -5,6 +5,7 @@"chat": "chat/completions","models": "models","embedding": "embeddings",+ "rerank": "rerank","asr": "audio/transcriptions","tts": "audio/speech"},diff --git a/internal/entity/models/model_test.go b/internal/entity/models/model_test.goindex e48924ee0b..8ebab3172a 100644--- a/internal/entity/models/model_test.go+++ b/internal/entity/models/model_test.go@@ -61,6 +61,21 @@ func readProviderConfig(t *testing.T, fileName string) []byte {return nil}+// A provider that supports rerank must declare its rerank url_suffix. Without it+// the request is posted to the base URL (".../") instead of "/rerank", which the+// server answers with a generic 500 — the failure reported in #16033.+func TestOpenAIAPICompatibleDeclaresRerankSuffix(t *testing.T) {+ var cfg struct {+ URLSuffix URLSuffix `json:"url_suffix"`+ }+ if err := json.Unmarshal(readProviderConfig(t, "openai-api-compatible.json"), &cfg); err != nil {+ t.Fatalf("parse openai-api-compatible.json: %v", err)+ }+ if strings.TrimSpace(cfg.URLSuffix.Rerank) == "" {+ t.Fatal("openai-api-compatible.json declares no rerank url_suffix")+ }+}+// setupProviderTestDir creates a temporary directory populated with provider// config files and conf/all_models.json, then changes the working directory to// it. InitProviderManager hardcodes a read of conf/all_models.json relative todiff --git a/internal/entity/models/non_chat_usage_test.go b/internal/entity/models/non_chat_usage_test.goindex a677af4f43..7bd8f3f25b 100644--- a/internal/entity/models/non_chat_usage_test.go+++ b/internal/entity/models/non_chat_usage_test.go@@ -133,6 +133,18 @@ func TestProviderEmbeddingAndRerankUsage(t *testing.T) {embedInput: 7,embedTotal: 7,},+ {+ // The OpenAI-API-Compatible provider reuses the vLLM driver, which is+ // how a locally served embedding model (llama-server, vLLM) is added.+ // It must report usage so the ingest path can calibrate its tokenizer.+ name: "openai-api-compatible (vLLM)",+ newModel: func(baseURL string) ModelDriver {+ return NewOpenAIAPICompatibleModel(map[string]string{"default": baseURL}, URLSuffix{Embedding: "embedding"})+ },+ embeddingBody: `{"id":"embed-vllm","data":[{"embedding":[0.1,0.2],"index":0}],"usage":{"prompt_tokens":7,"total_tokens":7}}`,+ embedInput: 7,+ embedTotal: 7,+ },}for _, tc := range cases {diff --git a/internal/entity/models/rerank_model_test.go b/internal/entity/models/rerank_model_test.goindex 2dc12d1700..7319925696 100644--- a/internal/entity/models/rerank_model_test.go+++ b/internal/entity/models/rerank_model_test.go@@ -89,3 +89,158 @@ func TestNewRerankModelDefaultsMaxTokens(t *testing.T) {t.Fatalf("MaxTokens = %d, want %d", model.MaxTokens, defaultMaxRerankTokens)}}++// scriptedRerankDriver returns whatever fail produces for each call, and records+// every request's documents so a test can see whether they shrank.+type scriptedRerankDriver struct {+ ModelDriver+ fail func(request RerankRequest, call int) error+ calls int+ docs [][]string+}++func (d *scriptedRerankDriver) Rerank(_ context.Context, _ *string, request RerankRequest, _ *APIConfig, _ *RerankConfig, _ *common.ModelUsage) (*RerankResponse, error) {+ d.calls+++ d.docs = append(d.docs, request.Documents)+ if d.fail != nil {+ if err := d.fail(request, d.calls); err != nil {+ return nil, err+ }+ }+ return &RerankResponse{Data: []RerankResult{{Index: 0, RelevanceScore: 1}}}, nil+}++func llamaRerankOverLimit() error {+ return errors.New(`vLLM rerank API error: 500 Internal Server Error, body: {"error":{"message":"input is too large to process. increase the physical batch size"}}`)+}++// A provider that rejects a window we computed from its own declared limit must+// not fail the call: the documents are re-cut with a smaller budget and the+// calibration is told, exactly as the embedding path does.+func TestRerankModelShrinksOnOverLimit(t *testing.T) {+ t.Setenv(common.EnvRerankTokenLimitMode, "truncate")+ document := strings.Repeat("token ", 2000)+ driver := &scriptedRerankDriver{+ fail: func(request RerankRequest, _ int) error {+ if len(request.Documents[0]) > 1024 {+ return llamaRerankOverLimit()+ }+ return nil+ },+ }+ apiKey := t.Name()+ model := NewRerankModel(driver, nil, &APIConfig{ApiKey: &apiKey}, 512)+ if _, err := model.Rerank(t.Context(), RerankRequest{Query: "q", Documents: []string{document}}, nil, nil); err != nil {+ t.Fatalf("Rerank() error = %v, want success after shrinking", err)+ }+ if driver.calls < 2 {+ t.Fatalf("driver calls = %d, want a retry with a smaller budget", driver.calls)+ }+ first, last := driver.docs[0][0], driver.docs[len(driver.docs)-1][0]+ if len(last) >= len(first) {+ t.Fatalf("documents did not shrink across attempts: first=%d bytes last=%d bytes", len(first), len(last))+ }+ // The rejection must teach the calibration from the attempt actually sent: the+ // count of the trimmed documents, not the longer original, which would+ // understate the ratio and leave it at the 1.05 default.+ ratio, _, rejects, ok := tokenizer.DefaultCalibration().Stats(model.QuotaKey())+ if !ok || rejects == 0 {+ t.Fatalf("calibration saw no over-limit observation: ok=%t rejects=%d", ok, rejects)+ }+ if ratio <= tokenizer.DefaultUncountedRatioUpper {+ t.Fatalf("calibration ratio = %.3f, want above the %.2f default", ratio, tokenizer.DefaultUncountedRatioUpper)+ }+}++// An exact tokenizer has no calibration: Limiter.Calibration() returns nil. A+// provider can still reject such an input (its physical batch may be smaller than+// the declared window), and the retry must keep working - ObserveOverLimit is+// nil-safe, so there is nothing to guard.+func TestRerankModelRetriesWithExactCounter(t *testing.T) {+ t.Setenv(common.EnvRerankTokenLimitMode, "truncate")+ t.Setenv("TOKENIZER_EMBEDDING_TOKENIZER", tokenizer.CounterCL100K)+ document := strings.Repeat("token ", 2000)+ driver := &scriptedRerankDriver{+ fail: func(request RerankRequest, _ int) error {+ if len(request.Documents[0]) > 1024 {+ return llamaRerankOverLimit()+ }+ return nil+ },+ }+ name := "cl100k-model"+ model := NewRerankModel(driver, &name, nil, 512)+ if cal, _ := model.limiter.Calibration(); cal != nil {+ t.Fatalf("expected an exact limiter (nil calibration), got %v", cal)+ }+ if _, err := model.Rerank(t.Context(), RerankRequest{Query: "q", Documents: []string{document}}, nil, nil); err != nil {+ t.Fatalf("Rerank() error = %v, want success after shrinking", err)+ }+ if driver.calls < 2 {+ t.Fatalf("driver calls = %d, want a retry with a smaller budget", driver.calls)+ }+}++// Only over-limit rejections are retried; a genuine failure is returned at once.+func TestRerankModelDoesNotRetryOtherErrors(t *testing.T) {+ t.Setenv(common.EnvRerankTokenLimitMode, "truncate")+ driver := &scriptedRerankDriver{+ fail: func(RerankRequest, int) error { return errors.New("connection refused") },+ }+ model := NewRerankModel(driver, nil, nil, 512)+ _, err := model.Rerank(t.Context(), RerankRequest{Query: "q", Documents: []string{"doc"}}, nil, nil)+ if err == nil || !strings.Contains(err.Error(), "connection refused") {+ t.Fatalf("Rerank() error = %v, want the driver error", err)+ }+ if driver.calls != 1 {+ t.Fatalf("driver calls = %d, want 1 (no retry for a non-size error)", driver.calls)+ }+}++func TestRerankModelQuotaKeyNamesDeployment(t *testing.T) {+ baseA, baseB, apiKey, name := "http://a/v1", "http://b/v1", "k", "bge-reranker-v2-m3"+ keyA := NewRerankModel(&captureRerankDriver{}, &name, &APIConfig{BaseURL: &baseA, ApiKey: &apiKey}, 10).QuotaKey()+ keyB := NewRerankModel(&captureRerankDriver{}, &name, &APIConfig{BaseURL: &baseB, ApiKey: &apiKey}, 10).QuotaKey()+ if keyA == "" {+ t.Fatal("QuotaKey() is empty")+ }+ if keyA == keyB {+ t.Fatalf("QuotaKey() = %q for two endpoints, want them distinct", keyA)+ }+ unnamed := NewRerankModel(&captureRerankDriver{}, nil, &APIConfig{BaseURL: &baseA, ApiKey: &apiKey}, 10).QuotaKey()+ if unnamed == keyA {+ t.Fatalf("QuotaKey() ignored the model name: %q", unnamed)+ }+ // With no BaseURL the driver falls back to its provider's default endpoint, so+ // the provider name is what keeps two deployments apart.+ noURL := &APIConfig{ApiKey: &apiKey}+ provA := NewRerankModelWithInfo(&captureRerankDriver{}, &name, noURL, 10, &ModelInfo{ProviderName: "VLLM"}).QuotaKey()+ provB := NewRerankModelWithInfo(&captureRerankDriver{}, &name, noURL, 10, &ModelInfo{ProviderName: "OpenAI-API-Compatible"}).QuotaKey()+ if provA == provB {+ t.Fatalf("QuotaKey() ignored the provider when BaseURL is empty: %q", provA)+ }+}++// passthrough sends the input without measuring it, so it must not be blocked by a+// missing tokenizer asset; the modes that do count still refuse.+func TestRerankModelPassthroughIgnoresUnavailableTokenizer(t *testing.T) {+ document := strings.Repeat("token ", 40)++ t.Setenv(common.EnvRerankTokenLimitMode, "passthrough")+ driver := &captureRerankDriver{}+ model := NewRerankModel(driver, nil, nil, 10)+ model.limiterErr = errors.New("model tokenizer is declared but its asset is unavailable")+ if _, err := model.Rerank(t.Context(), RerankRequest{Query: "q", Documents: []string{document}}, nil, nil); err != nil {+ t.Fatalf("passthrough must not require the tokenizer counter: %v", err)+ }+ if driver.calls != 1 {+ t.Fatalf("driver calls = %d, want 1", driver.calls)+ }++ t.Setenv(common.EnvRerankTokenLimitMode, "truncate")+ refusing := NewRerankModel(&captureRerankDriver{}, nil, nil, 10)+ refusing.limiterErr = errors.New("model tokenizer is declared but its asset is unavailable")+ if _, err := refusing.Rerank(t.Context(), RerankRequest{Query: "q", Documents: []string{document}}, nil, nil); err == nil {+ t.Fatal("truncate must refuse when the declared tokenizer asset is unavailable")+ }+}diff --git a/internal/entity/models/types.go b/internal/entity/models/types.goindex 111e47dd15..a7df795561 100644--- a/internal/entity/models/types.go+++ b/internal/entity/models/types.go@@ -354,34 +354,58 @@ func (m *EmbeddingModel) ResolveTokenizerID() string {return GetModelTokenizer(name)}-// QuotaKey names the deployment this embedding model counts against: endpoint,-// region, model name and an API-key prefix. The tokenizer belongs to the model, but-// what a provider accepts is per deployment - the same model behind two endpoints-// can have different windows - so everything that learns a real/own token ratio-// (the ingest embedder, the dataset-nav embedder, the knowledge-compiler embedder)-// has to key that ratio the same way; otherwise each path re-learns the same-// rejection and none of them tightens for the others.+// QuotaKey names the deployment this embedding model counts against: provider,+// endpoint, region, model name and an API-key prefix. The tokenizer belongs to the+// model, but what a provider accepts is per deployment - the same model behind two+// endpoints can have different windows - so everything that learns a real/own token+// ratio (the ingest embedder, the dataset-nav embedder, the knowledge-compiler+// embedder) has to key that ratio the same way; otherwise each path re-learns the+// same rejection and none of them tightens for the others.func (m *EmbeddingModel) QuotaKey() string {if m == nil {return ""}- var baseURL, region, apiKey, modelName string- if cfg := m.APIConfig; cfg != nil {- if cfg.BaseURL != nil {- baseURL = *cfg.BaseURL+ return quotaKeyFor(providerNameOf(m.info), m.APIConfig, m.ModelName)+}++// providerNameOf is the provider a model was resolved from, or "" when it was built+// without catalog metadata (tests, direct construction).+func providerNameOf(info *ModelInfo) string {+ if info == nil {+ return ""+ }+ return info.ProviderName+}++// quotaKeyFor is the shared key used by every model that calibrates a real/own+// token ratio against a provider deployment: provider, resolved endpoint, region,+// model name and a short API-key digest. The tokenizer belongs to the model, but the+// accepted window belongs to the deployment, and the digest keeps two tenants of the+// same endpoint from sharing one learned ratio.+//+// providerName matters when BaseURL is empty: the driver then falls back to the+// endpoint its provider config supplies for that region, so provider + region is what+// identifies the deployment. Without it two providers that resolve to different+// default endpoints but agree on region, model and key would share one learned ratio,+// and the tighter window would over-truncate the other's requests.+func quotaKeyFor(providerName string, apiConfig *APIConfig, modelName *string) string {+ var baseURL, region, apiKey, name string+ if apiConfig != nil {+ if apiConfig.BaseURL != nil {+ baseURL = *apiConfig.BaseURL}- if cfg.Region != nil {- region = *cfg.Region+ if apiConfig.Region != nil {+ region = *apiConfig.Region}- if cfg.ApiKey != nil {- apiKey = *cfg.ApiKey+ if apiConfig.ApiKey != nil {+ apiKey = *apiConfig.ApiKey}}- if m.ModelName != nil {- modelName = *m.ModelName+ if modelName != nil {+ name = *modelName}sum := sha256.Sum256([]byte(apiKey))- return fmt.Sprintf("%s|%s|%s|%x", baseURL, region, modelName, sum[:8])+ return fmt.Sprintf("%s|%s|%s|%s|%x", providerName, baseURL, region, name, sum[:8])}// RerankModel wraps a ModelDriver with rerank-specific configuration@@ -401,27 +425,43 @@ func NewRerankModel(driver ModelDriver, modelName *string, apiConfig *APIConfig,if maxTokens <= 0 {maxTokens = defaultMaxRerankTokens}- tokenizerID := ""- if modelName != nil {- tokenizerID = GetModelTokenizer(*modelName)- }- return &RerankModel{+ model := &RerankModel{ModelDriver: driver,ModelName: modelName,APIConfig: apiConfig,MaxTokens: maxTokens,- limiter: tokenizer.LimiterFor(tokenizerID, "", tokenizer.DefaultCalibration()),- limiterErr: tokenizer.RefuseUnavailableCounter(tokenizerID, "rerank"),}+ model.initLimiter()+ return model}// NewRerankModelWithInfo creates a rerank model with resolved metadata.func NewRerankModelWithInfo(driver ModelDriver, modelName *string, apiConfig *APIConfig, maxTokens int, info *ModelInfo) *RerankModel {model := NewRerankModel(driver, modelName, apiConfig, maxTokens)model.info = cloneModelInfo(info)+ // Rebuild the limiter now that the provider is known: it is part of the+ // calibration key (see quotaKeyFor), and it was built without it above.+ model.initLimiter()return model}+// initLimiter binds the model's declared tokenizer and its deployment calibration+// key. Safe to call more than once; the constructors call it after the model's+// metadata is set so the key can include the provider.+func (r *RerankModel) initLimiter() {+ tokenizerID := ""+ if r.ModelName != nil {+ tokenizerID = GetModelTokenizer(*r.ModelName)+ }+ // The calibration key is the deployment, not the model: rerank and embedding+ // that share an endpoint still learn separate ratios, but every rerank caller+ // of one deployment keys the same ratio and tightens it together (see+ // RerankModel.Rerank, which records over-limit rejections against it).+ quotaKey := r.QuotaKey()+ r.limiter = tokenizer.LimiterFor(tokenizerID, quotaKey, tokenizer.DefaultCalibration())+ r.limiterErr = tokenizer.RefuseUnavailableCounter(tokenizerID, quotaKey)+}+// Info returns metadata for this selected rerank model.func (r *RerankModel) Info() *ModelInfo {if r == nil {@@ -430,8 +470,26 @@ func (r *RerankModel) Info() *ModelInfo {return cloneModelInfo(r.info)}+// QuotaKey names the deployment this rerank model counts against, so its+// calibration is keyed the same way as an embedding model on the same endpoint.+func (r *RerankModel) QuotaKey() string {+ if r == nil {+ return ""+ }+ return quotaKeyFor(providerNameOf(r.info), r.APIConfig, r.ModelName)+}+// Rerank calculates similarity between query and texts. Rerank input is measured// and truncated with the model's declared tokenizer, just like embedding input.+// An over-limit rejection is recovered like the embedding path: the documents are+// re-cut with a smaller budget and the calibration is told, as an attempt - if every+// budget is rejected the provider's error is still returned.+//+// The window bounds the query PLUS each document, not a single text, so only the+// documents can shrink - the query is fixed. RERANK_TOKEN_LIMIT_MODE decides+// whether that is wanted at all: "passthrough" sends the input untouched, and+// "raise_error" reports an oversized document instead of trimming it, so neither+// retries; only "truncate" (the default) shrinks-and-retries.func (r *RerankModel) Rerank(ctx context.Context, request RerankRequest, rerankConfig *RerankConfig, modelUsage *common.ModelUsage) (*RerankResponse, error) {if r == nil || r.ModelDriver == nil {return nil, errors.New("rerank model: driver is nil")@@ -440,12 +498,6 @@ func (r *RerankModel) Rerank(ctx context.Context, request RerankRequest, rerankCif maxTokens <= 0 {maxTokens = defaultMaxRerankTokens}- if r.limiterErr != nil {- return nil, r.limiterErr- }- limiter := r.limiter- counter := limiter.Counter()- effectiveMaxTokens := limiter.Limit(maxTokens)mode := strings.ToLower(strings.TrimSpace(common.GetEnv(common.EnvRerankTokenLimitMode)))if mode == "" {mode = "truncate"@@ -453,26 +505,62 @@ func (r *RerankModel) Rerank(ctx context.Context, request RerankRequest, rerankCif mode != "truncate" && mode != "passthrough" && mode != "raise_error" {return nil, fmt.Errorf("%w: invalid %s %q; expected %q, %q, or %q", ErrRerankTokenLimitPolicy, common.EnvRerankTokenLimitMode, mode, "truncate", "passthrough", "raise_error")}- if mode != "passthrough" && len(request.Documents) > 0 {- queryTokens := counter.Count(request.Query)- if mode == "truncate" {- documentBudget := max(effectiveMaxTokens-queryTokens, 0)- documentTokens := tokenizer.OverLimitLadder(documentBudget)[0]- documents := make([]string, len(request.Documents))- for i, document := range request.Documents {- documents[i] = counter.TrimToLimit(document, documentTokens)- }- request.Documents = documents- } else {- for i, document := range request.Documents {- inputTokens := queryTokens + counter.Count(document)- if inputTokens > effectiveMaxTokens {- return nil, fmt.Errorf("%w: rerank input at document index %d has %d tokens, exceeding the configured maximum of %d", ErrRerankTokenLimitPolicy, i, inputTokens, effectiveMaxTokens)- }+ // passthrough opted out of measuring the input; an empty document list has+ // nothing to cut. Neither needs the tokenizer counter, so the "declared+ // tokenizer asset is missing" refusal does not apply to them.+ if mode == "passthrough" || len(request.Documents) == 0 {+ return r.ModelDriver.Rerank(ctx, r.ModelName, request, r.APIConfig, rerankConfig, modelUsage)+ }+ // Both remaining modes count tokens, so an unavailable declared tokenizer is a+ // refusal rather than a silent fallback to the calibrated estimate.+ if r.limiterErr != nil {+ return nil, r.limiterErr+ }+ limiter := r.limiter+ counter := limiter.Counter()+ effectiveMaxTokens := limiter.Limit(maxTokens)++ queryTokens := counter.Count(request.Query)+ documentBudget := max(effectiveMaxTokens-queryTokens, 0)++ if mode == "raise_error" {+ for i, document := range request.Documents {+ inputTokens := queryTokens + counter.Count(document)+ if inputTokens > effectiveMaxTokens {+ return nil, fmt.Errorf("%w: rerank input at document index %d has %d tokens, exceeding the configured maximum of %d", ErrRerankTokenLimitPolicy, i, inputTokens, effectiveMaxTokens)}}+ return r.ModelDriver.Rerank(ctx, r.ModelName, request, r.APIConfig, rerankConfig, modelUsage)+ }++ // mode == "truncate": pre-cut each document to the window, then shrink and+ // retry if the provider still rejects an input as over its window. A provider+ // that rejects a window computed from its own declared limit is telling us the+ // declared limit was too optimistic (an estimate, or a smaller real window);+ // re-cutting keeps the call alive, and the observation tightens the next one.+ original := request.Documents+ cal, calKey := limiter.Calibration()+ var lastErr error+ for _, budget := range tokenizer.OverLimitLadder(documentBudget) {+ attempt := request+ attempt.Documents = make([]string, len(original))+ for i, document := range original {+ attempt.Documents[i] = counter.TrimToLimit(document, budget)+ }+ resp, err := r.ModelDriver.Rerank(ctx, r.ModelName, attempt, r.APIConfig, rerankConfig, modelUsage)+ if err == nil {+ return resp, nil+ }+ if !tokenizer.IsOverLimitError(err) {+ return nil, err+ }+ // The provider rejected the documents we actually sent, so the true ratio is+ // at least maxTokens / our count of that attempt's largest pair - not the+ // original's, which is longer and would understate the ratio.+ cal.ObserveOverLimit(calKey, queryTokens+tokenizer.OwnTokenMax(attempt.Documents, counter), maxTokens)+ lastErr = err}- return r.ModelDriver.Rerank(ctx, r.ModelName, request, r.APIConfig, rerankConfig, modelUsage)+ return nil, lastErr}// OCRModel wraps a ModelDriver with optical-character-recognition configuration.diff --git a/internal/entity/models/vllm.go b/internal/entity/models/vllm.goindex 0d40701005..b0d91d89da 100644--- a/internal/entity/models/vllm.go+++ b/internal/entity/models/vllm.go@@ -153,10 +153,16 @@ func (v *VllmModel) ChatStreamlyWithSender(ctx context.Context, modelName string// Encode encodes a list of texts into embeddingstype vllmEmbeddingResponse struct {+ ID string `json:"id"`Data []struct {Index int `json:"index"`Embedding []float64 `json:"embedding"`} `json:"data"`+ // llama-server and vLLM report the request's token usage alongside the vectors.+ Usage struct {+ PromptTokens int `json:"prompt_tokens"`+ TotalTokens int `json:"total_tokens"`+ } `json:"usage"`}// Embed embeds a list of texts into embeddings@@ -241,6 +247,15 @@ func (v *VllmModel) Embed(ctx context.Context, modelName *string, request EmbedRembeddings = append(embeddings, embeddingData)}+ // The provider's own token count is the oracle the ingest path calibrates its+ // tokenizer against (see internal/tokenizer: ObserveUsage). Without it a+ // llama-server deployment that counts with the calibrated estimate never leaves+ // its default ratio. This mirrors OpenAIModel.Embed.+ recordResponseUsage(modelUsage, parsed.ID, &TokenUsage{+ PromptTokens: parsed.Usage.PromptTokens,+ TotalTokens: parsed.Usage.TotalTokens,+ }, "embedding")+return embeddings, nil}@@ -335,10 +350,16 @@ type vllmRerankRequest struct {// field is intentionally ignored — callers reconstruct text from the// original input via Index.type vllmRerankResponse struct {+ ID string `json:"id"`Results []struct {Index int `json:"index"`RelevanceScore float64 `json:"relevance_score"`} `json:"results"`+ // vLLM reports the request's token usage alongside the scores.+ Usage struct {+ PromptTokens int `json:"prompt_tokens"`+ TotalTokens int `json:"total_tokens"`+ } `json:"usage"`}// Rerank scores documents against the query using a vLLM rerank model@@ -439,6 +460,13 @@ func (v *VllmModel) Rerank(ctx context.Context, modelName *string, request Reran})}+ // Report the provider's token usage, mirroring OpenAIModel.Embed and this+ // driver's Embed, so rerank calls are counted instead of always reporting zero.+ recordResponseUsage(modelUsage, parsed.ID, &TokenUsage{+ PromptTokens: parsed.Usage.PromptTokens,+ TotalTokens: parsed.Usage.TotalTokens,+ }, "rerank")+return &rerankResponse, nil}diff --git a/internal/entity/models/vllm_rerank_test.go b/internal/entity/models/vllm_rerank_test.goindex 03be2f9f44..b63f51fe08 100644--- a/internal/entity/models/vllm_rerank_test.go+++ b/internal/entity/models/vllm_rerank_test.go@@ -7,6 +7,8 @@ import ("net/http/httptest""strings""testing"++ "ragflow/internal/common")func newVllmRerankServer(t *testing.T, expectAuth string, handler func(t *testing.T, body map[string]interface{}, w http.ResponseWriter)) *httptest.Server {@@ -224,3 +226,32 @@ func TestVllmRerankRejectsOutOfRangeIndex(t *testing.T) {t.Errorf("expected out-of-range error, got %v", err)}}++// The rerank response's usage is the oracle a local reranker's tokenizer+// calibration has; without it a llama-server/vLLM deployment reports 0 tokens.+func TestVllmRerankReportsUsage(t *testing.T) {+ withSSRFBypass(t)+ ctx := t.Context()+ srv := newVllmRerankServer(t, "Bearer test-key", func(t *testing.T, body map[string]interface{}, w http.ResponseWriter) {+ _ = json.NewEncoder(w).Encode(map[string]interface{}{+ "id": "rerank-1",+ "results": []map[string]interface{}{{"index": 0, "relevance_score": 0.5}},+ "usage": map[string]interface{}{"prompt_tokens": 7, "total_tokens": 7},+ })+ })+ defer srv.Close()++ model := newVllmModelForTest(srv.URL)+ apiKey := "test-key"+ modelName := "BAAI/bge-reranker-v2-m3"+ usage := &common.ModelUsage{}+ if _, err := model.Rerank(ctx, &modelName, RerankRequest{Query: "q", Documents: []string{"a"}}, &APIConfig{ApiKey: &apiKey}, &RerankConfig{}, usage); err != nil {+ t.Fatalf("Rerank failed: %v", err)+ }+ if usage.InputTokens != 7 || usage.TotalTokens != 7 {+ t.Fatalf("usage=(%d,%d), want (7,7)", usage.InputTokens, usage.TotalTokens)+ }+ if usage.Type != "rerank" {+ t.Fatalf("usage.Type=%q, want rerank", usage.Type)+ }+}diff --git a/internal/tokenizer/embedlimit.go b/internal/tokenizer/embedlimit.goindex 18b56075d0..45d753d639 100644--- a/internal/tokenizer/embedlimit.go+++ b/internal/tokenizer/embedlimit.go@@ -555,6 +555,11 @@ var OverLimitMarkers = []string{"token limit","reduce the length","maximum allowed",+ // llama-server's other embedding rejection: a 400 with type+ // "exceed_context_size_error" when the input fits the physical batch but not the+ // slot's context. Its wording ("larger than the max context size") matches none of+ // the phrases above, so the 4xx status alone would not classify it.+ "max context size",}// OverLimitCodes are the provider error codes that mean the same thing. 20015 is@@ -566,9 +571,46 @@ var OverLimitCodes = []string{"20015"}// Request" matches, "1400" and "4000" do not.var overLimitStatus = regexp.MustCompile(`\b(?:400|413|422)\b`)+// serverOverLimitStatus matches an HTTP 500 as a delimited number. It must be+// applied to statusField(msg), not the whole message: a token count inside the+// response body ("input (500 tokens) is too large to process") is not a status.+var serverOverLimitStatus = regexp.MustCompile(`\b500\b`)++// statusField returns the part of a driver error that carries the HTTP status:+// everything before the response body. Drivers format the error as+// "<provider> API error: <status>, body: <body>" (or "... API error: status <N>,+// body: <body>"), so the status is always in the head. Matching the whole message+// would read a token count in <body> as the status — e.g. a 503 whose body says+// "input (500 tokens) is too large to process" would be treated as a 500.+// A message without a body marker is returned unchanged.+func statusField(msg string) string {+ if i := strings.Index(msg, "body:"); i >= 0 {+ return msg[:i]+ }+ return msg+}++// serverOverLimitMarkers are the wordings that make a 5xx an over-limit rejection+// after all. They are matched ONLY together with serverOverLimitStatus, so a genuine+// server failure (500 without this wording, or any other 5xx) still returns at once.+//+// llama-server reports an embedding input over its physical batch (-ub) as HTTP 500+// (llama.cpp maps ERROR_TYPE_SERVER to a server error) even though a shorter input+// succeeds. Its message carries one of these phrases. Treating it as over-limit is+// what lets the calibration learn from it and the input be re-cut; the generic "a+// 5xx is the provider's problem" rule would fail the whole document instead.+var serverOverLimitMarkers = []string{+ "is too large to process",+}+// IsOverLimitError reports whether err is an over-limit rejection rather than a-// rate limit or a genuine failure. Only 4xx rejections qualify: a 5xx is the-// provider's problem and a shorter input would not fix it.+// rate limit or a genuine failure.+//+// A 4xx (400/413/422) qualifies when its message carries a known phrase or provider+// code. A 5xx does NOT, with one narrow exception: llama-server answers an input over+// its physical batch with 500 plus a specific phrase, and a shorter input does fix it+// (see serverOverLimitMarkers). The 500 is matched only in the status field, and any+// other 5xx is still the provider's problem.//// Status codes and provider codes are matched as DELIMITED numbers. A substring test// would read provider code 120015 as SiliconFlow's 20015 (and 1400 as 400), and the@@ -580,14 +622,25 @@ func IsOverLimitError(err error) bool {}msg := strings.ToLower(err.Error())if !overLimitStatus.MatchString(msg) {- return false+ if !serverOverLimitStatus.MatchString(statusField(msg)) {+ return false+ }+ return containsAnyMarker(msg, serverOverLimitMarkers)}- for _, marker := range OverLimitMarkers {+ return containsAnyMarker(msg, OverLimitMarkers) || containsAnyCode(msg, OverLimitCodes)+}++func containsAnyMarker(msg string, markers []string) bool {+ for _, marker := range markers {if strings.Contains(msg, marker) {return true}}- for _, code := range OverLimitCodes {+ return false+}++func containsAnyCode(msg string, codes []string) bool {+ for _, code := range codes {if hasDelimitedNumber(msg, code) {return true}diff --git a/internal/tokenizer/embedlimit_test.go b/internal/tokenizer/embedlimit_test.goindex 8bc24eb489..d36fe6892d 100644--- a/internal/tokenizer/embedlimit_test.go+++ b/internal/tokenizer/embedlimit_test.go@@ -304,6 +304,31 @@ func TestIsOverLimitErrorMatchesDelimitedNumbers(t *testing.T) {{"code 120015", `SILICONFLOW API error: 400 Bad Request, body: {"code":120015}`, false},{"code 200150", `SILICONFLOW API error: 400 Bad Request, body: {"code":200150}`, false},{"status 1400", `API error: 1400 Bad Request, body: too long`, false},+ // llama-server's two embedding rejections. The first is wrongly reported as+ // HTTP 500 but is fixed by a shorter input; the second is a 400 whose wording+ // matches none of the generic phrases, so both need explicit handling.+ {+ "llama-server over physical batch (500)",+ `vLLM embeddings API error: 500 Internal Server Error, body: {"error":{"code":500,"message":"input (2068 tokens) is too large to process. increase the physical batch size (current batch size: 2048)","type":"server_error"}}`,+ true,+ },+ {+ "llama-server wording under another 5xx",+ `vLLM embeddings API error: 503 Service Unavailable, body: {"error":{"code":503,"message":"input (2068 tokens) is too large to process. increase the physical batch size (current batch size: 2048)","type":"server_error"}}`,+ false,+ },+ {+ // A 500 in the body (a token count) must not be read as the HTTP status:+ // only the status field counts.+ "llama-server wording under 503 with 500 in the body",+ `vLLM embeddings API error: 503 Service Unavailable, body: {"error":{"code":503,"message":"input (500 tokens) is too large to process. increase the physical batch size","type":"server_error"}}`,+ false,+ },+ {+ "llama-server over slot context (400)",+ `vLLM embeddings API error: 400 Bad Request, body: {"error":{"code":400,"message":"input (2068 tokens) is larger than the max context size (2048 tokens). skipping","type":"exceed_context_size_error"}}`,+ true,+ },// Rate limits and provider failures must not be mistaken for size.{"rate limit", `SILICONFLOW API error: 429 Too Many Requests, body: {"message":"Request was rejected due to rate limiting. Details: TPM limit reached."}`, false},{"server error", `SILICONFLOW API error: 500 Internal Server Error, body: too long`, false},