Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(rerank): describe score scales by endpoint output (#4701)

Wang Xinmingcfb2dda03913
diff --git a/internal/models/legacy_keys.go b/internal/models/legacy_keys.go
index 96b371689..1dfc2a8cd 100644
--- a/internal/models/legacy_keys.go
+++ b/internal/models/legacy_keys.go
@@ -21,8 +21,8 @@ const (
LegacyTruncatePromptTokens = 511
// ExtraScoreScale overrides the vendor rerank score scale for one row.
// A self-hosted gateway serves whatever reranker was deployed behind it
- // and the two families disagree: BGE-class models answer a 0..1
- // probability, Qwen3-Reranker-class models an unbounded score. A vendor
- // can only state what its own documentation shows.
+ // and normalization depends on the serving configuration, not the
+ // model family. Already-normalized 0..1 relevance uses probability;
+ // raw logits use logit. A vendor can only state its documented default.
ExtraScoreScale = "score_scale"
)
diff --git a/internal/models/parity/rerank_test.go b/internal/models/parity/rerank_test.go
index dfad69445..8bcfcff68 100644
--- a/internal/models/parity/rerank_test.go
+++ b/internal/models/parity/rerank_test.go
@@ -441,13 +441,13 @@ func TestGatewayRerankEndpoints(t *testing.T) {
// TestScoreScaleIsOverridablePerRow covers the gateways, where the vendor
// cannot know the answer: GPUStack and a generic endpoint serve whatever
-// reranker was deployed behind them, and the two families disagree — BGE
-// answers a 0..1 probability, Qwen3-Reranker an unbounded score. Converting a
+// reranker was deployed behind them. Its serving configuration determines
+// whether the endpoint returns raw logits or probabilities. Converting a
// probability as if it were a logit squeezes every score into [0.5, 0.73],
// which makes a relevance threshold meaningless, so the operator needs a way
// to say which one is actually there.
func TestScoreScaleIsOverridablePerRow(t *testing.T) {
- for _, id := range []string{"gpustack", "generic"} {
+ for _, id := range []string{"gpustack", "generic", "litellm"} {
t.Run(id, func(t *testing.T) {
base := "http://127.0.0.1:9/v1"
overridden, err := modelruntime.Resolve(modelruntime.Ref{
diff --git a/internal/models/providers/generic.go b/internal/models/providers/generic.go
index f8f774250..80c803235 100644
--- a/internal/models/providers/generic.go
+++ b/internal/models/providers/generic.go
@@ -71,21 +71,21 @@ func newGenericProvider() *Definition {
Type: "select",
Options: []ExtraFieldOption{
{
- Label: "Unbounded score (Qwen3-Reranker class)",
- Labels: map[string]string{"zh-CN": "无界分数(Qwen3-Reranker 一类)"},
+ Label: "Unnormalized logits (apply sigmoid)",
+ Labels: map[string]string{"zh-CN": "未归一化的 logit(应用 sigmoid)"},
Value: "logit",
},
{
- Label: "0..1 relevance (BGE class)",
- Labels: map[string]string{"zh-CN": "0~1 相关度(BGE 一类)"},
+ Label: "Already normalized relevance (0..1)",
+ Labels: map[string]string{"zh-CN": "已归一化的相关度(0~1)"},
Value: "probability",
},
},
- Placeholder: "match the reranker actually deployed behind this endpoint",
- Placeholders: map[string]string{"zh-CN": "按这个端点后面实际部署的重排模型选择"},
+ Placeholder: "match the endpoint output; vLLM Qwen3 no/yes recipe: probability",
+ Placeholders: map[string]string{"zh-CN": "按端点输出选择;vLLM Qwen3 no/yes 配方选择 probability"},
// A gateway serves whatever reranker was deployed behind it, and
- // the two families disagree. The vendor default follows the
- // documentation; the operator knows which model is actually there.
+ // serving configuration determines its output scale. A model
+ // family alone does not identify whether normalization was applied.
Required: false,
ModelTypes: []types.ModelType{types.ModelTypeRerank},
}, {
diff --git a/internal/models/providers/gpustack.go b/internal/models/providers/gpustack.go
index 0062b6ecd..edd2ae625 100644
--- a/internal/models/providers/gpustack.go
+++ b/internal/models/providers/gpustack.go
@@ -85,21 +85,21 @@ func newGpustackProvider() *Definition {
Type: "select",
Options: []ExtraFieldOption{
{
- Label: "Unbounded score (Qwen3-Reranker class)",
- Labels: map[string]string{"zh-CN": "无界分数(Qwen3-Reranker 一类)"},
+ Label: "Unnormalized logits (apply sigmoid)",
+ Labels: map[string]string{"zh-CN": "未归一化的 logit(应用 sigmoid)"},
Value: "logit",
},
{
- Label: "0..1 relevance (BGE class)",
- Labels: map[string]string{"zh-CN": "0~1 相关度(BGE 一类)"},
+ Label: "Already normalized relevance (0..1)",
+ Labels: map[string]string{"zh-CN": "已归一化的相关度(0~1)"},
Value: "probability",
},
},
- Placeholder: "match the reranker actually deployed behind this endpoint",
- Placeholders: map[string]string{"zh-CN": "按这个端点后面实际部署的重排模型选择"},
+ Placeholder: "match the endpoint output; vLLM Qwen3 no/yes recipe: probability",
+ Placeholders: map[string]string{"zh-CN": "按端点输出选择;vLLM Qwen3 no/yes 配方选择 probability"},
// A gateway serves whatever reranker was deployed behind it, and
- // the two families disagree. The vendor default follows the
- // documentation; the operator knows which model is actually there.
+ // serving configuration determines its output scale. A model
+ // family alone does not identify whether normalization was applied.
Required: false,
ModelTypes: []types.ModelType{types.ModelTypeRerank},
}, {
@@ -138,7 +138,9 @@ func newGpustackProvider() *Definition {
// VoxBox, so the vLLM rerank extension reaches the model.
AcceptsTruncatePromptTokens: api.Ptr(true),
// The field is spelled relevance_score like the Cohere shape, but
- // it is not a probability: the rerank API page's own example
+ // its documented BGE example returns logits. This is not a
+ // model-family guarantee; probability-serving endpoints must
+ // override score_scale. The rerank API page's example
// returns 1.951932668685913, -3.7347371578216553 and
// -6.157620906829834. Read as a probability, every negative score
// falls below a relevance threshold tuned for 0..1.
diff --git a/internal/models/providers/litellm.go b/internal/models/providers/litellm.go
index 4d30d963a..e43b8de36 100644
--- a/internal/models/providers/litellm.go
+++ b/internal/models/providers/litellm.go
@@ -83,21 +83,21 @@ func newLitellmProvider() *Definition {
Type: "select",
Options: []ExtraFieldOption{
{
- Label: "Unbounded score (Qwen3-Reranker class)",
- Labels: map[string]string{"zh-CN": "无界分数(Qwen3-Reranker 一类)"},
+ Label: "Unnormalized logits (apply sigmoid)",
+ Labels: map[string]string{"zh-CN": "未归一化的 logit(应用 sigmoid)"},
Value: "logit",
},
{
- Label: "0..1 relevance (BGE class)",
- Labels: map[string]string{"zh-CN": "0~1 相关度(BGE 一类)"},
+ Label: "Already normalized relevance (0..1)",
+ Labels: map[string]string{"zh-CN": "已归一化的相关度(0~1)"},
Value: "probability",
},
},
- Placeholder: "match the reranker actually deployed behind this endpoint",
- Placeholders: map[string]string{"zh-CN": "按这个端点后面实际部署的重排模型选择"},
+ Placeholder: "match the endpoint output; vLLM Qwen3 no/yes recipe: probability",
+ Placeholders: map[string]string{"zh-CN": "按端点输出选择;vLLM Qwen3 no/yes 配方选择 probability"},
// A gateway serves whatever reranker was deployed behind it, and
- // the two families disagree. The vendor default follows the
- // documentation; the operator knows which model is actually there.
+ // serving configuration determines its output scale. A model
+ // family alone does not identify whether normalization was applied.
Required: false,
ModelTypes: []types.ModelType{types.ModelTypeRerank},
}},
diff --git a/internal/models/rerank/protocol_test.go b/internal/models/rerank/protocol_test.go
index b9c545164..4f7b55059 100644
--- a/internal/models/rerank/protocol_test.go
+++ b/internal/models/rerank/protocol_test.go
@@ -135,6 +135,10 @@ func TestLogitConversionPreservesOrder(t *testing.T) {
func TestProbabilityScoresPassThroughUntouched(t *testing.T) {
assert.Equal(t, 0.9819, normalizeScore(0.9819, api.ScoreProbability))
assert.Equal(t, 0.9819, normalizeScore(0.9819, ""))
+ // Already-normalized Qwen3 no/yes endpoint scores must not be sigmoid-ed twice.
+ for _, score := range []float64{0.0177, 0.9995, 0.9996} {
+ assert.Equal(t, score, normalizeScore(score, api.ScoreProbability))
+ }
}
// A vendor that caps the query cannot be satisfied by splitting documents,
diff --git a/website-docs/03-features/06-models.md b/website-docs/03-features/06-models.md
index ef1e4a810..9f81bd001 100644
--- a/website-docs/03-features/06-models.md
+++ b/website-docs/03-features/06-models.md
@@ -416,7 +416,7 @@ const (
| Azure OpenAI | `api_version` | 全部 | 留空走 `/openai/v1` 数据面;填写版本号(如 `2025-04-01-preview`)则走旧的 `/openai/deployments/{部署名}` 路径 |
| 腾讯云 LKEAP | SecretKey(必填,按密钥加密保存)、`region`(默认 `ap-guangzhou`) | 重排 | 重排接口使用 TC3 签名,API Key 一栏填 SecretId |
| 火山引擎 | SecretKey(必填,按密钥加密保存)、`region`(默认 `cn-beijing`)、`instruction` | 重排 | 重排接口使用 AK/SK 签名,API Key 一栏填 Access Key ID;`instruction` 默认为控制台原文 |
-| 自定义、GPUStack、LiteLLM | `score_scale` | 重排 | 「Rerank 分数标度」:`probability`(0~1 相关度,BGE 一类)或 `logit`(无界分数,Qwen3-Reranker 一类,会换算到 0~1 后再与重排阈值比较)。按端点后实际部署的模型选择 |
+| 自定义、GPUStack、LiteLLM | `score_scale` | 重排 | 「Rerank 分数标度」:端点已返回归一化的 0~1 相关度时选 `probability`;端点返回原始 logit 时选 `logit`(应用 sigmoid 后再与重排阈值比较)。按服务端输出约定选择,不能仅凭模型族判断 |
| 自定义、GPUStack | `truncate_prompt_tokens` | 重排 | vLLM 扩展参数,默认不发送;只在后端因文档过长报错时填写 |
#### 管理 API(`internal/router/routes_infra.go`)
@@ -668,3 +668,11 @@ return wrapChatConcurrency(c, config.MaxConcurrency, err)
仓库根目录的 `rerank_server_demo.py` 是一个**自托管 Rerank 服务的最小参考实现**:FastAPI + HuggingFace `AutoModelForSequenceClassification`,暴露 `POST /rerank`,请求体 `{query, documents}`,返回 `{"results": [{index, document: {text}, score}]}`。
示例服务返回 `score` 字段,可用于验证客户端兼容性。`cohererank` 协议(`internal/models/api/cohererank`)优先读取 `relevance_score`,缺失时读取 `score`;`document` 同时接受字符串和 `{text}` 对象。遵循此协议的私有重排服务可通过 `generic` provider 接入。
+
+### 自建重排端点的分数标度
+
+[vLLM 的 Qwen3-Reranker 官方配方](https://github.com/vllm-project/vllm/blob/main/examples/pooling/score/qwen3_reranker_online.py)使用 `classifier_from_token: ["no", "yes"]` 和 `is_original_qwen3_reranker: true`。这类端点的相关度已归一化,应选择 `score_scale=probability`,不要再次做 sigmoid。例如 0.9996 的概率误选 `logit` 后会变为约 0.731,影响阈值和与检索分数的融合。
+
+GPUStack 的默认值仍为 `logit`,依据其[重排 API 文档](https://docs.gpustack.ai/2.0/user-guide/rerank-api/)中 `bge-reranker-v2-m3` 返回负数及大于 1 的示例;这不代表所有 BGE 或 Qwen3 部署都输出 logit。如果该端点已经输出概率,请在模型配置中显式选择 `probability`。同样的规则适用于自定义和 LiteLLM 端点。原始 logit 也可能恰好落在 0~1,不能只看单次响应的取值范围自动判断。
+
+本修正不自动修改已保存的 `score_scale`,也不改变提供商默认值。受影响的既有模型需检查服务端输出并手动调整分数标度;不会因此新增模型调用或自动重建向量。