Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(rerank): accept bare array responses from self-hosted runtimes (#4518)

u7-u781f0dc2d3676
diff --git a/internal/models/api/cohererank/client.go b/internal/models/api/cohererank/client.go
index 921d24acc..34aacdce7 100644
--- a/internal/models/api/cohererank/client.go
+++ b/internal/models/api/cohererank/client.go
@@ -1,6 +1,8 @@
// Package cohererank implements the rerank shape Cohere defined and almost
// every other vendor copied: POST {base}/rerank with {model, query,
// documents} answering {results: [{index, relevance_score, document}]}.
+// A bare array of the same result objects is accepted too: self-hosted
+// runtimes such as SGLang return that shape instead of the envelope.
//
// Jina, Zhipu, SiliconFlow, Qianfan, GPUStack, WeKnora Cloud and any
// OpenAI-compatible gateway that serves rerank all speak it. Their
@@ -10,6 +12,7 @@
package cohererank
import (
+ "bytes"
"context"
"encoding/json"
"fmt"
@@ -185,12 +188,20 @@ func (c *Client) RerankImages(ctx context.Context, query string, images []api.Em
}
func (c *Client) post(ctx context.Context, body map[string]any, count int) ([]api.RerankResult, error) {
- var decoded response
- if err := c.cfg.Endpoint.PostJSON(ctx, c.url(), body, &decoded); err != nil {
+ // Decode into raw JSON first: some Cohere-shaped runtimes (SGLang, for
+ // one) answer the same request with a bare array instead of the
+ // {"results": [...]} envelope. Both shapes mean the same thing, so the
+ // envelope is not part of the contract callers may rely on.
+ var raw json.RawMessage
+ if err := c.cfg.Endpoint.PostJSON(ctx, c.url(), body, &raw); err != nil {
return nil, err
}
- out := make([]api.RerankResult, 0, len(decoded.Results))
- for _, item := range decoded.Results {
+ results, err := decodeResults(raw)
+ if err != nil {
+ return nil, err
+ }
+ out := make([]api.RerankResult, 0, len(results))
+ for _, item := range results {
if item.Index < 0 || item.Index >= count {
return nil, fmt.Errorf("rerank index %d out of range for %d documents", item.Index, count)
}
@@ -203,6 +214,24 @@ func (c *Client) post(ctx context.Context, body map[string]any, count int) ([]ap
return out, nil
}
+// decodeResults accepts both documented envelopes: the Cohere object
+// {"results": [...]} and the bare array some self-hosted runtimes return.
+func decodeResults(raw []byte) ([]result, error) {
+ trimmed := bytes.TrimSpace(raw)
+ if len(trimmed) > 0 && trimmed[0] == '[' {
+ var results []result
+ if err := json.Unmarshal(trimmed, &results); err != nil {
+ return nil, fmt.Errorf("decode response: %w", err)
+ }
+ return results, nil
+ }
+ var decoded response
+ if err := json.Unmarshal(trimmed, &decoded); err != nil {
+ return nil, fmt.Errorf("decode response: %w", err)
+ }
+ return decoded.Results, nil
+}
+
// firstNonNil picks relevance_score, falling back to score: the field is
// spelled both ways across vendors that otherwise share this shape.
func firstNonNil(values ...*float64) float64 {
diff --git a/internal/models/api/cohererank/golden_test.go b/internal/models/api/cohererank/golden_test.go
index 6fb2dd521..6d9a89fe9 100644
--- a/internal/models/api/cohererank/golden_test.go
+++ b/internal/models/api/cohererank/golden_test.go
@@ -156,6 +156,47 @@ func TestDecodesTheScoreAlias(t *testing.T) {
assert.InDelta(t, 0.42, got[0].Score, 1e-9)
}
+// sglangBareArrayResponse is what SGLang's /v1/rerank answers to the same
+// request: a bare array with score/document instead of the Cohere envelope.
+// Self-hosted Qwen3-Reranker deployments are commonly served this way.
+const sglangBareArrayResponse = `[
+ {"score": 0.0015978454940173438, "document": "doc one", "index": 0, "meta_info": null},
+ {"score": 0.0011695108220694027, "document": "doc two", "index": 1, "meta_info": null}
+]`
+
+func TestDecodesABareArrayResponse(t *testing.T) {
+ server, _, _ := serve(t, sglangBareArrayResponse)
+ defer server.Close()
+
+ c := newClient(t, server.URL+"/v1", api.RerankSettings{})
+ got, err := c.Rerank(context.Background(), "q", []string{"doc one", "doc two"})
+ require.NoError(t, err)
+ assert.Equal(t, []api.RerankResult{
+ {Index: 0, Score: 0.0015978454940173438, Text: "doc one"},
+ {Index: 1, Score: 0.0011695108220694027, Text: "doc two"},
+ }, got)
+}
+
+func TestBareArrayResponseStillValidatesIndexes(t *testing.T) {
+ server, _, _ := serve(t, `[{"index":5,"score":0.5,"document":"d"}]`)
+ defer server.Close()
+
+ c := newClient(t, server.URL, api.RerankSettings{})
+ _, err := c.Rerank(context.Background(), "q", []string{"d"})
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "out of range")
+}
+
+func TestRejectsAnUnsupportedResponseEnvelope(t *testing.T) {
+ server, _, _ := serve(t, `"not a rerank response"`)
+ defer server.Close()
+
+ c := newClient(t, server.URL, api.RerankSettings{})
+ _, err := c.Rerank(context.Background(), "q", []string{"d"})
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "decode response")
+}
+
func TestAppendsTheRerankPathToABareBaseURL(t *testing.T) {
server, path, _ := serve(t, `{"results":[]}`)
defer server.Close()