Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(retrieval): floor ES script_score cosine at 0 to stop 400s on negative similarity

dzc-bitf8daa8d785cd
diff --git a/internal/application/repository/retriever/elasticsearch/v7/repository.go b/internal/application/repository/retriever/elasticsearch/v7/repository.go
index 755483467..eee56bc26 100644
--- a/internal/application/repository/retriever/elasticsearch/v7/repository.go
+++ b/internal/application/repository/retriever/elasticsearch/v7/repository.go
@@ -627,6 +627,17 @@ func (e *elasticsearchRepository) VectorRetrieve(ctx context.Context,
}, nil
}
+// vectorScoreScriptSource scores every document by its cosine similarity to the
+// query vector, floored at 0. Lucene rejects negative final script_score values
+// ("script_score script returned an invalid score ... Must be a non-negative
+// score!"), so an unclamped negative cosine — which occurs whenever any stored
+// vector points away from the query — fails the entire search request with a
+// 400 (all shards failed) instead of merely ranking that document last. The
+// floor keeps the score in the [0, 1] range the shared retriever score
+// normalizer documents for this engine; documents clamped to 0 fall below any
+// positive min_score threshold.
+var vectorScoreScriptSource = "Math.max(cosineSimilarity(params.query_vector,'embedding'), 0.0)"
+
// buildVectorSearchQuery builds the vector search query JSON
func (e *elasticsearchRepository) buildVectorSearchQuery(ctx context.Context,
params typesLocal.RetrieveParams,
@@ -651,7 +662,7 @@ func (e *elasticsearchRepository) buildVectorSearchQuery(ctx context.Context,
},
},
"script": map[string]interface{}{
- "source": "cosineSimilarity(params.query_vector,'embedding')",
+ "source": vectorScoreScriptSource,
"params": map[string]interface{}{
"query_vector": params.Embedding,
},
diff --git a/internal/application/repository/retriever/elasticsearch/v7/vector_score_script_test.go b/internal/application/repository/retriever/elasticsearch/v7/vector_score_script_test.go
new file mode 100644
index 000000000..070e97b00
--- /dev/null
+++ b/internal/application/repository/retriever/elasticsearch/v7/vector_score_script_test.go
@@ -0,0 +1,52 @@
+package v7
+
+import (
+ "context"
+ "encoding/json"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ typesLocal "github.com/Tencent/WeKnora/internal/types"
+)
+
+// Regression test for #3156: the script_score source must floor the cosine
+// similarity at 0. An unclamped negative cosine makes Lucene reject the whole
+// search request ("script_score script returned an invalid score ... Must be a
+// non-negative score!"), turning one dissimilar document into a full vector
+// retrieval outage (all shards failed).
+func TestVectorScoreScriptSourceClampsNegativeCosine(t *testing.T) {
+ require.Equal(t,
+ "Math.max(cosineSimilarity(params.query_vector,'embedding'), 0.0)",
+ vectorScoreScriptSource)
+}
+
+// The marshaled query body must carry the clamped script source end to end.
+func TestBuildVectorSearchQueryClampsScriptScore(t *testing.T) {
+ e := &elasticsearchRepository{}
+ params := typesLocal.RetrieveParams{
+ Embedding: []float32{-1, 0},
+ Threshold: 0.15,
+ TopK: 5,
+ KnowledgeBaseIDs: []string{"kb-1"},
+ }
+
+ query, err := e.buildVectorSearchQuery(context.Background(), params)
+ require.NoError(t, err)
+
+ var body struct {
+ Query struct {
+ ScriptScore struct {
+ Script struct {
+ Source string `json:"source"`
+ } `json:"script"`
+ MinScore float64 `json:"min_score"`
+ } `json:"script_score"`
+ } `json:"query"`
+ }
+ require.NoError(t, json.Unmarshal([]byte(query), &body))
+ require.Equal(t,
+ "Math.max(cosineSimilarity(params.query_vector,'embedding'), 0.0)",
+ body.Query.ScriptScore.Script.Source)
+ require.InDelta(t, 0.15, body.Query.ScriptScore.MinScore, 1e-6)
+}
diff --git a/internal/application/repository/retriever/elasticsearch/v8/repository.go b/internal/application/repository/retriever/elasticsearch/v8/repository.go
index 57ce52adf..63134307b 100644
--- a/internal/application/repository/retriever/elasticsearch/v8/repository.go
+++ b/internal/application/repository/retriever/elasticsearch/v8/repository.go
@@ -403,35 +403,52 @@ func (e *elasticsearchRepository) Retrieve(ctx context.Context,
return nil, err
}
-// VectorRetrieve performs vector similarity search using cosine similarity
-// Returns a slice of RetrieveResult containing matching documents
-func (e *elasticsearchRepository) VectorRetrieve(ctx context.Context,
+// vectorScoreScriptSource scores every document by its cosine similarity to the
+// query vector, floored at 0. Lucene rejects negative final script_score values
+// ("script_score script returned an invalid score ... Must be a non-negative
+// score!"), so an unclamped negative cosine — which occurs whenever any stored
+// vector points away from the query — fails the entire search request with a
+// 400 (all shards failed) instead of merely ranking that document last. The
+// floor keeps the score in the [0, 1] range the shared retriever score
+// normalizer documents for this engine; documents clamped to 0 fall below any
+// positive min_score threshold.
+var vectorScoreScriptSource = "Math.max(cosineSimilarity(params.query_vector, 'embedding'), 0.0)"
+
+// buildVectorScriptScoreQuery wraps the cosine-similarity scoring script in a
+// script_score query over the request's base filter conditions.
+func (e *elasticsearchRepository) buildVectorScriptScoreQuery(
params typesLocal.RetrieveParams,
-) ([]*typesLocal.RetrieveResult, error) {
- log := logger.GetLogger(ctx)
- log.Infof("[Elasticsearch] Vector retrieval: dim=%d, topK=%d, threshold=%.4f",
- len(params.Embedding), params.TopK, params.Threshold)
-
- filter := e.getBaseConds(params)
-
- // Build script scoring query with cosine similarity
+) (*types.ScriptScoreQuery, error) {
queryVectorJSON, err := json.Marshal(params.Embedding)
if err != nil {
- log.Errorf("[Elasticsearch] Failed to marshal query vector: %v", err)
return nil, fmt.Errorf("failed to marshal query embedding: %w", err)
}
-
- scoreSource := "cosineSimilarity(params.query_vector, 'embedding')"
minScore := float32(params.Threshold)
- scriptScore := &types.ScriptScoreQuery{
- Query: types.Query{Bool: &types.BoolQuery{Filter: filter}},
+ return &types.ScriptScoreQuery{
+ Query: types.Query{Bool: &types.BoolQuery{Filter: e.getBaseConds(params)}},
Script: types.Script{
- Source: &scoreSource,
+ Source: &vectorScoreScriptSource,
Params: map[string]json.RawMessage{
"query_vector": json.RawMessage(queryVectorJSON),
},
},
MinScore: &minScore,
+ }, nil
+}
+
+// VectorRetrieve performs vector similarity search using cosine similarity
+// Returns a slice of RetrieveResult containing matching documents
+func (e *elasticsearchRepository) VectorRetrieve(ctx context.Context,
+ params typesLocal.RetrieveParams,
+) ([]*typesLocal.RetrieveResult, error) {
+ log := logger.GetLogger(ctx)
+ log.Infof("[Elasticsearch] Vector retrieval: dim=%d, topK=%d, threshold=%.4f",
+ len(params.Embedding), params.TopK, params.Threshold)
+
+ scriptScore, err := e.buildVectorScriptScoreQuery(params)
+ if err != nil {
+ log.Errorf("[Elasticsearch] Failed to marshal query vector: %v", err)
+ return nil, err
}
// Exclude embedding field from source to reduce response size
sourceFilter := &types.SourceFilter{
diff --git a/internal/application/repository/retriever/elasticsearch/v8/vector_score_script_test.go b/internal/application/repository/retriever/elasticsearch/v8/vector_score_script_test.go
new file mode 100644
index 000000000..9cafdbe72
--- /dev/null
+++ b/internal/application/repository/retriever/elasticsearch/v8/vector_score_script_test.go
@@ -0,0 +1,49 @@
+package v8
+
+import (
+ "encoding/json"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ typesLocal "github.com/Tencent/WeKnora/internal/types"
+)
+
+// Regression test for #3156: the script_score source must floor the cosine
+// similarity at 0. An unclamped negative cosine makes Lucene reject the whole
+// search request ("script_score script returned an invalid score ... Must be a
+// non-negative score!"), turning one dissimilar document into a full vector
+// retrieval outage (all shards failed).
+func TestVectorScoreScriptSourceClampsNegativeCosine(t *testing.T) {
+ require.Equal(t,
+ "Math.max(cosineSimilarity(params.query_vector, 'embedding'), 0.0)",
+ vectorScoreScriptSource)
+}
+
+func TestBuildVectorScriptScoreQuery(t *testing.T) {
+ e := &elasticsearchRepository{}
+ params := typesLocal.RetrieveParams{
+ Embedding: []float32{1, 0},
+ Threshold: 0.15,
+ TopK: 10,
+ KnowledgeBaseIDs: []string{"kb-1"},
+ }
+
+ q, err := e.buildVectorScriptScoreQuery(params)
+ require.NoError(t, err)
+ require.NotNil(t, q)
+
+ require.NotNil(t, q.Script.Source)
+ require.Equal(t, vectorScoreScriptSource, *q.Script.Source)
+
+ require.NotNil(t, q.MinScore)
+ require.InDelta(t, 0.15, float64(*q.MinScore), 1e-6)
+
+ require.NotNil(t, q.Query.Bool)
+ require.Len(t, q.Query.Bool.Filter, 1)
+
+ var queryVector []float32
+ require.NotNil(t, q.Script.Params["query_vector"])
+ require.NoError(t, json.Unmarshal(q.Script.Params["query_vector"], &queryVector))
+ require.Equal(t, []float32{1, 0}, queryVector)
+}