Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(retrieval): floor ES script_score cosine at 0 to stop 400s on negative similarity
dzc-bitf8daa8d785cd
diff --git a/internal/application/repository/retriever/elasticsearch/v7/repository.go b/internal/application/repository/retriever/elasticsearch/v7/repository.goindex 755483467..eee56bc26 100644--- a/internal/application/repository/retriever/elasticsearch/v7/repository.go+++ b/internal/application/repository/retriever/elasticsearch/v7/repository.go@@ -627,6 +627,17 @@ func (e *elasticsearchRepository) VectorRetrieve(ctx context.Context,}, nil}+// vectorScoreScriptSource scores every document by its cosine similarity to the+// query vector, floored at 0. Lucene rejects negative final script_score values+// ("script_score script returned an invalid score ... Must be a non-negative+// score!"), so an unclamped negative cosine — which occurs whenever any stored+// vector points away from the query — fails the entire search request with a+// 400 (all shards failed) instead of merely ranking that document last. The+// floor keeps the score in the [0, 1] range the shared retriever score+// normalizer documents for this engine; documents clamped to 0 fall below any+// positive min_score threshold.+var vectorScoreScriptSource = "Math.max(cosineSimilarity(params.query_vector,'embedding'), 0.0)"+// buildVectorSearchQuery builds the vector search query JSONfunc (e *elasticsearchRepository) buildVectorSearchQuery(ctx context.Context,params typesLocal.RetrieveParams,@@ -651,7 +662,7 @@ func (e *elasticsearchRepository) buildVectorSearchQuery(ctx context.Context,},},"script": map[string]interface{}{- "source": "cosineSimilarity(params.query_vector,'embedding')",+ "source": vectorScoreScriptSource,"params": map[string]interface{}{"query_vector": params.Embedding,},diff --git a/internal/application/repository/retriever/elasticsearch/v7/vector_score_script_test.go b/internal/application/repository/retriever/elasticsearch/v7/vector_score_script_test.gonew file mode 100644index 000000000..070e97b00--- /dev/null+++ b/internal/application/repository/retriever/elasticsearch/v7/vector_score_script_test.go@@ -0,0 +1,52 @@+package v7++import (+ "context"+ "encoding/json"+ "testing"++ "github.com/stretchr/testify/require"++ typesLocal "github.com/Tencent/WeKnora/internal/types"+)++// Regression test for #3156: the script_score source must floor the cosine+// similarity at 0. An unclamped negative cosine makes Lucene reject the whole+// search request ("script_score script returned an invalid score ... Must be a+// non-negative score!"), turning one dissimilar document into a full vector+// retrieval outage (all shards failed).+func TestVectorScoreScriptSourceClampsNegativeCosine(t *testing.T) {+ require.Equal(t,+ "Math.max(cosineSimilarity(params.query_vector,'embedding'), 0.0)",+ vectorScoreScriptSource)+}++// The marshaled query body must carry the clamped script source end to end.+func TestBuildVectorSearchQueryClampsScriptScore(t *testing.T) {+ e := &elasticsearchRepository{}+ params := typesLocal.RetrieveParams{+ Embedding: []float32{-1, 0},+ Threshold: 0.15,+ TopK: 5,+ KnowledgeBaseIDs: []string{"kb-1"},+ }++ query, err := e.buildVectorSearchQuery(context.Background(), params)+ require.NoError(t, err)++ var body struct {+ Query struct {+ ScriptScore struct {+ Script struct {+ Source string `json:"source"`+ } `json:"script"`+ MinScore float64 `json:"min_score"`+ } `json:"script_score"`+ } `json:"query"`+ }+ require.NoError(t, json.Unmarshal([]byte(query), &body))+ require.Equal(t,+ "Math.max(cosineSimilarity(params.query_vector,'embedding'), 0.0)",+ body.Query.ScriptScore.Script.Source)+ require.InDelta(t, 0.15, body.Query.ScriptScore.MinScore, 1e-6)+}diff --git a/internal/application/repository/retriever/elasticsearch/v8/repository.go b/internal/application/repository/retriever/elasticsearch/v8/repository.goindex 57ce52adf..63134307b 100644--- a/internal/application/repository/retriever/elasticsearch/v8/repository.go+++ b/internal/application/repository/retriever/elasticsearch/v8/repository.go@@ -403,35 +403,52 @@ func (e *elasticsearchRepository) Retrieve(ctx context.Context,return nil, err}-// VectorRetrieve performs vector similarity search using cosine similarity-// Returns a slice of RetrieveResult containing matching documents-func (e *elasticsearchRepository) VectorRetrieve(ctx context.Context,+// vectorScoreScriptSource scores every document by its cosine similarity to the+// query vector, floored at 0. Lucene rejects negative final script_score values+// ("script_score script returned an invalid score ... Must be a non-negative+// score!"), so an unclamped negative cosine — which occurs whenever any stored+// vector points away from the query — fails the entire search request with a+// 400 (all shards failed) instead of merely ranking that document last. The+// floor keeps the score in the [0, 1] range the shared retriever score+// normalizer documents for this engine; documents clamped to 0 fall below any+// positive min_score threshold.+var vectorScoreScriptSource = "Math.max(cosineSimilarity(params.query_vector, 'embedding'), 0.0)"++// buildVectorScriptScoreQuery wraps the cosine-similarity scoring script in a+// script_score query over the request's base filter conditions.+func (e *elasticsearchRepository) buildVectorScriptScoreQuery(params typesLocal.RetrieveParams,-) ([]*typesLocal.RetrieveResult, error) {- log := logger.GetLogger(ctx)- log.Infof("[Elasticsearch] Vector retrieval: dim=%d, topK=%d, threshold=%.4f",- len(params.Embedding), params.TopK, params.Threshold)-- filter := e.getBaseConds(params)-- // Build script scoring query with cosine similarity+) (*types.ScriptScoreQuery, error) {queryVectorJSON, err := json.Marshal(params.Embedding)if err != nil {- log.Errorf("[Elasticsearch] Failed to marshal query vector: %v", err)return nil, fmt.Errorf("failed to marshal query embedding: %w", err)}-- scoreSource := "cosineSimilarity(params.query_vector, 'embedding')"minScore := float32(params.Threshold)- scriptScore := &types.ScriptScoreQuery{- Query: types.Query{Bool: &types.BoolQuery{Filter: filter}},+ return &types.ScriptScoreQuery{+ Query: types.Query{Bool: &types.BoolQuery{Filter: e.getBaseConds(params)}},Script: types.Script{- Source: &scoreSource,+ Source: &vectorScoreScriptSource,Params: map[string]json.RawMessage{"query_vector": json.RawMessage(queryVectorJSON),},},MinScore: &minScore,+ }, nil+}++// VectorRetrieve performs vector similarity search using cosine similarity+// Returns a slice of RetrieveResult containing matching documents+func (e *elasticsearchRepository) VectorRetrieve(ctx context.Context,+ params typesLocal.RetrieveParams,+) ([]*typesLocal.RetrieveResult, error) {+ log := logger.GetLogger(ctx)+ log.Infof("[Elasticsearch] Vector retrieval: dim=%d, topK=%d, threshold=%.4f",+ len(params.Embedding), params.TopK, params.Threshold)++ scriptScore, err := e.buildVectorScriptScoreQuery(params)+ if err != nil {+ log.Errorf("[Elasticsearch] Failed to marshal query vector: %v", err)+ return nil, err}// Exclude embedding field from source to reduce response sizesourceFilter := &types.SourceFilter{diff --git a/internal/application/repository/retriever/elasticsearch/v8/vector_score_script_test.go b/internal/application/repository/retriever/elasticsearch/v8/vector_score_script_test.gonew file mode 100644index 000000000..9cafdbe72--- /dev/null+++ b/internal/application/repository/retriever/elasticsearch/v8/vector_score_script_test.go@@ -0,0 +1,49 @@+package v8++import (+ "encoding/json"+ "testing"++ "github.com/stretchr/testify/require"++ typesLocal "github.com/Tencent/WeKnora/internal/types"+)++// Regression test for #3156: the script_score source must floor the cosine+// similarity at 0. An unclamped negative cosine makes Lucene reject the whole+// search request ("script_score script returned an invalid score ... Must be a+// non-negative score!"), turning one dissimilar document into a full vector+// retrieval outage (all shards failed).+func TestVectorScoreScriptSourceClampsNegativeCosine(t *testing.T) {+ require.Equal(t,+ "Math.max(cosineSimilarity(params.query_vector, 'embedding'), 0.0)",+ vectorScoreScriptSource)+}++func TestBuildVectorScriptScoreQuery(t *testing.T) {+ e := &elasticsearchRepository{}+ params := typesLocal.RetrieveParams{+ Embedding: []float32{1, 0},+ Threshold: 0.15,+ TopK: 10,+ KnowledgeBaseIDs: []string{"kb-1"},+ }++ q, err := e.buildVectorScriptScoreQuery(params)+ require.NoError(t, err)+ require.NotNil(t, q)++ require.NotNil(t, q.Script.Source)+ require.Equal(t, vectorScoreScriptSource, *q.Script.Source)++ require.NotNil(t, q.MinScore)+ require.InDelta(t, 0.15, float64(*q.MinScore), 1e-6)++ require.NotNil(t, q.Query.Bool)+ require.Len(t, q.Query.Bool.Filter, 1)++ var queryVector []float32+ require.NotNil(t, q.Script.Params["query_vector"])+ require.NoError(t, json.Unmarshal(q.Script.Params["query_vector"], &queryVector))+ require.Equal(t, []float32{1, 0}, queryVector)+}