RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
serenedb: stop compiling the similarity threshold into the ANN scan (#20222)
deadtrickster4e38aa47c23f
diff --git a/internal/engine/serenedb/integration_test.go b/internal/engine/serenedb/integration_test.goindex 05d29a9479..3bdfa35de8 100644--- a/internal/engine/serenedb/integration_test.go+++ b/internal/engine/serenedb/integration_test.go@@ -158,6 +158,25 @@ func TestIntegrationChunkLifecycle(t *testing.T) {}})+ // The builder tests assert the SQL TEXT; they say nothing about whether Search+ // actually applies the post-filter it moved the threshold into. Without a+ // threshold on the request, deleting the filterByScore call leaves every test+ // green. This one asks for a cutoff only 'a' can clear.+ t.Run("vector_threshold", func(t *testing.T) {+ res := waitForFulltext(t, e, req([]interface{}{+ &types.MatchDenseExpr{+ VectorColumnName: "q_4_vec",+ EmbeddingData: []float64{1, 0, 0, 0},+ TopN: 10,+ ExtraOptions: map[string]interface{}{"similarity": 0.999},+ },+ }, nil))+ ids := chunkIDs(res)+ if len(ids) != 1 || ids[0] != "a" {+ t.Fatalf("similarity 0.999 should leave only 'a', got %v", ids)+ }+ })+t.Run("fusion", func(t *testing.T) {res := waitForFulltext(t, e, req([]interface{}{&types.MatchTextExpr{MatchingText: "alpha", TopN: 10},@@ -170,6 +189,36 @@ func TestIntegrationChunkLifecycle(t *testing.T) {}})+ // Same gap on the hybrid path, and it is the one that actually regressed: the+ // FULL OUTER JOIN can return a vector-only candidate with no term match, so a+ // sub-threshold row reaches the fusion unless the vec branch filters its own+ // limited candidates.+ //+ // The term is "beta", which ONLY 'a' carries - not "alpha", which 'c' also has.+ // With "alpha" the lex branch returns 'c' legitimately and the assertion below+ // fails against correct behaviour; that is a property of the fixture, not of+ // the threshold. 'c' is [0.9,0.1,0,0], similarity 0.9939 against the query, so+ // 0.999 excludes it from the vec branch and nothing else can bring it in.+ t.Run("fusion_threshold", func(t *testing.T) {+ res := waitForFulltext(t, e, req([]interface{}{+ &types.MatchTextExpr{MatchingText: "beta", TopN: 10},+ &types.MatchDenseExpr{+ VectorColumnName: "q_4_vec",+ EmbeddingData: []float64{1, 0, 0, 0},+ TopN: 10,+ ExtraOptions: map[string]interface{}{"similarity": 0.999},+ },+ &types.FusionExpr{Method: "weighted_sum", FusionParams: map[string]interface{}{"weights": "0.3,0.7"}},+ }, nil))+ // Exactly one, not "nothing unexpected": a range over an empty result makes+ // no assertion at all, so the weaker form passes when the hybrid search+ // returns nothing - including when a regression drops the qualifying row.+ ids := chunkIDs(res)+ if len(ids) != 1 || ids[0] != "a" {+ t.Fatalf("fusion at similarity 0.999 should leave only 'a', got %v", ids)+ }+ })+t.Run("filter_only", func(t *testing.T) {res, err := e.Search(ctx, req(nil, map[string]interface{}{"doc_id": "d2"}))if err != nil {diff --git a/internal/engine/serenedb/search.go b/internal/engine/serenedb/search.goindex 0f8629c0a0..e51d5a9e15 100644--- a/internal/engine/serenedb/search.go+++ b/internal/engine/serenedb/search.go@@ -38,14 +38,15 @@ const (const pagerankExpr = "COALESCE(" + pagerankField + ", 0) / 100.0"type parsedMatch struct {- textQuery string- textTopN int- vectorData []float64- vectorTopN int- vecThreshold float64- vectorWeight float64- hasText bool- hasVector bool+ textQuery string+ textTopN int+ vectorData []float64+ vectorTopN int+ vecThreshold float64+ hasVecThreshold bool+ vectorWeight float64+ hasText bool+ hasVector bool}// Search runs the fulltext, vector, hybrid-fusion, or filter-only query implied@@ -101,6 +102,22 @@ func (e *serenedbEngine) Search(ctx context.Context, req *types.SearchRequest) (if err != nil {return nil, fmt.Errorf("serenedb: search %s: %w", tableName, err)}+ // Taking the threshold out of the WHERE clause moved it here, on the pure-vector+ // branch only. Elasticsearch applies `similarity` the same way, as a post-filter on+ // the k nearest. Without this every Go consumer of Search - SearchCompiled among+ // them - would start seeing rows below the threshold it asked for.+ if pm.hasVector && !pm.hasText {+ // Whenever a threshold was GIVEN, not only a positive one: similarity is+ // -(v <#> q) and is legitimately negative, so 0 and below are real cutoffs.+ if pm.hasVecThreshold {+ rows = filterByScore(rows, pm.vecThreshold)+ } else {+ // No threshold still means no vectorless rows: the ANN scan sorts a NULL+ // distance last and returns it when fewer than k rows have a vector.+ rows = dropNullSim(rows)+ }+ stripVecSim(rows)+ }result.Chunks = append(result.Chunks, rows...)}@@ -197,17 +214,33 @@ func buildFulltextSQL(tableName, fieldsExpr, where, textQuery string, limit, offfieldsExpr, idx, pagerankExpr, idx, where, match, limit, offset)}-// buildVectorSQL runs the ANN scan on the normalized shadow column. The-// similarity threshold goes straight in the WHERE (relies on SereneDB 26.07.4).+// buildVectorSQL runs the ANN scan on the normalized shadow column.+//+// The similarity threshold is deliberately NOT in the WHERE clause. There it+// compiles to a radius search (`Vector Range / Radius <= -0`) and enumerates+// every row inside the radius rather than letting the IVF index return top-k:+// measured 104,095 ms vs 626 ms on 42.8M rows, 166x. No sibling backend does+// this either - Elasticsearch passes `similarity` as a post-filter on the k+// nearest, OpenSearch drops it, Infinity hands it to the engine natively - and+// RAGFlow re-applies it afterwards against the hybrid score anyway.func buildVectorSQL(tableName, fieldsExpr, where string, pm parsedMatch, limit, offset int) string {idx := indexRelation(tableName)vecN := normColumn(len(pm.vectorData))qv := vectorLiteral(pm.vectorData)sim := fmt.Sprintf("-(%s <#> %s)", vecN, qv)+ // _vec_sim is the RAW similarity: _score adds pagerank, so a 0.75 similarity with 0.10 of+ // pagerank would pass a 0.80 threshold. Subquery so the distance is computed once.+ // pagerank_fea is itself selectable; adding it twice makes the outer reference ambiguous.+ innerExtra := ", " + pagerankField+ if strings.Contains(fieldsExpr, pagerankField) {+ innerExtra = ""+ }return fmt.Sprintf(- "SELECT %s, %s + %s AS _score FROM %s WHERE %s AND %s >= %s "+- "ORDER BY %s <#> %s LIMIT %d OFFSET %d",- fieldsExpr, sim, pagerankExpr, idx, where, sim, formatFloat(pm.vecThreshold),+ "SELECT %s, %s, %s + %s AS _score FROM ("++ "SELECT %s%s, %s AS %s FROM %s WHERE %s "++ "ORDER BY %s <#> %s LIMIT %d OFFSET %d) t",+ fieldsExpr, vecSimColumn, vecSimColumn, pagerankExpr,+ fieldsExpr, innerExtra, sim, vecSimColumn, idx, where,vecN, qv, limit, offset)}@@ -237,15 +270,30 @@ func buildFusionSQL(tableName, fieldsExpr string, outputFields []string, where sprefixed[i] = "t." + f}vw := pm.vectorWeight+ // Top-k first, threshold second. In the WHERE it compiles to a radius search (93.9s+ // against 0.12s on the 111M-chunk corpus), so it cannot go there. It still has to be+ // applied though: the branch has only vectorTopN slots, and a sub-threshold candidate+ // spends one of them before fusion ever sees it, displacing a candidate the caller+ // would have accepted. Elasticsearch likewise applies `similarity` to the knn clause+ // of a hybrid query. Filtering the already-limited candidates gets both.+ vecBranch := fmt.Sprintf(`SELECT id, -(%s <#> %s) AS sim+ FROM %s WHERE %s+ ORDER BY %s <#> %s LIMIT %d`, vecN, qv, idx, where, vecN, qv, vN)+ // Always wrapped, because the normalized vector column is nullable and a NULL+ // similarity would otherwise reach COALESCE(v.sim, 0) and score as zero. `sim >= t`+ // excludes NULL on its own under three-valued logic.+ simCond := "sim IS NOT NULL"+ if pm.hasVecThreshold {+ simCond = fmt.Sprintf("sim >= %s", formatFloat(pm.vecThreshold))+ }+ vecBranch = fmt.Sprintf(`SELECT id, sim FROM (%s) c WHERE %s`, vecBranch, simCond)return fmt.Sprintf(`WITH lex AS (SELECT id, BM25(%s.tableoid) AS sFROM %s WHERE %s AND (%s)ORDER BY s DESC LIMIT %d),lexn AS (SELECT id, s / NULLIF(MAX(s) OVER (), 0) AS sn FROM lex),vec AS (- SELECT id, -(%s <#> %s) AS sim- FROM %s WHERE %s AND -(%s <#> %s) >= %s- ORDER BY %s <#> %s LIMIT %d),+ %s),fused AS (SELECT COALESCE(l.id, v.id) AS id,COALESCE(l.sn, 0) * %s + COALESCE(v.sim, 0) * %s AS fs@@ -254,7 +302,7 @@ SELECT %s, f.fs + COALESCE(t.%s, 0) / 100.0 AS _scoreFROM fused f JOIN %s t ON t.id = f.idORDER BY _score DESC LIMIT %d OFFSET %d`,idx, idx, where, match, lexN,- vecN, qv, idx, where, vecN, qv, formatFloat(pm.vecThreshold), vecN, qv, vN,+ vecBranch,formatWeight(1.0-vw), formatWeight(vw),strings.Join(prefixed, ", "), pagerankField, tableName, n, offset)}@@ -320,7 +368,7 @@ func parseMatchExprs(exprs []interface{}) parsedMatch {if len(expr.EmbeddingData) > 0 {pm.vectorData = expr.EmbeddingDatapm.vectorTopN = expr.TopN- pm.vecThreshold = denseThreshold(expr.ExtraOptions)+ pm.vecThreshold, pm.hasVecThreshold = denseThreshold(expr.ExtraOptions)pm.hasVector = true}case *types.FusionExpr:@@ -332,22 +380,61 @@ func parseMatchExprs(exprs []interface{}) parsedMatch {return pm}-func denseThreshold(opts map[string]interface{}) float64 {+// vecSimColumn carries the raw ANN similarity from buildVectorSQL to the post-filter.+const vecSimColumn = "_vec_sim"++// filterByScore drops rows whose raw similarity is below the requested one. The fusion+// branch does not come through here - its _score is the hybrid score, so it filters its+// own ANN candidates inside buildFusionSQL instead.+// dropNullSim removes rows the ANN scan returned with no vector at all.+func dropNullSim(rows []map[string]interface{}) []map[string]interface{} {+ kept := rows[:0]+ for _, row := range rows {+ if sim, ok := row[vecSimColumn]; ok && sim != nil {+ kept = append(kept, row)+ }+ }+ return kept+}++func filterByScore(rows []map[string]interface{}, threshold float64) []map[string]interface{} {+ kept := rows[:0]+ for _, row := range rows {+ sim, ok := row[vecSimColumn]+ // No similarity means DROPPED: the vector columns are nullable, and a chunk with no+ // vector must not survive a threshold.+ if !ok || sim == nil || toFloat64(sim) < threshold {+ continue+ }+ kept = append(kept, row)+ }+ return kept+}++// stripVecSim removes the helper column before the rows leave the engine, so the caller sees+// exactly the fields it selected.+func stripVecSim(rows []map[string]interface{}) {+ for _, row := range rows {+ delete(row, vecSimColumn)+ }+}++func denseThreshold(opts map[string]interface{}) (float64, bool) {if opts == nil {- return 0.0+ return 0.0, false}switch v := opts["similarity"].(type) {case float64:- return v+ return v, truecase string:- f, _ := strconv.ParseFloat(v, 64)- return f+ f, err := strconv.ParseFloat(v, 64)+ return f, err == nil}if s, ok := opts["threshold"].(string); ok {- f, _ := strconv.ParseFloat(s, 64)- return f+ f, err := strconv.ParseFloat(s, 64)+ return f, err == nil}- return 0.0+ return 0.0, false}// fusionVectorWeight reads the vector weight (second element of the weightsdiff --git a/internal/engine/serenedb/serenedb_test.go b/internal/engine/serenedb/serenedb_test.goindex 1418d8efc2..9772a1725e 100644--- a/internal/engine/serenedb/serenedb_test.go+++ b/internal/engine/serenedb/serenedb_test.go@@ -205,10 +205,19 @@ func TestBuildFulltextSQLSingleColumn(t *testing.T) {mustNotContain(t, sql, "title_tks @@")}-func TestBuildVectorSQLThresholdInWhere(t *testing.T) {- pm := parsedMatch{vectorData: []float64{3, 4}, vecThreshold: 0.2}+// The similarity threshold must NOT reach the WHERE clause: there it compiles to+// a radius search instead of a top-k lookup (173,001 ms vs 136 ms at k=1024 on+// 111.8M chunks). Elasticsearch applies it as a post-filter, OpenSearch drops+// it, and RAGFlow re-applies it against the hybrid score afterwards.+//+// hasVecThreshold must be SET here. Without it the fixture describes a request+// that supplied no threshold at all, so the assertion holds trivially and the+// test would still pass if buildVectorSQL started emitting the radius predicate+// for a threshold that WAS given - the only case it is meant to guard.+func TestBuildVectorSQLThresholdNotInWhere(t *testing.T) {+ pm := parsedMatch{vectorData: []float64{3, 4}, vecThreshold: 0.2, hasVecThreshold: true}sql := buildVectorSQL("t", "id", "TRUE", pm, 10, 5)- mustContain(t, sql, "-(q_2_vec_n <#> ARRAY[0.6,0.8]::FLOAT[2]) >= 0.2")+ mustNotContain(t, sql, ">= 0.2")mustContain(t, sql, "ORDER BY q_2_vec_n <#> ARRAY[0.6,0.8]::FLOAT[2] LIMIT 10 OFFSET 5")}@@ -227,6 +236,43 @@ func TestBuildFusionSQLShape(t *testing.T) {mustContain(t, sql, "t.id, t.content_ltks") // output fields prefixed with t.}+func TestBuildFusionSQLThresholdFiltersAfterLimit(t *testing.T) {+ pm := parsedMatch{+ textQuery: "q", vectorData: []float64{3, 4},+ textTopN: 200, vectorTopN: 200, vectorWeight: 0.95,+ vecThreshold: 0.2, hasVecThreshold: true,+ }+ sql := buildFusionSQL("t", "id", []string{"id"}, "TRUE", pm, 0, 30)+ // Applied to the already-limited candidates, so the IVF scan still returns top-k...+ mustContain(t, sql, "LIMIT 200) c WHERE sim >= 0.2")+ // ...and never as a radius predicate on the scan itself.+ mustNotContain(t, sql, "AS sim\n FROM idx_t WHERE TRUE AND")+}++func TestBuildFusionSQLNoThresholdStillExcludesNulls(t *testing.T) {+ pm := parsedMatch{+ textQuery: "q", vectorData: []float64{3, 4},+ textTopN: 200, vectorTopN: 200, vectorWeight: 0.95,+ }+ sql := buildFusionSQL("t", "id", []string{"id"}, "TRUE", pm, 0, 30)+ mustNotContain(t, sql, "WHERE sim >=")+ // COALESCE(v.sim, 0) would otherwise score a vectorless row as zero.+ mustContain(t, sql, "WHERE sim IS NOT NULL")+}++func TestDropNullSimRemovesVectorlessRows(t *testing.T) {+ rows := []map[string]interface{}{+ {"id": "a", vecSimColumn: 0.9},+ {"id": "b", vecSimColumn: nil},+ {"id": "c"},+ {"id": "d", vecSimColumn: -0.4},+ }+ got := dropNullSim(rows)+ if len(got) != 2 || got[0]["id"] != "a" || got[1]["id"] != "d" {+ t.Errorf("dropNullSim = %v, want the two rows carrying a similarity", got)+ }+}+func TestBuildUpsert(t *testing.T) {q, args := buildUpsert("t", []string{"id", "doc_id"}, [][]interface{}{{"1", "d1"}, {"2", "d2"}})mustContain(t, q, "INSERT INTO t (id, doc_id) VALUES ($1, $2), ($3, $4)")