Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

serenedb: stop compiling the similarity threshold into the ANN scan (#20222)

deadtrickster4e38aa47c23f
diff --git a/internal/engine/serenedb/integration_test.go b/internal/engine/serenedb/integration_test.go
index 05d29a9479..3bdfa35de8 100644
--- a/internal/engine/serenedb/integration_test.go
+++ b/internal/engine/serenedb/integration_test.go
@@ -158,6 +158,25 @@ func TestIntegrationChunkLifecycle(t *testing.T) {
}
})
+ // The builder tests assert the SQL TEXT; they say nothing about whether Search
+ // actually applies the post-filter it moved the threshold into. Without a
+ // threshold on the request, deleting the filterByScore call leaves every test
+ // green. This one asks for a cutoff only 'a' can clear.
+ t.Run("vector_threshold", func(t *testing.T) {
+ res := waitForFulltext(t, e, req([]interface{}{
+ &types.MatchDenseExpr{
+ VectorColumnName: "q_4_vec",
+ EmbeddingData: []float64{1, 0, 0, 0},
+ TopN: 10,
+ ExtraOptions: map[string]interface{}{"similarity": 0.999},
+ },
+ }, nil))
+ ids := chunkIDs(res)
+ if len(ids) != 1 || ids[0] != "a" {
+ t.Fatalf("similarity 0.999 should leave only 'a', got %v", ids)
+ }
+ })
+
t.Run("fusion", func(t *testing.T) {
res := waitForFulltext(t, e, req([]interface{}{
&types.MatchTextExpr{MatchingText: "alpha", TopN: 10},
@@ -170,6 +189,36 @@ func TestIntegrationChunkLifecycle(t *testing.T) {
}
})
+ // Same gap on the hybrid path, and it is the one that actually regressed: the
+ // FULL OUTER JOIN can return a vector-only candidate with no term match, so a
+ // sub-threshold row reaches the fusion unless the vec branch filters its own
+ // limited candidates.
+ //
+ // The term is "beta", which ONLY 'a' carries - not "alpha", which 'c' also has.
+ // With "alpha" the lex branch returns 'c' legitimately and the assertion below
+ // fails against correct behaviour; that is a property of the fixture, not of
+ // the threshold. 'c' is [0.9,0.1,0,0], similarity 0.9939 against the query, so
+ // 0.999 excludes it from the vec branch and nothing else can bring it in.
+ t.Run("fusion_threshold", func(t *testing.T) {
+ res := waitForFulltext(t, e, req([]interface{}{
+ &types.MatchTextExpr{MatchingText: "beta", TopN: 10},
+ &types.MatchDenseExpr{
+ VectorColumnName: "q_4_vec",
+ EmbeddingData: []float64{1, 0, 0, 0},
+ TopN: 10,
+ ExtraOptions: map[string]interface{}{"similarity": 0.999},
+ },
+ &types.FusionExpr{Method: "weighted_sum", FusionParams: map[string]interface{}{"weights": "0.3,0.7"}},
+ }, nil))
+ // Exactly one, not "nothing unexpected": a range over an empty result makes
+ // no assertion at all, so the weaker form passes when the hybrid search
+ // returns nothing - including when a regression drops the qualifying row.
+ ids := chunkIDs(res)
+ if len(ids) != 1 || ids[0] != "a" {
+ t.Fatalf("fusion at similarity 0.999 should leave only 'a', got %v", ids)
+ }
+ })
+
t.Run("filter_only", func(t *testing.T) {
res, err := e.Search(ctx, req(nil, map[string]interface{}{"doc_id": "d2"}))
if err != nil {
diff --git a/internal/engine/serenedb/search.go b/internal/engine/serenedb/search.go
index 0f8629c0a0..e51d5a9e15 100644
--- a/internal/engine/serenedb/search.go
+++ b/internal/engine/serenedb/search.go
@@ -38,14 +38,15 @@ const (
const pagerankExpr = "COALESCE(" + pagerankField + ", 0) / 100.0"
type parsedMatch struct {
- textQuery string
- textTopN int
- vectorData []float64
- vectorTopN int
- vecThreshold float64
- vectorWeight float64
- hasText bool
- hasVector bool
+ textQuery string
+ textTopN int
+ vectorData []float64
+ vectorTopN int
+ vecThreshold float64
+ hasVecThreshold bool
+ vectorWeight float64
+ hasText bool
+ hasVector bool
}
// Search runs the fulltext, vector, hybrid-fusion, or filter-only query implied
@@ -101,6 +102,22 @@ func (e *serenedbEngine) Search(ctx context.Context, req *types.SearchRequest) (
if err != nil {
return nil, fmt.Errorf("serenedb: search %s: %w", tableName, err)
}
+ // Taking the threshold out of the WHERE clause moved it here, on the pure-vector
+ // branch only. Elasticsearch applies `similarity` the same way, as a post-filter on
+ // the k nearest. Without this every Go consumer of Search - SearchCompiled among
+ // them - would start seeing rows below the threshold it asked for.
+ if pm.hasVector && !pm.hasText {
+ // Whenever a threshold was GIVEN, not only a positive one: similarity is
+ // -(v <#> q) and is legitimately negative, so 0 and below are real cutoffs.
+ if pm.hasVecThreshold {
+ rows = filterByScore(rows, pm.vecThreshold)
+ } else {
+ // No threshold still means no vectorless rows: the ANN scan sorts a NULL
+ // distance last and returns it when fewer than k rows have a vector.
+ rows = dropNullSim(rows)
+ }
+ stripVecSim(rows)
+ }
result.Chunks = append(result.Chunks, rows...)
}
@@ -197,17 +214,33 @@ func buildFulltextSQL(tableName, fieldsExpr, where, textQuery string, limit, off
fieldsExpr, idx, pagerankExpr, idx, where, match, limit, offset)
}
-// buildVectorSQL runs the ANN scan on the normalized shadow column. The
-// similarity threshold goes straight in the WHERE (relies on SereneDB 26.07.4).
+// buildVectorSQL runs the ANN scan on the normalized shadow column.
+//
+// The similarity threshold is deliberately NOT in the WHERE clause. There it
+// compiles to a radius search (`Vector Range / Radius <= -0`) and enumerates
+// every row inside the radius rather than letting the IVF index return top-k:
+// measured 104,095 ms vs 626 ms on 42.8M rows, 166x. No sibling backend does
+// this either - Elasticsearch passes `similarity` as a post-filter on the k
+// nearest, OpenSearch drops it, Infinity hands it to the engine natively - and
+// RAGFlow re-applies it afterwards against the hybrid score anyway.
func buildVectorSQL(tableName, fieldsExpr, where string, pm parsedMatch, limit, offset int) string {
idx := indexRelation(tableName)
vecN := normColumn(len(pm.vectorData))
qv := vectorLiteral(pm.vectorData)
sim := fmt.Sprintf("-(%s <#> %s)", vecN, qv)
+ // _vec_sim is the RAW similarity: _score adds pagerank, so a 0.75 similarity with 0.10 of
+ // pagerank would pass a 0.80 threshold. Subquery so the distance is computed once.
+ // pagerank_fea is itself selectable; adding it twice makes the outer reference ambiguous.
+ innerExtra := ", " + pagerankField
+ if strings.Contains(fieldsExpr, pagerankField) {
+ innerExtra = ""
+ }
return fmt.Sprintf(
- "SELECT %s, %s + %s AS _score FROM %s WHERE %s AND %s >= %s "+
- "ORDER BY %s <#> %s LIMIT %d OFFSET %d",
- fieldsExpr, sim, pagerankExpr, idx, where, sim, formatFloat(pm.vecThreshold),
+ "SELECT %s, %s, %s + %s AS _score FROM ("+
+ "SELECT %s%s, %s AS %s FROM %s WHERE %s "+
+ "ORDER BY %s <#> %s LIMIT %d OFFSET %d) t",
+ fieldsExpr, vecSimColumn, vecSimColumn, pagerankExpr,
+ fieldsExpr, innerExtra, sim, vecSimColumn, idx, where,
vecN, qv, limit, offset)
}
@@ -237,15 +270,30 @@ func buildFusionSQL(tableName, fieldsExpr string, outputFields []string, where s
prefixed[i] = "t." + f
}
vw := pm.vectorWeight
+ // Top-k first, threshold second. In the WHERE it compiles to a radius search (93.9s
+ // against 0.12s on the 111M-chunk corpus), so it cannot go there. It still has to be
+ // applied though: the branch has only vectorTopN slots, and a sub-threshold candidate
+ // spends one of them before fusion ever sees it, displacing a candidate the caller
+ // would have accepted. Elasticsearch likewise applies `similarity` to the knn clause
+ // of a hybrid query. Filtering the already-limited candidates gets both.
+ vecBranch := fmt.Sprintf(`SELECT id, -(%s <#> %s) AS sim
+ FROM %s WHERE %s
+ ORDER BY %s <#> %s LIMIT %d`, vecN, qv, idx, where, vecN, qv, vN)
+ // Always wrapped, because the normalized vector column is nullable and a NULL
+ // similarity would otherwise reach COALESCE(v.sim, 0) and score as zero. `sim >= t`
+ // excludes NULL on its own under three-valued logic.
+ simCond := "sim IS NOT NULL"
+ if pm.hasVecThreshold {
+ simCond = fmt.Sprintf("sim >= %s", formatFloat(pm.vecThreshold))
+ }
+ vecBranch = fmt.Sprintf(`SELECT id, sim FROM (%s) c WHERE %s`, vecBranch, simCond)
return fmt.Sprintf(`WITH lex AS (
SELECT id, BM25(%s.tableoid) AS s
FROM %s WHERE %s AND (%s)
ORDER BY s DESC LIMIT %d),
lexn AS (SELECT id, s / NULLIF(MAX(s) OVER (), 0) AS sn FROM lex),
vec AS (
- SELECT id, -(%s <#> %s) AS sim
- FROM %s WHERE %s AND -(%s <#> %s) >= %s
- ORDER BY %s <#> %s LIMIT %d),
+ %s),
fused AS (
SELECT COALESCE(l.id, v.id) AS id,
COALESCE(l.sn, 0) * %s + COALESCE(v.sim, 0) * %s AS fs
@@ -254,7 +302,7 @@ SELECT %s, f.fs + COALESCE(t.%s, 0) / 100.0 AS _score
FROM fused f JOIN %s t ON t.id = f.id
ORDER BY _score DESC LIMIT %d OFFSET %d`,
idx, idx, where, match, lexN,
- vecN, qv, idx, where, vecN, qv, formatFloat(pm.vecThreshold), vecN, qv, vN,
+ vecBranch,
formatWeight(1.0-vw), formatWeight(vw),
strings.Join(prefixed, ", "), pagerankField, tableName, n, offset)
}
@@ -320,7 +368,7 @@ func parseMatchExprs(exprs []interface{}) parsedMatch {
if len(expr.EmbeddingData) > 0 {
pm.vectorData = expr.EmbeddingData
pm.vectorTopN = expr.TopN
- pm.vecThreshold = denseThreshold(expr.ExtraOptions)
+ pm.vecThreshold, pm.hasVecThreshold = denseThreshold(expr.ExtraOptions)
pm.hasVector = true
}
case *types.FusionExpr:
@@ -332,22 +380,61 @@ func parseMatchExprs(exprs []interface{}) parsedMatch {
return pm
}
-func denseThreshold(opts map[string]interface{}) float64 {
+// vecSimColumn carries the raw ANN similarity from buildVectorSQL to the post-filter.
+const vecSimColumn = "_vec_sim"
+
+// filterByScore drops rows whose raw similarity is below the requested one. The fusion
+// branch does not come through here - its _score is the hybrid score, so it filters its
+// own ANN candidates inside buildFusionSQL instead.
+// dropNullSim removes rows the ANN scan returned with no vector at all.
+func dropNullSim(rows []map[string]interface{}) []map[string]interface{} {
+ kept := rows[:0]
+ for _, row := range rows {
+ if sim, ok := row[vecSimColumn]; ok && sim != nil {
+ kept = append(kept, row)
+ }
+ }
+ return kept
+}
+
+func filterByScore(rows []map[string]interface{}, threshold float64) []map[string]interface{} {
+ kept := rows[:0]
+ for _, row := range rows {
+ sim, ok := row[vecSimColumn]
+ // No similarity means DROPPED: the vector columns are nullable, and a chunk with no
+ // vector must not survive a threshold.
+ if !ok || sim == nil || toFloat64(sim) < threshold {
+ continue
+ }
+ kept = append(kept, row)
+ }
+ return kept
+}
+
+// stripVecSim removes the helper column before the rows leave the engine, so the caller sees
+// exactly the fields it selected.
+func stripVecSim(rows []map[string]interface{}) {
+ for _, row := range rows {
+ delete(row, vecSimColumn)
+ }
+}
+
+func denseThreshold(opts map[string]interface{}) (float64, bool) {
if opts == nil {
- return 0.0
+ return 0.0, false
}
switch v := opts["similarity"].(type) {
case float64:
- return v
+ return v, true
case string:
- f, _ := strconv.ParseFloat(v, 64)
- return f
+ f, err := strconv.ParseFloat(v, 64)
+ return f, err == nil
}
if s, ok := opts["threshold"].(string); ok {
- f, _ := strconv.ParseFloat(s, 64)
- return f
+ f, err := strconv.ParseFloat(s, 64)
+ return f, err == nil
}
- return 0.0
+ return 0.0, false
}
// fusionVectorWeight reads the vector weight (second element of the weights
diff --git a/internal/engine/serenedb/serenedb_test.go b/internal/engine/serenedb/serenedb_test.go
index 1418d8efc2..9772a1725e 100644
--- a/internal/engine/serenedb/serenedb_test.go
+++ b/internal/engine/serenedb/serenedb_test.go
@@ -205,10 +205,19 @@ func TestBuildFulltextSQLSingleColumn(t *testing.T) {
mustNotContain(t, sql, "title_tks @@")
}
-func TestBuildVectorSQLThresholdInWhere(t *testing.T) {
- pm := parsedMatch{vectorData: []float64{3, 4}, vecThreshold: 0.2}
+// The similarity threshold must NOT reach the WHERE clause: there it compiles to
+// a radius search instead of a top-k lookup (173,001 ms vs 136 ms at k=1024 on
+// 111.8M chunks). Elasticsearch applies it as a post-filter, OpenSearch drops
+// it, and RAGFlow re-applies it against the hybrid score afterwards.
+//
+// hasVecThreshold must be SET here. Without it the fixture describes a request
+// that supplied no threshold at all, so the assertion holds trivially and the
+// test would still pass if buildVectorSQL started emitting the radius predicate
+// for a threshold that WAS given - the only case it is meant to guard.
+func TestBuildVectorSQLThresholdNotInWhere(t *testing.T) {
+ pm := parsedMatch{vectorData: []float64{3, 4}, vecThreshold: 0.2, hasVecThreshold: true}
sql := buildVectorSQL("t", "id", "TRUE", pm, 10, 5)
- mustContain(t, sql, "-(q_2_vec_n <#> ARRAY[0.6,0.8]::FLOAT[2]) >= 0.2")
+ mustNotContain(t, sql, ">= 0.2")
mustContain(t, sql, "ORDER BY q_2_vec_n <#> ARRAY[0.6,0.8]::FLOAT[2] LIMIT 10 OFFSET 5")
}
@@ -227,6 +236,43 @@ func TestBuildFusionSQLShape(t *testing.T) {
mustContain(t, sql, "t.id, t.content_ltks") // output fields prefixed with t.
}
+func TestBuildFusionSQLThresholdFiltersAfterLimit(t *testing.T) {
+ pm := parsedMatch{
+ textQuery: "q", vectorData: []float64{3, 4},
+ textTopN: 200, vectorTopN: 200, vectorWeight: 0.95,
+ vecThreshold: 0.2, hasVecThreshold: true,
+ }
+ sql := buildFusionSQL("t", "id", []string{"id"}, "TRUE", pm, 0, 30)
+ // Applied to the already-limited candidates, so the IVF scan still returns top-k...
+ mustContain(t, sql, "LIMIT 200) c WHERE sim >= 0.2")
+ // ...and never as a radius predicate on the scan itself.
+ mustNotContain(t, sql, "AS sim\n FROM idx_t WHERE TRUE AND")
+}
+
+func TestBuildFusionSQLNoThresholdStillExcludesNulls(t *testing.T) {
+ pm := parsedMatch{
+ textQuery: "q", vectorData: []float64{3, 4},
+ textTopN: 200, vectorTopN: 200, vectorWeight: 0.95,
+ }
+ sql := buildFusionSQL("t", "id", []string{"id"}, "TRUE", pm, 0, 30)
+ mustNotContain(t, sql, "WHERE sim >=")
+ // COALESCE(v.sim, 0) would otherwise score a vectorless row as zero.
+ mustContain(t, sql, "WHERE sim IS NOT NULL")
+}
+
+func TestDropNullSimRemovesVectorlessRows(t *testing.T) {
+ rows := []map[string]interface{}{
+ {"id": "a", vecSimColumn: 0.9},
+ {"id": "b", vecSimColumn: nil},
+ {"id": "c"},
+ {"id": "d", vecSimColumn: -0.4},
+ }
+ got := dropNullSim(rows)
+ if len(got) != 2 || got[0]["id"] != "a" || got[1]["id"] != "d" {
+ t.Errorf("dropNullSim = %v, want the two rows carrying a similarity", got)
+ }
+}
+
func TestBuildUpsert(t *testing.T) {
q, args := buildUpsert("t", []string{"id", "doc_id"}, [][]interface{}{{"1", "d1"}, {"2", "d2"}})
mustContain(t, q, "INSERT INTO t (id, doc_id) VALUES ($1, $2), ($3, $4)")