AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(query): implement rank_feature saturation/log/sigmoid/linear
xerj-orgd96cdd6c83e7
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex 7e67d34c9..3de10cdf1 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -16171,6 +16171,45 @@ fn compute_field_value_factor(fvf: &FieldValueFactor, source: &Value) -> f32 {(modified * fvf.factor as f64) as f32}+/// Compute the score contribution from a `rank_feature` function over a+/// numeric feature field. Implements the four ES functions exactly:+///+/// * saturation: `v / (v + pivot)` (pivot defaults to 1.0 when omitted)+/// * log: `log10(scaling_factor + v)`+/// * sigmoid: `v^exponent / (v^exponent + pivot^exponent)`+/// * linear: `v`+///+/// A missing or non-numeric field contributes 0 (the surrounding Exists+/// query already filters docs lacking the field).+fn compute_rank_feature_score(rf: &xerj_query::ast::RankFeature, source: &Value) -> f32 {+ use xerj_query::ast::RankFeatureFn;+ let Some(v) = get_field_value(source, &rf.field).and_then(|x| x.as_f64()) else {+ return 0.0;+ };+ let s = match &rf.function {+ RankFeatureFn::Saturation { pivot } => {+ let p = pivot.unwrap_or(1.0);+ if v + p == 0.0 {+ 0.0+ } else {+ v / (v + p)+ }+ }+ RankFeatureFn::Log { scaling_factor } => (scaling_factor + v).log10(),+ RankFeatureFn::Sigmoid { pivot, exponent } => {+ let ve = v.powf(*exponent);+ let pe = pivot.powf(*exponent);+ if ve + pe == 0.0 {+ 0.0+ } else {+ ve / (ve + pe)+ }+ }+ RankFeatureFn::Linear => v,+ };+ s as f32+}+/// Compute the score contribution from a `distance_feature` function./// Formula: `pivot / (pivot + distance(origin, field_value))`.///@@ -16422,6 +16461,12 @@ fn apply_function_score(score = ds;}+ if let Some(rf) = &func.rank_feature {+ // rank_feature emits its own score; the query `boost` (carried in+ // `weight`, default 1.0) multiplies it, matching ES.+ score *= compute_rank_feature_score(rf, source);+ }+fn_scores.push(score);}@@ -16987,3 +17032,58 @@ mod terms_set_tests {assert!(!doc_matches_query(&q, &json!({ "codes": ["a", "b"] })));}}++#[cfg(test)]+mod rank_feature_tests {+ use super::*;+ use serde_json::json;+ use xerj_query::ast::{RankFeature, RankFeatureFn};++ fn score(function: RankFeatureFn, v: f64) -> f32 {+ let rf = RankFeature {+ field: "pagerank".into(),+ function,+ };+ compute_rank_feature_score(&rf, &json!({ "pagerank": v }))+ }++ #[test]+ fn saturation_formula_and_ordering() {+ // v / (v + pivot); pivot = 8.+ let f = |v| score(RankFeatureFn::Saturation { pivot: Some(8.0) }, v);+ assert!((f(8.0) - 0.5).abs() < 1e-6, "8/(8+8) = 0.5");+ assert!((f(2.0) - 0.2).abs() < 1e-6, "2/(2+8) = 0.2");+ // Monotonic increasing.+ assert!(f(2.0) < f(8.0) && f(8.0) < f(100.0));+ }++ #[test]+ fn linear_is_identity() {+ assert!((score(RankFeatureFn::Linear, 42.0) - 42.0).abs() < 1e-4);+ assert!((score(RankFeatureFn::Linear, 3.5) - 3.5).abs() < 1e-4);+ }++ #[test]+ fn log_is_monotonic() {+ let f = |v| {+ score(+ RankFeatureFn::Log {+ scaling_factor: 4.0,+ },+ v,+ )+ };+ // log10(4 + 6) = 1.0.+ assert!((f(6.0) - 1.0).abs() < 1e-6);+ assert!(f(1.0) < f(6.0) && f(6.0) < f(96.0));+ }++ #[test]+ fn missing_field_scores_zero() {+ let rf = RankFeature {+ field: "pagerank".into(),+ function: RankFeatureFn::Linear,+ };+ assert_eq!(compute_rank_feature_score(&rf, &json!({ "other": 1 })), 0.0);+ }+}diff --git a/engine/crates/xerj-query/src/ast.rs b/engine/crates/xerj-query/src/ast.rsindex 86824c088..c3616f2a0 100644--- a/engine/crates/xerj-query/src/ast.rs+++ b/engine/crates/xerj-query/src/ast.rs@@ -224,6 +224,37 @@ pub struct ScoreFunction {/// the pivot is a duration (for date fields) or distance (for geo_point).#[serde(skip_serializing_if = "Option::is_none")]pub distance_feature: Option<DistanceFeature>,+ /// `rank_feature` scoring — saturation / log / sigmoid / linear proximity+ /// scoring over a numeric `rank_feature` field.+ #[serde(skip_serializing_if = "Option::is_none")]+ pub rank_feature: Option<RankFeature>,+}++/// Payload for a `rank_feature` function.+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+pub struct RankFeature {+ /// The rank_feature field to score on.+ pub field: String,+ /// Which of the four ES functions to apply.+ pub function: RankFeatureFn,+}++/// The four Elasticsearch `rank_feature` scoring functions.+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum RankFeatureFn {+ /// `v / (v + pivot)`. When `pivot` is omitted ES uses the field's+ /// approximate value distribution; we default it to 1.0.+ Saturation {+ #[serde(skip_serializing_if = "Option::is_none")]+ pivot: Option<f64>,+ },+ /// `log10(scaling_factor + v)`.+ Log { scaling_factor: f64 },+ /// `v^exponent / (v^exponent + pivot^exponent)`.+ Sigmoid { pivot: f64, exponent: f64 },+ /// `v` (identity).+ Linear,}/// Payload for a `distance_feature` function.diff --git a/engine/crates/xerj-query/src/parser.rs b/engine/crates/xerj-query/src/parser.rsindex d317a542f..94f7d9b10 100644--- a/engine/crates/xerj-query/src/parser.rs+++ b/engine/crates/xerj-query/src/parser.rs@@ -2462,6 +2462,7 @@ fn parse_score_function_inline(obj: &serde_json::Map<String, Value>) -> Result<Sscript_params,name,distance_feature: None,+ rank_feature: None,})}@@ -3583,6 +3584,7 @@ fn parse_script_score(params: &Value) -> Result<QueryNode> {script_params: s_params,name: None,distance_feature: None,+ rank_feature: None,};Ok(QueryNode::FunctionScore {@@ -3637,6 +3639,7 @@ fn parse_distance_feature(params: &Value) -> Result<QueryNode> {script_params: None,name: None,distance_feature: Some(df),+ rank_feature: None,}],score_mode: ScoreMode::Multiply,boost_mode: BoostMode::Replace,@@ -3650,6 +3653,8 @@ fn parse_distance_feature(params: &Value) -> Result<QueryNode> {////// ES docs: <https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-rank-feature-query.html>fn parse_rank_feature(params: &Value) -> Result<QueryNode> {+ use crate::ast::{RankFeature, RankFeatureFn};+let obj = params.as_object().ok_or_else(|| qerr("`rank_feature` must be an object"))?;@@ -3657,30 +3662,57 @@ fn parse_rank_feature(params: &Value) -> Result<QueryNode> {let field = string_field(obj, "field")?;let boost = obj.get("boost").and_then(|v| v.as_f64()).map(|b| b as f32);- // log(pivot + value) / log(pivot + max_value) approximation — we use- // a field_value_factor with log1p modifier as a close stand-in.- let fvf = FieldValueFactor {+ // Detect which of the four ES functions is requested. Default to+ // `saturation` (with the field's default pivot) when none is given.+ let function = if let Some(sat) = obj.get("saturation") {+ let pivot = sat.get("pivot").and_then(|v| v.as_f64());+ RankFeatureFn::Saturation { pivot }+ } else if let Some(log) = obj.get("log") {+ let scaling_factor = log+ .get("scaling_factor")+ .and_then(|v| v.as_f64())+ .ok_or_else(|| qerr("`rank_feature.log` requires a `scaling_factor`"))?;+ RankFeatureFn::Log { scaling_factor }+ } else if let Some(sig) = obj.get("sigmoid") {+ let pivot = sig+ .get("pivot")+ .and_then(|v| v.as_f64())+ .ok_or_else(|| qerr("`rank_feature.sigmoid` requires a `pivot`"))?;+ let exponent = sig+ .get("exponent")+ .and_then(|v| v.as_f64())+ .ok_or_else(|| qerr("`rank_feature.sigmoid` requires an `exponent`"))?;+ RankFeatureFn::Sigmoid { pivot, exponent }+ } else if obj.contains_key("linear") {+ RankFeatureFn::Linear+ } else {+ RankFeatureFn::Saturation { pivot: None }+ };++ let rank_feature = RankFeature {field: field.clone(),- factor: 1.0,- modifier: Modifier::Log1p,- missing: Some(0.0),+ function,};Ok(QueryNode::FunctionScore {+ // The inner Exists query supplies the candidate set (all docs that+ // have the rank_feature field); `boost_mode: replace` substitutes+ // the feature's own score for the query score.query: Box::new(QueryNode::Exists { field }),functions: vec![ScoreFunction {filter: None,weight: boost,- field_value_factor: Some(fvf),+ field_value_factor: None,random_score: None,script_score: None,script_source: None,script_params: None,name: None,distance_feature: None,+ rank_feature: Some(rank_feature),}],score_mode: ScoreMode::Multiply,- boost_mode: BoostMode::Multiply,+ boost_mode: BoostMode::Replace,max_boost: None,})}@@ -4635,4 +4667,76 @@ mod tests {other => panic!("expected bool w/ script msm, got {:?}", other),}}++ // ── rank_feature ────────────────────────────────────────────────────────++ fn rank_feature_fn(body: serde_json::Value) -> crate::ast::RankFeatureFn {+ use crate::ast::QueryNode;+ match q(body) {+ QueryNode::FunctionScore {+ functions,+ boost_mode,+ ..+ } => {+ assert_eq!(boost_mode, crate::ast::BoostMode::Replace);+ functions[0]+ .rank_feature+ .as_ref()+ .expect("rank_feature payload present")+ .function+ .clone()+ }+ other => panic!("expected function_score, got {:?}", other),+ }+ }++ #[test]+ fn test_rank_feature_saturation_with_pivot() {+ let f = rank_feature_fn(json!({+ "rank_feature": { "field": "pagerank", "saturation": { "pivot": 8 } }+ }));+ assert_eq!(+ f,+ crate::ast::RankFeatureFn::Saturation { pivot: Some(8.0) }+ );+ }++ #[test]+ fn test_rank_feature_defaults_to_saturation() {+ let f = rank_feature_fn(json!({ "rank_feature": { "field": "pagerank" } }));+ assert_eq!(f, crate::ast::RankFeatureFn::Saturation { pivot: None });+ }++ #[test]+ fn test_rank_feature_log_sigmoid_linear() {+ assert_eq!(+ rank_feature_fn(json!({+ "rank_feature": { "field": "pr", "log": { "scaling_factor": 4 } }+ })),+ crate::ast::RankFeatureFn::Log {+ scaling_factor: 4.0+ }+ );+ assert_eq!(+ rank_feature_fn(json!({+ "rank_feature": { "field": "pr", "sigmoid": { "pivot": 7, "exponent": 0.6 } }+ })),+ crate::ast::RankFeatureFn::Sigmoid {+ pivot: 7.0,+ exponent: 0.6+ }+ );+ assert_eq!(+ rank_feature_fn(json!({ "rank_feature": { "field": "pr", "linear": {} } })),+ crate::ast::RankFeatureFn::Linear+ );+ }++ #[test]+ fn test_rank_feature_log_missing_scaling_is_400() {+ assert!(parse_query(&json!({+ "rank_feature": { "field": "pr", "log": {} }+ }))+ .is_err());+ }}diff --git a/engine/tests/es-compat-yaml/yaml/search/255_rank_feature.yml b/engine/tests/es-compat-yaml/yaml/search/255_rank_feature.ymlnew file mode 100644index 000000000..adac34570--- /dev/null+++ b/engine/tests/es-compat-yaml/yaml/search/255_rank_feature.yml@@ -0,0 +1,81 @@+setup:+ - do:+ indices.create:+ index: rf+ body:+ mappings:+ properties:+ pagerank:+ type: rank_feature++ - do:+ bulk:+ refresh: true+ body:+ - '{ "index": { "_index": "rf", "_id": "1" } }'+ - '{ "pagerank": 10 }'+ - '{ "index": { "_index": "rf", "_id": "2" } }'+ - '{ "pagerank": 100 }'+ - '{ "index": { "_index": "rf", "_id": "3" } }'+ - '{ "pagerank": 1 }'+ - '{ "index": { "_index": "rf", "_id": "4" } }'+ - '{ "pagerank": 50 }'++---+"rank_feature saturation orders by v/(v+pivot) and yields that score":++ - do:+ search:+ rest_total_hits_as_int: true+ index: rf+ body:+ query:+ rank_feature:+ field: pagerank+ saturation:+ pivot: 8++ - length: { hits.hits: 4 }+ - match: { hits.hits.0._id: "2" }+ - match: { hits.hits.1._id: "4" }+ - match: { hits.hits.2._id: "1" }+ - match: { hits.hits.3._id: "3" }+ # S = 10 / (10 + 8) = 0.5555556 for doc 1.+ - match: { hits.hits.2._score: 0.5555556 }++---+"rank_feature linear yields _score == value":++ - do:+ search:+ rest_total_hits_as_int: true+ index: rf+ body:+ query:+ rank_feature:+ field: pagerank+ linear: {}++ - match: { hits.hits.0._id: "2" }+ - match: { hits.hits.0._score: 100.0 }+ - match: { hits.hits.1._id: "4" }+ - match: { hits.hits.1._score: 50.0 }++---+"rank_feature log ordering is monotonic":++ - do:+ search:+ rest_total_hits_as_int: true+ index: rf+ body:+ query:+ rank_feature:+ field: pagerank+ log:+ scaling_factor: 4++ - match: { hits.hits.0._id: "2" }+ - match: { hits.hits.1._id: "4" }+ - match: { hits.hits.2._id: "1" }+ - match: { hits.hits.3._id: "3" }