Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(query): implement rank_feature saturation/log/sigmoid/linear

xerj-orgd96cdd6c83e7
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index 7e67d34c9..3de10cdf1 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -16171,6 +16171,45 @@ fn compute_field_value_factor(fvf: &FieldValueFactor, source: &Value) -> f32 {
(modified * fvf.factor as f64) as f32
}
+/// Compute the score contribution from a `rank_feature` function over a
+/// numeric feature field. Implements the four ES functions exactly:
+///
+/// * saturation: `v / (v + pivot)` (pivot defaults to 1.0 when omitted)
+/// * log: `log10(scaling_factor + v)`
+/// * sigmoid: `v^exponent / (v^exponent + pivot^exponent)`
+/// * linear: `v`
+///
+/// A missing or non-numeric field contributes 0 (the surrounding Exists
+/// query already filters docs lacking the field).
+fn compute_rank_feature_score(rf: &xerj_query::ast::RankFeature, source: &Value) -> f32 {
+ use xerj_query::ast::RankFeatureFn;
+ let Some(v) = get_field_value(source, &rf.field).and_then(|x| x.as_f64()) else {
+ return 0.0;
+ };
+ let s = match &rf.function {
+ RankFeatureFn::Saturation { pivot } => {
+ let p = pivot.unwrap_or(1.0);
+ if v + p == 0.0 {
+ 0.0
+ } else {
+ v / (v + p)
+ }
+ }
+ RankFeatureFn::Log { scaling_factor } => (scaling_factor + v).log10(),
+ RankFeatureFn::Sigmoid { pivot, exponent } => {
+ let ve = v.powf(*exponent);
+ let pe = pivot.powf(*exponent);
+ if ve + pe == 0.0 {
+ 0.0
+ } else {
+ ve / (ve + pe)
+ }
+ }
+ RankFeatureFn::Linear => v,
+ };
+ s as f32
+}
+
/// Compute the score contribution from a `distance_feature` function.
/// Formula: `pivot / (pivot + distance(origin, field_value))`.
///
@@ -16422,6 +16461,12 @@ fn apply_function_score(
score = ds;
}
+ if let Some(rf) = &func.rank_feature {
+ // rank_feature emits its own score; the query `boost` (carried in
+ // `weight`, default 1.0) multiplies it, matching ES.
+ score *= compute_rank_feature_score(rf, source);
+ }
+
fn_scores.push(score);
}
@@ -16987,3 +17032,58 @@ mod terms_set_tests {
assert!(!doc_matches_query(&q, &json!({ "codes": ["a", "b"] })));
}
}
+
+#[cfg(test)]
+mod rank_feature_tests {
+ use super::*;
+ use serde_json::json;
+ use xerj_query::ast::{RankFeature, RankFeatureFn};
+
+ fn score(function: RankFeatureFn, v: f64) -> f32 {
+ let rf = RankFeature {
+ field: "pagerank".into(),
+ function,
+ };
+ compute_rank_feature_score(&rf, &json!({ "pagerank": v }))
+ }
+
+ #[test]
+ fn saturation_formula_and_ordering() {
+ // v / (v + pivot); pivot = 8.
+ let f = |v| score(RankFeatureFn::Saturation { pivot: Some(8.0) }, v);
+ assert!((f(8.0) - 0.5).abs() < 1e-6, "8/(8+8) = 0.5");
+ assert!((f(2.0) - 0.2).abs() < 1e-6, "2/(2+8) = 0.2");
+ // Monotonic increasing.
+ assert!(f(2.0) < f(8.0) && f(8.0) < f(100.0));
+ }
+
+ #[test]
+ fn linear_is_identity() {
+ assert!((score(RankFeatureFn::Linear, 42.0) - 42.0).abs() < 1e-4);
+ assert!((score(RankFeatureFn::Linear, 3.5) - 3.5).abs() < 1e-4);
+ }
+
+ #[test]
+ fn log_is_monotonic() {
+ let f = |v| {
+ score(
+ RankFeatureFn::Log {
+ scaling_factor: 4.0,
+ },
+ v,
+ )
+ };
+ // log10(4 + 6) = 1.0.
+ assert!((f(6.0) - 1.0).abs() < 1e-6);
+ assert!(f(1.0) < f(6.0) && f(6.0) < f(96.0));
+ }
+
+ #[test]
+ fn missing_field_scores_zero() {
+ let rf = RankFeature {
+ field: "pagerank".into(),
+ function: RankFeatureFn::Linear,
+ };
+ assert_eq!(compute_rank_feature_score(&rf, &json!({ "other": 1 })), 0.0);
+ }
+}
diff --git a/engine/crates/xerj-query/src/ast.rs b/engine/crates/xerj-query/src/ast.rs
index 86824c088..c3616f2a0 100644
--- a/engine/crates/xerj-query/src/ast.rs
+++ b/engine/crates/xerj-query/src/ast.rs
@@ -224,6 +224,37 @@ pub struct ScoreFunction {
/// the pivot is a duration (for date fields) or distance (for geo_point).
#[serde(skip_serializing_if = "Option::is_none")]
pub distance_feature: Option<DistanceFeature>,
+ /// `rank_feature` scoring — saturation / log / sigmoid / linear proximity
+ /// scoring over a numeric `rank_feature` field.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub rank_feature: Option<RankFeature>,
+}
+
+/// Payload for a `rank_feature` function.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct RankFeature {
+ /// The rank_feature field to score on.
+ pub field: String,
+ /// Which of the four ES functions to apply.
+ pub function: RankFeatureFn,
+}
+
+/// The four Elasticsearch `rank_feature` scoring functions.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum RankFeatureFn {
+ /// `v / (v + pivot)`. When `pivot` is omitted ES uses the field's
+ /// approximate value distribution; we default it to 1.0.
+ Saturation {
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pivot: Option<f64>,
+ },
+ /// `log10(scaling_factor + v)`.
+ Log { scaling_factor: f64 },
+ /// `v^exponent / (v^exponent + pivot^exponent)`.
+ Sigmoid { pivot: f64, exponent: f64 },
+ /// `v` (identity).
+ Linear,
}
/// Payload for a `distance_feature` function.
diff --git a/engine/crates/xerj-query/src/parser.rs b/engine/crates/xerj-query/src/parser.rs
index d317a542f..94f7d9b10 100644
--- a/engine/crates/xerj-query/src/parser.rs
+++ b/engine/crates/xerj-query/src/parser.rs
@@ -2462,6 +2462,7 @@ fn parse_score_function_inline(obj: &serde_json::Map<String, Value>) -> Result<S
script_params,
name,
distance_feature: None,
+ rank_feature: None,
})
}
@@ -3583,6 +3584,7 @@ fn parse_script_score(params: &Value) -> Result<QueryNode> {
script_params: s_params,
name: None,
distance_feature: None,
+ rank_feature: None,
};
Ok(QueryNode::FunctionScore {
@@ -3637,6 +3639,7 @@ fn parse_distance_feature(params: &Value) -> Result<QueryNode> {
script_params: None,
name: None,
distance_feature: Some(df),
+ rank_feature: None,
}],
score_mode: ScoreMode::Multiply,
boost_mode: BoostMode::Replace,
@@ -3650,6 +3653,8 @@ fn parse_distance_feature(params: &Value) -> Result<QueryNode> {
///
/// ES docs: <https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-rank-feature-query.html>
fn parse_rank_feature(params: &Value) -> Result<QueryNode> {
+ use crate::ast::{RankFeature, RankFeatureFn};
+
let obj = params
.as_object()
.ok_or_else(|| qerr("`rank_feature` must be an object"))?;
@@ -3657,30 +3662,57 @@ fn parse_rank_feature(params: &Value) -> Result<QueryNode> {
let field = string_field(obj, "field")?;
let boost = obj.get("boost").and_then(|v| v.as_f64()).map(|b| b as f32);
- // log(pivot + value) / log(pivot + max_value) approximation — we use
- // a field_value_factor with log1p modifier as a close stand-in.
- let fvf = FieldValueFactor {
+ // Detect which of the four ES functions is requested. Default to
+ // `saturation` (with the field's default pivot) when none is given.
+ let function = if let Some(sat) = obj.get("saturation") {
+ let pivot = sat.get("pivot").and_then(|v| v.as_f64());
+ RankFeatureFn::Saturation { pivot }
+ } else if let Some(log) = obj.get("log") {
+ let scaling_factor = log
+ .get("scaling_factor")
+ .and_then(|v| v.as_f64())
+ .ok_or_else(|| qerr("`rank_feature.log` requires a `scaling_factor`"))?;
+ RankFeatureFn::Log { scaling_factor }
+ } else if let Some(sig) = obj.get("sigmoid") {
+ let pivot = sig
+ .get("pivot")
+ .and_then(|v| v.as_f64())
+ .ok_or_else(|| qerr("`rank_feature.sigmoid` requires a `pivot`"))?;
+ let exponent = sig
+ .get("exponent")
+ .and_then(|v| v.as_f64())
+ .ok_or_else(|| qerr("`rank_feature.sigmoid` requires an `exponent`"))?;
+ RankFeatureFn::Sigmoid { pivot, exponent }
+ } else if obj.contains_key("linear") {
+ RankFeatureFn::Linear
+ } else {
+ RankFeatureFn::Saturation { pivot: None }
+ };
+
+ let rank_feature = RankFeature {
field: field.clone(),
- factor: 1.0,
- modifier: Modifier::Log1p,
- missing: Some(0.0),
+ function,
};
Ok(QueryNode::FunctionScore {
+ // The inner Exists query supplies the candidate set (all docs that
+ // have the rank_feature field); `boost_mode: replace` substitutes
+ // the feature's own score for the query score.
query: Box::new(QueryNode::Exists { field }),
functions: vec![ScoreFunction {
filter: None,
weight: boost,
- field_value_factor: Some(fvf),
+ field_value_factor: None,
random_score: None,
script_score: None,
script_source: None,
script_params: None,
name: None,
distance_feature: None,
+ rank_feature: Some(rank_feature),
}],
score_mode: ScoreMode::Multiply,
- boost_mode: BoostMode::Multiply,
+ boost_mode: BoostMode::Replace,
max_boost: None,
})
}
@@ -4635,4 +4667,76 @@ mod tests {
other => panic!("expected bool w/ script msm, got {:?}", other),
}
}
+
+ // ── rank_feature ────────────────────────────────────────────────────────
+
+ fn rank_feature_fn(body: serde_json::Value) -> crate::ast::RankFeatureFn {
+ use crate::ast::QueryNode;
+ match q(body) {
+ QueryNode::FunctionScore {
+ functions,
+ boost_mode,
+ ..
+ } => {
+ assert_eq!(boost_mode, crate::ast::BoostMode::Replace);
+ functions[0]
+ .rank_feature
+ .as_ref()
+ .expect("rank_feature payload present")
+ .function
+ .clone()
+ }
+ other => panic!("expected function_score, got {:?}", other),
+ }
+ }
+
+ #[test]
+ fn test_rank_feature_saturation_with_pivot() {
+ let f = rank_feature_fn(json!({
+ "rank_feature": { "field": "pagerank", "saturation": { "pivot": 8 } }
+ }));
+ assert_eq!(
+ f,
+ crate::ast::RankFeatureFn::Saturation { pivot: Some(8.0) }
+ );
+ }
+
+ #[test]
+ fn test_rank_feature_defaults_to_saturation() {
+ let f = rank_feature_fn(json!({ "rank_feature": { "field": "pagerank" } }));
+ assert_eq!(f, crate::ast::RankFeatureFn::Saturation { pivot: None });
+ }
+
+ #[test]
+ fn test_rank_feature_log_sigmoid_linear() {
+ assert_eq!(
+ rank_feature_fn(json!({
+ "rank_feature": { "field": "pr", "log": { "scaling_factor": 4 } }
+ })),
+ crate::ast::RankFeatureFn::Log {
+ scaling_factor: 4.0
+ }
+ );
+ assert_eq!(
+ rank_feature_fn(json!({
+ "rank_feature": { "field": "pr", "sigmoid": { "pivot": 7, "exponent": 0.6 } }
+ })),
+ crate::ast::RankFeatureFn::Sigmoid {
+ pivot: 7.0,
+ exponent: 0.6
+ }
+ );
+ assert_eq!(
+ rank_feature_fn(json!({ "rank_feature": { "field": "pr", "linear": {} } })),
+ crate::ast::RankFeatureFn::Linear
+ );
+ }
+
+ #[test]
+ fn test_rank_feature_log_missing_scaling_is_400() {
+ assert!(parse_query(&json!({
+ "rank_feature": { "field": "pr", "log": {} }
+ }))
+ .is_err());
+ }
}
diff --git a/engine/tests/es-compat-yaml/yaml/search/255_rank_feature.yml b/engine/tests/es-compat-yaml/yaml/search/255_rank_feature.yml
new file mode 100644
index 000000000..adac34570
--- /dev/null
+++ b/engine/tests/es-compat-yaml/yaml/search/255_rank_feature.yml
@@ -0,0 +1,81 @@
+setup:
+ - do:
+ indices.create:
+ index: rf
+ body:
+ mappings:
+ properties:
+ pagerank:
+ type: rank_feature
+
+ - do:
+ bulk:
+ refresh: true
+ body:
+ - '{ "index": { "_index": "rf", "_id": "1" } }'
+ - '{ "pagerank": 10 }'
+ - '{ "index": { "_index": "rf", "_id": "2" } }'
+ - '{ "pagerank": 100 }'
+ - '{ "index": { "_index": "rf", "_id": "3" } }'
+ - '{ "pagerank": 1 }'
+ - '{ "index": { "_index": "rf", "_id": "4" } }'
+ - '{ "pagerank": 50 }'
+
+---
+"rank_feature saturation orders by v/(v+pivot) and yields that score":
+
+ - do:
+ search:
+ rest_total_hits_as_int: true
+ index: rf
+ body:
+ query:
+ rank_feature:
+ field: pagerank
+ saturation:
+ pivot: 8
+
+ - length: { hits.hits: 4 }
+ - match: { hits.hits.0._id: "2" }
+ - match: { hits.hits.1._id: "4" }
+ - match: { hits.hits.2._id: "1" }
+ - match: { hits.hits.3._id: "3" }
+ # S = 10 / (10 + 8) = 0.5555556 for doc 1.
+ - match: { hits.hits.2._score: 0.5555556 }
+
+---
+"rank_feature linear yields _score == value":
+
+ - do:
+ search:
+ rest_total_hits_as_int: true
+ index: rf
+ body:
+ query:
+ rank_feature:
+ field: pagerank
+ linear: {}
+
+ - match: { hits.hits.0._id: "2" }
+ - match: { hits.hits.0._score: 100.0 }
+ - match: { hits.hits.1._id: "4" }
+ - match: { hits.hits.1._score: 50.0 }
+
+---
+"rank_feature log ordering is monotonic":
+
+ - do:
+ search:
+ rest_total_hits_as_int: true
+ index: rf
+ body:
+ query:
+ rank_feature:
+ field: pagerank
+ log:
+ scaling_factor: 4
+
+ - match: { hits.hits.0._id: "2" }
+ - match: { hits.hits.1._id: "4" }
+ - match: { hits.hits.2._id: "1" }
+ - match: { hits.hits.3._id: "3" }