Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

ci: pass the fmt and clippy gates — first run was on the pushed branch

xerj-org3192686fead5
diff --git a/engine/crates/xerj-api/src/systemone_api.rs b/engine/crates/xerj-api/src/systemone_api.rs
index ef090b614..1e6ab5a7f 100644
--- a/engine/crates/xerj-api/src/systemone_api.rs
+++ b/engine/crates/xerj-api/src/systemone_api.rs
@@ -140,8 +140,10 @@ pub async fn systemone(
);
}
Kind::Choice { options } => {
- let criteria_total: f64 =
- options.iter().map(|o| weight_by_label.get(o.as_str()).copied().unwrap_or(0.0)).sum();
+ let criteria_total: f64 = options
+ .iter()
+ .map(|o| weight_by_label.get(o.as_str()).copied().unwrap_or(0.0))
+ .sum();
if criteria_total <= 0.0 {
unsupported.push(id.clone());
continue;
@@ -150,7 +152,9 @@ pub async fn systemone(
for option in options {
probabilities.insert(
option.clone(),
- json!(round6(weight_by_label.get(option.as_str()).copied().unwrap_or(0.0) / total)),
+ json!(round6(
+ weight_by_label.get(option.as_str()).copied().unwrap_or(0.0) / total
+ )),
);
}
let (winner, _) = options
@@ -187,7 +191,11 @@ pub async fn systemone(
return no_support(&unsupported, &cfg);
}
- let _ = state.metrics.record_query(&cfg.index, "systemone_vote", started.elapsed().as_secs_f64());
+ state.metrics.record_query(
+ &cfg.index,
+ "systemone_vote",
+ started.elapsed().as_secs_f64(),
+ );
Json(json!({
"model": MODEL_ID,
"answers": Value::Object(answers),
@@ -264,11 +272,11 @@ pub async fn decide(
cfg.label_field = defaults.label_field.clone();
cfg.text_field = defaults.text_field.clone();
- let neighbours = match vote_neighbours(&state, &cfg, &clip(&question, MAX_VOTE_TEXT_CHARS)).await
- {
- Ok(n) => n,
- Err(resp) => return resp,
- };
+ let neighbours =
+ match vote_neighbours(&state, &cfg, &clip(&question, MAX_VOTE_TEXT_CHARS)).await {
+ Ok(n) => n,
+ Err(resp) => return resp,
+ };
let total: f64 = neighbours.iter().map(|(w, _, _)| *w).sum();
let mut weight_by_label: BTreeMap<String, f64> = BTreeMap::new();
for (w, label, _) in &neighbours {
@@ -311,7 +319,7 @@ pub async fn decide(
})
})
.collect();
- let _ = state
+ state
.metrics
.record_query(&index, "decide", started.elapsed().as_secs_f64());
@@ -355,7 +363,9 @@ fn parse_question(id: &str, q: &Value, state_val: &Value) -> Result<Question, St
let criteria = obj
.get("criteria")
.and_then(Value::as_object)
- .ok_or_else(|| format!("question `{id}` is a choice and needs `criteria` naming its options"))?;
+ .ok_or_else(|| {
+ format!("question `{id}` is a choice and needs `criteria` naming its options")
+ })?;
if criteria.is_empty() || criteria.len() > MAX_CHOICE_OPTIONS {
return Err(format!(
"question `{id}`: `criteria` must name 1..{MAX_CHOICE_OPTIONS} options (got {})",
@@ -434,7 +444,11 @@ async fn vote_neighbours(
});
let search_req = match parse_request(&query_body) {
Ok(r) => r,
- Err(e) => return Err(unprocessable(&format!("internal query would not parse: {e}"))),
+ Err(e) => {
+ return Err(unprocessable(&format!(
+ "internal query would not parse: {e}"
+ )))
+ }
};
let result = match idx.search(&search_req).await {
Ok(r) => r,
@@ -568,7 +582,10 @@ fn no_support(ids: &[String], cfg: &DecisionsConfig) -> axum::response::Response
cfg.index,
cfg.k,
if ids.len() == 1 { "" } else { "s" },
- ids.iter().map(|i| format!("`{i}`")).collect::<Vec<_>>().join(", ")
+ ids.iter()
+ .map(|i| format!("`{i}`"))
+ .collect::<Vec<_>>()
+ .join(", ")
),
)
}
diff --git a/engine/crates/xerj-api/tests/rerank_stage_http.rs b/engine/crates/xerj-api/tests/rerank_stage_http.rs
index d5e9cefa8..694c6b3a7 100644
--- a/engine/crates/xerj-api/tests/rerank_stage_http.rs
+++ b/engine/crates/xerj-api/tests/rerank_stage_http.rs
@@ -1030,7 +1030,9 @@ async fn wire_shape_bearer_auth_model_and_one_noul_per_document_keyed_identicall
"{key}: the question must name the field it judges: {text}"
);
assert!(
- instructions["document"].as_str().is_some_and(|d| !d.is_empty()),
+ instructions["document"]
+ .as_str()
+ .is_some_and(|d| !d.is_empty()),
"{key}: the candidate itself travels with its question"
);
}
@@ -2216,8 +2218,13 @@ async fn the_matching_passage_is_judgeable_and_goes_first() {
// tail and never the part that matched.
let (title, rest) = text.split_once(". ").unwrap_or(("", text));
assert!(
- ["Bone health basics", "Trial results", "Cooking", "Density of materials"]
- .contains(&title),
+ [
+ "Bone health basics",
+ "Trial results",
+ "Cooking",
+ "Density of materials"
+ ]
+ .contains(&title),
"{key}: the returned title joins its document: {text:?}"
);
let first_line = rest.lines().next().unwrap_or_default();
diff --git a/engine/crates/xerj-api/tests/systemone_http.rs b/engine/crates/xerj-api/tests/systemone_http.rs
index edaf81b21..734ad0f91 100644
--- a/engine/crates/xerj-api/tests/systemone_http.rs
+++ b/engine/crates/xerj-api/tests/systemone_http.rs
@@ -79,7 +79,13 @@ async fn gate_node() -> Node {
}
impl Node {
- async fn call(&self, router: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {
+ async fn call(
+ &self,
+ router: &axum::Router,
+ method: &str,
+ path: &str,
+ body: Value,
+ ) -> (StatusCode, Value) {
let response = router
.clone()
.oneshot(
@@ -103,10 +109,12 @@ impl Node {
}
async fn systemone(&self, body: Value) -> (StatusCode, Value) {
- self.call(&self.native.clone(), "POST", "/v1/systemone", body).await
+ self.call(&self.native.clone(), "POST", "/v1/systemone", body)
+ .await
}
async fn models(&self) -> (StatusCode, Value) {
- self.call(&self.native.clone(), "GET", "/v1/models", json!({})).await
+ self.call(&self.native.clone(), "GET", "/v1/models", json!({}))
+ .await
}
async fn decide(&self, body: Value) -> (StatusCode, Value) {
self.call(&self.app.clone(), "POST", "/_decide", body).await
@@ -139,13 +147,21 @@ async fn seed(node: &Node, index: &str, docs: &[(&str, &str, &str)]) {
assert!(st.is_success(), "index {index}/{id}: {st} {b}");
}
let (st, b) = node
- .call(&node.app.clone(), "POST", &format!("/{index}/_refresh"), json!({}))
+ .call(
+ &node.app.clone(),
+ "POST",
+ &format!("/{index}/_refresh"),
+ json!({}),
+ )
.await;
assert!(st.is_success(), "refresh {index}: {st} {b}");
}
fn reason(r: &Value) -> String {
- r["error"]["reason"].as_str().unwrap_or_default().to_string()
+ r["error"]["reason"]
+ .as_str()
+ .unwrap_or_default()
+ .to_string()
}
// ─────────────────────────────────────────────────────────────────────────────
@@ -205,7 +221,10 @@ async fn answers_a_jev_reranker_shaped_request_with_backtick_references() {
for id in ["d0", "d1"] {
let a = answers.get(id).unwrap_or(&Value::Null);
// Clients parse answers strictly: exactly the documented fields.
- let keys: Vec<&str> = a.as_object().map(|o| o.keys().map(String::as_str).collect()).unwrap_or_default();
+ let keys: Vec<&str> = a
+ .as_object()
+ .map(|o| o.keys().map(String::as_str).collect())
+ .unwrap_or_default();
assert_eq!(keys, vec!["type", "noul"], "{id}: {a}");
assert_eq!(a["type"], "noul");
}
@@ -285,7 +304,10 @@ async fn choice_answers_carry_a_probability_for_every_option() {
keys.sort_unstable();
assert_eq!(keys, vec!["billing", "tech"], "{probs:?}");
let sum: f64 = probs.values().filter_map(Value::as_f64).sum();
- assert!((sum - 1.0).abs() < 1e-4, "probabilities sum to {sum}: {probs:?}");
+ assert!(
+ (sum - 1.0).abs() < 1e-4,
+ "probabilities sum to {sum}: {probs:?}"
+ );
// The choice is the argmax of the reported probabilities, and the
// confidence is that option's own probability.
let (argmax, _) = probs
@@ -299,7 +321,9 @@ async fn choice_answers_carry_a_probability_for_every_option() {
// The same vote through /_decide names its neighbours; the probabilities
// above must be the weighted shares of exactly those neighbours.
- let (st2, d) = node.decide(json!({"index": "mixed", "question": vote})).await;
+ let (st2, d) = node
+ .decide(json!({"index": "mixed", "question": vote}))
+ .await;
assert!(st2.is_success(), "{d}");
let total: f64 = d["neighbours"]
.as_array()
@@ -362,7 +386,10 @@ async fn zero_support_is_an_error_naming_the_question_not_a_half() {
assert!(why.contains("`d0`"), "{why}");
assert!(why.contains("`d1`"), "{why}");
assert!(!r.to_string().contains("0.5"), "no fabricated halves: {r}");
- assert!(r["answers"].is_null(), "no half-answered judgement history: {r}");
+ assert!(
+ r["answers"].is_null(),
+ "no half-answered judgement history: {r}"
+ );
}
/// Until `[decisions] index` names an index, the endpoint says 503 — not a
@@ -427,7 +454,10 @@ async fn more_than_300_questions_is_refused() {
let node = gate_node().await;
let mut questions = serde_json::Map::new();
for i in 0..301 {
- questions.insert(format!("d{i}"), json!({"type": "noul", "instructions": "refund"}));
+ questions.insert(
+ format!("d{i}"),
+ json!({"type": "noul", "instructions": "refund"}),
+ );
}
let (st, r) = node
.systemone(json!({"state": "refund", "questions": Value::Object(questions)}))
@@ -454,7 +484,10 @@ async fn decide_returns_its_neighbours_and_honours_per_request_k() {
assert_eq!(r["index"], "history");
assert_eq!(r["k"], 1);
assert_eq!(r["label"], "true");
- assert!((r["confidence"].as_f64().unwrap_or(0.0) - 1.0).abs() < 1e-9, "{r}");
+ assert!(
+ (r["confidence"].as_f64().unwrap_or(0.0) - 1.0).abs() < 1e-9,
+ "{r}"
+ );
assert_eq!(r["abstain"], json!(false));
let neighbours = r["neighbours"].as_array().expect("neighbours");
assert_eq!(neighbours.len(), 1, "k=1: {neighbours:?}");
@@ -462,7 +495,10 @@ async fn decide_returns_its_neighbours_and_honours_per_request_k() {
assert_eq!(n["_id"], "h1");
assert_eq!(n["label"], "true");
assert!(n["_score"].as_f64().is_some(), "engine score: {n}");
- assert!(n["text"].as_str().is_some_and(|t| t.contains("refund")), "{n}");
+ assert!(
+ n["text"].as_str().is_some_and(|t| t.contains("refund")),
+ "{n}"
+ );
// Full k: both refund docs are neighbours, weights reciprocal in rank.
let (_, r) = node
@@ -500,7 +536,12 @@ async fn decide_abstains_below_min_confidence_and_on_no_neighbours() {
.await;
assert!(st.is_success(), "{r}");
assert_eq!(r["abstain"], json!(true));
- assert!(r["reason"].as_str().is_some_and(|s| s.contains("min_confidence")), "{r}");
+ assert!(
+ r["reason"]
+ .as_str()
+ .is_some_and(|s| s.contains("min_confidence")),
+ "{r}"
+ );
// Nothing matches at all: abstain with the no-neighbour reason.
let (st, r) = strict
@@ -509,7 +550,12 @@ async fn decide_abstains_below_min_confidence_and_on_no_neighbours() {
assert!(st.is_success(), "{r}");
assert_eq!(r["abstain"], json!(true));
assert!(r["label"].is_null(), "{r}");
- assert!(r["reason"].as_str().is_some_and(|s| s.contains("no labelled neighbour")), "{r}");
+ assert!(
+ r["reason"]
+ .as_str()
+ .is_some_and(|s| s.contains("no labelled neighbour")),
+ "{r}"
+ );
}
#[tokio::test]
diff --git a/engine/crates/xerj-common/src/config.rs b/engine/crates/xerj-common/src/config.rs
index 8ecc4ccbb..b95d8558c 100644
--- a/engine/crates/xerj-common/src/config.rs
+++ b/engine/crates/xerj-common/src/config.rs
@@ -2292,7 +2292,6 @@ impl RerankProviderConfig {
}
}
-
/// Typed decisions (`/v1/systemone`, `/_decide`) answered locally by a
/// weighted nearest-neighbour vote over a labelled-history index — the
/// retrieval analogue of a judge model, with the evidence staying on the node.
diff --git a/engine/crates/xerj-fts/src/search.rs b/engine/crates/xerj-fts/src/search.rs
index 7f4b24491..b47aee5cd 100644
--- a/engine/crates/xerj-fts/src/search.rs
+++ b/engine/crates/xerj-fts/src/search.rs
@@ -667,11 +667,7 @@ impl FtsSearcher {
/// narrower contract `execute_bool`'s `run_clauses` already upholds for
/// abandoned `should` clauses, surfaced as `timed_out: true` by
/// [`Self::deadline_tripped`].
- fn wand_should_bool(
- &self,
- query: &Query,
- cap: usize,
- ) -> Result<Option<(Vec<ScoredHit>, u64)>> {
+ fn wand_should_bool(&self, query: &Query, cap: usize) -> Result<Option<(Vec<ScoredHit>, u64)>> {
let Query::Bool(bq) = query else {
return Ok(None);
};
@@ -724,7 +720,9 @@ impl FtsSearcher {
if ord & 63 == 0 && self.deadline_hit() {
break;
}
- let Query::Term(tq) = q else { unreachable!("checked above") };
+ let Query::Term(tq) = q else {
+ unreachable!("checked above")
+ };
let Some(tp) = self.reader.lookup_term(&tq.field, &tq.term) else {
// Term absent from this segment: the clause matches nothing
// and contributes no score — exactly the generic path's
@@ -820,13 +818,13 @@ impl FtsSearcher {
// pop off after the re-sort. `sort_unstable` never allocates the
// scratch buffer a stable sort spills to on >20-element slices —
// this re-sort runs once per DOC, so that alloc was per-doc too.
- for i in 0..k {
- match scorers[i].reader.next() {
+ for s in scorers.iter_mut().take(k) {
+ match s.reader.next() {
Some(next) => {
- scorers[i].doc = next.doc_id;
- scorers[i].tf = next.term_freq;
+ s.doc = next.doc_id;
+ s.tf = next.term_freq;
}
- None => scorers[i].doc = u32::MAX,
+ None => s.doc = u32::MAX,
}
}
scorers.sort_unstable_by_key(|s| (s.doc, s.ord));
@@ -3340,7 +3338,9 @@ mod tests {
format!("the pad{i} filler{}", i % 7)
};
let fields: HashMap<String, FieldValues> =
- [("body".to_owned(), FieldValues::from(text))].into_iter().collect();
+ [("body".to_owned(), FieldValues::from(text))]
+ .into_iter()
+ .collect();
writer.add_document(i, &fields);
}
writer.finish().unwrap();
@@ -3462,11 +3462,7 @@ mod tests {
assert!(!full.is_empty(), "{name}: fixture must match something");
for cap in [0usize, 1, 2, 3, 10, full.len(), full.len() + 5] {
let (bounded, total) = searcher.search_bounded(q, cap, false).unwrap();
- assert_eq!(
- total,
- full.len() as u64,
- "{name}: total at cap={cap}"
- );
+ assert_eq!(total, full.len() as u64, "{name}: total at cap={cap}");
let expected: Vec<(u32, u32)> = full
.iter()
.take(cap)
@@ -3533,7 +3529,10 @@ mod tests {
.iter()
.map(|h| (h.doc_id, h.score.to_bits()))
.collect();
- assert_eq!(got, expected, "top-{cap} with duplicate/boosted/absent clauses");
+ assert_eq!(
+ got, expected,
+ "top-{cap} with duplicate/boosted/absent clauses"
+ );
}
}
@@ -3571,9 +3570,14 @@ mod tests {
// Terms of full df (the "stopwords"), a graded mid-df term, and
// unique padding for realistic lengths.
let mid = if i % 4 == 0 { "mid" } else { "pad" };
- let text = format!("{} {mid} u{i}", stops[..nclauses.min(stops.len())].join(" "));
+ let text = format!(
+ "{} {mid} u{i}",
+ stops[..nclauses.min(stops.len())].join(" ")
+ );
let fields: HashMap<String, FieldValues> =
- [("body".to_owned(), FieldValues::from(text))].into_iter().collect();
+ [("body".to_owned(), FieldValues::from(text))]
+ .into_iter()
+ .collect();
writer.add_document(i, &fields);
}
writer.finish().unwrap();
@@ -3593,7 +3597,9 @@ mod tests {
let _ = searcher.search_bounded(&q, cap, false).unwrap();
wand_us += t0.elapsed().as_secs_f64() * 1e6;
}
- let (g_hits, g_total) = searcher.search_bounded_observed(&q, cap, false, |_| {}).unwrap();
+ let (g_hits, g_total) = searcher
+ .search_bounded_observed(&q, cap, false, |_| {})
+ .unwrap();
let mut gen_us = 0f64;
for _ in 0..REPS {
let t0 = std::time::Instant::now();
@@ -3605,8 +3611,14 @@ mod tests {
let (w_hits, w_total) = searcher.search_bounded(&q, cap, false).unwrap();
assert_eq!(w_total, g_total);
assert_eq!(
- w_hits.iter().map(|h| (h.doc_id, h.score.to_bits())).collect::<Vec<_>>(),
- g_hits.iter().map(|h| (h.doc_id, h.score.to_bits())).collect::<Vec<_>>(),
+ w_hits
+ .iter()
+ .map(|h| (h.doc_id, h.score.to_bits()))
+ .collect::<Vec<_>>(),
+ g_hits
+ .iter()
+ .map(|h| (h.doc_id, h.score.to_bits()))
+ .collect::<Vec<_>>(),
);
println!(
"{nclauses}-stopword disjunction over {N} docs (df={N} each), cap={cap}: \
diff --git a/engine/crates/xerj-rerank/src/lib.rs b/engine/crates/xerj-rerank/src/lib.rs
index 5702f6788..86ab036c4 100644
--- a/engine/crates/xerj-rerank/src/lib.rs
+++ b/engine/crates/xerj-rerank/src/lib.rs
@@ -552,7 +552,6 @@ impl RerankConfig {
Ok(cfg)
}
-
}
/// Truncate on a char boundary. Byte slicing a `&str` here would panic on any