Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(api): #310 pierce — resolve a named companion through fields under default _source (WIP)

xerj-org50913f5815c7
diff --git a/engine/crates/xerj-api/src/es_compat.rs b/engine/crates/xerj-api/src/es_compat.rs
index add4719db..27f3bed72 100644
--- a/engine/crates/xerj-api/src/es_compat.rs
+++ b/engine/crates/xerj-api/src/es_compat.rs
@@ -5992,6 +5992,47 @@ fn parse_time_to_millis(s: &str) -> Option<u64> {
Some(n.saturating_mul(mul_ms).max(1))
}
+/// Every field name this request pulls a VALUE out of the stored document for
+/// by a route other than `_source` — the `fields` clause and `docvalue_fields`.
+///
+/// Used once per request, before the engine runs, to decide whether the default
+/// `_source` projection has to be pierced (#310). It deliberately re-parses the
+/// two clauses instead of the response-time lists built later in `search_impl`:
+/// the decision has to be taken BEFORE the search, and the URL-param forms
+/// (`?fields=`, `?docvalue_fields=`) have already been promoted into `body` by
+/// then, so this sees exactly what the response layer will.
+fn value_bearing_field_names(body: &EsSearchBody, fields: &[String]) -> Vec<String> {
+ fn push_names(v: Option<&Value>, out: &mut Vec<String>) {
+ match v {
+ Some(Value::Array(arr)) => {
+ for entry in arr {
+ match entry {
+ Value::String(s) => out.push(s.clone()),
+ Value::Object(o) => {
+ if let Some(name) = o.get("field").and_then(Value::as_str) {
+ out.push(name.to_string());
+ }
+ }
+ _ => {}
+ }
+ }
+ }
+ Some(Value::String(s)) => out.push(s.clone()),
+ _ => {}
+ }
+ }
+ let mut names = Vec::new();
+ match &body.fields {
+ Some(v @ (Value::Array(_) | Value::String(_))) => push_names(Some(v), &mut names),
+ // Same fallback the response-time `field_specs` builder takes: anything
+ // else (absent, or a shape the body parser does not recognise) leaves
+ // the typed request's list.
+ _ => names.extend(fields.iter().cloned()),
+ }
+ push_names(body.docvalue_fields.as_ref(), &mut names);
+ names
+}
+
/// Build a `SearchRequest` from the ES body, forwarding all relevant options.
fn build_search_request(
body: &EsSearchBody,
@@ -10784,7 +10825,7 @@ async fn search_impl(
.filter(|n| !value_type_failed_indices.contains(*n))
.collect();
- let search_req = match build_search_request(&body, aggs_value) {
+ let mut search_req = match build_search_request(&body, aggs_value) {
Ok(mut r) => {
// ES 7.16 leaf ordering for the implicit auto-@timestamp hint
// (see the injection block above) — internal field, never on
@@ -10798,6 +10839,57 @@ async fn search_impl(
Err(e) => return ApiError::new(e).into_response(),
};
+ // ── The default `_source` projection, and who has to see past it (#310) ──
+ //
+ // Since #309 a request with no `_source` clause gets `SourceFilter::Default`
+ // and the engine drops the generated embedding companions before the
+ // response layer ever sees a hit. But `_source` is not the only consumer of
+ // the stored document: `fields` and `docvalue_fields` resolve their values
+ // out of that same `hit.source`, AFTER the projection has narrowed it. So
+ // `{"fields": ["body_vector"]}` came back with nothing — silently, because an
+ // unresolvable `fields` entry is legally omitted — while the very same clause
+ // under `"_source": false` returned the whole vector, since `Enabled(false)`
+ // deliberately keeps the raw source for exactly this resolution. Two spellings
+ // of "I do not want `_source`", opposite answers.
+ //
+ // The fix: if a caller named a companion in `fields`/`docvalue_fields` and
+ // wrote no `_source` clause, ask the engine for the intact source and narrow
+ // it back to the default at each emission site below. Wire bytes are
+ // unchanged; the caller gets the value they explicitly asked for. Decided in
+ // the API (the only layer that sees the `fields` clause), not the engine —
+ // teaching the engine to treat `Default` as `Enabled(true)` would put
+ // companions back into every internal `SearchRequest::default()`.
+ let default_source_projection = body.source.is_none();
+ let value_bearing_names = if default_source_projection {
+ value_bearing_field_names(&body, &search_req.fields)
+ } else {
+ Vec::new()
+ };
+ // One schema read-lock per participating index, and only for a request that
+ // could actually consume the answer.
+ let mut companions_by_index: HashMap<String, HashSet<String>> = HashMap::new();
+ if default_source_projection && (!value_bearing_names.is_empty() || body.collapse.is_some()) {
+ for idx_name in &index_names {
+ let Ok(idx) = state.engine.get_index(idx_name) else {
+ continue;
+ };
+ let companions = idx.embedding_companion_fields().await;
+ if !companions.is_empty() {
+ companions_by_index.insert((*idx_name).to_string(), companions);
+ }
+ }
+ }
+ // The pierce, and the obligation it creates: every site that emits a
+ // `_source` for this response must undo it (top-level hit, collapse
+ // `inner_hits`, scroll snapshot).
+ let pierce_default_source = value_bearing_names
+ .iter()
+ .any(|f| companions_by_index.values().any(|c| c.contains(f)));
+ if pierce_default_source {
+ search_req.source = xerj_query::ast::SourceFilter::Enabled(true);
+ }
+ let search_req = search_req;
+
// Execute search on each index and merge results.
let mut merged_hits: Vec<(String, xerj_query::executor::Hit)> = Vec::new(); // (index_name, hit)
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index b68e9f9a3..a3601e9f7 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -8405,6 +8405,20 @@ impl Index {
/// RAM. This keeps memory usage bounded regardless of dataset size.
///
/// Search reads: active memtable (RAM) + disk segments (file I/O).
+ /// The engine-generated embedding companion field names for this index —
+ /// exactly the keys the default `_source` projection omits (#309).
+ ///
+ /// Published for the API layer, which has to know the same set for a reason
+ /// the engine cannot serve: only the API sees the `fields`/`docvalue_fields`
+ /// clauses, and only it can decide that a caller who named a companion there
+ /// must get the intact source from the engine and the narrowed one on the
+ /// wire (#310). The mapping rule stays in `generated_embedding_companion_fields`;
+ /// this is a lock-scoped accessor for it, not a second implementation.
+ pub async fn embedding_companion_fields(&self) -> HashSet<String> {
+ let schema = self.schema.read().await;
+ generated_embedding_companion_fields(&schema.schema)
+ }
+
pub async fn refresh(&self) -> Result<()> {
{
let mem = &*self.memtable;