AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
wip(second-brain/core): engine graph traversal + graph_api + stage-0 cleanup
xerj-orgd39ccd8ec0b7
diff --git a/engine/crates/xerj-ai/src/lib.rs b/engine/crates/xerj-ai/src/lib.rsindex 9e3e3c134..fa4b0f91c 100644--- a/engine/crates/xerj-ai/src/lib.rs+++ b/engine/crates/xerj-ai/src/lib.rs@@ -11,13 +11,17 @@//! - `onnx` — Experimental MiniLM-compatible FP32 ONNX backend//! (feature `onnx-experimental`; server feature + explicit runtime selection required)//! - [`chunker`] — Text chunking with sentence-aware splitting and overlap-//! - [`memory`] — Agent memory: semantic dedup + recency-blended recall+//!+//! Agent memory does NOT live here. The real memory store is index-backed+//! (`/_memory/*` in xerj-api over ordinary XERJ indices — durable, WAL-replayed,+//! kNN/BM25-searchable). An earlier in-RAM `AgentMemory` (O(N²) dedup scan, no+//! durability, process-lifetime only) was deleted once it reached zero callers:+//! keeping it around shadowed the real API and invited accidental use.pub mod chunker;pub mod embed;pub mod embedder;pub mod local;-pub mod memory;#[cfg(feature = "neural")]pub mod neural;#[cfg(feature = "onnx-experimental")]@@ -27,6 +31,5 @@ pub use chunker::{Chunk, TextChunker};pub use embed::{EmbeddingProxy, EmbeddingProxyConfig};pub use embedder::Embedder;pub use local::{local_embed, DEFAULT_DIMS};-pub use memory::{AgentMemory, MemoryEntry};pub use xerj_common::Result;diff --git a/engine/crates/xerj-ai/src/memory.rs b/engine/crates/xerj-ai/src/memory.rsdeleted file mode 100644index 2f26e0dca..000000000--- a/engine/crates/xerj-ai/src/memory.rs+++ /dev/null@@ -1,327 +0,0 @@-//! Agent memory: append-only semantic memory with dedup and recency blending.-//!-//! Stores text snippets with embeddings. On recall, blends cosine similarity-//! with recency using a configurable weight, so recently stored memories are-//! surfaced even if slightly less semantically similar than older ones.--use serde::{Deserialize, Serialize};-use serde_json::Value;-use std::sync::Mutex;-use tracing::debug;-use xerj_common::XerjError;--/// Result alias.-pub type Result<T> = std::result::Result<T, XerjError>;--// ─────────────────────────────────────────────────────────────────────────────-// MemoryEntry-// ─────────────────────────────────────────────────────────────────────────────--/// A stored memory entry.-#[derive(Debug, Clone, Serialize, Deserialize)]-pub struct MemoryEntry {- /// Unique ID (monotonically assigned).- pub id: u64,- /// The text content.- pub text: String,- /// Arbitrary metadata (tags, source, etc.).- pub metadata: Value,- /// Dense embedding vector.- pub embedding: Vec<f32>,- /// Unix timestamp (seconds) when this was stored.- pub stored_at: i64,-}--impl MemoryEntry {- /// Blended score combining similarity and recency.- ///- /// `recency_weight` ∈ [0, 1]:- /// - 0.0 → pure semantic similarity- /// - 1.0 → pure recency- pub fn score(&self, similarity: f32, now_secs: i64, recency_weight: f32) -> f32 {- let age_secs = (now_secs - self.stored_at).max(0) as f32;- // Decay: e^(-age / half_life) where half_life = 7 days in seconds- let half_life = 7.0 * 86_400.0f32;- let recency = (-age_secs / half_life).exp();-- let semantic = (1.0 - recency_weight).max(0.0);- let recency_w = recency_weight.min(1.0);-- similarity * semantic + recency * recency_w- }-}--// ─────────────────────────────────────────────────────────────────────────────-// Similarity helpers-// ─────────────────────────────────────────────────────────────────────────────--fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {- let mut dot = 0.0f32;- let mut na = 0.0f32;- let mut nb = 0.0f32;- for (&ai, &bi) in a.iter().zip(b.iter()) {- dot += ai * bi;- na += ai * ai;- nb += bi * bi;- }- let denom = (na * nb).sqrt();- if denom == 0.0 {- 0.0- } else {- dot / denom- }-}--// ─────────────────────────────────────────────────────────────────────────────-// AgentMemory-// ─────────────────────────────────────────────────────────────────────────────--/// Append-only in-memory store with semantic dedup.-///-/// Store: embeds the text, checks similarity against existing entries, and-/// skips insertion if the closest match exceeds the dedup threshold.-///-/// Recall: scores all entries by blended similarity + recency, returns top-k.-///-/// # Note on embeddings-///-/// `AgentMemory` stores pre-computed embeddings. The caller is responsible for-/// embedding the text (e.g., via [`EmbeddingProxy`]) before calling-/// [`store_embedded`] or [`store`] with a pre-computed vector.-pub struct AgentMemory {- entries: Mutex<Vec<MemoryEntry>>,- next_id: Mutex<u64>,- /// Cosine similarity threshold above which a new entry is considered- /// duplicate and discarded.- dedup_threshold: f32,-}--impl AgentMemory {- /// Create a new memory store with the given dedup threshold.- ///- /// `dedup_threshold` ∈ [0, 1]: entries with similarity above this value- /// are considered duplicates. Recommended: 0.95.- pub fn new(dedup_threshold: f32) -> Self {- Self {- entries: Mutex::new(Vec::new()),- next_id: Mutex::new(0),- dedup_threshold,- }- }-- /// Store a text with a pre-computed embedding.- ///- /// Returns `None` if the entry was deduplicated (too similar to existing).- pub fn store_embedded(- &self,- text: impl Into<String>,- embedding: Vec<f32>,- metadata: Value,- stored_at_secs: i64,- ) -> Option<u64> {- let text = text.into();- let mut entries = self.entries.lock().unwrap();-- // Semantic dedup check- for existing in entries.iter() {- if existing.embedding.len() == embedding.len() {- let sim = cosine_similarity(&existing.embedding, &embedding);- if sim >= self.dedup_threshold {- debug!(- "memory dedup: skipping '{}' (sim={:.3} vs '{}')",- &text[..text.len().min(40)],- sim,- &existing.text[..existing.text.len().min(40)]- );- return None;- }- }- }-- let id = {- let mut next = self.next_id.lock().unwrap();- let id = *next;- *next += 1;- id- };-- entries.push(MemoryEntry {- id,- text,- metadata,- embedding,- stored_at: stored_at_secs,- });-- Some(id)- }-- /// Recall the top-k most relevant memories for a query embedding.- ///- /// `recency_weight` ∈ [0, 1]:- /// - 0.0 → pure semantic similarity ordering- /// - 1.0 → pure recency ordering- pub fn recall(- &self,- query_embedding: &[f32],- k: usize,- recency_weight: f32,- now_secs: i64,- ) -> Vec<MemoryEntry> {- let entries = self.entries.lock().unwrap();-- let mut scored: Vec<(f32, &MemoryEntry)> = entries- .iter()- .filter_map(|e| {- if e.embedding.len() != query_embedding.len() {- return None;- }- let sim = cosine_similarity(query_embedding, &e.embedding);- let score = e.score(sim, now_secs, recency_weight);- Some((score, e))- })- .collect();-- // Sort descending by score- scored.sort_unstable_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));-- scored.into_iter().take(k).map(|(_, e)| e.clone()).collect()- }-- /// Number of stored entries.- pub fn len(&self) -> usize {- self.entries.lock().unwrap().len()- }-- pub fn is_empty(&self) -> bool {- self.len() == 0- }-- /// Clear all stored memories.- pub fn clear(&self) {- self.entries.lock().unwrap().clear();- *self.next_id.lock().unwrap() = 0;- }-- /// Return all stored entries (for inspection/serialization).- pub fn all_entries(&self) -> Vec<MemoryEntry> {- self.entries.lock().unwrap().clone()- }-}--// ─────────────────────────────────────────────────────────────────────────────-// Tests-// ─────────────────────────────────────────────────────────────────────────────--#[cfg(test)]-mod tests {- use super::*;- use serde_json::json;-- fn vec_a() -> Vec<f32> {- vec![1.0, 0.0, 0.0, 0.0]- }- fn vec_b() -> Vec<f32> {- vec![0.0, 1.0, 0.0, 0.0]- }- fn vec_c() -> Vec<f32> {- vec![0.0, 0.0, 1.0, 0.0]- }- // Very similar to vec_a- fn vec_a_near() -> Vec<f32> {- vec![0.999, 0.001, 0.0, 0.0]- }-- #[test]- fn store_and_recall_basic() {- let mem = AgentMemory::new(0.95);- let now = 1_700_000_000i64;-- mem.store_embedded("memory about cats", vec_a(), json!({}), now);- mem.store_embedded("memory about dogs", vec_b(), json!({}), now);- mem.store_embedded("memory about fish", vec_c(), json!({}), now);-- assert_eq!(mem.len(), 3);-- let results = mem.recall(&vec_a(), 1, 0.0, now);- assert_eq!(results.len(), 1);- assert_eq!(results[0].text, "memory about cats");- }-- #[test]- fn dedup_skips_near_duplicate() {- let mem = AgentMemory::new(0.95);- let now = 1_700_000_000i64;-- let id1 = mem.store_embedded("cats", vec_a(), json!({}), now);- let id2 = mem.store_embedded("cats (near duplicate)", vec_a_near(), json!({}), now);-- assert!(id1.is_some());- assert!(id2.is_none(), "near-duplicate should be deduped");- assert_eq!(mem.len(), 1);- }-- #[test]- fn dedup_allows_distinct_entries() {- let mem = AgentMemory::new(0.95);- let now = 1_700_000_000i64;-- mem.store_embedded("cats", vec_a(), json!({}), now);- mem.store_embedded("dogs", vec_b(), json!({}), now);-- assert_eq!(mem.len(), 2);- }-- #[test]- fn recency_weight_affects_ordering() {- let mem = AgentMemory::new(0.5);- let base = 1_700_000_000i64;-- // Store old entry with matching vector and recent entry with different vector- mem.store_embedded("old cats", vec_a(), json!({}), base - 30 * 86_400); // 30 days ago- mem.store_embedded("recent dogs", vec_b(), json!({}), base); // now-- // Query close to vec_a- let query = vec![0.9, 0.1, 0.0, 0.0f32];-- // Pure similarity: old cats should rank first (closer to query)- let by_sim = mem.recall(&query, 2, 0.0, base);- assert_eq!(by_sim[0].text, "old cats");-- // Pure recency: recent dogs should rank first- let by_recency = mem.recall(&query, 2, 1.0, base);- assert_eq!(by_recency[0].text, "recent dogs");- }-- #[test]- fn recall_empty_memory() {- let mem = AgentMemory::new(0.95);- let results = mem.recall(&vec_a(), 5, 0.5, 1_700_000_000);- assert!(results.is_empty());- }-- #[test]- fn clear_resets_state() {- let mem = AgentMemory::new(0.95);- mem.store_embedded("a", vec_a(), json!({}), 1_700_000_000);- mem.clear();- assert_eq!(mem.len(), 0);- }-- #[test]- fn ids_are_unique() {- // Use threshold 2.0 (above max cosine similarity of 1.0) to disable dedup- let mem = AgentMemory::new(2.0);- let ids: Vec<u64> = (0..5)- .map(|i| {- // Each vector points in a completely different direction- let mut v = vec![0.0f32; 5];- v[i] = 1.0;- mem.store_embedded(format!("entry {i}"), v, json!({}), 1_700_000_000)- .unwrap()- })- .collect();- let unique: std::collections::HashSet<u64> = ids.into_iter().collect();- assert_eq!(unique.len(), 5);- }-}diff --git a/engine/crates/xerj-engine/src/graph.rs b/engine/crates/xerj-engine/src/graph.rsnew file mode 100644index 000000000..9d193f07a--- /dev/null+++ b/engine/crates/xerj-engine/src/graph.rs@@ -0,0 +1,649 @@+//! Graph expansion (second brain) — batched, columnar, bounded.+//!+//! A relationship layer over documents that already exist. Edges are ordinary+//! documents in a reserved edges index (schema: SECOND_BRAIN contract §2); this+//! module is the ONLY engine-side traversal over them. It is deliberately not a+//! graph database: no query language, no shortest-path, no PageRank, and no+//! unbounded traversal — expansion is a bounded, batched read (≤2 hops per+//! call) that composes client-side by feeding `reachable` back in as the next+//! call's frontier.+//!+//! ## Why a column scan instead of per-node queries+//!+//! Expanding a frontier of F nodes as F term queries costs F query setups and F+//! segment walks. Instead, each hop resolves the WHOLE frontier to per-segment+//! keyword ordinals (one O(log T) FST probe per frontier id via+//! [`xerj_storage::doc_values::KeywordColumn::ord_for_term`]) and then makes+//! ONE pass over the segment's `src` ordinal array (plus the `dst` array for+//! direction `both`) with O(1) hash-set membership per row — cost is+//! frontier-size-independent. Everything an admitted edge needs (`edge_id`,+//! `src`, `dst`, `type`, `weight`, `valid_at`, `invalid_at`) is random-accessed+//! from doc-values columns; the hop path NEVER touches `_source` (no+//! `StoredSlices`, no JSON parse, no hydration — evidence is hydrated by the+//! HTTP layer AFTER traversal, on the bounded result, via an `ids` query).+//!+//! ## Correctness sources (same machinery the columnar search paths use)+//!+//! - Segment list: [`Index::store_snapshot`] once per request.+//! - Columns: `Index::dv_columns_for` (cached per-segment sidecar).+//! - Liveness under updates/deletes: `VersionMap::ghost_events` gate ++//! `Index::ghost_positions_for` word-bitmaps — the exact source of truth+//! `scored_columnar` uses, so a soft-invalidated edge re-indexed under the+//! same `_id` never double-surfaces from its stale segment row.+//! - Unflushed edges: a bounded walk of the sharded FTS memtable+//! (`ShardedFtsMemtable::all_docs_with_sources_arc`) per hop — correctness,+//! not speed; the memtable is bounded by the flush threshold and the walk is+//! reported in-band (`memtable_docs_scanned`).+//!+//! ## Honesty in-band+//!+//! Every response carries [`GraphExpandStats`]: clipped frontiers, clipped+//! edges, bi-temporally excluded edges, type-filtered edges, segments skipped+//! for missing columns (the hop never silently falls back to hydration —+//! skipped means reported), and scan-cost counters.++use std::collections::{HashMap, HashSet};++use serde_json::Value;+use xerj_common::XerjError;++use super::Index;+use xerj_storage::doc_values::{Column, KeywordColumn, NumericColumn};++/// Hard ceiling on expansion depth. Depth beyond this composes client-side:+/// feed `reachable` back in as the next call's frontier.+pub const GRAPH_MAX_HOPS: u8 = 2;+/// Frontier ids admitted per hop; excess is dropped and counted+/// (`stats.frontier_clipped`). 4096 ords is a ~32KB hash set — cheap — while+/// bounding the per-segment FST-probe phase.+pub const GRAPH_MAX_FRONTIER: usize = 4096;+/// Result-edge ceiling per request; collection stops (scan continues to the+/// end of the current segment, then aborts remaining segments) and+/// `stats.edges_clipped` reports the overflow.+pub const GRAPH_MAX_RESULT_EDGES: usize = 100_000;++/// The not-a-graph-database refusal, verbatim (contract §4.6). Shared with the+/// HTTP layer so the engine error reason and the API 400 body carry the exact+/// same sentence — the wording is normative, not cosmetic: it tells the caller+/// the supported alternative (iterate on `reachable`).+pub const GRAPH_HOPS_CAP_REASON: &str = "hops is capped at 2: XERJ's second brain is a \+ relationship layer over documents, not a graph database (no Cypher, no shortest-path, \+ no variable-depth traversal). Iterate: expand again from this response's 'reachable' ids.";++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum GraphDirection {+ /// Follow src → dst (scan the `src` column).+ Out,+ /// Follow dst → src (scan the `dst` column).+ In,+ /// Both directions (two column scans per segment). The default everywhere.+ Both,+}++#[derive(Debug, Clone)]+pub struct GraphExpandRequest {+ /// Node ids to expand from. Deduped internally; clipped to+ /// GRAPH_MAX_FRONTIER (clip counted, never silent).+ pub frontier: Vec<String>,+ /// 1 or 2. 0 or >2 → XerjError::InvalidQuery (see module docs).+ pub hops: u8,+ pub direction: GraphDirection,+ /// Edge-type allowlist; None = all types.+ pub types: Option<Vec<String>>,+ /// Bi-temporal cut: an edge is visible iff+ /// valid_at <= as_of_ms AND (invalid_at absent OR invalid_at > as_of_ms).+ pub as_of_ms: i64,+ /// true = ignore invalid_at (return soft-deleted edges too; used by+ /// ego?include_expired=true). Excluded edges are counted either way.+ pub include_expired: bool,+ /// Per-request result cap, clamped to 1..=GRAPH_MAX_RESULT_EDGES.+ pub max_result_edges: usize,+}++/// One edge as read ENTIRELY from doc-values columns — no `_source` access.+/// `weight` is f64 because numeric columns store f64 bits (see+/// `build_doc_value_columns`); timestamps are exact for |v| < 2^53 (epoch-ms+/// until year ~287396).+#[derive(Debug, Clone)]+pub struct GraphEdgeLite {+ pub edge_id: String,+ pub src: String,+ pub dst: String,+ pub edge_type: String,+ pub weight: f64,+ pub valid_at_ms: i64,+ pub invalid_at_ms: Option<i64>,+ /// Which hop discovered this edge (1 or 2).+ pub hop: u8,+}++/// In-band honesty: what the expansion did NOT show, and what it cost.+#[derive(Debug, Clone, Default, PartialEq, Eq)]+pub struct GraphExpandStats {+ pub frontier_clipped: u64,+ pub edges_clipped: u64,+ /// Edges skipped by the bi-temporal cut (valid_at in the future of as_of,+ /// or invalidated at/before as_of) — NOT counting include_expired=true.+ pub expired_excluded: u64,+ /// Edges skipped by the `types` allowlist.+ pub type_filtered: u64,+ pub segments_scanned: u64,+ /// Segments skipped because they lack a dv sidecar or a `src`/`dst`+ /// keyword column (legacy segment / meta-only segment / poisoned column).+ /// The hop NEVER falls back to hydration — skipped means reported.+ pub segments_without_columns: u64,+ pub memtable_docs_scanned: u64,+}++#[derive(Debug, Clone)]+pub struct GraphExpandResult {+ /// Deduped by edge_id, sorted (hop asc, weight desc, edge_id asc) — the+ /// stable order every response surface and the fixture assert on.+ pub edges: Vec<GraphEdgeLite>,+ /// Frontier ids (that were admitted) first, in input order, then newly+ /// reached ids in first-discovery order following the sorted edge list.+ /// Deduped.+ pub reachable: Vec<String>,+ pub stats: GraphExpandStats,+}++/// Get a keyword column by name, or `None` when absent or numeric-typed+/// (numeric-typed = a writer violated the §2.2 type discipline; the segment is+/// treated as column-less rather than guessed at).+fn kw_col<'a>(cols: &'a super::DocValueMap, name: &str) -> Option<&'a KeywordColumn> {+ match cols.get(name) {+ Some(Column::Keyword(k)) => Some(k),+ _ => None,+ }+}++/// Get a numeric column by name (same honesty rule as [`kw_col`]).+fn num_col<'a>(cols: &'a super::DocValueMap, name: &str) -> Option<&'a NumericColumn> {+ match cols.get(name) {+ Some(Column::Numeric(n)) => Some(n),+ _ => None,+ }+}++/// Numeric columns store f64 BITS in an i64 slot (one representation for the+/// whole column, integers exact to 2^53). Decode before use — using the raw+/// i64 silently corrupts every timestamp/weight compare.+#[inline]+fn bits_to_f64(v: i64) -> f64 {+ f64::from_bits(v as u64)+}++/// Epoch-ms from a memtable JSON number. Writers emit integer epoch-ms, but a+/// float that reaches us (e.g. a client JSON-encoding 1.7536e12) still means+/// the same instant, so accept it rather than mis-classifying the row.+#[inline]+fn json_ms(v: &Value) -> Option<i64> {+ v.as_i64().or_else(|| v.as_f64().map(|f| f as i64))+}++/// Bi-temporal visibility (contract §3.2). Returns `false` (and the caller+/// counts `expired_excluded`) for edges asserted after `as_of` or — unless+/// `include_expired` — invalidated at/before `as_of`. `include_expired` only+/// disables the invalidation half: a future-asserted edge is invisible either+/// way (it did not exist yet in the world being asked about).+#[inline]+fn bitemporal_visible(+ valid_at_ms: i64,+ invalid_at_ms: Option<i64>,+ as_of_ms: i64,+ include_expired: bool,+) -> bool {+ if valid_at_ms > as_of_ms {+ return false;+ }+ if include_expired {+ return true;+ }+ match invalid_at_ms {+ Some(inv) => inv > as_of_ms,+ None => true,+ }+}++/// Collected-edge bookkeeping: `from_memtable` implements the dedupe rule that+/// a memtable row wins over any segment row with the same edge_id (it is the+/// newer version — relevant only in the conservative ghost-bitmap-unavailable+/// path; when ghost bitmaps resolve, the stale segment row never surfaces).+struct Collected {+ edge: GraphEdgeLite,+ from_memtable: bool,+}++impl Index {+ /// Batched graph expansion over an edges index (contract §2 schema).+ ///+ /// Sync fn (no await points): all reads are memtable walks + doc-values+ /// column access — the same blocking pattern the columnar agg/search fast+ /// paths already use from async handlers.+ ///+ /// Explicitly forbidden on this path: `StoredSlices` access, JSON parsing+ /// of stored docs, hydration helpers, per-frontier-id queries, and any+ /// per-edge `_source` read. Evidence hydration happens AFTER traversal in+ /// the HTTP layer, on the bounded result set, via an `ids` query.+ pub fn graph_expand(&self, req: &GraphExpandRequest) -> xerj_common::Result<GraphExpandResult> {+ if req.hops == 0 || req.hops > GRAPH_MAX_HOPS {+ // Same normative sentence for both bounds: the caller's fix+ // (iterate on `reachable`) is identical, and the HTTP layer+ // forwards this reason verbatim as its 400 body.+ let reason = if req.hops == 0 {+ format!("hops must be at least 1. {GRAPH_HOPS_CAP_REASON}")+ } else {+ GRAPH_HOPS_CAP_REASON.to_string()+ };+ return Err(XerjError::invalid_query(reason));+ }++ let mut stats = GraphExpandStats::default();++ // Dedupe the request frontier preserving input order (the order is+ // part of the `reachable` contract), then clip with the clip counted.+ let mut frontier: Vec<String> = Vec::with_capacity(req.frontier.len().min(16));+ {+ let mut seen: HashSet<&str> = HashSet::with_capacity(req.frontier.len());+ for id in &req.frontier {+ if seen.insert(id.as_str()) {+ frontier.push(id.clone());+ }+ }+ }+ if frontier.is_empty() {+ return Err(XerjError::invalid_query(+ "graph expansion requires a non-empty frontier (after dedup)",+ ));+ }+ if frontier.len() > GRAPH_MAX_FRONTIER {+ stats.frontier_clipped += (frontier.len() - GRAPH_MAX_FRONTIER) as u64;+ frontier.truncate(GRAPH_MAX_FRONTIER);+ }+ let admitted_frontier = frontier.clone();++ let cap = req.max_result_edges.clamp(1, GRAPH_MAX_RESULT_EDGES);+ let scan_src = matches!(req.direction, GraphDirection::Out | GraphDirection::Both);+ let scan_dst = matches!(req.direction, GraphDirection::In | GraphDirection::Both);++ // Point-in-time capture once per request: segment list + "any ghosts?"+ // gate — the same pair `scored_columnar` snapshots, so liveness+ // decisions are consistent across every segment of this expansion.+ let snapshot = self.store_snapshot();+ let segments_dir = self.data_dir.join("segments");+ let any_ghosts = self.store.version_map.ghost_events() > 0;++ // Every node ever admitted to a frontier or discovered as an endpoint;+ // hop-2 frontier construction subtracts it ("minus already-visited").+ let mut visited: HashSet<String> = frontier.iter().cloned().collect();+ let mut collected: HashMap<String, Collected> = HashMap::new();+ // True once the result cap overflowed: the current container finishes+ // (counting overflow into `edges_clipped`) and everything after —+ // remaining segments, the memtable walk, further hops — is aborted.+ let mut capped = false;++ for hop in 1..=req.hops {+ if capped || frontier.is_empty() {+ break;+ }+ let frontier_set: HashSet<&str> = frontier.iter().map(String::as_str).collect();+ // edge_ids first collected during THIS hop, for next-frontier+ // construction (order does not matter here; the walk below sorts).+ let mut hop_new: Vec<String> = Vec::new();++ // ── Segments: one columnar pass per segment for the whole frontier ──+ for meta in &snapshot.segments {+ if capped {+ break;+ }+ let Some(cols) = self.dv_columns_for(&segments_dir, &meta.id) else {+ stats.segments_without_columns += 1;+ continue;+ };+ let cols = cols.value();+ // Required columns. Missing/mistyped `src`/`dst` (needed for+ // the scan AND for emission) or `edge_id`/`type`/`weight`/+ // `valid_at` (needed for emission/filtering) → the segment is+ // schema-corrupt or edge-free (e.g. meta-doc-only): skip it,+ // reported — never guess, never hydrate. A missing+ // `invalid_at` column is NORMAL (no edge in this segment was+ // ever invalidated) and means "no edge here is invalidated".+ let (Some(k_src), Some(k_dst), Some(k_edge), Some(k_type)) = (+ kw_col(cols, "src"),+ kw_col(cols, "dst"),+ kw_col(cols, "edge_id"),+ kw_col(cols, "type"),+ ) else {+ stats.segments_without_columns += 1;+ continue;+ };+ let (Some(n_weight), Some(n_valid)) =+ (num_col(cols, "weight"), num_col(cols, "valid_at"))+ else {+ stats.segments_without_columns += 1;+ continue;+ };+ let n_invalid = match cols.get("invalid_at") {+ None => None,+ Some(Column::Numeric(n)) => Some(n),+ // Present but keyword-typed = a writer stored ISO strings;+ // an as-of compare against strings would be a lie.+ Some(Column::Keyword(_)) => {+ stats.segments_without_columns += 1;+ continue;+ }+ };++ // Frontier → per-segment ordinal sets (one O(log T) FST probe+ // per frontier id). Both empty → nothing of the frontier+ // exists in this segment; skip without scanning.+ let src_ords: HashSet<u32> = if scan_src {+ frontier+ .iter()+ .filter_map(|id| k_src.ord_for_term(id))+ .collect()+ } else {+ HashSet::new()+ };+ let dst_ords: HashSet<u32> = if scan_dst {+ frontier+ .iter()+ .filter_map(|id| k_dst.ord_for_term(id))+ .collect()+ } else {+ HashSet::new()+ };+ if src_ords.is_empty() && dst_ords.is_empty() {+ continue;+ }++ // Liveness bitmap (deleted + superseded rows). Unbuildable →+ // conservatively skip the whole segment, reported — admitting+ // possibly-stale rows would silently resurrect overwritten+ // edges (e.g. a soft-invalidation would un-happen).+ let ghosts = if any_ghosts {+ match self.ghost_positions_for(meta.id.as_str(), meta.doc_count) {+ Some(bm) => Some(bm),+ None => {+ stats.segments_without_columns += 1;+ continue;+ }+ }+ } else {+ None+ };++ // Per-segment type-allowlist ordinal set. A term absent from+ // this segment's dictionary matches nothing here.+ let type_ords: Option<HashSet<u32>> = req.types.as_ref().map(|ts| {+ ts.iter()+ .filter_map(|t| k_type.ord_for_term(t))+ .collect::<HashSet<u32>>()+ });++ stats.segments_scanned += 1;++ // THE scan: one pass over the ordinal array(s) for the WHOLE+ // frontier, O(1) membership per row. Direction `both` reads+ // both ords arrays in the same position loop — cost-identical+ // to two passes, and an edge admitted by both sides is+ // processed exactly once.+ let doc_count = k_src.doc_count;+ for pos in 0..doc_count {+ let via_src = scan_src+ && k_src.ord_for(pos).is_some_and(|o| src_ords.contains(&o));+ let admitted = via_src+ || (scan_dst+ && k_dst.ord_for(pos).is_some_and(|o| dst_ords.contains(&o)));+ if !admitted {+ continue;+ }+ // Liveness before anything else (a ghost row is not an+ // edge at all — its live version surfaces elsewhere).+ if let Some(bm) = &ghosts {+ let p = pos as usize;+ if (bm[p / 64] >> (p % 64)) & 1 == 1 {+ continue;+ }+ }+ // Type filter (counted).+ if let Some(t_ords) = &type_ords {+ match k_type.ord_for(pos) {+ Some(o) if t_ords.contains(&o) => {}+ _ => {+ stats.type_filtered += 1;+ continue;+ }+ }+ }+ // Bi-temporal cut (counted). A null valid_at is a+ // schema-corrupt row (writers always emit it); such a row+ // cannot be placed on the timeline, so it is skipped.+ let Some(valid_at_ms) = n_valid.get(pos).map(|v| bits_to_f64(v) as i64)+ else {+ continue;+ };+ let invalid_at_ms = n_invalid+ .and_then(|c| c.get(pos))+ .map(|v| bits_to_f64(v) as i64);+ if !bitemporal_visible(+ valid_at_ms,+ invalid_at_ms,+ req.as_of_ms,+ req.include_expired,+ ) {+ stats.expired_excluded += 1;+ continue;+ }+ // Emit — keyword random access only, still no `_source`.+ let Some(edge_id) = k_edge.ord_for(pos).and_then(|o| k_edge.term_for_ord(o))+ else {+ continue;+ };+ if collected.contains_key(edge_id) {+ continue; // rediscovered (earlier hop or the other column) — no cap cost+ }+ if collected.len() >= cap {+ stats.edges_clipped += 1;+ capped = true; // finish counting this segment, then abort+ continue;+ }+ let (Some(src), Some(dst), Some(edge_type), Some(weight)) = (+ k_src.ord_for(pos).and_then(|o| k_src.term_for_ord(o)),+ k_dst.ord_for(pos).and_then(|o| k_dst.term_for_ord(o)),+ k_type.ord_for(pos).and_then(|o| k_type.term_for_ord(o)),+ n_weight.get(pos).map(bits_to_f64),+ ) else {+ continue;+ };+ let edge_id = edge_id.to_string();+ hop_new.push(edge_id.clone());+ collected.insert(+ edge_id.clone(),+ Collected {+ edge: GraphEdgeLite {+ edge_id,+ src: src.to_string(),+ dst: dst.to_string(),+ edge_type: edge_type.to_string(),+ weight,+ valid_at_ms,+ invalid_at_ms,+ hop,+ },+ from_memtable: false,+ },+ );+ }+ }++ // ── Memtable: unflushed edges (correctness, not speed) ────────────+ // Bounded by the flush threshold; the walk cost is reported+ // in-band. Values come from the in-memory source JSON — that is+ // not `_source` hydration (no segment I/O, no stored-section+ // parse), it IS the memtable's native representation.+ // Once `capped` flips mid-walk the loop still finishes the+ // memtable (it is one container — same rule as a segment):+ // remaining admits fall into the `len() >= cap` arm and are+ // counted into `edges_clipped`.+ if !capped {+ for (id, source) in self.memtable.all_docs_with_sources_arc() {+ stats.memtable_docs_scanned += 1;+ // Tombstoned (deleted-by-API) rows are not edges.+ if self+ .store+ .version_map+ .get(&id)+ .is_some_and(|v| v.deleted)+ {+ continue;+ }+ let src = source.get("src").and_then(Value::as_str);+ let dst = source.get("dst").and_then(Value::as_str);+ let via_src = scan_src && src.is_some_and(|s| frontier_set.contains(s));+ let admitted = via_src+ || (scan_dst && dst.is_some_and(|d| frontier_set.contains(d)));+ if !admitted {+ continue;+ }+ // Emission needs both endpoints + the envelope fields; a+ // row missing any of them (only the brain-meta doc or a+ // non-§2 writer) is not an edge.+ let (Some(src), Some(dst)) = (src, dst) else {+ continue;+ };+ let Some(edge_id) = source.get("edge_id").and_then(Value::as_str) else {+ continue;+ };+ let Some(edge_type) = source.get("type").and_then(Value::as_str) else {+ continue;+ };+ if let Some(allow) = &req.types {+ if !allow.iter().any(|t| t == edge_type) {+ stats.type_filtered += 1;+ continue;+ }+ }+ let Some(valid_at_ms) = source.get("valid_at").and_then(json_ms) else {+ continue;+ };+ let invalid_at_ms = source.get("invalid_at").and_then(json_ms);+ if !bitemporal_visible(+ valid_at_ms,+ invalid_at_ms,+ req.as_of_ms,+ req.include_expired,+ ) {+ stats.expired_excluded += 1;+ continue;+ }+ let Some(weight) = source.get("weight").and_then(Value::as_f64) else {+ continue;+ };+ let edge = GraphEdgeLite {+ edge_id: edge_id.to_string(),+ src: src.to_string(),+ dst: dst.to_string(),+ edge_type: edge_type.to_string(),+ weight,+ valid_at_ms,+ invalid_at_ms,+ hop,+ };+ match collected.get_mut(edge_id) {+ Some(existing) => {+ // Memtable wins over a segment row with the same+ // edge_id (it is the newer version); the earliest+ // discovery hop is kept so the sort/frontier+ // semantics don't shift.+ if !existing.from_memtable {+ let keep_hop = existing.edge.hop.min(hop);+ existing.edge = GraphEdgeLite {+ hop: keep_hop,+ ..edge+ };+ existing.from_memtable = true;+ }+ }+ None => {+ if collected.len() >= cap {+ stats.edges_clipped += 1;+ capped = true;+ continue;+ }+ hop_new.push(edge.edge_id.clone());+ collected.insert(+ edge.edge_id.clone(),+ Collected {+ edge,+ from_memtable: true,+ },+ );+ }+ }+ }+ }++ // ── Next frontier: endpoints newly discovered this hop ───────────+ // Walked in the hop's stable order (weight desc, edge_id asc — hop+ // is constant within the block) so a hop-2 frontier clip drops the+ // same ids on every run.+ if hop < req.hops && !capped {+ let mut hop_edges: Vec<&GraphEdgeLite> = hop_new+ .iter()+ .filter_map(|eid| collected.get(eid).map(|c| &c.edge))+ .collect();+ hop_edges.sort_by(|a, b| {+ b.weight+ .total_cmp(&a.weight)+ .then_with(|| a.edge_id.cmp(&b.edge_id))+ });+ let mut next: Vec<String> = Vec::new();+ for e in hop_edges {+ for id in [&e.src, &e.dst] {+ if !visited.contains(id) {+ visited.insert(id.clone());+ next.push(id.clone());+ }+ }+ }+ if next.len() > GRAPH_MAX_FRONTIER {+ stats.frontier_clipped += (next.len() - GRAPH_MAX_FRONTIER) as u64;+ next.truncate(GRAPH_MAX_FRONTIER);+ }+ frontier = next;+ }+ }++ // ── Final order + reachable ──────────────────────────────────────────+ let mut edges: Vec<GraphEdgeLite> =+ collected.into_values().map(|c| c.edge).collect();+ edges.sort_by(|a, b| {+ a.hop+ .cmp(&b.hop)+ .then_with(|| b.weight.total_cmp(&a.weight))+ .then_with(|| a.edge_id.cmp(&b.edge_id))+ });++ let mut reachable: Vec<String> = Vec::with_capacity(admitted_frontier.len() + edges.len());+ let mut seen: HashSet<String> = HashSet::with_capacity(admitted_frontier.len());+ let push_unique = |out: &mut Vec<String>, seen: &mut HashSet<String>, id: &str| {+ if seen.insert(id.to_string()) {+ out.push(id.to_string());+ }+ };+ for id in &admitted_frontier {+ push_unique(&mut reachable, &mut seen, id);+ }+ for e in &edges {+ push_unique(&mut reachable, &mut seen, &e.src);+ push_unique(&mut reachable, &mut seen, &e.dst);+ }++ Ok(GraphExpandResult {+ edges,+ reachable,+ stats,+ })+ }+}diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rsindex e0019ba00..7410ef33f 100644--- a/engine/crates/xerj-engine/src/index.rs+++ b/engine/crates/xerj-engine/src/index.rs@@ -21576,8 +21576,6 @@ fn is_doc_scan_query(q: &QueryNode) -> bool {| QueryNode::SpanFirst { .. }| QueryNode::SpanContaining { .. }| QueryNode::SpanWithin { .. }- | QueryNode::HasChild { .. }- | QueryNode::HasParent { .. }| QueryNode::Constant { .. }| QueryNode::Boosted { .. }| QueryNode::Intervals { .. }@@ -21798,9 +21796,6 @@ fn query_needs_id_injection(q: &QueryNode) -> bool {} => query_needs_id_injection(positive) || query_needs_id_injection(negative),QueryNode::DisMax { queries, .. } => queries.iter().any(query_needs_id_injection),QueryNode::Pinned { organic, .. } => query_needs_id_injection(organic),- QueryNode::HasChild { query, .. } | QueryNode::HasParent { query, .. } => {- query_needs_id_injection(query)- }QueryNode::Knn { filter, .. } | QueryNode::SemanticSearch { filter, .. } => {filter.as_deref().is_some_and(query_needs_id_injection)}@@ -22960,15 +22955,9 @@ fn doc_matches_query(q: &QueryNode, source: &Value) -> bool {.any(|b| littles.iter().any(|l| b.start <= l.start && l.end <= b.end))}- // ── Join queries — UNREACHABLE ─────────────────────────────────────────- // has_child/has_parent are rejected with a 400 at parse time- // (see xerj-query parser.rs::parse_has_child), so these AST variants- // are never built and these branches never execute. Previously they- // silently ran the inner query on the flat doc set, which returned- // wrong results (no parent/child join is materialized). Kept for- // exhaustiveness / a future real join executor.- QueryNode::HasChild { query, .. } => doc_matches_query(query, source),- QueryNode::HasParent { query, .. } => doc_matches_query(query, source),+ // (has_child/has_parent have no AST variants: the parser 400s them, and+ // the flat-match arms that would run the inner query on an unjoined doc+ // set — wrong results — were removed along with the variants.)// ── Geo shape queries ─────────────────────────────────────────────────@@ -30508,3 +30497,14 @@ mod flush_memory_integration_tests {run_post_warm_failure_case("flush-post-warm-panic", true).await;}}++// ─────────────────────────────────────────────────────────────────────────────+// Graph expansion (second brain) — batched, columnar, bounded+// ─────────────────────────────────────────────────────────────────────────────+// Declared as a CHILD of this module (via `#[path]`) rather than a sibling:+// the hop path needs `dv_columns_for`, `ghost_positions_for`, `self.store`,+// `self.memtable`, `self.data_dir` — all private to `index`. A child module+// sees them without widening any visibility; the implementation still lives in+// its own file (src/graph.rs) instead of growing this one.+#[path = "graph.rs"]+pub mod graph;diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rsindex 06d9c0711..be0b21f85 100644--- a/engine/crates/xerj-engine/src/lib.rs+++ b/engine/crates/xerj-engine/src/lib.rs@@ -43,6 +43,10 @@ pub use index::{detect_log_format, resolve_date_math, resolve_field_alias, EnrichTable, FieldEncodingInfo,Index, IndexResponse, IndexStats, LogFormat,};+// Second-brain graph expansion (bounded columnar traversal over an edges+// index). Lives under `index` for private-field access; re-exported at the+// crate root so callers write `xerj_engine::graph::GraphExpandRequest`.+pub use index::graph;pub use memtable::FtsMemtable;// ── Process-global metrics handle ─────────────────────────────────────────────diff --git a/engine/crates/xerj-engine/tests/graph_expand.rs b/engine/crates/xerj-engine/tests/graph_expand.rsnew file mode 100644index 000000000..8aab3b6e3--- /dev/null+++ b/engine/crates/xerj-engine/tests/graph_expand.rs@@ -0,0 +1,378 @@+//! Engine-level tests for `Index::graph_expand` (second brain, contract §3.6).+//!+//! Every test builds the contract §8 fixture — the five `notes/` files' edge+//! set (4 wikilink + 4 same_dir edges, all valid_at = created_at =+//! 1753600000000) — via `index_document` on a `.xerj-memory-notes-edges`+//! index, then asserts the §8.4 expansions:+//!+//! - pre-flush (memtable path) and post-flush (columnar path) results are+//! identical modulo the two flush-state cost counters,+//! - soft invalidation is time-travelable via `as_of` (and exercises the+//! ghost-bitmap integration: the stale pre-invalidation segment row must+//! NOT resurface),+//! - the hop-2 block ordering, the result-edge cap, and the hops bounds.++use serde_json::{json, Value};+use tempfile::TempDir;+use xerj_common::config::Config;+use xerj_engine::graph::{+ GraphDirection, GraphExpandRequest, GraphExpandResult, GRAPH_HOPS_CAP_REASON,+};+use xerj_engine::{Engine, Index};++const EDGES_INDEX: &str = ".xerj-memory-notes-edges";+const T0: i64 = 1_753_600_000_000; // fixture valid_at / created_at+const AS_OF: i64 = 1_753_700_000_000; // fixture as-of instant++// §8.3 edge ids (computed with the contract's edge_id fn; pinned there).+const E_ALPHA_BETA_WIKI: &str = "bef814a75bd3d914c3e561f610154304";+const E_ALPHA_GAMMA_WIKI: &str = "11c2d0ef216cd6e99a3907a0b53c1452";+const E_BETA_GAMMA_WIKI: &str = "9bbf7d2068321ac0fa71d95e21fae2fd";+const E_DELTA_ALPHA_WIKI: &str = "cead55986c364ad5ff6f0894daf61f77";+const E_ALPHA_BETA_DIR: &str = "63b747655365aa16d38188aa49966f40";+const E_BETA_DELTA_DIR: &str = "a61e6caacb5e485baf6d45184f23ec67";+const E_DELTA_EPSILON_DIR: &str = "3efff61b58c978943e6fd2a1e4eeaee8";+const E_EPSILON_GAMMA_DIR: &str = "7c07cdc441f0a3faa29be8946df3e7a4";++fn make_engine(dir: &TempDir) -> Engine {+ let mut config = Config::default();+ config.server.data_dir = dir.path().to_str().unwrap().to_string();+ Engine::new(config).expect("engine::new")+}++/// One §2.4-shaped edge document. Timestamps are epoch-ms NUMBERS and+/// `invalid_at`/`expired_at` are omitted entirely (never null) — the omission+/// is what lands the row in the column null_bitmap, which is the "still valid"+/// signal the hop reads.+#[allow(clippy::too_many_arguments)]+fn edge_doc(+ edge_id: &str,+ src: &str,+ dst: &str,+ edge_type: &str,+ weight: f64,+ confidence: f64,+ detector: &str,+ quote: &str,+ source: &str,+ offset: u64,+) -> Value {+ json!({+ "edge_id": edge_id,+ "src": src,+ "dst": dst,+ "type": edge_type,+ "weight": weight,+ "valid_at": T0,+ "created_at": T0,+ "detector": detector,+ "confidence": confidence,+ "schema_version": 1,+ "src_file": source,+ "evidence": { "quote": quote, "source": source, "offset": offset }+ })+}++/// The eight §8.3 fixture edges, in table order.+fn fixture_edges() -> Vec<Value> {+ let alpha_line = "Alpha is the hub note. It links to [[beta]] and [[gamma]].";+ vec![+ edge_doc(E_ALPHA_BETA_WIKI, "note-alpha", "note-beta", "wikilink", 1.0, 0.95,+ "wikilink@1", alpha_line, "alpha.md", 35),+ edge_doc(E_ALPHA_GAMMA_WIKI, "note-alpha", "note-gamma", "wikilink", 1.0, 0.95,+ "wikilink@1", alpha_line, "alpha.md", 48),+ edge_doc(E_BETA_GAMMA_WIKI, "note-beta", "note-gamma", "wikilink", 1.0, 0.95,+ "wikilink@1", "Beta continues the thread and references [[gamma]].", "beta.md", 41),+ edge_doc(E_DELTA_ALPHA_WIKI, "note-delta", "note-alpha", "wikilink", 1.0, 0.95,+ "wikilink@1", "Delta cites [[alpha]] as its source.", "delta.md", 12),+ edge_doc(E_ALPHA_BETA_DIR, "note-alpha", "note-beta", "same_dir", 0.3, 0.4,+ "samedir@1", "alpha.md and beta.md share directory .", "alpha.md", 0),+ edge_doc(E_BETA_DELTA_DIR, "note-beta", "note-delta", "same_dir", 0.3, 0.4,+ "samedir@1", "beta.md and delta.md share directory .", "beta.md", 0),+ edge_doc(E_DELTA_EPSILON_DIR, "note-delta", "note-epsilon", "same_dir", 0.3, 0.4,+ "samedir@1", "delta.md and epsilon.md share directory .", "delta.md", 0),+ edge_doc(E_EPSILON_GAMMA_DIR, "note-epsilon", "note-gamma", "same_dir", 0.3, 0.4,+ "samedir@1", "epsilon.md and gamma.md share directory .", "epsilon.md", 0),+ ]+}++async fn fixture_index(engine: &Engine) -> std::sync::Arc<Index> {+ engine+ .create_index(EDGES_INDEX, xerj_common::types::Schema::empty())+ .expect("create edges index");+ let idx = engine.get_index(EDGES_INDEX).expect("get edges index");+ for doc in fixture_edges() {+ let id = doc["edge_id"].as_str().unwrap().to_string();+ idx.index_document(Some(id), doc).await.expect("index edge");+ }+ idx+}++fn expand_req(frontier: &[&str], hops: u8) -> GraphExpandRequest {+ GraphExpandRequest {+ frontier: frontier.iter().map(|s| s.to_string()).collect(),+ hops,+ direction: GraphDirection::Both,+ types: None,+ as_of_ms: AS_OF,+ include_expired: false,+ max_result_edges: 1000,+ }+}++/// (edge_id, src, dst, type, weight, hop) tuples — the §8.4 comparison shape.+fn tuples(res: &GraphExpandResult) -> Vec<(String, String, String, String, f64, u8)> {+ res.edges+ .iter()+ .map(|e| {+ (+ e.edge_id.clone(),+ e.src.clone(),+ e.dst.clone(),+ e.edge_type.clone(),+ e.weight,+ e.hop,+ )+ })+ .collect()+}++fn assert_hop1_fixture(res: &GraphExpandResult) {+ // §8.4: EXACTLY this order (hop asc, weight desc, edge_id asc).+ let got = tuples(res);+ let want = vec![+ (E_ALPHA_GAMMA_WIKI.into(), "note-alpha".into(), "note-gamma".into(), "wikilink".into(), 1.0, 1u8),+ (E_ALPHA_BETA_WIKI.into(), "note-alpha".into(), "note-beta".into(), "wikilink".into(), 1.0, 1u8),+ (E_DELTA_ALPHA_WIKI.into(), "note-delta".into(), "note-alpha".into(), "wikilink".into(), 1.0, 1u8),+ (E_ALPHA_BETA_DIR.into(), "note-alpha".into(), "note-beta".into(), "same_dir".into(), 0.3, 1u8),+ ];+ assert_eq!(got, want, "hop-1 edge list must match §8.4 exactly");+ assert_eq!(+ res.reachable,+ vec!["note-alpha", "note-gamma", "note-beta", "note-delta"],+ "§8.4 reachable order: frontier first, then first-discovery over sorted edges"+ );+ // Every stats field except the two flush-state cost counters is exact.+ assert_eq!(res.stats.frontier_clipped, 0);+ assert_eq!(res.stats.edges_clipped, 0);+ assert_eq!(res.stats.expired_excluded, 0);+ assert_eq!(res.stats.type_filtered, 0);+ assert_eq!(res.stats.segments_without_columns, 0);+}++/// §3.6 (a) + (b): the §8.4 hop-1 expansion is exact on the memtable path and+/// identical after `flush()` on the columnar path.+#[tokio::test]+async fn graph_expand_fixture_hop1_pre_and_post_flush() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = fixture_index(&engine).await;+ let req = expand_req(&["note-alpha"], 1);++ // Pre-flush: everything comes from the memtable walk.+ let pre = idx.graph_expand(&req).expect("pre-flush expand");+ assert_hop1_fixture(&pre);+ assert_eq!(pre.stats.segments_scanned, 0, "no segments exist pre-flush");+ assert!(pre.stats.memtable_docs_scanned >= 8);++ // Post-flush: everything comes from doc-values columns.+ idx.flush().await.expect("flush");+ let post = idx.graph_expand(&req).expect("post-flush expand");+ assert_hop1_fixture(&post);+ assert!(post.stats.segments_scanned >= 1, "columnar path must run");+ assert_eq!(+ post.stats.memtable_docs_scanned, 0,+ "flush drained the memtable"+ );+ assert_eq!(tuples(&pre), tuples(&post), "flush must not change results");+ assert_eq!(pre.reachable, post.reachable);+}++/// §8.4 hop-2: the hop-2 block (frontier = gamma, beta, delta) appends the+/// remaining four edges in (weight desc, edge_id asc) order and reachable+/// gains note-epsilon.+#[tokio::test]+async fn graph_expand_fixture_hop2_block_order() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = fixture_index(&engine).await;+ let req = expand_req(&["note-alpha"], 2);++ for flushed in [false, true] {+ if flushed {+ idx.flush().await.expect("flush");+ }+ let res = idx.graph_expand(&req).expect("expand hops=2");+ let ids: Vec<&str> = res.edges.iter().map(|e| e.edge_id.as_str()).collect();+ assert_eq!(+ ids,+ vec![+ // hop-1 block (§8.4)+ E_ALPHA_GAMMA_WIKI,+ E_ALPHA_BETA_WIKI,+ E_DELTA_ALPHA_WIKI,+ E_ALPHA_BETA_DIR,+ // hop-2 block: 9bbf (w 1.0), then w-0.3 ties by edge_id asc+ E_BETA_GAMMA_WIKI,+ E_DELTA_EPSILON_DIR,+ E_EPSILON_GAMMA_DIR,+ E_BETA_DELTA_DIR,+ ],+ "hop-2 block order (flushed={flushed})"+ );+ assert!(res.edges[..4].iter().all(|e| e.hop == 1));+ assert!(res.edges[4..].iter().all(|e| e.hop == 2));+ assert_eq!(+ res.reachable,+ vec![+ "note-alpha",+ "note-gamma",+ "note-beta",+ "note-delta",+ "note-epsilon"+ ],+ "hop 2 reaches epsilon (flushed={flushed})"+ );+ }+}++/// §3.6 (c) + §8.6: soft invalidation is a re-index under the same `_id` with+/// `invalid_at`/`expired_at` added. Expanding at `as_of < invalid_at` still+/// sees the edge (belief last Tuesday intact); at `as_of >= invalid_at` it is+/// gone with `expired_excluded == 1`; `include_expired` brings it back.+///+/// The first round runs with the STALE row in a flushed segment and the+/// invalidated row in the memtable — the mixed state that only works when the+/// ghost-bitmap machinery suppresses the superseded segment row.+#[tokio::test]+async fn graph_expand_soft_invalidate_time_travel() {+ const INVALID_AT: i64 = 1_753_650_000_000;+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = fixture_index(&engine).await;+ idx.flush().await.expect("flush fixture");++ // Soft-invalidate alpha→beta wikilink: same doc, same `_id`, two added+ // fields — exactly what `DELETE /_graph/{brain}/link/{edge_id}` re-indexes.+ let mut invalidated = fixture_edges()[0].clone();+ invalidated["invalid_at"] = json!(INVALID_AT);+ invalidated["expired_at"] = json!(INVALID_AT);+ idx.index_document(Some(E_ALPHA_BETA_WIKI.to_string()), invalidated)+ .await+ .expect("re-index invalidated edge");++ for (round, flushed) in [(1, false), (2, true)] {+ if flushed {+ idx.flush().await.expect("flush invalidated version");+ }++ // Belief BEFORE the invalidation instant: all four §8.4 edges.+ let mut req = expand_req(&["note-alpha"], 1);+ req.as_of_ms = 1_753_640_000_000;+ let before = idx.graph_expand(&req).expect("expand before invalid_at");+ assert_hop1_fixture(&before);++ // Belief AFTER: the invalidated edge is excluded and counted.+ req.as_of_ms = AS_OF;+ let after = idx.graph_expand(&req).expect("expand after invalid_at");+ let ids: Vec<&str> = after.edges.iter().map(|e| e.edge_id.as_str()).collect();+ assert_eq!(+ ids,+ vec![E_ALPHA_GAMMA_WIKI, E_DELTA_ALPHA_WIKI, E_ALPHA_BETA_DIR],+ "round {round}: invalidated edge must disappear at as_of >= invalid_at"+ );+ assert_eq!(+ after.stats.expired_excluded, 1,+ "round {round}: the exclusion is counted, not silent"+ );++ // include_expired=true: the edge returns, carrying its invalid_at.+ req.include_expired = true;+ let all = idx.graph_expand(&req).expect("expand include_expired");+ let inv = all+ .edges+ .iter()+ .find(|e| e.edge_id == E_ALPHA_BETA_WIKI)+ .expect("include_expired returns the invalidated edge");+ assert_eq!(inv.invalid_at_ms, Some(INVALID_AT));+ assert_eq!(all.edges.len(), 4);+ }+}++/// §3.6 (d) + §3.5: result cap reports clipping; hops out of bounds and an+/// empty frontier are loud `InvalidQuery` errors carrying the §4.6 sentence.+#[tokio::test]+async fn graph_expand_caps_and_bounds() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = fixture_index(&engine).await;++ for flushed in [false, true] {+ if flushed {+ idx.flush().await.expect("flush");+ }+ let mut req = expand_req(&["note-alpha"], 2);+ req.max_result_edges = 2;+ let res = idx.graph_expand(&req).expect("capped expand");+ assert_eq!(res.edges.len(), 2, "cap bounds the result (flushed={flushed})");+ assert!(+ res.stats.edges_clipped > 0,+ "overflow must be counted, never silent (flushed={flushed})"+ );+ }++ for bad_hops in [0u8, 3] {+ let err = idx+ .graph_expand(&expand_req(&["note-alpha"], bad_hops))+ .expect_err("hops out of bounds must fail loud");+ let msg = err.to_string();+ assert!(+ msg.contains(GRAPH_HOPS_CAP_REASON),+ "hops={bad_hops} error must carry the §4.6 sentence, got: {msg}"+ );+ }++ let err = idx+ .graph_expand(&expand_req(&[], 1))+ .expect_err("empty frontier must fail loud");+ assert!(err.to_string().contains("non-empty frontier"));+}++/// Direction and type filters: `Out`/`In` scan a single column; the `types`+/// allowlist counts what it drops.+#[tokio::test]+async fn graph_expand_direction_and_type_filter() {+ let dir = TempDir::new().unwrap();+ let engine = make_engine(&dir);+ let idx = fixture_index(&engine).await;+ idx.flush().await.expect("flush");++ let mut req = expand_req(&["note-alpha"], 1);+ req.direction = GraphDirection::Out;+ let out = idx.graph_expand(&req).expect("out expand");+ let ids: Vec<&str> = out.edges.iter().map(|e| e.edge_id.as_str()).collect();+ assert_eq!(+ ids,+ vec![E_ALPHA_GAMMA_WIKI, E_ALPHA_BETA_WIKI, E_ALPHA_BETA_DIR],+ "direction=out follows src → dst only"+ );++ req.direction = GraphDirection::In;+ let inn = idx.graph_expand(&req).expect("in expand");+ let ids: Vec<&str> = inn.edges.iter().map(|e| e.edge_id.as_str()).collect();+ assert_eq!(ids, vec![E_DELTA_ALPHA_WIKI], "direction=in follows dst → src only");++ req.direction = GraphDirection::Both;+ req.types = Some(vec!["wikilink".to_string()]);+ let typed = idx.graph_expand(&req).expect("typed expand");+ let ids: Vec<&str> = typed.edges.iter().map(|e| e.edge_id.as_str()).collect();+ assert_eq!(+ ids,+ vec![E_ALPHA_GAMMA_WIKI, E_ALPHA_BETA_WIKI, E_DELTA_ALPHA_WIKI],+ "types allowlist keeps only wikilink edges"+ );+ assert_eq!(+ typed.stats.type_filtered, 1,+ "the dropped same_dir edge is counted"+ );+}diff --git a/engine/crates/xerj-query/src/ast.rs b/engine/crates/xerj-query/src/ast.rsindex bc7830d8b..af8787ee0 100644--- a/engine/crates/xerj-query/src/ast.rs+++ b/engine/crates/xerj-query/src/ast.rs@@ -662,20 +662,12 @@ pub enum QueryNode {big: Box<QueryNode>,},- // ── Join queries ──────────────────────────────────────────────────────────- /// Has-child — matches parent documents that have child documents matching the query.- HasChild {- child_type: String,- query: Box<QueryNode>,- score_mode: Option<String>,- },-- /// Has-parent — matches child documents that have parent documents matching the query.- HasParent {- parent_type: String,- query: Box<QueryNode>,- score: bool,- },+ // NOTE: there are deliberately NO join-query variants (has_child /+ // has_parent). XERJ never materializes a parent/child join, so an AST+ // variant here would only exist to be flat-matched — silently wrong+ // results — if anyone re-enabled parsing. The parser rejects both with a+ // 400 (parser.rs::parse_has_child / parse_has_parent); the corpse variants+ // and their dead planner/executor arms were removed.// ── Geo shape queries ─────────────────────────────────────────────────────/// Geo polygon — matches documents whose geo_point falls within the polygon.diff --git a/engine/crates/xerj-query/src/planner.rs b/engine/crates/xerj-query/src/planner.rsindex d1b905664..ffda47fa0 100644--- a/engine/crates/xerj-query/src/planner.rs+++ b/engine/crates/xerj-query/src/planner.rs@@ -648,21 +648,6 @@ fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {plan_node(*big, schema)}- // ── Join queries — UNREACHABLE ────────────────────────────────────────- // has_child/has_parent are rejected with a 400 at parse time- // (see parser.rs::parse_has_child), so these AST variants are never- // built and this branch is never taken. Kept for exhaustiveness and- // in case the AST variant is ever wired to a real join executor.- QueryNode::HasChild { query, .. } | QueryNode::HasParent { query, .. } => {- let inner = plan_node(*query, schema);- let cost = inner.cost() + 10.0;- ExecutionPlan::Boost {- boost: 1.0,- inner: Box::new(inner),- cost,- }- }-// ── Geo shape queries — treat as doc-scans ────────────────────────────QueryNode::GeoPolygon { field, .. } => ExecutionPlan::ExistsScan { field },QueryNode::GeoShape { field, .. } => ExecutionPlan::ExistsScan { field },