Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

wip(second-brain/core): engine graph traversal + graph_api + stage-0 cleanup

xerj-orgd39ccd8ec0b7
diff --git a/engine/crates/xerj-ai/src/lib.rs b/engine/crates/xerj-ai/src/lib.rs
index 9e3e3c134..fa4b0f91c 100644
--- a/engine/crates/xerj-ai/src/lib.rs
+++ b/engine/crates/xerj-ai/src/lib.rs
@@ -11,13 +11,17 @@
//! - `onnx` — Experimental MiniLM-compatible FP32 ONNX backend
//! (feature `onnx-experimental`; server feature + explicit runtime selection required)
//! - [`chunker`] — Text chunking with sentence-aware splitting and overlap
-//! - [`memory`] — Agent memory: semantic dedup + recency-blended recall
+//!
+//! Agent memory does NOT live here. The real memory store is index-backed
+//! (`/_memory/*` in xerj-api over ordinary XERJ indices — durable, WAL-replayed,
+//! kNN/BM25-searchable). An earlier in-RAM `AgentMemory` (O(N²) dedup scan, no
+//! durability, process-lifetime only) was deleted once it reached zero callers:
+//! keeping it around shadowed the real API and invited accidental use.
pub mod chunker;
pub mod embed;
pub mod embedder;
pub mod local;
-pub mod memory;
#[cfg(feature = "neural")]
pub mod neural;
#[cfg(feature = "onnx-experimental")]
@@ -27,6 +31,5 @@ pub use chunker::{Chunk, TextChunker};
pub use embed::{EmbeddingProxy, EmbeddingProxyConfig};
pub use embedder::Embedder;
pub use local::{local_embed, DEFAULT_DIMS};
-pub use memory::{AgentMemory, MemoryEntry};
pub use xerj_common::Result;
diff --git a/engine/crates/xerj-ai/src/memory.rs b/engine/crates/xerj-ai/src/memory.rs
deleted file mode 100644
index 2f26e0dca..000000000
--- a/engine/crates/xerj-ai/src/memory.rs
+++ /dev/null
@@ -1,327 +0,0 @@
-//! Agent memory: append-only semantic memory with dedup and recency blending.
-//!
-//! Stores text snippets with embeddings. On recall, blends cosine similarity
-//! with recency using a configurable weight, so recently stored memories are
-//! surfaced even if slightly less semantically similar than older ones.
-
-use serde::{Deserialize, Serialize};
-use serde_json::Value;
-use std::sync::Mutex;
-use tracing::debug;
-use xerj_common::XerjError;
-
-/// Result alias.
-pub type Result<T> = std::result::Result<T, XerjError>;
-
-// ─────────────────────────────────────────────────────────────────────────────
-// MemoryEntry
-// ─────────────────────────────────────────────────────────────────────────────
-
-/// A stored memory entry.
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct MemoryEntry {
- /// Unique ID (monotonically assigned).
- pub id: u64,
- /// The text content.
- pub text: String,
- /// Arbitrary metadata (tags, source, etc.).
- pub metadata: Value,
- /// Dense embedding vector.
- pub embedding: Vec<f32>,
- /// Unix timestamp (seconds) when this was stored.
- pub stored_at: i64,
-}
-
-impl MemoryEntry {
- /// Blended score combining similarity and recency.
- ///
- /// `recency_weight` ∈ [0, 1]:
- /// - 0.0 → pure semantic similarity
- /// - 1.0 → pure recency
- pub fn score(&self, similarity: f32, now_secs: i64, recency_weight: f32) -> f32 {
- let age_secs = (now_secs - self.stored_at).max(0) as f32;
- // Decay: e^(-age / half_life) where half_life = 7 days in seconds
- let half_life = 7.0 * 86_400.0f32;
- let recency = (-age_secs / half_life).exp();
-
- let semantic = (1.0 - recency_weight).max(0.0);
- let recency_w = recency_weight.min(1.0);
-
- similarity * semantic + recency * recency_w
- }
-}
-
-// ─────────────────────────────────────────────────────────────────────────────
-// Similarity helpers
-// ─────────────────────────────────────────────────────────────────────────────
-
-fn cosine_similarity(a: &[f32], b: &[f32]) -> f32 {
- let mut dot = 0.0f32;
- let mut na = 0.0f32;
- let mut nb = 0.0f32;
- for (&ai, &bi) in a.iter().zip(b.iter()) {
- dot += ai * bi;
- na += ai * ai;
- nb += bi * bi;
- }
- let denom = (na * nb).sqrt();
- if denom == 0.0 {
- 0.0
- } else {
- dot / denom
- }
-}
-
-// ─────────────────────────────────────────────────────────────────────────────
-// AgentMemory
-// ─────────────────────────────────────────────────────────────────────────────
-
-/// Append-only in-memory store with semantic dedup.
-///
-/// Store: embeds the text, checks similarity against existing entries, and
-/// skips insertion if the closest match exceeds the dedup threshold.
-///
-/// Recall: scores all entries by blended similarity + recency, returns top-k.
-///
-/// # Note on embeddings
-///
-/// `AgentMemory` stores pre-computed embeddings. The caller is responsible for
-/// embedding the text (e.g., via [`EmbeddingProxy`]) before calling
-/// [`store_embedded`] or [`store`] with a pre-computed vector.
-pub struct AgentMemory {
- entries: Mutex<Vec<MemoryEntry>>,
- next_id: Mutex<u64>,
- /// Cosine similarity threshold above which a new entry is considered
- /// duplicate and discarded.
- dedup_threshold: f32,
-}
-
-impl AgentMemory {
- /// Create a new memory store with the given dedup threshold.
- ///
- /// `dedup_threshold` ∈ [0, 1]: entries with similarity above this value
- /// are considered duplicates. Recommended: 0.95.
- pub fn new(dedup_threshold: f32) -> Self {
- Self {
- entries: Mutex::new(Vec::new()),
- next_id: Mutex::new(0),
- dedup_threshold,
- }
- }
-
- /// Store a text with a pre-computed embedding.
- ///
- /// Returns `None` if the entry was deduplicated (too similar to existing).
- pub fn store_embedded(
- &self,
- text: impl Into<String>,
- embedding: Vec<f32>,
- metadata: Value,
- stored_at_secs: i64,
- ) -> Option<u64> {
- let text = text.into();
- let mut entries = self.entries.lock().unwrap();
-
- // Semantic dedup check
- for existing in entries.iter() {
- if existing.embedding.len() == embedding.len() {
- let sim = cosine_similarity(&existing.embedding, &embedding);
- if sim >= self.dedup_threshold {
- debug!(
- "memory dedup: skipping '{}' (sim={:.3} vs '{}')",
- &text[..text.len().min(40)],
- sim,
- &existing.text[..existing.text.len().min(40)]
- );
- return None;
- }
- }
- }
-
- let id = {
- let mut next = self.next_id.lock().unwrap();
- let id = *next;
- *next += 1;
- id
- };
-
- entries.push(MemoryEntry {
- id,
- text,
- metadata,
- embedding,
- stored_at: stored_at_secs,
- });
-
- Some(id)
- }
-
- /// Recall the top-k most relevant memories for a query embedding.
- ///
- /// `recency_weight` ∈ [0, 1]:
- /// - 0.0 → pure semantic similarity ordering
- /// - 1.0 → pure recency ordering
- pub fn recall(
- &self,
- query_embedding: &[f32],
- k: usize,
- recency_weight: f32,
- now_secs: i64,
- ) -> Vec<MemoryEntry> {
- let entries = self.entries.lock().unwrap();
-
- let mut scored: Vec<(f32, &MemoryEntry)> = entries
- .iter()
- .filter_map(|e| {
- if e.embedding.len() != query_embedding.len() {
- return None;
- }
- let sim = cosine_similarity(query_embedding, &e.embedding);
- let score = e.score(sim, now_secs, recency_weight);
- Some((score, e))
- })
- .collect();
-
- // Sort descending by score
- scored.sort_unstable_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
-
- scored.into_iter().take(k).map(|(_, e)| e.clone()).collect()
- }
-
- /// Number of stored entries.
- pub fn len(&self) -> usize {
- self.entries.lock().unwrap().len()
- }
-
- pub fn is_empty(&self) -> bool {
- self.len() == 0
- }
-
- /// Clear all stored memories.
- pub fn clear(&self) {
- self.entries.lock().unwrap().clear();
- *self.next_id.lock().unwrap() = 0;
- }
-
- /// Return all stored entries (for inspection/serialization).
- pub fn all_entries(&self) -> Vec<MemoryEntry> {
- self.entries.lock().unwrap().clone()
- }
-}
-
-// ─────────────────────────────────────────────────────────────────────────────
-// Tests
-// ─────────────────────────────────────────────────────────────────────────────
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use serde_json::json;
-
- fn vec_a() -> Vec<f32> {
- vec![1.0, 0.0, 0.0, 0.0]
- }
- fn vec_b() -> Vec<f32> {
- vec![0.0, 1.0, 0.0, 0.0]
- }
- fn vec_c() -> Vec<f32> {
- vec![0.0, 0.0, 1.0, 0.0]
- }
- // Very similar to vec_a
- fn vec_a_near() -> Vec<f32> {
- vec![0.999, 0.001, 0.0, 0.0]
- }
-
- #[test]
- fn store_and_recall_basic() {
- let mem = AgentMemory::new(0.95);
- let now = 1_700_000_000i64;
-
- mem.store_embedded("memory about cats", vec_a(), json!({}), now);
- mem.store_embedded("memory about dogs", vec_b(), json!({}), now);
- mem.store_embedded("memory about fish", vec_c(), json!({}), now);
-
- assert_eq!(mem.len(), 3);
-
- let results = mem.recall(&vec_a(), 1, 0.0, now);
- assert_eq!(results.len(), 1);
- assert_eq!(results[0].text, "memory about cats");
- }
-
- #[test]
- fn dedup_skips_near_duplicate() {
- let mem = AgentMemory::new(0.95);
- let now = 1_700_000_000i64;
-
- let id1 = mem.store_embedded("cats", vec_a(), json!({}), now);
- let id2 = mem.store_embedded("cats (near duplicate)", vec_a_near(), json!({}), now);
-
- assert!(id1.is_some());
- assert!(id2.is_none(), "near-duplicate should be deduped");
- assert_eq!(mem.len(), 1);
- }
-
- #[test]
- fn dedup_allows_distinct_entries() {
- let mem = AgentMemory::new(0.95);
- let now = 1_700_000_000i64;
-
- mem.store_embedded("cats", vec_a(), json!({}), now);
- mem.store_embedded("dogs", vec_b(), json!({}), now);
-
- assert_eq!(mem.len(), 2);
- }
-
- #[test]
- fn recency_weight_affects_ordering() {
- let mem = AgentMemory::new(0.5);
- let base = 1_700_000_000i64;
-
- // Store old entry with matching vector and recent entry with different vector
- mem.store_embedded("old cats", vec_a(), json!({}), base - 30 * 86_400); // 30 days ago
- mem.store_embedded("recent dogs", vec_b(), json!({}), base); // now
-
- // Query close to vec_a
- let query = vec![0.9, 0.1, 0.0, 0.0f32];
-
- // Pure similarity: old cats should rank first (closer to query)
- let by_sim = mem.recall(&query, 2, 0.0, base);
- assert_eq!(by_sim[0].text, "old cats");
-
- // Pure recency: recent dogs should rank first
- let by_recency = mem.recall(&query, 2, 1.0, base);
- assert_eq!(by_recency[0].text, "recent dogs");
- }
-
- #[test]
- fn recall_empty_memory() {
- let mem = AgentMemory::new(0.95);
- let results = mem.recall(&vec_a(), 5, 0.5, 1_700_000_000);
- assert!(results.is_empty());
- }
-
- #[test]
- fn clear_resets_state() {
- let mem = AgentMemory::new(0.95);
- mem.store_embedded("a", vec_a(), json!({}), 1_700_000_000);
- mem.clear();
- assert_eq!(mem.len(), 0);
- }
-
- #[test]
- fn ids_are_unique() {
- // Use threshold 2.0 (above max cosine similarity of 1.0) to disable dedup
- let mem = AgentMemory::new(2.0);
- let ids: Vec<u64> = (0..5)
- .map(|i| {
- // Each vector points in a completely different direction
- let mut v = vec![0.0f32; 5];
- v[i] = 1.0;
- mem.store_embedded(format!("entry {i}"), v, json!({}), 1_700_000_000)
- .unwrap()
- })
- .collect();
- let unique: std::collections::HashSet<u64> = ids.into_iter().collect();
- assert_eq!(unique.len(), 5);
- }
-}
diff --git a/engine/crates/xerj-engine/src/graph.rs b/engine/crates/xerj-engine/src/graph.rs
new file mode 100644
index 000000000..9d193f07a
--- /dev/null
+++ b/engine/crates/xerj-engine/src/graph.rs
@@ -0,0 +1,649 @@
+//! Graph expansion (second brain) — batched, columnar, bounded.
+//!
+//! A relationship layer over documents that already exist. Edges are ordinary
+//! documents in a reserved edges index (schema: SECOND_BRAIN contract §2); this
+//! module is the ONLY engine-side traversal over them. It is deliberately not a
+//! graph database: no query language, no shortest-path, no PageRank, and no
+//! unbounded traversal — expansion is a bounded, batched read (≤2 hops per
+//! call) that composes client-side by feeding `reachable` back in as the next
+//! call's frontier.
+//!
+//! ## Why a column scan instead of per-node queries
+//!
+//! Expanding a frontier of F nodes as F term queries costs F query setups and F
+//! segment walks. Instead, each hop resolves the WHOLE frontier to per-segment
+//! keyword ordinals (one O(log T) FST probe per frontier id via
+//! [`xerj_storage::doc_values::KeywordColumn::ord_for_term`]) and then makes
+//! ONE pass over the segment's `src` ordinal array (plus the `dst` array for
+//! direction `both`) with O(1) hash-set membership per row — cost is
+//! frontier-size-independent. Everything an admitted edge needs (`edge_id`,
+//! `src`, `dst`, `type`, `weight`, `valid_at`, `invalid_at`) is random-accessed
+//! from doc-values columns; the hop path NEVER touches `_source` (no
+//! `StoredSlices`, no JSON parse, no hydration — evidence is hydrated by the
+//! HTTP layer AFTER traversal, on the bounded result, via an `ids` query).
+//!
+//! ## Correctness sources (same machinery the columnar search paths use)
+//!
+//! - Segment list: [`Index::store_snapshot`] once per request.
+//! - Columns: `Index::dv_columns_for` (cached per-segment sidecar).
+//! - Liveness under updates/deletes: `VersionMap::ghost_events` gate +
+//! `Index::ghost_positions_for` word-bitmaps — the exact source of truth
+//! `scored_columnar` uses, so a soft-invalidated edge re-indexed under the
+//! same `_id` never double-surfaces from its stale segment row.
+//! - Unflushed edges: a bounded walk of the sharded FTS memtable
+//! (`ShardedFtsMemtable::all_docs_with_sources_arc`) per hop — correctness,
+//! not speed; the memtable is bounded by the flush threshold and the walk is
+//! reported in-band (`memtable_docs_scanned`).
+//!
+//! ## Honesty in-band
+//!
+//! Every response carries [`GraphExpandStats`]: clipped frontiers, clipped
+//! edges, bi-temporally excluded edges, type-filtered edges, segments skipped
+//! for missing columns (the hop never silently falls back to hydration —
+//! skipped means reported), and scan-cost counters.
+
+use std::collections::{HashMap, HashSet};
+
+use serde_json::Value;
+use xerj_common::XerjError;
+
+use super::Index;
+use xerj_storage::doc_values::{Column, KeywordColumn, NumericColumn};
+
+/// Hard ceiling on expansion depth. Depth beyond this composes client-side:
+/// feed `reachable` back in as the next call's frontier.
+pub const GRAPH_MAX_HOPS: u8 = 2;
+/// Frontier ids admitted per hop; excess is dropped and counted
+/// (`stats.frontier_clipped`). 4096 ords is a ~32KB hash set — cheap — while
+/// bounding the per-segment FST-probe phase.
+pub const GRAPH_MAX_FRONTIER: usize = 4096;
+/// Result-edge ceiling per request; collection stops (scan continues to the
+/// end of the current segment, then aborts remaining segments) and
+/// `stats.edges_clipped` reports the overflow.
+pub const GRAPH_MAX_RESULT_EDGES: usize = 100_000;
+
+/// The not-a-graph-database refusal, verbatim (contract §4.6). Shared with the
+/// HTTP layer so the engine error reason and the API 400 body carry the exact
+/// same sentence — the wording is normative, not cosmetic: it tells the caller
+/// the supported alternative (iterate on `reachable`).
+pub const GRAPH_HOPS_CAP_REASON: &str = "hops is capped at 2: XERJ's second brain is a \
+ relationship layer over documents, not a graph database (no Cypher, no shortest-path, \
+ no variable-depth traversal). Iterate: expand again from this response's 'reachable' ids.";
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum GraphDirection {
+ /// Follow src → dst (scan the `src` column).
+ Out,
+ /// Follow dst → src (scan the `dst` column).
+ In,
+ /// Both directions (two column scans per segment). The default everywhere.
+ Both,
+}
+
+#[derive(Debug, Clone)]
+pub struct GraphExpandRequest {
+ /// Node ids to expand from. Deduped internally; clipped to
+ /// GRAPH_MAX_FRONTIER (clip counted, never silent).
+ pub frontier: Vec<String>,
+ /// 1 or 2. 0 or >2 → XerjError::InvalidQuery (see module docs).
+ pub hops: u8,
+ pub direction: GraphDirection,
+ /// Edge-type allowlist; None = all types.
+ pub types: Option<Vec<String>>,
+ /// Bi-temporal cut: an edge is visible iff
+ /// valid_at <= as_of_ms AND (invalid_at absent OR invalid_at > as_of_ms).
+ pub as_of_ms: i64,
+ /// true = ignore invalid_at (return soft-deleted edges too; used by
+ /// ego?include_expired=true). Excluded edges are counted either way.
+ pub include_expired: bool,
+ /// Per-request result cap, clamped to 1..=GRAPH_MAX_RESULT_EDGES.
+ pub max_result_edges: usize,
+}
+
+/// One edge as read ENTIRELY from doc-values columns — no `_source` access.
+/// `weight` is f64 because numeric columns store f64 bits (see
+/// `build_doc_value_columns`); timestamps are exact for |v| < 2^53 (epoch-ms
+/// until year ~287396).
+#[derive(Debug, Clone)]
+pub struct GraphEdgeLite {
+ pub edge_id: String,
+ pub src: String,
+ pub dst: String,
+ pub edge_type: String,
+ pub weight: f64,
+ pub valid_at_ms: i64,
+ pub invalid_at_ms: Option<i64>,
+ /// Which hop discovered this edge (1 or 2).
+ pub hop: u8,
+}
+
+/// In-band honesty: what the expansion did NOT show, and what it cost.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct GraphExpandStats {
+ pub frontier_clipped: u64,
+ pub edges_clipped: u64,
+ /// Edges skipped by the bi-temporal cut (valid_at in the future of as_of,
+ /// or invalidated at/before as_of) — NOT counting include_expired=true.
+ pub expired_excluded: u64,
+ /// Edges skipped by the `types` allowlist.
+ pub type_filtered: u64,
+ pub segments_scanned: u64,
+ /// Segments skipped because they lack a dv sidecar or a `src`/`dst`
+ /// keyword column (legacy segment / meta-only segment / poisoned column).
+ /// The hop NEVER falls back to hydration — skipped means reported.
+ pub segments_without_columns: u64,
+ pub memtable_docs_scanned: u64,
+}
+
+#[derive(Debug, Clone)]
+pub struct GraphExpandResult {
+ /// Deduped by edge_id, sorted (hop asc, weight desc, edge_id asc) — the
+ /// stable order every response surface and the fixture assert on.
+ pub edges: Vec<GraphEdgeLite>,
+ /// Frontier ids (that were admitted) first, in input order, then newly
+ /// reached ids in first-discovery order following the sorted edge list.
+ /// Deduped.
+ pub reachable: Vec<String>,
+ pub stats: GraphExpandStats,
+}
+
+/// Get a keyword column by name, or `None` when absent or numeric-typed
+/// (numeric-typed = a writer violated the §2.2 type discipline; the segment is
+/// treated as column-less rather than guessed at).
+fn kw_col<'a>(cols: &'a super::DocValueMap, name: &str) -> Option<&'a KeywordColumn> {
+ match cols.get(name) {
+ Some(Column::Keyword(k)) => Some(k),
+ _ => None,
+ }
+}
+
+/// Get a numeric column by name (same honesty rule as [`kw_col`]).
+fn num_col<'a>(cols: &'a super::DocValueMap, name: &str) -> Option<&'a NumericColumn> {
+ match cols.get(name) {
+ Some(Column::Numeric(n)) => Some(n),
+ _ => None,
+ }
+}
+
+/// Numeric columns store f64 BITS in an i64 slot (one representation for the
+/// whole column, integers exact to 2^53). Decode before use — using the raw
+/// i64 silently corrupts every timestamp/weight compare.
+#[inline]
+fn bits_to_f64(v: i64) -> f64 {
+ f64::from_bits(v as u64)
+}
+
+/// Epoch-ms from a memtable JSON number. Writers emit integer epoch-ms, but a
+/// float that reaches us (e.g. a client JSON-encoding 1.7536e12) still means
+/// the same instant, so accept it rather than mis-classifying the row.
+#[inline]
+fn json_ms(v: &Value) -> Option<i64> {
+ v.as_i64().or_else(|| v.as_f64().map(|f| f as i64))
+}
+
+/// Bi-temporal visibility (contract §3.2). Returns `false` (and the caller
+/// counts `expired_excluded`) for edges asserted after `as_of` or — unless
+/// `include_expired` — invalidated at/before `as_of`. `include_expired` only
+/// disables the invalidation half: a future-asserted edge is invisible either
+/// way (it did not exist yet in the world being asked about).
+#[inline]
+fn bitemporal_visible(
+ valid_at_ms: i64,
+ invalid_at_ms: Option<i64>,
+ as_of_ms: i64,
+ include_expired: bool,
+) -> bool {
+ if valid_at_ms > as_of_ms {
+ return false;
+ }
+ if include_expired {
+ return true;
+ }
+ match invalid_at_ms {
+ Some(inv) => inv > as_of_ms,
+ None => true,
+ }
+}
+
+/// Collected-edge bookkeeping: `from_memtable` implements the dedupe rule that
+/// a memtable row wins over any segment row with the same edge_id (it is the
+/// newer version — relevant only in the conservative ghost-bitmap-unavailable
+/// path; when ghost bitmaps resolve, the stale segment row never surfaces).
+struct Collected {
+ edge: GraphEdgeLite,
+ from_memtable: bool,
+}
+
+impl Index {
+ /// Batched graph expansion over an edges index (contract §2 schema).
+ ///
+ /// Sync fn (no await points): all reads are memtable walks + doc-values
+ /// column access — the same blocking pattern the columnar agg/search fast
+ /// paths already use from async handlers.
+ ///
+ /// Explicitly forbidden on this path: `StoredSlices` access, JSON parsing
+ /// of stored docs, hydration helpers, per-frontier-id queries, and any
+ /// per-edge `_source` read. Evidence hydration happens AFTER traversal in
+ /// the HTTP layer, on the bounded result set, via an `ids` query.
+ pub fn graph_expand(&self, req: &GraphExpandRequest) -> xerj_common::Result<GraphExpandResult> {
+ if req.hops == 0 || req.hops > GRAPH_MAX_HOPS {
+ // Same normative sentence for both bounds: the caller's fix
+ // (iterate on `reachable`) is identical, and the HTTP layer
+ // forwards this reason verbatim as its 400 body.
+ let reason = if req.hops == 0 {
+ format!("hops must be at least 1. {GRAPH_HOPS_CAP_REASON}")
+ } else {
+ GRAPH_HOPS_CAP_REASON.to_string()
+ };
+ return Err(XerjError::invalid_query(reason));
+ }
+
+ let mut stats = GraphExpandStats::default();
+
+ // Dedupe the request frontier preserving input order (the order is
+ // part of the `reachable` contract), then clip with the clip counted.
+ let mut frontier: Vec<String> = Vec::with_capacity(req.frontier.len().min(16));
+ {
+ let mut seen: HashSet<&str> = HashSet::with_capacity(req.frontier.len());
+ for id in &req.frontier {
+ if seen.insert(id.as_str()) {
+ frontier.push(id.clone());
+ }
+ }
+ }
+ if frontier.is_empty() {
+ return Err(XerjError::invalid_query(
+ "graph expansion requires a non-empty frontier (after dedup)",
+ ));
+ }
+ if frontier.len() > GRAPH_MAX_FRONTIER {
+ stats.frontier_clipped += (frontier.len() - GRAPH_MAX_FRONTIER) as u64;
+ frontier.truncate(GRAPH_MAX_FRONTIER);
+ }
+ let admitted_frontier = frontier.clone();
+
+ let cap = req.max_result_edges.clamp(1, GRAPH_MAX_RESULT_EDGES);
+ let scan_src = matches!(req.direction, GraphDirection::Out | GraphDirection::Both);
+ let scan_dst = matches!(req.direction, GraphDirection::In | GraphDirection::Both);
+
+ // Point-in-time capture once per request: segment list + "any ghosts?"
+ // gate — the same pair `scored_columnar` snapshots, so liveness
+ // decisions are consistent across every segment of this expansion.
+ let snapshot = self.store_snapshot();
+ let segments_dir = self.data_dir.join("segments");
+ let any_ghosts = self.store.version_map.ghost_events() > 0;
+
+ // Every node ever admitted to a frontier or discovered as an endpoint;
+ // hop-2 frontier construction subtracts it ("minus already-visited").
+ let mut visited: HashSet<String> = frontier.iter().cloned().collect();
+ let mut collected: HashMap<String, Collected> = HashMap::new();
+ // True once the result cap overflowed: the current container finishes
+ // (counting overflow into `edges_clipped`) and everything after —
+ // remaining segments, the memtable walk, further hops — is aborted.
+ let mut capped = false;
+
+ for hop in 1..=req.hops {
+ if capped || frontier.is_empty() {
+ break;
+ }
+ let frontier_set: HashSet<&str> = frontier.iter().map(String::as_str).collect();
+ // edge_ids first collected during THIS hop, for next-frontier
+ // construction (order does not matter here; the walk below sorts).
+ let mut hop_new: Vec<String> = Vec::new();
+
+ // ── Segments: one columnar pass per segment for the whole frontier ──
+ for meta in &snapshot.segments {
+ if capped {
+ break;
+ }
+ let Some(cols) = self.dv_columns_for(&segments_dir, &meta.id) else {
+ stats.segments_without_columns += 1;
+ continue;
+ };
+ let cols = cols.value();
+ // Required columns. Missing/mistyped `src`/`dst` (needed for
+ // the scan AND for emission) or `edge_id`/`type`/`weight`/
+ // `valid_at` (needed for emission/filtering) → the segment is
+ // schema-corrupt or edge-free (e.g. meta-doc-only): skip it,
+ // reported — never guess, never hydrate. A missing
+ // `invalid_at` column is NORMAL (no edge in this segment was
+ // ever invalidated) and means "no edge here is invalidated".
+ let (Some(k_src), Some(k_dst), Some(k_edge), Some(k_type)) = (
+ kw_col(cols, "src"),
+ kw_col(cols, "dst"),
+ kw_col(cols, "edge_id"),
+ kw_col(cols, "type"),
+ ) else {
+ stats.segments_without_columns += 1;
+ continue;
+ };
+ let (Some(n_weight), Some(n_valid)) =
+ (num_col(cols, "weight"), num_col(cols, "valid_at"))
+ else {
+ stats.segments_without_columns += 1;
+ continue;
+ };
+ let n_invalid = match cols.get("invalid_at") {
+ None => None,
+ Some(Column::Numeric(n)) => Some(n),
+ // Present but keyword-typed = a writer stored ISO strings;
+ // an as-of compare against strings would be a lie.
+ Some(Column::Keyword(_)) => {
+ stats.segments_without_columns += 1;
+ continue;
+ }
+ };
+
+ // Frontier → per-segment ordinal sets (one O(log T) FST probe
+ // per frontier id). Both empty → nothing of the frontier
+ // exists in this segment; skip without scanning.
+ let src_ords: HashSet<u32> = if scan_src {
+ frontier
+ .iter()
+ .filter_map(|id| k_src.ord_for_term(id))
+ .collect()
+ } else {
+ HashSet::new()
+ };
+ let dst_ords: HashSet<u32> = if scan_dst {
+ frontier
+ .iter()
+ .filter_map(|id| k_dst.ord_for_term(id))
+ .collect()
+ } else {
+ HashSet::new()
+ };
+ if src_ords.is_empty() && dst_ords.is_empty() {
+ continue;
+ }
+
+ // Liveness bitmap (deleted + superseded rows). Unbuildable →
+ // conservatively skip the whole segment, reported — admitting
+ // possibly-stale rows would silently resurrect overwritten
+ // edges (e.g. a soft-invalidation would un-happen).
+ let ghosts = if any_ghosts {
+ match self.ghost_positions_for(meta.id.as_str(), meta.doc_count) {
+ Some(bm) => Some(bm),
+ None => {
+ stats.segments_without_columns += 1;
+ continue;
+ }
+ }
+ } else {
+ None
+ };
+
+ // Per-segment type-allowlist ordinal set. A term absent from
+ // this segment's dictionary matches nothing here.
+ let type_ords: Option<HashSet<u32>> = req.types.as_ref().map(|ts| {
+ ts.iter()
+ .filter_map(|t| k_type.ord_for_term(t))
+ .collect::<HashSet<u32>>()
+ });
+
+ stats.segments_scanned += 1;
+
+ // THE scan: one pass over the ordinal array(s) for the WHOLE
+ // frontier, O(1) membership per row. Direction `both` reads
+ // both ords arrays in the same position loop — cost-identical
+ // to two passes, and an edge admitted by both sides is
+ // processed exactly once.
+ let doc_count = k_src.doc_count;
+ for pos in 0..doc_count {
+ let via_src = scan_src
+ && k_src.ord_for(pos).is_some_and(|o| src_ords.contains(&o));
+ let admitted = via_src
+ || (scan_dst
+ && k_dst.ord_for(pos).is_some_and(|o| dst_ords.contains(&o)));
+ if !admitted {
+ continue;
+ }
+ // Liveness before anything else (a ghost row is not an
+ // edge at all — its live version surfaces elsewhere).
+ if let Some(bm) = &ghosts {
+ let p = pos as usize;
+ if (bm[p / 64] >> (p % 64)) & 1 == 1 {
+ continue;
+ }
+ }
+ // Type filter (counted).
+ if let Some(t_ords) = &type_ords {
+ match k_type.ord_for(pos) {
+ Some(o) if t_ords.contains(&o) => {}
+ _ => {
+ stats.type_filtered += 1;
+ continue;
+ }
+ }
+ }
+ // Bi-temporal cut (counted). A null valid_at is a
+ // schema-corrupt row (writers always emit it); such a row
+ // cannot be placed on the timeline, so it is skipped.
+ let Some(valid_at_ms) = n_valid.get(pos).map(|v| bits_to_f64(v) as i64)
+ else {
+ continue;
+ };
+ let invalid_at_ms = n_invalid
+ .and_then(|c| c.get(pos))
+ .map(|v| bits_to_f64(v) as i64);
+ if !bitemporal_visible(
+ valid_at_ms,
+ invalid_at_ms,
+ req.as_of_ms,
+ req.include_expired,
+ ) {
+ stats.expired_excluded += 1;
+ continue;
+ }
+ // Emit — keyword random access only, still no `_source`.
+ let Some(edge_id) = k_edge.ord_for(pos).and_then(|o| k_edge.term_for_ord(o))
+ else {
+ continue;
+ };
+ if collected.contains_key(edge_id) {
+ continue; // rediscovered (earlier hop or the other column) — no cap cost
+ }
+ if collected.len() >= cap {
+ stats.edges_clipped += 1;
+ capped = true; // finish counting this segment, then abort
+ continue;
+ }
+ let (Some(src), Some(dst), Some(edge_type), Some(weight)) = (
+ k_src.ord_for(pos).and_then(|o| k_src.term_for_ord(o)),
+ k_dst.ord_for(pos).and_then(|o| k_dst.term_for_ord(o)),
+ k_type.ord_for(pos).and_then(|o| k_type.term_for_ord(o)),
+ n_weight.get(pos).map(bits_to_f64),
+ ) else {
+ continue;
+ };
+ let edge_id = edge_id.to_string();
+ hop_new.push(edge_id.clone());
+ collected.insert(
+ edge_id.clone(),
+ Collected {
+ edge: GraphEdgeLite {
+ edge_id,
+ src: src.to_string(),
+ dst: dst.to_string(),
+ edge_type: edge_type.to_string(),
+ weight,
+ valid_at_ms,
+ invalid_at_ms,
+ hop,
+ },
+ from_memtable: false,
+ },
+ );
+ }
+ }
+
+ // ── Memtable: unflushed edges (correctness, not speed) ────────────
+ // Bounded by the flush threshold; the walk cost is reported
+ // in-band. Values come from the in-memory source JSON — that is
+ // not `_source` hydration (no segment I/O, no stored-section
+ // parse), it IS the memtable's native representation.
+ // Once `capped` flips mid-walk the loop still finishes the
+ // memtable (it is one container — same rule as a segment):
+ // remaining admits fall into the `len() >= cap` arm and are
+ // counted into `edges_clipped`.
+ if !capped {
+ for (id, source) in self.memtable.all_docs_with_sources_arc() {
+ stats.memtable_docs_scanned += 1;
+ // Tombstoned (deleted-by-API) rows are not edges.
+ if self
+ .store
+ .version_map
+ .get(&id)
+ .is_some_and(|v| v.deleted)
+ {
+ continue;
+ }
+ let src = source.get("src").and_then(Value::as_str);
+ let dst = source.get("dst").and_then(Value::as_str);
+ let via_src = scan_src && src.is_some_and(|s| frontier_set.contains(s));
+ let admitted = via_src
+ || (scan_dst && dst.is_some_and(|d| frontier_set.contains(d)));
+ if !admitted {
+ continue;
+ }
+ // Emission needs both endpoints + the envelope fields; a
+ // row missing any of them (only the brain-meta doc or a
+ // non-§2 writer) is not an edge.
+ let (Some(src), Some(dst)) = (src, dst) else {
+ continue;
+ };
+ let Some(edge_id) = source.get("edge_id").and_then(Value::as_str) else {
+ continue;
+ };
+ let Some(edge_type) = source.get("type").and_then(Value::as_str) else {
+ continue;
+ };
+ if let Some(allow) = &req.types {
+ if !allow.iter().any(|t| t == edge_type) {
+ stats.type_filtered += 1;
+ continue;
+ }
+ }
+ let Some(valid_at_ms) = source.get("valid_at").and_then(json_ms) else {
+ continue;
+ };
+ let invalid_at_ms = source.get("invalid_at").and_then(json_ms);
+ if !bitemporal_visible(
+ valid_at_ms,
+ invalid_at_ms,
+ req.as_of_ms,
+ req.include_expired,
+ ) {
+ stats.expired_excluded += 1;
+ continue;
+ }
+ let Some(weight) = source.get("weight").and_then(Value::as_f64) else {
+ continue;
+ };
+ let edge = GraphEdgeLite {
+ edge_id: edge_id.to_string(),
+ src: src.to_string(),
+ dst: dst.to_string(),
+ edge_type: edge_type.to_string(),
+ weight,
+ valid_at_ms,
+ invalid_at_ms,
+ hop,
+ };
+ match collected.get_mut(edge_id) {
+ Some(existing) => {
+ // Memtable wins over a segment row with the same
+ // edge_id (it is the newer version); the earliest
+ // discovery hop is kept so the sort/frontier
+ // semantics don't shift.
+ if !existing.from_memtable {
+ let keep_hop = existing.edge.hop.min(hop);
+ existing.edge = GraphEdgeLite {
+ hop: keep_hop,
+ ..edge
+ };
+ existing.from_memtable = true;
+ }
+ }
+ None => {
+ if collected.len() >= cap {
+ stats.edges_clipped += 1;
+ capped = true;
+ continue;
+ }
+ hop_new.push(edge.edge_id.clone());
+ collected.insert(
+ edge.edge_id.clone(),
+ Collected {
+ edge,
+ from_memtable: true,
+ },
+ );
+ }
+ }
+ }
+ }
+
+ // ── Next frontier: endpoints newly discovered this hop ───────────
+ // Walked in the hop's stable order (weight desc, edge_id asc — hop
+ // is constant within the block) so a hop-2 frontier clip drops the
+ // same ids on every run.
+ if hop < req.hops && !capped {
+ let mut hop_edges: Vec<&GraphEdgeLite> = hop_new
+ .iter()
+ .filter_map(|eid| collected.get(eid).map(|c| &c.edge))
+ .collect();
+ hop_edges.sort_by(|a, b| {
+ b.weight
+ .total_cmp(&a.weight)
+ .then_with(|| a.edge_id.cmp(&b.edge_id))
+ });
+ let mut next: Vec<String> = Vec::new();
+ for e in hop_edges {
+ for id in [&e.src, &e.dst] {
+ if !visited.contains(id) {
+ visited.insert(id.clone());
+ next.push(id.clone());
+ }
+ }
+ }
+ if next.len() > GRAPH_MAX_FRONTIER {
+ stats.frontier_clipped += (next.len() - GRAPH_MAX_FRONTIER) as u64;
+ next.truncate(GRAPH_MAX_FRONTIER);
+ }
+ frontier = next;
+ }
+ }
+
+ // ── Final order + reachable ──────────────────────────────────────────
+ let mut edges: Vec<GraphEdgeLite> =
+ collected.into_values().map(|c| c.edge).collect();
+ edges.sort_by(|a, b| {
+ a.hop
+ .cmp(&b.hop)
+ .then_with(|| b.weight.total_cmp(&a.weight))
+ .then_with(|| a.edge_id.cmp(&b.edge_id))
+ });
+
+ let mut reachable: Vec<String> = Vec::with_capacity(admitted_frontier.len() + edges.len());
+ let mut seen: HashSet<String> = HashSet::with_capacity(admitted_frontier.len());
+ let push_unique = |out: &mut Vec<String>, seen: &mut HashSet<String>, id: &str| {
+ if seen.insert(id.to_string()) {
+ out.push(id.to_string());
+ }
+ };
+ for id in &admitted_frontier {
+ push_unique(&mut reachable, &mut seen, id);
+ }
+ for e in &edges {
+ push_unique(&mut reachable, &mut seen, &e.src);
+ push_unique(&mut reachable, &mut seen, &e.dst);
+ }
+
+ Ok(GraphExpandResult {
+ edges,
+ reachable,
+ stats,
+ })
+ }
+}
diff --git a/engine/crates/xerj-engine/src/index.rs b/engine/crates/xerj-engine/src/index.rs
index e0019ba00..7410ef33f 100644
--- a/engine/crates/xerj-engine/src/index.rs
+++ b/engine/crates/xerj-engine/src/index.rs
@@ -21576,8 +21576,6 @@ fn is_doc_scan_query(q: &QueryNode) -> bool {
| QueryNode::SpanFirst { .. }
| QueryNode::SpanContaining { .. }
| QueryNode::SpanWithin { .. }
- | QueryNode::HasChild { .. }
- | QueryNode::HasParent { .. }
| QueryNode::Constant { .. }
| QueryNode::Boosted { .. }
| QueryNode::Intervals { .. }
@@ -21798,9 +21796,6 @@ fn query_needs_id_injection(q: &QueryNode) -> bool {
} => query_needs_id_injection(positive) || query_needs_id_injection(negative),
QueryNode::DisMax { queries, .. } => queries.iter().any(query_needs_id_injection),
QueryNode::Pinned { organic, .. } => query_needs_id_injection(organic),
- QueryNode::HasChild { query, .. } | QueryNode::HasParent { query, .. } => {
- query_needs_id_injection(query)
- }
QueryNode::Knn { filter, .. } | QueryNode::SemanticSearch { filter, .. } => {
filter.as_deref().is_some_and(query_needs_id_injection)
}
@@ -22960,15 +22955,9 @@ fn doc_matches_query(q: &QueryNode, source: &Value) -> bool {
.any(|b| littles.iter().any(|l| b.start <= l.start && l.end <= b.end))
}
- // ── Join queries — UNREACHABLE ─────────────────────────────────────────
- // has_child/has_parent are rejected with a 400 at parse time
- // (see xerj-query parser.rs::parse_has_child), so these AST variants
- // are never built and these branches never execute. Previously they
- // silently ran the inner query on the flat doc set, which returned
- // wrong results (no parent/child join is materialized). Kept for
- // exhaustiveness / a future real join executor.
- QueryNode::HasChild { query, .. } => doc_matches_query(query, source),
- QueryNode::HasParent { query, .. } => doc_matches_query(query, source),
+ // (has_child/has_parent have no AST variants: the parser 400s them, and
+ // the flat-match arms that would run the inner query on an unjoined doc
+ // set — wrong results — were removed along with the variants.)
// ── Geo shape queries ─────────────────────────────────────────────────
@@ -30508,3 +30497,14 @@ mod flush_memory_integration_tests {
run_post_warm_failure_case("flush-post-warm-panic", true).await;
}
}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Graph expansion (second brain) — batched, columnar, bounded
+// ─────────────────────────────────────────────────────────────────────────────
+// Declared as a CHILD of this module (via `#[path]`) rather than a sibling:
+// the hop path needs `dv_columns_for`, `ghost_positions_for`, `self.store`,
+// `self.memtable`, `self.data_dir` — all private to `index`. A child module
+// sees them without widening any visibility; the implementation still lives in
+// its own file (src/graph.rs) instead of growing this one.
+#[path = "graph.rs"]
+pub mod graph;
diff --git a/engine/crates/xerj-engine/src/lib.rs b/engine/crates/xerj-engine/src/lib.rs
index 06d9c0711..be0b21f85 100644
--- a/engine/crates/xerj-engine/src/lib.rs
+++ b/engine/crates/xerj-engine/src/lib.rs
@@ -43,6 +43,10 @@ pub use index::{
detect_log_format, resolve_date_math, resolve_field_alias, EnrichTable, FieldEncodingInfo,
Index, IndexResponse, IndexStats, LogFormat,
};
+// Second-brain graph expansion (bounded columnar traversal over an edges
+// index). Lives under `index` for private-field access; re-exported at the
+// crate root so callers write `xerj_engine::graph::GraphExpandRequest`.
+pub use index::graph;
pub use memtable::FtsMemtable;
// ── Process-global metrics handle ─────────────────────────────────────────────
diff --git a/engine/crates/xerj-engine/tests/graph_expand.rs b/engine/crates/xerj-engine/tests/graph_expand.rs
new file mode 100644
index 000000000..8aab3b6e3
--- /dev/null
+++ b/engine/crates/xerj-engine/tests/graph_expand.rs
@@ -0,0 +1,378 @@
+//! Engine-level tests for `Index::graph_expand` (second brain, contract §3.6).
+//!
+//! Every test builds the contract §8 fixture — the five `notes/` files' edge
+//! set (4 wikilink + 4 same_dir edges, all valid_at = created_at =
+//! 1753600000000) — via `index_document` on a `.xerj-memory-notes-edges`
+//! index, then asserts the §8.4 expansions:
+//!
+//! - pre-flush (memtable path) and post-flush (columnar path) results are
+//! identical modulo the two flush-state cost counters,
+//! - soft invalidation is time-travelable via `as_of` (and exercises the
+//! ghost-bitmap integration: the stale pre-invalidation segment row must
+//! NOT resurface),
+//! - the hop-2 block ordering, the result-edge cap, and the hops bounds.
+
+use serde_json::{json, Value};
+use tempfile::TempDir;
+use xerj_common::config::Config;
+use xerj_engine::graph::{
+ GraphDirection, GraphExpandRequest, GraphExpandResult, GRAPH_HOPS_CAP_REASON,
+};
+use xerj_engine::{Engine, Index};
+
+const EDGES_INDEX: &str = ".xerj-memory-notes-edges";
+const T0: i64 = 1_753_600_000_000; // fixture valid_at / created_at
+const AS_OF: i64 = 1_753_700_000_000; // fixture as-of instant
+
+// §8.3 edge ids (computed with the contract's edge_id fn; pinned there).
+const E_ALPHA_BETA_WIKI: &str = "bef814a75bd3d914c3e561f610154304";
+const E_ALPHA_GAMMA_WIKI: &str = "11c2d0ef216cd6e99a3907a0b53c1452";
+const E_BETA_GAMMA_WIKI: &str = "9bbf7d2068321ac0fa71d95e21fae2fd";
+const E_DELTA_ALPHA_WIKI: &str = "cead55986c364ad5ff6f0894daf61f77";
+const E_ALPHA_BETA_DIR: &str = "63b747655365aa16d38188aa49966f40";
+const E_BETA_DELTA_DIR: &str = "a61e6caacb5e485baf6d45184f23ec67";
+const E_DELTA_EPSILON_DIR: &str = "3efff61b58c978943e6fd2a1e4eeaee8";
+const E_EPSILON_GAMMA_DIR: &str = "7c07cdc441f0a3faa29be8946df3e7a4";
+
+fn make_engine(dir: &TempDir) -> Engine {
+ let mut config = Config::default();
+ config.server.data_dir = dir.path().to_str().unwrap().to_string();
+ Engine::new(config).expect("engine::new")
+}
+
+/// One §2.4-shaped edge document. Timestamps are epoch-ms NUMBERS and
+/// `invalid_at`/`expired_at` are omitted entirely (never null) — the omission
+/// is what lands the row in the column null_bitmap, which is the "still valid"
+/// signal the hop reads.
+#[allow(clippy::too_many_arguments)]
+fn edge_doc(
+ edge_id: &str,
+ src: &str,
+ dst: &str,
+ edge_type: &str,
+ weight: f64,
+ confidence: f64,
+ detector: &str,
+ quote: &str,
+ source: &str,
+ offset: u64,
+) -> Value {
+ json!({
+ "edge_id": edge_id,
+ "src": src,
+ "dst": dst,
+ "type": edge_type,
+ "weight": weight,
+ "valid_at": T0,
+ "created_at": T0,
+ "detector": detector,
+ "confidence": confidence,
+ "schema_version": 1,
+ "src_file": source,
+ "evidence": { "quote": quote, "source": source, "offset": offset }
+ })
+}
+
+/// The eight §8.3 fixture edges, in table order.
+fn fixture_edges() -> Vec<Value> {
+ let alpha_line = "Alpha is the hub note. It links to [[beta]] and [[gamma]].";
+ vec![
+ edge_doc(E_ALPHA_BETA_WIKI, "note-alpha", "note-beta", "wikilink", 1.0, 0.95,
+ "wikilink@1", alpha_line, "alpha.md", 35),
+ edge_doc(E_ALPHA_GAMMA_WIKI, "note-alpha", "note-gamma", "wikilink", 1.0, 0.95,
+ "wikilink@1", alpha_line, "alpha.md", 48),
+ edge_doc(E_BETA_GAMMA_WIKI, "note-beta", "note-gamma", "wikilink", 1.0, 0.95,
+ "wikilink@1", "Beta continues the thread and references [[gamma]].", "beta.md", 41),
+ edge_doc(E_DELTA_ALPHA_WIKI, "note-delta", "note-alpha", "wikilink", 1.0, 0.95,
+ "wikilink@1", "Delta cites [[alpha]] as its source.", "delta.md", 12),
+ edge_doc(E_ALPHA_BETA_DIR, "note-alpha", "note-beta", "same_dir", 0.3, 0.4,
+ "samedir@1", "alpha.md and beta.md share directory .", "alpha.md", 0),
+ edge_doc(E_BETA_DELTA_DIR, "note-beta", "note-delta", "same_dir", 0.3, 0.4,
+ "samedir@1", "beta.md and delta.md share directory .", "beta.md", 0),
+ edge_doc(E_DELTA_EPSILON_DIR, "note-delta", "note-epsilon", "same_dir", 0.3, 0.4,
+ "samedir@1", "delta.md and epsilon.md share directory .", "delta.md", 0),
+ edge_doc(E_EPSILON_GAMMA_DIR, "note-epsilon", "note-gamma", "same_dir", 0.3, 0.4,
+ "samedir@1", "epsilon.md and gamma.md share directory .", "epsilon.md", 0),
+ ]
+}
+
+async fn fixture_index(engine: &Engine) -> std::sync::Arc<Index> {
+ engine
+ .create_index(EDGES_INDEX, xerj_common::types::Schema::empty())
+ .expect("create edges index");
+ let idx = engine.get_index(EDGES_INDEX).expect("get edges index");
+ for doc in fixture_edges() {
+ let id = doc["edge_id"].as_str().unwrap().to_string();
+ idx.index_document(Some(id), doc).await.expect("index edge");
+ }
+ idx
+}
+
+fn expand_req(frontier: &[&str], hops: u8) -> GraphExpandRequest {
+ GraphExpandRequest {
+ frontier: frontier.iter().map(|s| s.to_string()).collect(),
+ hops,
+ direction: GraphDirection::Both,
+ types: None,
+ as_of_ms: AS_OF,
+ include_expired: false,
+ max_result_edges: 1000,
+ }
+}
+
+/// (edge_id, src, dst, type, weight, hop) tuples — the §8.4 comparison shape.
+fn tuples(res: &GraphExpandResult) -> Vec<(String, String, String, String, f64, u8)> {
+ res.edges
+ .iter()
+ .map(|e| {
+ (
+ e.edge_id.clone(),
+ e.src.clone(),
+ e.dst.clone(),
+ e.edge_type.clone(),
+ e.weight,
+ e.hop,
+ )
+ })
+ .collect()
+}
+
+fn assert_hop1_fixture(res: &GraphExpandResult) {
+ // §8.4: EXACTLY this order (hop asc, weight desc, edge_id asc).
+ let got = tuples(res);
+ let want = vec![
+ (E_ALPHA_GAMMA_WIKI.into(), "note-alpha".into(), "note-gamma".into(), "wikilink".into(), 1.0, 1u8),
+ (E_ALPHA_BETA_WIKI.into(), "note-alpha".into(), "note-beta".into(), "wikilink".into(), 1.0, 1u8),
+ (E_DELTA_ALPHA_WIKI.into(), "note-delta".into(), "note-alpha".into(), "wikilink".into(), 1.0, 1u8),
+ (E_ALPHA_BETA_DIR.into(), "note-alpha".into(), "note-beta".into(), "same_dir".into(), 0.3, 1u8),
+ ];
+ assert_eq!(got, want, "hop-1 edge list must match §8.4 exactly");
+ assert_eq!(
+ res.reachable,
+ vec!["note-alpha", "note-gamma", "note-beta", "note-delta"],
+ "§8.4 reachable order: frontier first, then first-discovery over sorted edges"
+ );
+ // Every stats field except the two flush-state cost counters is exact.
+ assert_eq!(res.stats.frontier_clipped, 0);
+ assert_eq!(res.stats.edges_clipped, 0);
+ assert_eq!(res.stats.expired_excluded, 0);
+ assert_eq!(res.stats.type_filtered, 0);
+ assert_eq!(res.stats.segments_without_columns, 0);
+}
+
+/// §3.6 (a) + (b): the §8.4 hop-1 expansion is exact on the memtable path and
+/// identical after `flush()` on the columnar path.
+#[tokio::test]
+async fn graph_expand_fixture_hop1_pre_and_post_flush() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = fixture_index(&engine).await;
+ let req = expand_req(&["note-alpha"], 1);
+
+ // Pre-flush: everything comes from the memtable walk.
+ let pre = idx.graph_expand(&req).expect("pre-flush expand");
+ assert_hop1_fixture(&pre);
+ assert_eq!(pre.stats.segments_scanned, 0, "no segments exist pre-flush");
+ assert!(pre.stats.memtable_docs_scanned >= 8);
+
+ // Post-flush: everything comes from doc-values columns.
+ idx.flush().await.expect("flush");
+ let post = idx.graph_expand(&req).expect("post-flush expand");
+ assert_hop1_fixture(&post);
+ assert!(post.stats.segments_scanned >= 1, "columnar path must run");
+ assert_eq!(
+ post.stats.memtable_docs_scanned, 0,
+ "flush drained the memtable"
+ );
+ assert_eq!(tuples(&pre), tuples(&post), "flush must not change results");
+ assert_eq!(pre.reachable, post.reachable);
+}
+
+/// §8.4 hop-2: the hop-2 block (frontier = gamma, beta, delta) appends the
+/// remaining four edges in (weight desc, edge_id asc) order and reachable
+/// gains note-epsilon.
+#[tokio::test]
+async fn graph_expand_fixture_hop2_block_order() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = fixture_index(&engine).await;
+ let req = expand_req(&["note-alpha"], 2);
+
+ for flushed in [false, true] {
+ if flushed {
+ idx.flush().await.expect("flush");
+ }
+ let res = idx.graph_expand(&req).expect("expand hops=2");
+ let ids: Vec<&str> = res.edges.iter().map(|e| e.edge_id.as_str()).collect();
+ assert_eq!(
+ ids,
+ vec![
+ // hop-1 block (§8.4)
+ E_ALPHA_GAMMA_WIKI,
+ E_ALPHA_BETA_WIKI,
+ E_DELTA_ALPHA_WIKI,
+ E_ALPHA_BETA_DIR,
+ // hop-2 block: 9bbf (w 1.0), then w-0.3 ties by edge_id asc
+ E_BETA_GAMMA_WIKI,
+ E_DELTA_EPSILON_DIR,
+ E_EPSILON_GAMMA_DIR,
+ E_BETA_DELTA_DIR,
+ ],
+ "hop-2 block order (flushed={flushed})"
+ );
+ assert!(res.edges[..4].iter().all(|e| e.hop == 1));
+ assert!(res.edges[4..].iter().all(|e| e.hop == 2));
+ assert_eq!(
+ res.reachable,
+ vec![
+ "note-alpha",
+ "note-gamma",
+ "note-beta",
+ "note-delta",
+ "note-epsilon"
+ ],
+ "hop 2 reaches epsilon (flushed={flushed})"
+ );
+ }
+}
+
+/// §3.6 (c) + §8.6: soft invalidation is a re-index under the same `_id` with
+/// `invalid_at`/`expired_at` added. Expanding at `as_of < invalid_at` still
+/// sees the edge (belief last Tuesday intact); at `as_of >= invalid_at` it is
+/// gone with `expired_excluded == 1`; `include_expired` brings it back.
+///
+/// The first round runs with the STALE row in a flushed segment and the
+/// invalidated row in the memtable — the mixed state that only works when the
+/// ghost-bitmap machinery suppresses the superseded segment row.
+#[tokio::test]
+async fn graph_expand_soft_invalidate_time_travel() {
+ const INVALID_AT: i64 = 1_753_650_000_000;
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = fixture_index(&engine).await;
+ idx.flush().await.expect("flush fixture");
+
+ // Soft-invalidate alpha→beta wikilink: same doc, same `_id`, two added
+ // fields — exactly what `DELETE /_graph/{brain}/link/{edge_id}` re-indexes.
+ let mut invalidated = fixture_edges()[0].clone();
+ invalidated["invalid_at"] = json!(INVALID_AT);
+ invalidated["expired_at"] = json!(INVALID_AT);
+ idx.index_document(Some(E_ALPHA_BETA_WIKI.to_string()), invalidated)
+ .await
+ .expect("re-index invalidated edge");
+
+ for (round, flushed) in [(1, false), (2, true)] {
+ if flushed {
+ idx.flush().await.expect("flush invalidated version");
+ }
+
+ // Belief BEFORE the invalidation instant: all four §8.4 edges.
+ let mut req = expand_req(&["note-alpha"], 1);
+ req.as_of_ms = 1_753_640_000_000;
+ let before = idx.graph_expand(&req).expect("expand before invalid_at");
+ assert_hop1_fixture(&before);
+
+ // Belief AFTER: the invalidated edge is excluded and counted.
+ req.as_of_ms = AS_OF;
+ let after = idx.graph_expand(&req).expect("expand after invalid_at");
+ let ids: Vec<&str> = after.edges.iter().map(|e| e.edge_id.as_str()).collect();
+ assert_eq!(
+ ids,
+ vec![E_ALPHA_GAMMA_WIKI, E_DELTA_ALPHA_WIKI, E_ALPHA_BETA_DIR],
+ "round {round}: invalidated edge must disappear at as_of >= invalid_at"
+ );
+ assert_eq!(
+ after.stats.expired_excluded, 1,
+ "round {round}: the exclusion is counted, not silent"
+ );
+
+ // include_expired=true: the edge returns, carrying its invalid_at.
+ req.include_expired = true;
+ let all = idx.graph_expand(&req).expect("expand include_expired");
+ let inv = all
+ .edges
+ .iter()
+ .find(|e| e.edge_id == E_ALPHA_BETA_WIKI)
+ .expect("include_expired returns the invalidated edge");
+ assert_eq!(inv.invalid_at_ms, Some(INVALID_AT));
+ assert_eq!(all.edges.len(), 4);
+ }
+}
+
+/// §3.6 (d) + §3.5: result cap reports clipping; hops out of bounds and an
+/// empty frontier are loud `InvalidQuery` errors carrying the §4.6 sentence.
+#[tokio::test]
+async fn graph_expand_caps_and_bounds() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = fixture_index(&engine).await;
+
+ for flushed in [false, true] {
+ if flushed {
+ idx.flush().await.expect("flush");
+ }
+ let mut req = expand_req(&["note-alpha"], 2);
+ req.max_result_edges = 2;
+ let res = idx.graph_expand(&req).expect("capped expand");
+ assert_eq!(res.edges.len(), 2, "cap bounds the result (flushed={flushed})");
+ assert!(
+ res.stats.edges_clipped > 0,
+ "overflow must be counted, never silent (flushed={flushed})"
+ );
+ }
+
+ for bad_hops in [0u8, 3] {
+ let err = idx
+ .graph_expand(&expand_req(&["note-alpha"], bad_hops))
+ .expect_err("hops out of bounds must fail loud");
+ let msg = err.to_string();
+ assert!(
+ msg.contains(GRAPH_HOPS_CAP_REASON),
+ "hops={bad_hops} error must carry the §4.6 sentence, got: {msg}"
+ );
+ }
+
+ let err = idx
+ .graph_expand(&expand_req(&[], 1))
+ .expect_err("empty frontier must fail loud");
+ assert!(err.to_string().contains("non-empty frontier"));
+}
+
+/// Direction and type filters: `Out`/`In` scan a single column; the `types`
+/// allowlist counts what it drops.
+#[tokio::test]
+async fn graph_expand_direction_and_type_filter() {
+ let dir = TempDir::new().unwrap();
+ let engine = make_engine(&dir);
+ let idx = fixture_index(&engine).await;
+ idx.flush().await.expect("flush");
+
+ let mut req = expand_req(&["note-alpha"], 1);
+ req.direction = GraphDirection::Out;
+ let out = idx.graph_expand(&req).expect("out expand");
+ let ids: Vec<&str> = out.edges.iter().map(|e| e.edge_id.as_str()).collect();
+ assert_eq!(
+ ids,
+ vec![E_ALPHA_GAMMA_WIKI, E_ALPHA_BETA_WIKI, E_ALPHA_BETA_DIR],
+ "direction=out follows src → dst only"
+ );
+
+ req.direction = GraphDirection::In;
+ let inn = idx.graph_expand(&req).expect("in expand");
+ let ids: Vec<&str> = inn.edges.iter().map(|e| e.edge_id.as_str()).collect();
+ assert_eq!(ids, vec![E_DELTA_ALPHA_WIKI], "direction=in follows dst → src only");
+
+ req.direction = GraphDirection::Both;
+ req.types = Some(vec!["wikilink".to_string()]);
+ let typed = idx.graph_expand(&req).expect("typed expand");
+ let ids: Vec<&str> = typed.edges.iter().map(|e| e.edge_id.as_str()).collect();
+ assert_eq!(
+ ids,
+ vec![E_ALPHA_GAMMA_WIKI, E_ALPHA_BETA_WIKI, E_DELTA_ALPHA_WIKI],
+ "types allowlist keeps only wikilink edges"
+ );
+ assert_eq!(
+ typed.stats.type_filtered, 1,
+ "the dropped same_dir edge is counted"
+ );
+}
diff --git a/engine/crates/xerj-query/src/ast.rs b/engine/crates/xerj-query/src/ast.rs
index bc7830d8b..af8787ee0 100644
--- a/engine/crates/xerj-query/src/ast.rs
+++ b/engine/crates/xerj-query/src/ast.rs
@@ -662,20 +662,12 @@ pub enum QueryNode {
big: Box<QueryNode>,
},
- // ── Join queries ──────────────────────────────────────────────────────────
- /// Has-child — matches parent documents that have child documents matching the query.
- HasChild {
- child_type: String,
- query: Box<QueryNode>,
- score_mode: Option<String>,
- },
-
- /// Has-parent — matches child documents that have parent documents matching the query.
- HasParent {
- parent_type: String,
- query: Box<QueryNode>,
- score: bool,
- },
+ // NOTE: there are deliberately NO join-query variants (has_child /
+ // has_parent). XERJ never materializes a parent/child join, so an AST
+ // variant here would only exist to be flat-matched — silently wrong
+ // results — if anyone re-enabled parsing. The parser rejects both with a
+ // 400 (parser.rs::parse_has_child / parse_has_parent); the corpse variants
+ // and their dead planner/executor arms were removed.
// ── Geo shape queries ─────────────────────────────────────────────────────
/// Geo polygon — matches documents whose geo_point falls within the polygon.
diff --git a/engine/crates/xerj-query/src/planner.rs b/engine/crates/xerj-query/src/planner.rs
index d1b905664..ffda47fa0 100644
--- a/engine/crates/xerj-query/src/planner.rs
+++ b/engine/crates/xerj-query/src/planner.rs
@@ -648,21 +648,6 @@ fn plan_node(query: QueryNode, schema: &Schema) -> ExecutionPlan {
plan_node(*big, schema)
}
- // ── Join queries — UNREACHABLE ────────────────────────────────────────
- // has_child/has_parent are rejected with a 400 at parse time
- // (see parser.rs::parse_has_child), so these AST variants are never
- // built and this branch is never taken. Kept for exhaustiveness and
- // in case the AST variant is ever wired to a real join executor.
- QueryNode::HasChild { query, .. } | QueryNode::HasParent { query, .. } => {
- let inner = plan_node(*query, schema);
- let cost = inner.cost() + 10.0;
- ExecutionPlan::Boost {
- boost: 1.0,
- inner: Box::new(inner),
- cost,
- }
- }
-
// ── Geo shape queries — treat as doc-scans ────────────────────────────
QueryNode::GeoPolygon { field, .. } => ExecutionPlan::ExistsScan { field },
QueryNode::GeoShape { field, .. } => ExecutionPlan::ExistsScan { field },