AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
wip(second-brain/autoindex): autoindex link detectors
xerj-org3772201eab63
diff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rsindex 09436f00c..ee81b6a7b 100644--- a/engine/crates/xerj-autoindex/src/cli.rs+++ b/engine/crates/xerj-autoindex/src/cli.rs@@ -19,6 +19,10 @@ pub struct IndexCfg {pub max_file_gb: u64,pub sample: usize,pub no_semantic: bool,+ /// Second-brain name; None derives it from the root folder basename.+ pub brain: Option<String>,+ /// Disable edge detection entirely (no `.xerj-memory-*-edges` writes).+ pub no_graph: bool,pub dry_run: bool,pub json: bool,pub quiet: bool,@@ -75,6 +79,10 @@ pub fn print_help() {--max-file-gb <N> skip+record oversized non-streamable files (default 2)\n\--sample <N> records sampled per file for inference (default 500)\n\--no-semantic skip semantic_text on body fields (pure BM25+keyword)\n\+ --brain <NAME> second-brain name; relationship edges land in\n\+ .xerj-memory-<NAME>-edges (default: folder name slug)\n\+ --no-graph skip relationship detection (wikilinks, local links,\n\+ section order, directory chains) — no edges are written\n\--dry-run walk+sniff+infer, print the plan, index nothing\n\--json machine-readable output (map: raw catalog docs)\n\--quiet errors only\n\@@ -132,6 +140,8 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {let mut max_file_gb = 2u64;let mut sample = 500usize;let mut no_semantic = false;+ let mut brain: Option<String> = None;+ let mut no_graph = false;let mut dry_run = false;let mut json = false;let mut quiet = false;@@ -198,6 +208,15 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {.ok_or("--sample needs a number")?}"--no-semantic" => no_semantic = true,+ "--brain" => {+ let name = it.next().ok_or("--brain needs a value")?;+ // Fail at parse time — a bad brain name after a long Phase A+ // would be a rude place to learn about the '-edges' rule.+ crate::detect::validate_brain(&name)+ .map_err(|reason| format!("--brain {name}: {reason}"))?;+ brain = Some(name);+ }+ "--no-graph" => no_graph = true,"--dry-run" => dry_run = true,"--json" => json = true,"--md" => json = false,@@ -254,6 +273,8 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {max_file_gb,sample: sample.max(50),no_semantic,+ brain,+ no_graph,dry_run,json,quiet,@@ -278,6 +299,32 @@ mod tests {assert_eq!(index(&["data"]).bulk_timeout_secs, 300);}+ #[test]+ fn graph_is_on_by_default_with_derived_brain() {+ let cfg = index(&["data"]);+ assert!(!cfg.no_graph);+ assert_eq!(cfg.brain, None, "brain derives from the folder at run time");+ assert!(index(&["data", "--no-graph"]).no_graph);+ assert_eq!(+ index(&["data", "--brain", "notes"]).brain.as_deref(),+ Some("notes")+ );+ }++ #[test]+ fn brain_names_are_validated_at_parse_time() {+ for bad in ["kb-edges", "Notes", "-x", "a..b"] {+ let err = parse(+ ["data", "--brain", bad]+ .into_iter()+ .map(str::to_string)+ .collect(),+ )+ .unwrap_err();+ assert!(err.contains("--brain"), "{err}");+ }+ }+#[test]fn bulk_timeout_accepts_custom_bounded_value() {assert_eq!(diff --git a/engine/crates/xerj-autoindex/src/detect/e2e.rs b/engine/crates/xerj-autoindex/src/detect/e2e.rsnew file mode 100644index 000000000..4d8f199b7--- /dev/null+++ b/engine/crates/xerj-autoindex/src/detect/e2e.rs@@ -0,0 +1,541 @@+//! End-to-end detector tests over the contract's §8 fixture: a real+//! `run_index` against a fake ES endpoint, asserting the EXACT edge set the+//! five notes must produce — including what must NOT be produced (dangling+//! links become counters, replaced files' prior edges become soft-invalidated+//! documents, and a re-run converges byte-identically modulo `created_at`).++use crate::cli::IndexCfg;+use crate::detect;+use serde_json::{json, Value};+use std::collections::{BTreeMap, HashMap};+use std::fs;+use std::io::{BufRead, BufReader, Read, Write};+use std::net::{TcpListener, TcpStream};+use std::path::Path;+use std::sync::{Arc, Mutex};+use std::thread;++/// index → (_id → _source). BTreeMaps keep every assertion order-stable.+type Docs = BTreeMap<String, BTreeMap<String, Value>>;++struct MockEs {+ url: String,+ docs: Arc<Mutex<Docs>>,+ stop: Arc<Mutex<bool>>,+ join: Option<thread::JoinHandle<()>>,+}++impl MockEs {+ fn start() -> Self {+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ listener.set_nonblocking(true).unwrap();+ let url = format!("http://{}", listener.local_addr().unwrap());+ let docs: Arc<Mutex<Docs>> = Arc::default();+ let stop = Arc::new(Mutex::new(false));+ let (sd, ss) = (Arc::clone(&docs), Arc::clone(&stop));+ let join = thread::spawn(move || loop {+ match listener.accept() {+ Ok((stream, _)) => handle(stream, &sd),+ Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {+ if *ss.lock().unwrap() {+ break;+ }+ thread::yield_now();+ }+ Err(e) => panic!("mock accept: {e}"),+ }+ });+ Self {+ url,+ docs,+ stop,+ join: Some(join),+ }+ }++ fn index(&self, name: &str) -> BTreeMap<String, Value> {+ self.docs.lock().unwrap().get(name).cloned().unwrap_or_default()+ }++ /// rel path → anchor node `_id`, straight from the published node docs so+ /// the expectation uses exactly the ids the pipeline derived.+ fn anchors(&self) -> HashMap<String, String> {+ let mut out = HashMap::new();+ for (index, docs) in self.docs.lock().unwrap().iter() {+ if index.starts_with('.') {+ continue; // catalog + edges indices are not node datasets+ }+ for (id, source) in docs {+ if source.get("ax_locator").and_then(Value::as_str) == Some("s0") {+ if let Some(rel) = source.get("ax_path").and_then(Value::as_str) {+ out.insert(rel.to_string(), id.clone());+ }+ }+ }+ }+ out+ }+}++impl Drop for MockEs {+ fn drop(&mut self) {+ *self.stop.lock().unwrap() = true;+ let _ = TcpStream::connect(self.url.trim_start_matches("http://"));+ self.join.take().unwrap().join().unwrap();+ }+}++fn respond(mut stream: TcpStream, status: u16, body: &Value) {+ let bytes = body.to_string();+ let reason = if status == 200 { "OK" } else { "Not Found" };+ write!(+ stream,+ "HTTP/1.1 {status} {reason}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",+ bytes.len(),+ bytes+ )+ .unwrap();+}++fn handle(stream: TcpStream, docs: &Arc<Mutex<Docs>>) {+ let clone = stream.try_clone().unwrap();+ let mut reader = BufReader::new(clone);+ let mut request_line = String::new();+ if reader.read_line(&mut request_line).unwrap() == 0 {+ return;+ }+ let mut content_length = 0usize;+ loop {+ let mut line = String::new();+ reader.read_line(&mut line).unwrap();+ if line == "\r\n" || line.is_empty() {+ break;+ }+ if let Some(v) = line+ .to_ascii_lowercase()+ .strip_prefix("content-length:")+ .map(str::trim)+ {+ content_length = v.parse().unwrap();+ }+ }+ let mut body = vec![0; content_length];+ reader.read_exact(&mut body).unwrap();+ let mut parts = request_line.split_whitespace();+ let method = parts.next().unwrap_or("");+ let path = parts.next().unwrap_or("").to_string();++ if method == "POST" && path == "/_bulk" {+ return respond(stream, 200, &bulk(&body, docs));+ }+ if let Some((index, id)) = doc_route(&path) {+ let locked = docs.lock().unwrap();+ return match locked.get(&index).and_then(|d| d.get(&id)) {+ Some(source) => respond(+ stream,+ 200,+ &json!({"found": true, "_id": id, "_source": source}),+ ),+ None => respond(stream, 404, &json!({"found": false})),+ };+ }+ if method == "POST" && path.contains("/_search") {+ let index = path.trim_start_matches('/').split('/').next().unwrap_or("");+ let query: Value = serde_json::from_slice(&body).unwrap_or(Value::Null);+ return respond(stream, 200, &search(index, &query, docs));+ }+ if method == "POST" && path.contains("/_delete_by_query") {+ let index = path.trim_start_matches('/').split('/').next().unwrap_or("");+ let query: Value = serde_json::from_slice(&body).unwrap_or(Value::Null);+ let key = query+ .pointer("/query/term/ax_file")+ .and_then(Value::as_str)+ .map(str::to_owned);+ let mut deleted = 0u64;+ if let (Some(key), Some(store)) = (key, docs.lock().unwrap().get_mut(index)) {+ let before = store.len();+ store.retain(|_, d| d.get("ax_file").and_then(Value::as_str) != Some(key.as_str()));+ deleted = (before - store.len()) as u64;+ }+ return respond(stream, 200, &json!({"deleted": deleted, "failures": []}));+ }+ respond(stream, 200, &json!({"acknowledged": true}));+}++fn doc_route(path: &str) -> Option<(String, String)> {+ let mut segs = path.trim_start_matches('/').split('/');+ let index = segs.next()?;+ if segs.next()? != "_doc" {+ return None;+ }+ let id = segs.next()?;+ segs.next().is_none().then(|| (index.to_string(), id.to_string()))+}++fn bulk(body: &[u8], docs: &Arc<Mutex<Docs>>) -> Value {+ let lines: Vec<&[u8]> = body+ .split(|b| *b == b'\n')+ .filter(|l| !l.is_empty())+ .collect();+ let mut locked = docs.lock().unwrap();+ let mut items = Vec::new();+ let mut errors = false;+ for pair in lines.chunks_exact(2) {+ let action: Value = serde_json::from_slice(pair[0]).unwrap();+ let doc: Value = serde_json::from_slice(pair[1]).unwrap();+ let (op, meta) = if let Some(m) = action.get("index") {+ ("index", m)+ } else if let Some(m) = action.get("create") {+ ("create", m)+ } else {+ continue;+ };+ let index = meta["_index"].as_str().unwrap().to_string();+ let id = meta["_id"].as_str().unwrap().to_string();+ let store = locked.entry(index).or_default();+ if op == "create" && store.contains_key(&id) {+ errors = true;+ items.push(json!({"create": {"status": 409,+ "error": {"type": "version_conflict_engine_exception", "reason": "exists"}}}));+ continue;+ }+ store.insert(id, doc);+ items.push(json!({op: {"status": 200}}));+ }+ json!({"errors": errors, "items": items})+}++fn search(index: &str, body: &Value, docs: &Arc<Mutex<Docs>>) -> Value {+ let locked = docs.lock().unwrap();+ let store = locked.get(index).cloned().unwrap_or_default();+ // The §6.6.3 invalidation query: live edges taught by one src_file.+ if let Some(rel) = body+ .pointer("/query/bool/filter/0/term/src_file")+ .and_then(Value::as_str)+ {+ let size = body.get("size").and_then(Value::as_u64).unwrap_or(10) as usize;+ let hits: Vec<Value> = store+ .iter()+ .filter(|(_, s)| {+ s.get("src_file").and_then(Value::as_str) == Some(rel)+ && s.get("invalid_at").is_none()+ })+ .take(size)+ .map(|(id, s)| json!({"_id": id, "_source": s}))+ .collect();+ return json!({"hits": {"total": {"value": hits.len()}, "hits": hits}});+ }+ // size:0 totals (dataset counts, map summary): total = matching docs.+ let live_only = body.pointer("/query/bool/must_not").is_some();+ let total = store+ .values()+ .filter(|s| !live_only || s.get("invalid_at").is_none())+ .count();+ json!({"hits": {"total": {"value": total}, "hits": []}, "aggregations": {}})+}++// ─── scenarios ───────────────────────────────────────────────────────────++const EDGES_INDEX: &str = ".xerj-memory-notes-edges";++fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {+ IndexCfg {+ root: root.to_owned(),+ url: url.to_owned(),+ api_key: None,+ workers: 1,+ pdf_workers: 1,+ pdf_timeout_secs: 30,+ bulk_mb: 8,+ bulk_timeout_secs: 300,+ prefix: "ax".into(),+ state_dir: Some(state_dir.to_owned()),+ fresh: false,+ follow_symlinks: false,+ max_file_gb: 1,+ sample: 50,+ no_semantic: true,+ brain: Some("notes".into()),+ no_graph: false,+ dry_run: false,+ json: false,+ quiet: true,+ }+}++/// §8.1 — all five bodies verbatim; the wikilink byte offsets in §8.2 are+/// offsets into these exact strings.+fn write_fixture(dir: &Path) {+ fs::write(+ dir.join("alpha.md"),+ "Alpha is the hub note. It links to [[beta]] and [[gamma]].",+ )+ .unwrap();+ fs::write(+ dir.join("beta.md"),+ "Beta continues the thread and references [[gamma]].",+ )+ .unwrap();+ fs::write(+ dir.join("gamma.md"),+ "Gamma is the sink note with no outgoing links.",+ )+ .unwrap();+ fs::write(dir.join("delta.md"), "Delta cites [[alpha]] as its source.").unwrap();+ fs::write(dir.join("epsilon.md"), "Epsilon stands alone.").unwrap();+}++fn journal_graph_summary(state_dir: &Path) -> Value {+ let journal = fs::read_to_string(state_dir.join("journal.ndjson")).unwrap();+ journal+ .lines()+ .filter_map(|l| serde_json::from_str::<Value>(l).ok())+ .filter(|v| v.get("kind").and_then(Value::as_str) == Some("finish"))+ .next_back()+ .and_then(|v| v.pointer("/summary/graph").cloned())+ .expect("finish event with graph summary")+}++/// Comparable edge shape: everything the contract §8.3 table pins for+/// autoindex output (node-id derivation included via the live anchors).+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]+struct EdgeKey {+ src: String,+ dst: String,+ edge_type: String,+ detector: String,+ weight: String,+ confidence: String,+ src_file: String,+ quote: String,+ offset: u64,+}++fn edge_keys(edges: &BTreeMap<String, Value>) -> Vec<EdgeKey> {+ let mut out: Vec<EdgeKey> = edges+ .iter()+ .filter(|(id, _)| id.as_str() != detect::BRAIN_META_ID)+ .map(|(id, s)| {+ // Identity discipline: _id == edge_id field == the §2.3 hash.+ assert_eq!(s["edge_id"].as_str().unwrap(), id);+ assert_eq!(+ detect::edge_id(+ s["src"].as_str().unwrap(),+ s["type"].as_str().unwrap(),+ s["dst"].as_str().unwrap(),+ s["valid_at"].as_i64().unwrap(),+ ),+ id.as_str()+ );+ assert_eq!(s["schema_version"], json!(1));+ EdgeKey {+ src: s["src"].as_str().unwrap().into(),+ dst: s["dst"].as_str().unwrap().into(),+ edge_type: s["type"].as_str().unwrap().into(),+ detector: s["detector"].as_str().unwrap().into(),+ weight: s["weight"].to_string(),+ confidence: s["confidence"].to_string(),+ src_file: s["src_file"].as_str().unwrap().into(),+ quote: s["evidence"]["quote"].as_str().unwrap().into(),+ offset: s["evidence"]["offset"].as_u64().unwrap(),+ }+ })+ .collect();+ out.sort();+ out+}++#[allow(clippy::too_many_arguments)]+fn expect(+ anchors: &HashMap<String, String>,+ src_rel: &str,+ dst_rel: &str,+ edge_type: &str,+ detector: &str,+ weight: &str,+ confidence: &str,+ quote: &str,+ offset: u64,+) -> EdgeKey {+ EdgeKey {+ src: anchors[src_rel].clone(),+ dst: anchors[dst_rel].clone(),+ edge_type: edge_type.into(),+ detector: detector.into(),+ weight: weight.into(),+ confidence: confidence.into(),+ src_file: src_rel.into(),+ quote: quote.into(),+ offset,+ }+}++#[test]+fn fixture_folder_end_to_end_matches_the_contract_edge_set() {+ let corpus = tempfile::tempdir().unwrap();+ let state = tempfile::tempdir().unwrap();+ write_fixture(corpus.path());+ let es = MockEs::start();++ assert_eq!(+ crate::run_index(cfg(corpus.path(), state.path(), &es.url)).unwrap(),+ 0+ );++ let anchors = es.anchors();+ assert_eq!(anchors.len(), 5, "five single-section note anchors");+ let edges = es.index(EDGES_INDEX);+ assert!(+ edges.contains_key(detect::BRAIN_META_ID),+ "brain meta doc must exist"+ );+ let meta = &edges[detect::BRAIN_META_ID];+ assert_eq!(meta["brain"], json!("notes"));+ assert_eq!(meta["meta_version"], json!(1));+ assert!(meta["nodes_index"].as_str().unwrap().starts_with("ax-"));++ let alpha_line = "Alpha is the hub note. It links to [[beta]] and [[gamma]].";+ let mut expected = vec![+ // §8.2/§8.3 wikilink edges — exact quotes and byte offsets.+ expect(&anchors, "alpha.md", "beta.md", "wikilink", "wikilink@1", "1.0", "0.95", alpha_line, 35),+ expect(&anchors, "alpha.md", "gamma.md", "wikilink", "wikilink@1", "1.0", "0.95", alpha_line, 48),+ expect(&anchors, "beta.md", "gamma.md", "wikilink", "wikilink@1", "1.0", "0.95",+ "Beta continues the thread and references [[gamma]].", 41),+ expect(&anchors, "delta.md", "alpha.md", "wikilink", "wikilink@1", "1.0", "0.95",+ "Delta cites [[alpha]] as its source.", 12),+ // §8.3 samedir chain over rel-sorted files — 4 edges, not a clique.+ expect(&anchors, "alpha.md", "beta.md", "same_dir", "samedir@1", "0.3", "0.4",+ "alpha.md and beta.md share directory .", 0),+ expect(&anchors, "beta.md", "delta.md", "same_dir", "samedir@1", "0.3", "0.4",+ "beta.md and delta.md share directory .", 0),+ expect(&anchors, "delta.md", "epsilon.md", "same_dir", "samedir@1", "0.3", "0.4",+ "delta.md and epsilon.md share directory .", 0),+ expect(&anchors, "epsilon.md", "gamma.md", "same_dir", "samedir@1", "0.3", "0.4",+ "epsilon.md and gamma.md share directory .", 0),+ ];+ expected.sort();+ assert_eq!(edge_keys(&edges), expected, "the exact §8.3 edge set");++ let g = journal_graph_summary(state.path());+ assert_eq!(g["edges_written"], json!(8));+ assert_eq!(g["by_detector"]["wikilink@1"], json!(4));+ assert_eq!(g["by_detector"]["samedir@1"], json!(4));+ assert_eq!(g["edges_unresolved"], json!(0));+ assert_eq!(g["edges_ambiguous"], json!(0));+ assert_eq!(g["edges_self_dropped"], json!(0));+ assert_eq!(g["edges_invalidated"], json!(0));++ // Determinism (§6.4): a fresh re-run over the unchanged corpus overwrites+ // every edge in place — byte-identical modulo created_at.+ let strip_created = |m: &BTreeMap<String, Value>| -> BTreeMap<String, Value> {+ m.iter()+ .map(|(id, s)| {+ let mut s = s.clone();+ if let Some(o) = s.as_object_mut() {+ o.remove("created_at");+ }+ (id.clone(), s)+ })+ .collect()+ };+ let first = strip_created(&edges);+ let state2 = tempfile::tempdir().unwrap();+ assert_eq!(+ crate::run_index(cfg(corpus.path(), state2.path(), &es.url)).unwrap(),+ 0+ );+ assert_eq!(+ strip_created(&es.index(EDGES_INDEX)),+ first,+ "re-run must converge on the identical edge set"+ );+}++#[test]+fn dangling_wikilink_is_counted_not_silently_dropped() {+ let corpus = tempfile::tempdir().unwrap();+ let state = tempfile::tempdir().unwrap();+ fs::write(+ corpus.path().join("zeta.md"),+ "Zeta links [[nowhere]] and [[eta]].",+ )+ .unwrap();+ fs::write(corpus.path().join("eta.md"), "Eta stands alone.").unwrap();+ let es = MockEs::start();++ assert_eq!(+ crate::run_index(cfg(corpus.path(), state.path(), &es.url)).unwrap(),+ 0+ );++ let edges = es.index(EDGES_INDEX);+ let keys = edge_keys(&edges);+ let types: Vec<(&str, &str)> = keys+ .iter()+ .map(|k| (k.edge_type.as_str(), k.detector.as_str()))+ .collect();+ // One resolved wikilink + one samedir chain edge; [[nowhere]] must not+ // fabricate an edge…+ assert_eq!(+ types,+ vec![("same_dir", "samedir@1"), ("wikilink", "wikilink@1")]+ );+ // …but it must not vanish either: it is a dangling-link fact of record.+ let g = journal_graph_summary(state.path());+ assert_eq!(g["edges_unresolved"], json!(1));+ assert_eq!(g["edges_written"], json!(2));+}++#[test]+fn replaced_file_soft_invalidates_its_prior_edges() {+ let corpus = tempfile::tempdir().unwrap();+ let state = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("alpha.md"), "Alpha cites [[beta]] today.").unwrap();+ fs::write(corpus.path().join("beta.md"), "Beta stands alone.").unwrap();+ let es = MockEs::start();+ assert_eq!(+ crate::run_index(cfg(corpus.path(), state.path(), &es.url)).unwrap(),+ 0+ );+ let before = es.index(EDGES_INDEX);+ let old_wikilink: Vec<String> = before+ .iter()+ .filter(|(_, s)| s["type"] == json!("wikilink"))+ .map(|(id, _)| id.clone())+ .collect();+ assert_eq!(old_wikilink.len(), 1);++ // Rewrite alpha at a strictly later mtime-ms: same fact, new valid_at →+ // a NEW edge_id; the old edge must survive as an invalidated document.+ std::thread::sleep(std::time::Duration::from_millis(15));+ fs::write(+ corpus.path().join("alpha.md"),+ "Alpha still cites [[beta]], reworded.",+ )+ .unwrap();+ assert_eq!(+ crate::run_index(cfg(corpus.path(), state.path(), &es.url)).unwrap(),+ 0+ );++ let after = es.index(EDGES_INDEX);+ let old = &after[&old_wikilink[0]];+ assert!(+ old.get("invalid_at").is_some() && old.get("expired_at").is_some(),+ "prior edge is soft-invalidated, not deleted: {old}"+ );+ let live_wikilinks: Vec<&Value> = after+ .values()+ .filter(|s| s["type"] == json!("wikilink") && s.get("invalid_at").is_none())+ .collect();+ assert_eq!(live_wikilinks.len(), 1, "exactly one live replacement edge");+ assert_eq!(+ live_wikilinks[0]["evidence"]["quote"],+ json!("Alpha still cites [[beta]], reworded.")+ );+ let g = journal_graph_summary(state.path());+ assert!(+ g["edges_invalidated"].as_u64().unwrap() >= 1,+ "invalidation surfaced in the run summary: {g}"+ );+}diff --git a/engine/crates/xerj-autoindex/src/detect/href.rs b/engine/crates/xerj-autoindex/src/detect/href.rsnew file mode 100644index 000000000..8c561df4b--- /dev/null+++ b/engine/crates/xerj-autoindex/src/detect/href.rs@@ -0,0 +1,228 @@+//! href@1 — `<a href="…">` to a corpus file, html-extracted files only.+//!+//! Weight 0.7: an HTML anchor is authored intent, but exported HTML (wikis,+//! doc generators) also mass-produces navigation links, so one anchor says+//! less than one hand-written markdown link.+//!+//! ## Why there is a raw-source pass+//! The HTML extractor strips markup before sectioning — an `<a>` tag never+//! survives into section text, so `detect_text` alone would make this+//! detector a no-op exactly where it matters. `detect_raw_html` therefore+//! scans the RAW decoded source once per html file (the pipeline feeds it in,+//! §6.6.2): edges anchor to the file's s0 node and carry offset 0, the+//! contract's escape hatch for "the extractor lost byte positions". The two+//! passes cannot double-fire: real tags exist only in the raw source, while+//! entity-escaped tags (`<a …>`) exist only in the decoded section+//! text.++use super::{+ line_at, resolve_local, CorpusFile, CorpusIndex, DetectorCounters, EdgeDetector, EdgeDraft,+ LinkTarget, SectionCtx,+};+use std::sync::atomic::{AtomicU64, Ordering};+use std::sync::OnceLock;++pub const TAG: &str = "href@1";+pub const EDGE_TYPE: &str = "href";+pub const WEIGHT: f32 = 0.7;+pub const CONFIDENCE: f32 = 0.85;++/// Family string (`Family::as_str`) this detector is scoped to. Sniffing+/// decides — a `.html` extension on a file that sniffed as prose does not+/// qualify, and an extension-less html export does.+const HTML_FAMILY: &str = "html";++#[derive(Default)]+pub struct Href {+ unresolved: AtomicU64,+}++/// Case-insensitive `<a … href=VALUE>`; the capture groups locate the VALUE+/// start byte so section-text hits (escaped markup) carry a real offset.+fn anchor_re() -> &'static regex::Regex {+ static RE: OnceLock<regex::Regex> = OnceLock::new();+ RE.get_or_init(|| {+ regex::Regex::new(r#"(?is)<a\s[^>]*?href\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))"#)+ .expect("static anchor regex")+ })+}++fn scan_anchors(text: &str, mut hit: impl FnMut(usize, &str)) {+ for caps in anchor_re().captures_iter(text) {+ let m = caps+ .get(1)+ .or_else(|| caps.get(2))+ .or_else(|| caps.get(3));+ if let Some(m) = m {+ hit(m.start(), m.as_str());+ }+ }+}++fn draft(+ file: &CorpusFile,+ src: &str,+ target: &CorpusFile,+ quote: String,+ offset: u64,+) -> EdgeDraft {+ EdgeDraft {+ src: src.to_string(),+ dst: target.anchor_doc_id.clone(),+ edge_type: EDGE_TYPE,+ weight: WEIGHT,+ confidence: CONFIDENCE,+ valid_at_ms: file.mtime_ms,+ src_file: file.rel.clone(),+ quote,+ offset,+ }+}++impl Href {+ /// The raw-source pass (see module docs). `src` is the file's anchor node:+ /// section attribution was lost with the markup, and honesty prefers a+ /// coarse-but-true source node over a guessed section.+ pub fn detect_raw_html(+ &self,+ corpus: &CorpusIndex,+ file: &CorpusFile,+ raw: &str,+ out: &mut Vec<EdgeDraft>,+ ) {+ if file.family != HTML_FAMILY {+ return;+ }+ scan_anchors(raw, |pos, url| {+ match resolve_local(corpus, &file.dir, url) {+ LinkTarget::Hit(target) => out.push(draft(+ file,+ &file.anchor_doc_id,+ target,+ line_at(raw, pos),+ // Byte positions did not survive extraction — the §6.5+ // "best effort" zero, NOT the raw-file offset, which no+ // reader could map back onto a section.+ 0,+ )),+ LinkTarget::External | LinkTarget::Empty => {}+ LinkTarget::Miss => {+ self.unresolved.fetch_add(1, Ordering::Relaxed);+ }+ }+ });+ }+}++impl EdgeDetector for Href {+ fn tag(&self) -> &'static str {+ TAG+ }++ /// Section-text pass: only fires when markup survived into the extracted+ /// text (entity-escaped anchors) — then the byte offset is real.+ fn detect_text(&self, ctx: &SectionCtx<'_>, out: &mut Vec<EdgeDraft>) {+ if ctx.file.family != HTML_FAMILY {+ return;+ }+ scan_anchors(ctx.text, |pos, url| {+ match resolve_local(ctx.corpus, &ctx.file.dir, url) {+ LinkTarget::Hit(target) => out.push(draft(+ ctx.file,+ ctx.section_doc_id,+ target,+ line_at(ctx.text, pos),+ pos as u64,+ )),+ LinkTarget::External | LinkTarget::Empty => {}+ LinkTarget::Miss => {+ self.unresolved.fetch_add(1, Ordering::Relaxed);+ }+ }+ });+ }++ fn counters(&self) -> DetectorCounters {+ DetectorCounters {+ unresolved: self.unresolved.load(Ordering::Relaxed),+ ambiguous: 0,+ }+ }+}++#[cfg(test)]+mod tests {+ use super::super::{corpus_file, CorpusIndex, SectionCtx};+ use super::*;++ fn corpus() -> CorpusIndex {+ CorpusIndex::build(vec![+ corpus_file("site/index.html", "k1", "d", "html", 10),+ corpus_file("site/about.html", "k2", "d", "html", 10),+ corpus_file("notes.md", "k3", "d", "txt-prose", 10),+ ])+ }++ #[test]+ fn raw_pass_resolves_relative_and_counts_misses() {+ let corpus = corpus();+ let det = Href::default();+ let mut out = Vec::new();+ det.detect_raw_html(+ &corpus,+ &corpus.files["site/index.html"],+ "<p><A HREF='about.html'>about</A> <a class=\"x\" href=\"https://e.co\">ext</a>\n\+ <a href=gone.html>miss</a></p>",+ &mut out,+ );+ assert_eq!(out.len(), 1);+ assert_eq!(out[0].dst, corpus.files["site/about.html"].anchor_doc_id);+ assert_eq!(out[0].src, corpus.files["site/index.html"].anchor_doc_id);+ assert_eq!(out[0].offset, 0, "raw pass lost byte positions by contract");+ assert!(out[0].quote.contains("about"));+ assert_eq!(det.counters().unresolved, 1);+ }++ #[test]+ fn non_html_families_are_out_of_scope() {+ let corpus = corpus();+ let det = Href::default();+ let mut out = Vec::new();+ det.detect_raw_html(+ &corpus,+ &corpus.files["notes.md"],+ "<a href=\"site/about.html\">x</a>",+ &mut out,+ );+ let ctx = SectionCtx {+ corpus: &corpus,+ file: &corpus.files["notes.md"],+ section_ordinal: 0,+ section_doc_id: "sec0",+ text: "<a href=\"site/about.html\">x</a>",+ };+ det.detect_text(&ctx, &mut out);+ assert!(out.is_empty());+ assert_eq!(det.counters().unresolved, 0);+ }++ #[test]+ fn section_text_pass_keeps_real_offsets() {+ let corpus = corpus();+ let det = Href::default();+ let text = "escaped <a href=\"about.html\">about</a> anchor";+ let ctx = SectionCtx {+ corpus: &corpus,+ file: &corpus.files["site/index.html"],+ section_ordinal: 0,+ section_doc_id: "sec0",+ text,+ };+ let mut out = Vec::new();+ det.detect_text(&ctx, &mut out);+ assert_eq!(out.len(), 1);+ assert_eq!(out[0].src, "sec0");+ assert_eq!(out[0].offset, 17, "offset of the href VALUE start");+ assert_eq!(&text[17..27], "about.html");+ }+}diff --git a/engine/crates/xerj-autoindex/src/detect/mdlink.rs b/engine/crates/xerj-autoindex/src/detect/mdlink.rsnew file mode 100644index 000000000..c8a524769--- /dev/null+++ b/engine/crates/xerj-autoindex/src/detect/mdlink.rs@@ -0,0 +1,170 @@+//! mdlink@1 — `[text](relative/path.md)` where the target is a corpus file.+//!+//! Slightly below wikilink (weight 0.9): an inline markdown link is still an+//! explicit authorial act, but it is also how people cite external material,+//! so a resolved local target carries marginally less "these notes belong+//! together" intent than a `[[wiki link]]`.++use super::{+ line_at, resolve_local, DetectorCounters, EdgeDetector, EdgeDraft, LinkTarget, SectionCtx,+};+use std::sync::atomic::{AtomicU64, Ordering};++pub const TAG: &str = "mdlink@1";+pub const EDGE_TYPE: &str = "mdlink";+pub const WEIGHT: f32 = 0.9;+pub const CONFIDENCE: f32 = 0.9;++#[derive(Default)]+pub struct Mdlink {+ unresolved: AtomicU64,+}++/// Find `[label](url)` occurrences. Hand scan, not regex: the grammar is two+/// delimiter searches, and the scan must never backtrack pathologically on a+/// section full of stray brackets. No nesting support — deterministic first+/// `]` / first `)` wins, which matches how common renderers degrade.+fn scan_links(text: &str, mut hit: impl FnMut(usize, &str)) {+ let bytes = text.as_bytes();+ let mut i = 0usize;+ while i < bytes.len() {+ let Some(open) = text[i..].find('[') else {+ break;+ };+ let open = i + open;+ let Some(close) = text[open + 1..].find(']') else {+ break;+ };+ let close = open + 1 + close;+ if bytes.get(close + 1) != Some(&b'(') {+ i = open + 1;+ continue;+ }+ let Some(paren) = text[close + 2..].find(')') else {+ break;+ };+ let paren = close + 2 + paren;+ // `[t](url "title")` — the optional title starts at the first+ // whitespace inside the parens.+ let raw = text[close + 2..paren].trim();+ let url = raw.split_whitespace().next().unwrap_or("");+ hit(open, url);+ i = paren + 1;+ }+}++impl EdgeDetector for Mdlink {+ fn tag(&self) -> &'static str {+ TAG+ }++ fn detect_text(&self, ctx: &SectionCtx<'_>, out: &mut Vec<EdgeDraft>) {+ scan_links(ctx.text, |open, url| {+ match resolve_local(ctx.corpus, &ctx.file.dir, url) {+ LinkTarget::Hit(target) => out.push(EdgeDraft {+ src: ctx.section_doc_id.to_string(),+ dst: target.anchor_doc_id.clone(),+ edge_type: EDGE_TYPE,+ weight: WEIGHT,+ confidence: CONFIDENCE,+ valid_at_ms: ctx.file.mtime_ms,+ src_file: ctx.file.rel.clone(),+ quote: line_at(ctx.text, open),+ offset: open as u64,+ }),+ // External links are what markdown links are FOR — skipping+ // them silently is correct, only local misses are dangling.+ LinkTarget::External | LinkTarget::Empty => {}+ LinkTarget::Miss => {+ self.unresolved.fetch_add(1, Ordering::Relaxed);+ }+ }+ });+ }++ fn counters(&self) -> DetectorCounters {+ DetectorCounters {+ unresolved: self.unresolved.load(Ordering::Relaxed),+ ambiguous: 0,+ }+ }+}++#[cfg(test)]+mod tests {+ use super::super::{corpus_file, CorpusIndex, SectionCtx};+ use super::*;++ fn corpus() -> CorpusIndex {+ CorpusIndex::build(vec![+ corpus_file("docs/guide.md", "k1", "d", "txt-prose", 10),+ corpus_file("readme.md", "k2", "d", "txt-prose", 10),+ ])+ }++ fn run(corpus: &CorpusIndex, from: &str, text: &str) -> (Vec<EdgeDraft>, DetectorCounters) {+ let det = Mdlink::default();+ let file = &corpus.files[from];+ let ctx = SectionCtx {+ corpus,+ file,+ section_ordinal: 0,+ section_doc_id: "sec0",+ text,+ };+ let mut out = Vec::new();+ det.detect_text(&ctx, &mut out);+ (out, det.counters())+ }++ #[test]+ fn resolves_dir_relative_then_root_relative() {+ let corpus = corpus();+ let (out, c) = run(+ &corpus,+ "docs/guide.md",+ "see [readme](../readme.md) and [self](guide.md) and [root](docs/guide.md)",+ );+ assert_eq!(out.len(), 3);+ assert_eq!(out[0].dst, corpus.files["readme.md"].anchor_doc_id);+ assert_eq!(out[1].dst, corpus.files["docs/guide.md"].anchor_doc_id);+ // "docs/guide.md" fails dir-relative (docs/docs/guide.md), resolves+ // root-relative.+ assert_eq!(out[2].dst, corpus.files["docs/guide.md"].anchor_doc_id);+ assert_eq!(c.unresolved, 0);+ assert_eq!(out[0].offset, 4);+ }++ #[test]+ fn external_links_skip_and_local_misses_count() {+ let corpus = corpus();+ let (out, c) = run(+ &corpus,+ "readme.md",+ "[web](https://example.com/a.md) [mail](mailto:[email protected]) [gone](missing.md) [frag](#top)",+ );+ assert!(out.is_empty());+ assert_eq!(c.unresolved, 1, "only the local miss is dangling");+ }++ #[test]+ fn title_and_fragment_are_stripped() {+ let corpus = corpus();+ let (out, _) = run(+ &corpus,+ "readme.md",+ "x [g](docs/guide.md#section \"The Guide\") y",+ );+ assert_eq!(out.len(), 1);+ assert_eq!(out[0].dst, corpus.files["docs/guide.md"].anchor_doc_id);+ assert_eq!(out[0].quote, "x [g](docs/guide.md#section \"The Guide\") y");+ }++ #[test]+ fn wikilinks_do_not_false_positive() {+ let corpus = corpus();+ let (out, c) = run(&corpus, "readme.md", "a [[guide]] wikilink, no mdlink here");+ assert!(out.is_empty());+ assert_eq!(c.unresolved, 0);+ }+}diff --git a/engine/crates/xerj-autoindex/src/detect/mod.rs b/engine/crates/xerj-autoindex/src/detect/mod.rsnew file mode 100644index 000000000..201f8d35f--- /dev/null+++ b/engine/crates/xerj-autoindex/src/detect/mod.rs@@ -0,0 +1,739 @@+//! Second-brain edge detectors: deterministic, versioned relationship+//! extraction over the corpus autoindex is already walking (contract:+//! SECOND_BRAIN_SPEC §6).+//!+//! Why deterministic instead of an LLM pass: the same corpus must produce a+//! byte-identical edge set on every run (given unchanged mtimes), because edge+//! identity — and therefore idempotent re-runs, kill -9 safety, and the+//! bi-temporal history — hangs off a content hash of (src, type, dst,+//! valid_at). A model in that loop would turn every re-run into a new belief+//! set. Every emitted edge instead carries the exact quote that taught it and+//! a `detector` tag like `wikilink@1`; bump the `@N` on ANY behavior change so+//! old edges remain attributable to the rules that produced them.+//!+//! Edges are ordinary documents in one reserved `.xerj-memory-{brain}-edges`+//! index per brain — no storage-format change, they ride the normal+//! WAL → memtable → segment path and are soft-invalidated (never deleted) so+//! "what did it believe last Tuesday" stays answerable.++pub mod href;+pub mod mdlink;+pub mod samedir;+pub mod sequence;+pub mod wikilink;++#[cfg(test)]+mod e2e;++use crate::esclient::Es;+use anyhow::{anyhow, Context, Result};+use serde_json::{json, Value};+use std::collections::BTreeMap;++/// Edge-schema version stamped into every emitted edge (`schema_version`).+pub const EDGE_SCHEMA_VERSION: u32 = 1;++/// Reserved `_id` of the per-brain meta document. It carries no `src`/`dst`,+/// so the hop path and every edge count (all filter on `exists src`) never+/// see it.+pub const BRAIN_META_ID: &str = "__xerj-brain-meta";++/// edge_id = xxh3_128("xg1\0" src "\0" type "\0" dst "\0" decimal(valid_at_ms)),+/// rendered as 32 lowercase hex chars ({:032x}).+pub fn edge_id(src: &str, edge_type: &str, dst: &str, valid_at_ms: i64) -> String {+ use xxhash_rust::xxh3::xxh3_128;+ let mut input = Vec::with_capacity(16 + src.len() + edge_type.len() + dst.len());+ input.extend_from_slice(b"xg1\x00");+ input.extend_from_slice(src.as_bytes());+ input.push(0);+ input.extend_from_slice(edge_type.as_bytes());+ input.push(0);+ input.extend_from_slice(dst.as_bytes());+ input.push(0);+ input.extend_from_slice(valid_at_ms.to_string().as_bytes());+ format!("{:032x}", xxh3_128(&input))+}++/// Edges index name for a brain (SECOND_BRAIN_SPEC §1). The leading dot keeps+/// it out of user index APIs and `_cat` listings.+pub fn edges_index_name(brain: &str) -> String {+ format!(".xerj-memory-{brain}-edges")+}++/// Brain-name validation: identical rules to the agent-memory namespace+/// (lowercase/digit start, `[a-z0-9._-]`, ≤200 chars, no `..`) plus the+/// `-edges` suffix rejection — without it a brain named `kb-edges` would+/// collide with brain `kb`'s edge index.+pub fn validate_brain(brain: &str) -> std::result::Result<(), String> {+ if brain.is_empty() {+ return Err("brain name must not be empty".into());+ }+ if brain.len() > 200 {+ return Err("brain name too long (max 200 chars)".into());+ }+ let first = brain.chars().next().unwrap();+ if !(first.is_ascii_lowercase() || first.is_ascii_digit()) {+ return Err("brain name must start with a lowercase letter or digit".into());+ }+ for c in brain.chars() {+ let ok = c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '_' | '-' | '.');+ if !ok {+ return Err(format!(+ "brain name contains illegal character '{c}' (allowed: a-z 0-9 _ - .)"+ ));+ }+ }+ if brain.contains("..") {+ return Err("brain name must not contain '..'".into());+ }+ if brain.ends_with("-edges") {+ return Err("namespace suffix '-edges' is reserved for graph edge indices".into());+ }+ Ok(())+}++/// The exact edges-index mapping body (SECOND_BRAIN_SPEC §2.1). The API+/// stream's `graph_api::link` sends the byte-identical body — the two writers+/// must agree or the first one to touch a brain decides the column types for+/// everyone.+pub fn edge_index_mapping() -> Value {+ json!({+ "mappings": {+ "properties": {+ "edge_id": { "type": "keyword" },+ "src": { "type": "keyword" },+ "dst": { "type": "keyword" },+ "type": { "type": "keyword" },+ "weight": { "type": "float" },+ "valid_at": { "type": "date", "format": "epoch_millis" },+ "invalid_at": { "type": "date", "format": "epoch_millis" },+ "created_at": { "type": "date", "format": "epoch_millis" },+ "expired_at": { "type": "date", "format": "epoch_millis" },+ "detector": { "type": "keyword" },+ "confidence": { "type": "float" },+ "schema_version": { "type": "integer" },+ "src_file": { "type": "keyword" },+ "evidence": {+ "properties": {+ "quote": { "type": "text" },+ "source": { "type": "keyword" },+ "offset": { "type": "long" }+ }+ }+ }+ }+ })+}++// ─── corpus resolution table ─────────────────────────────────────────────++/// Corpus-wide resolution table, built once after Phase A (plan assignment):+/// every indexed file's rel path → identity + the doc id of its first section.+pub struct CorpusIndex {+ /// key: root-relative path with forward slashes (`FileEntry.rel`).+ pub files: BTreeMap<String, CorpusFile>,+ /// lowercase file-stem → rel paths bearing it (wikilink resolution;+ /// BTreeMap + sorted values so ambiguity resolution is deterministic).+ pub by_stem: BTreeMap<String, Vec<String>>,+}++pub struct CorpusFile {+ pub rel: String,+ /// `ids::file_key` output.+ pub file_key: String,+ pub dataset_slug: String,+ /// `ids::doc_id(dataset_slug, file_key, "s0")` — the file's anchor node.+ pub anchor_doc_id: String,+ pub mtime_ms: i64,+ /// Parent dir rel path, "" for root.+ pub dir: String,+ /// Sniffed format family (`Family::as_str`), from the frozen plan — NOT+ /// guessed from the extension. href@1 gates on this: the contract scopes+ /// anchor detection to html-extracted files, and an `.html` extension is+ /// not proof of that (sniffing decides the family).+ pub family: String,+}++/// Build one [`CorpusFile`], deriving the anchor node id and parent dir from+/// the identity fields so every caller (pipeline and tests) derives them the+/// same way.+pub fn corpus_file(+ rel: &str,+ file_key: &str,+ dataset_slug: &str,+ family: &str,+ mtime_ms: i64,+) -> CorpusFile {+ CorpusFile {+ rel: rel.to_string(),+ file_key: file_key.to_string(),+ dataset_slug: dataset_slug.to_string(),+ anchor_doc_id: crate::ids::doc_id(dataset_slug, file_key, "s0"),+ mtime_ms,+ dir: rel.rsplit_once('/').map(|(d, _)| d.to_string()).unwrap_or_default(),+ family: family.to_string(),+ }+}++impl CorpusIndex {+ pub fn build(files: Vec<CorpusFile>) -> Self {+ let mut map: BTreeMap<String, CorpusFile> = BTreeMap::new();+ let mut by_stem: BTreeMap<String, Vec<String>> = BTreeMap::new();+ for f in files {+ by_stem+ .entry(stem_of(&f.rel).to_ascii_lowercase())+ .or_default()+ .push(f.rel.clone());+ map.insert(f.rel.clone(), f);+ }+ for rels in by_stem.values_mut() {+ rels.sort();+ rels.dedup();+ }+ CorpusIndex {+ files: map,+ by_stem,+ }+ }+}++/// File stem of a rel path: final path segment minus its last extension.+/// Dotfiles (".env") keep their full name — an empty stem would collide every+/// dotfile into one wikilink bucket.+pub(crate) fn stem_of(rel: &str) -> &str {+ let name = rel.rsplit('/').next().unwrap_or(rel);+ match name.rsplit_once('.') {+ Some((stem, _)) if !stem.is_empty() => stem,+ _ => name,+ }+}++/// Rel path minus its last extension (whole-path form used by wikilink's+/// "with or without extension" rule). "" stem is preserved as-is for dotfiles.+pub(crate) fn rel_without_ext(rel: &str) -> String {+ let (dir, name) = rel.rsplit_once('/').unwrap_or(("", rel));+ let stem = match name.rsplit_once('.') {+ Some((s, _)) if !s.is_empty() => s,+ _ => name,+ };+ if dir.is_empty() {+ stem.to_string()+ } else {+ format!("{dir}/{stem}")+ }+}++// ─── detector trait ──────────────────────────────────────────────────────++/// One detected edge before identity/envelope assembly.+pub struct EdgeDraft {+ /// Node doc id (usually the section containing the evidence).+ pub src: String,+ /// Node doc id (target file's anchor_doc_id).+ pub dst: String,+ pub edge_type: &'static str,+ pub weight: f32,+ pub confidence: f32,+ /// §6.4 determinism rule: mtime of `src_file`, never the wall clock —+ /// re-runs over an unchanged corpus reproduce identical edge_ids.+ pub valid_at_ms: i64,+ /// Rel path that taught this edge (top-level keyword — exists so+ /// replacement invalidation is a doc-values term query, not an+ /// `evidence.source` object scan).+ pub src_file: String,+ /// evidence.quote (≤240 chars, trimmed).+ pub quote: String,+ /// evidence.offset (byte offset in section text; 0 for structural).+ pub offset: u64,+}++/// Per-section textual context. `text` is the exact section string that became+/// the node doc's `body` (post `split_sections`).+pub struct SectionCtx<'a> {+ pub corpus: &'a CorpusIndex,+ pub file: &'a CorpusFile,+ pub section_ordinal: u32,+ pub section_doc_id: &'a str,+ pub text: &'a str,+}++/// What a resolving detector could not turn into edges. Surfaced in the run+/// summary so a dangling `[[link]]` is recorded, never silently dropped.+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]+pub struct DetectorCounters {+ /// Link targets that resolved to no corpus file (dangling).+ pub unresolved: u64,+ /// Link targets with >1 candidate; the lexicographically smallest rel won.+ pub ambiguous: u64,+}++/// Deterministic, versioned edge detector. NO network, NO LLM, NO clock reads+/// (all time comes from ctx mtimes) — same inputs must yield the same drafts.+pub trait EdgeDetector: Sync {+ /// Versioned tag stored in `detector`, e.g. "wikilink@1". Bump the @N on+ /// ANY behavior change.+ fn tag(&self) -> &'static str;+ /// Per-section textual detection. Default: no-op.+ fn detect_text(&self, _ctx: &SectionCtx<'_>, _out: &mut Vec<EdgeDraft>) {}+ /// Corpus-structural detection, called once after Phase A. Default: no-op.+ fn detect_structure(&self, _corpus: &CorpusIndex, _out: &mut Vec<EdgeDraft>) {}+ /// Run-summary honesty counters (default: nothing to report). Detectors+ /// that resolve link targets keep these in atomics so the counts survive+ /// the shared `&self` the parallel Phase B workers hold.+ fn counters(&self) -> DetectorCounters {+ DetectorCounters::default()+ }+}++/// The registry. Order is normative (fixture edge ordering depends on it only+/// via edge sort, so this is cosmetic — but keep it).+pub fn default_detectors() -> Vec<Box<dyn EdgeDetector>> {+ vec![+ Box::new(wikilink::Wikilink::default()),+ Box::new(mdlink::Mdlink::default()),+ Box::new(href::Href::default()),+ Box::new(sequence::Sequence),+ Box::new(samedir::SameDir),+ ]+}++/// Detector tag for an edge type. The draft deliberately carries only the+/// type (§6.3 shape); the 1:1 type→tag map lives here, next to the registry,+/// so a version bump touches exactly one detector file plus nothing else.+pub fn detector_tag_for(edge_type: &str) -> &'static str {+ match edge_type {+ wikilink::EDGE_TYPE => wikilink::TAG,+ mdlink::EDGE_TYPE => mdlink::TAG,+ href::EDGE_TYPE => href::TAG,+ sequence::EDGE_TYPE => sequence::TAG,+ samedir::EDGE_TYPE => samedir::TAG,+ _ => "unknown@0",+ }+}++// ─── shared text/link helpers ────────────────────────────────────────────++/// Evidence quote discipline: the full trimmed line containing `offset`,+/// clipped to 240 chars — enough to show a human (or agent) exactly what+/// taught the edge without dragging whole sections into every edge doc.+pub(crate) fn line_at(text: &str, offset: usize) -> String {+ let offset = offset.min(text.len());+ let start = text[..offset].rfind('\n').map(|i| i + 1).unwrap_or(0);+ let end = text[offset..]+ .find('\n')+ .map(|i| offset + i)+ .unwrap_or(text.len());+ clip_quote(&text[start..end])+}++pub(crate) fn clip_quote(s: &str) -> String {+ let t = s.trim();+ if t.chars().count() <= 240 {+ t.to_string()+ } else {+ t.chars().take(240).collect()+ }+}++/// Outcome of scheme-less link resolution (mdlink/href rule, §6.5).+pub(crate) enum LinkTarget<'a> {+ /// Has a scheme or `//` prefix — external by design, not dangling.+ External,+ /// Nothing left after stripping fragment/query (same-page anchor).+ Empty,+ /// Scheme-less but no corpus file matches — dangling, counted.+ Miss,+ Hit(&'a CorpusFile),+}++/// Resolve a scheme-less url against the corpus: relative to the containing+/// file's dir first, then as root-relative. Fragment (`#…`) and query (`?…`)+/// are stripped; percent-encoding is NOT decoded (a deliberate v1 limit —+/// decoding would need an escaping table and links in local corpora rarely+/// use it; a miss is counted, not hidden).+pub(crate) fn resolve_local<'a>(+ corpus: &'a CorpusIndex,+ containing_dir: &str,+ url: &str,+) -> LinkTarget<'a> {+ let url = url.trim();+ if url.is_empty() {+ return LinkTarget::Empty;+ }+ if url.starts_with("//") || has_scheme(url) {+ return LinkTarget::External;+ }+ let path = url.split(['#', '?']).next().unwrap_or("");+ if path.is_empty() {+ return LinkTarget::Empty;+ }+ let candidates: [Option<String>; 2] = if let Some(site_abs) = path.strip_prefix('/') {+ // Site-absolute → root-relative only; joining it under the containing+ // dir would invent a path the author never wrote.+ [normalize_rel(site_abs), None]+ } else {+ let joined = if containing_dir.is_empty() {+ path.to_string()+ } else {+ format!("{containing_dir}/{path}")+ };+ [normalize_rel(&joined), normalize_rel(path)]+ };+ for cand in candidates.into_iter().flatten() {+ if let Some(f) = corpus.files.get(&cand) {+ return LinkTarget::Hit(f);+ }+ }+ LinkTarget::Miss+}++/// Any ':' before the first '/' marks the target external ("https:", "mailto:",+/// "tel:", "host:8080"…). Corpus rel paths essentially never contain a colon,+/// and a rule this blunt is deterministic where scheme-grammar parsing would+/// accumulate special cases.+fn has_scheme(url: &str) -> bool {+ match (url.find(':'), url.find('/')) {+ (Some(c), Some(s)) => c < s,+ (Some(_), None) => true,+ (None, _) => false,+ }+}++/// Normalize `a/./b/../c` → `a/c`; `..` escaping the corpus root is a miss+/// (None), not a guess.+fn normalize_rel(path: &str) -> Option<String> {+ let mut parts: Vec<&str> = Vec::new();+ for seg in path.split('/') {+ match seg {+ "" | "." => {}+ ".." => {+ parts.pop()?;+ }+ s => parts.push(s),+ }+ }+ if parts.is_empty() {+ None+ } else {+ Some(parts.join("/"))+ }+}++// ─── envelope assembly + write path ──────────────────────────────────────++/// One edge ready for `_bulk`: the two ndjson lines plus the tag for+/// per-detector accounting.+pub struct AssembledEdge {+ pub detector: &'static str,+ pub edge_id: String,+ pub ndjson: Vec<u8>,+}++pub struct AssembleOutcome {+ pub edges: Vec<AssembledEdge>,+ /// Self-edges (src == dst) dropped by the assembler — counted, per §6.4.+ pub self_dropped: u64,+}++/// f32 → JSON number without widening artifacts: `0.3f32 as f64` is+/// 0.30000001192092896, which would poison every stored weight and the+/// doc-values compare downstream. The shortest f32 display ("0.3") parsed as+/// f64 is the number the contract fixtures assert.+fn clean_f64(v: f32) -> f64 {+ format!("{v}").parse().unwrap_or(f64::from(v))+}++/// Turn drafts into `_bulk` actions (§2.2 stored-type discipline: every+/// scalar a plain string/number, `invalid_at`/`expired_at` omitted — the+/// null_bitmap IS the "still valid" signal). `created_at_ms` is the one+/// non-deterministic field and never participates in `edge_id`.+pub fn assemble(drafts: &[EdgeDraft], edges_index: &str, created_at_ms: i64) -> AssembleOutcome {+ let mut edges = Vec::with_capacity(drafts.len());+ let mut self_dropped = 0u64;+ for d in drafts {+ if d.src == d.dst {+ self_dropped += 1;+ continue;+ }+ let id = edge_id(&d.src, d.edge_type, &d.dst, d.valid_at_ms);+ let tag = detector_tag_for(d.edge_type);+ let action = json!({"index": {"_index": edges_index, "_id": id}});+ let doc = json!({+ "edge_id": id,+ "src": d.src,+ "dst": d.dst,+ "type": d.edge_type,+ "weight": clean_f64(d.weight),+ "valid_at": d.valid_at_ms,+ "created_at": created_at_ms,+ "detector": tag,+ "confidence": clean_f64(d.confidence),+ "schema_version": EDGE_SCHEMA_VERSION,+ "src_file": d.src_file,+ "evidence": {+ "quote": d.quote,+ "source": d.src_file,+ "offset": d.offset,+ }+ });+ let mut ndjson = action.to_string().into_bytes();+ ndjson.push(b'\n');+ ndjson.extend_from_slice(doc.to_string().as_bytes());+ ndjson.push(b'\n');+ edges.push(AssembledEdge {+ detector: tag,+ edge_id: id,+ ndjson,+ });+ }+ AssembleOutcome {+ edges,+ self_dropped,+ }+}++/// Create-if-absent write of the §2.5 brain meta doc. `create` semantics via+/// a probing GET: a racing writer's meta doc must never be clobbered — it may+/// carry a different `nodes_index` some reader already resolved through.+pub fn ensure_brain_meta(+ es: &Es,+ edges_index: &str,+ brain: &str,+ nodes_index: &str,+ created_at_ms: i64,+) -> Result<()> {+ if es.get_doc(edges_index, BRAIN_META_ID)?.is_some() {+ return Ok(());+ }+ let action = json!({"create": {"_index": edges_index, "_id": BRAIN_META_ID}});+ let doc = json!({+ "meta_version": 1,+ "brain": brain,+ "nodes_index": nodes_index,+ "created_at": created_at_ms,+ });+ let mut body = action.to_string().into_bytes();+ body.push(b'\n');+ body.extend_from_slice(doc.to_string().as_bytes());+ body.push(b'\n');+ let outcome = es.bulk(body).context("write brain meta doc")?;+ if outcome.server_errors > 0 {+ return Err(anyhow!(+ "brain meta write failed: {}",+ outcome+ .first_server_error+ .as_deref()+ .unwrap_or("unknown server error")+ ));+ }+ // item_errors here can only be the create-conflict race — the doc exists,+ // which is exactly the state we want.+ Ok(())+}++/// Soft-invalidate every live edge taught by `rel` (replacement hook, §6.6.3).+/// Runs BEFORE any of this run's edges for that file are written, so the+/// `src_file` + no-`invalid_at` query can only match prior-generation edges.+/// Each pass re-indexes its hits with `invalid_at = expired_at = now` and+/// refreshes, so repeat-until-empty converges; the pass bound turns a broken+/// endpoint into an error instead of an infinite loop.+pub fn invalidate_prior_edges(+ es: &Es,+ edges_index: &str,+ rel: &str,+ now_ms: i64,+) -> Result<u64> {+ const MAX_PASSES: usize = 1_000;+ let query = json!({+ "query": {"bool": {+ "filter": [{"term": {"src_file": rel}}],+ "must_not": [{"exists": {"field": "invalid_at"}}]+ }},+ "size": 1000,+ "_source": true+ });+ let mut total = 0u64;+ for _ in 0..MAX_PASSES {+ let resp = es.search(edges_index, &query)?;+ let hits = resp+ .pointer("/hits/hits")+ .and_then(Value::as_array)+ .cloned()+ .unwrap_or_default();+ if hits.is_empty() {+ return Ok(total);+ }+ let mut body = Vec::new();+ for hit in &hits {+ let id = hit+ .get("_id")+ .and_then(Value::as_str)+ .ok_or_else(|| anyhow!("edge hit without _id in {edges_index}"))?;+ let mut source = hit+ .get("_source")+ .and_then(Value::as_object)+ .cloned()+ .ok_or_else(|| anyhow!("edge {id} returned without _source"))?;+ source.insert("invalid_at".into(), json!(now_ms));+ source.insert("expired_at".into(), json!(now_ms));+ body.extend_from_slice(+ json!({"index": {"_index": edges_index, "_id": id}})+ .to_string()+ .as_bytes(),+ );+ body.push(b'\n');+ body.extend_from_slice(Value::Object(source).to_string().as_bytes());+ body.push(b'\n');+ }+ let outcome = es.bulk(body).context("invalidate prior edges")?;+ if outcome.server_errors > 0 || outcome.item_errors > 0 {+ return Err(anyhow!(+ "edge invalidation for {} was partial: {}",+ rel,+ outcome+ .first_server_error+ .or(outcome.first_error)+ .unwrap_or_else(|| "unknown bulk error".into())+ ));+ }+ total += hits.len() as u64;+ es.refresh(edges_index)?;+ }+ Err(anyhow!(+ "edge invalidation for {rel} still found live edges after {MAX_PASSES} passes"+ ))+}++#[cfg(test)]+mod tests {+ use super::*;++ /// §2.3 pin vector — xerj-api carries a byte-identical copy of `edge_id`;+ /// this shared vector is what keeps the two in lockstep.+ #[test]+ fn edge_id_pin_vector() {+ assert_eq!(+ edge_id("note-alpha", "wikilink", "note-beta", 1753600000000),+ "bef814a75bd3d914c3e561f610154304"+ );+ }++ #[test]+ fn edge_id_is_valid_at_sensitive() {+ let a = edge_id("a", "t", "b", 1);+ assert_eq!(a, edge_id("a", "t", "b", 1));+ assert_ne!(a, edge_id("a", "t", "b", 2));+ assert_ne!(a, edge_id("a", "u", "b", 1));+ }++ #[test]+ fn brain_names_reject_the_edges_suffix_and_bad_shapes() {+ assert!(validate_brain("notes").is_ok());+ assert!(validate_brain("a.b-c_9").is_ok());+ assert!(validate_brain("kb-edges").is_err());+ assert!(validate_brain("").is_err());+ assert!(validate_brain("-notes").is_err());+ assert!(validate_brain("Notes").is_err());+ assert!(validate_brain("a..b").is_err());+ }++ #[test]+ fn assembler_drops_self_edges_and_stamps_the_envelope() {+ let drafts = vec![+ EdgeDraft {+ src: "n1".into(),+ dst: "n1".into(),+ edge_type: wikilink::EDGE_TYPE,+ weight: 1.0,+ confidence: 0.95,+ valid_at_ms: 5,+ src_file: "a.md".into(),+ quote: "self".into(),+ offset: 0,+ },+ EdgeDraft {+ src: "n1".into(),+ dst: "n2".into(),+ edge_type: samedir::EDGE_TYPE,+ weight: samedir::WEIGHT,+ confidence: samedir::CONFIDENCE,+ valid_at_ms: 5,+ src_file: "a.md".into(),+ quote: "a.md and b.md share directory .".into(),+ offset: 0,+ },+ ];+ let out = assemble(&drafts, ".xerj-memory-t-edges", 99);+ assert_eq!(out.self_dropped, 1);+ assert_eq!(out.edges.len(), 1);+ let lines: Vec<&str> = std::str::from_utf8(&out.edges[0].ndjson)+ .unwrap()+ .lines()+ .collect();+ let doc: Value = serde_json::from_str(lines[1]).unwrap();+ // f32 weights must land as their short decimal, not the widened f64.+ assert_eq!(doc["weight"], json!(0.3));+ assert_eq!(doc["confidence"], json!(0.4));+ assert_eq!(doc["detector"], json!(samedir::TAG));+ assert_eq!(doc["schema_version"], json!(1));+ assert_eq!(doc["created_at"], json!(99));+ assert_eq!(doc["evidence"]["source"], json!("a.md"));+ assert!(doc.get("invalid_at").is_none(), "unset keys must be omitted");+ assert_eq!(doc["edge_id"], json!(out.edges[0].edge_id));+ }++ #[test]+ fn assembly_is_deterministic_for_a_fixed_created_at() {+ let draft = EdgeDraft {+ src: "n1".into(),+ dst: "n2".into(),+ edge_type: mdlink::EDGE_TYPE,+ weight: mdlink::WEIGHT,+ confidence: mdlink::CONFIDENCE,+ valid_at_ms: 1753600000000,+ src_file: "a.md".into(),+ quote: "q".into(),+ offset: 3,+ };+ let a = assemble(std::slice::from_ref(&draft), "i", 7);+ let b = assemble(std::slice::from_ref(&draft), "i", 7);+ assert_eq!(a.edges[0].ndjson, b.edges[0].ndjson);+ }++ #[test]+ fn local_resolution_handles_relative_root_and_external_targets() {+ let corpus = CorpusIndex::build(vec![+ corpus_file("docs/a.md", "k1", "d", "txt-prose", 1),+ corpus_file("b.md", "k2", "d", "txt-prose", 1),+ ]);+ assert!(matches!(+ resolve_local(&corpus, "docs", "../b.md"),+ LinkTarget::Hit(f) if f.rel == "b.md"+ ));+ assert!(matches!(+ resolve_local(&corpus, "", "docs/a.md#frag?q=1"),+ LinkTarget::Hit(f) if f.rel == "docs/a.md"+ ));+ assert!(matches!(+ resolve_local(&corpus, "docs", "/b.md"),+ LinkTarget::Hit(f) if f.rel == "b.md"+ ));+ assert!(matches!(+ resolve_local(&corpus, "docs", "missing.md"),+ LinkTarget::Miss+ ));+ assert!(matches!(+ resolve_local(&corpus, "", "https://example.com/b.md"),+ LinkTarget::External+ ));+ assert!(matches!(+ resolve_local(&corpus, "", "//cdn.example.com/x"),+ LinkTarget::External+ ));+ assert!(matches!(resolve_local(&corpus, "", "#anchor"), LinkTarget::Empty));+ assert!(matches!(+ resolve_local(&corpus, "", "../../escape.md"),+ LinkTarget::Miss+ ));+ }+}diff --git a/engine/crates/xerj-autoindex/src/detect/samedir.rs b/engine/crates/xerj-autoindex/src/detect/samedir.rsnew file mode 100644index 000000000..6cc00624e--- /dev/null+++ b/engine/crates/xerj-autoindex/src/detect/samedir.rs@@ -0,0 +1,126 @@+//! samedir@1 — files sharing a directory, chained, never a clique.+//!+//! Directory co-location is weak evidence (weight 0.3, confidence 0.4 — the+//! floor of the detector set): it often means "same topic", but it is the+//! only detector that fires on a corpus with no links at all, so it is what+//! keeps an unlinked folder from being a graph of isolated points.+//!+//! ## The fan-out cap: a CHAIN over rel-sorted files+//! A clique is O(n²): one 500-file directory would emit 500·499/2 = 124,750+//! edges and drown every explicit-link signal under same_dir noise. The chain+//! emits exactly n-1 edges (499 for that directory) — O(n) — while directory+//! cohesion stays reachable inside the engine's 2-hop expansion cap for+//! small dirs (±2 rel-sorted neighbors per hop pair). Sorting by rel makes+//! the chain deterministic AND naturally follows name-ordered conventions+//! (01-intro, 02-setup, …), which is why this detector doubles as the+//! adjacent-file "sequence by naming" signal at the file level.++use super::{CorpusIndex, EdgeDetector, EdgeDraft};++pub const TAG: &str = "samedir@1";+pub const EDGE_TYPE: &str = "same_dir";+pub const WEIGHT: f32 = 0.3;+pub const CONFIDENCE: f32 = 0.4;++pub struct SameDir;++impl EdgeDetector for SameDir {+ fn tag(&self) -> &'static str {+ TAG+ }++ fn detect_structure(&self, corpus: &CorpusIndex, out: &mut Vec<EdgeDraft>) {+ // corpus.files is a BTreeMap keyed by rel: iteration is globally+ // rel-sorted, so per-directory order needs no extra sort.+ let mut by_dir: std::collections::BTreeMap<&str, Vec<&super::CorpusFile>> =+ std::collections::BTreeMap::new();+ for f in corpus.files.values() {+ by_dir.entry(f.dir.as_str()).or_default().push(f);+ }+ for (dir, files) in by_dir {+ let shown_dir = if dir.is_empty() { "." } else { dir };+ for pair in files.windows(2) {+ let (left, right) = (pair[0], pair[1]);+ out.push(EdgeDraft {+ src: left.anchor_doc_id.clone(),+ dst: right.anchor_doc_id.clone(),+ edge_type: EDGE_TYPE,+ weight: WEIGHT,+ confidence: CONFIDENCE,+ valid_at_ms: left.mtime_ms,+ src_file: left.rel.clone(),+ quote: format!(+ "{} and {} share directory {shown_dir}",+ left.rel, right.rel+ ),+ offset: 0,+ });+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::super::{corpus_file, CorpusIndex};+ use super::*;++ fn chain(rels: &[&str]) -> Vec<EdgeDraft> {+ let corpus = CorpusIndex::build(+ rels.iter()+ .enumerate()+ .map(|(i, rel)| corpus_file(rel, &format!("k{i}"), "notes", "txt-prose", 100 + i as i64))+ .collect(),+ );+ let mut out = Vec::new();+ SameDir.detect_structure(&corpus, &mut out);+ out+ }++ /// The §8 fixture directory: 5 files chain into exactly 4 edges in+ /// rel-sorted order — never the 10-edge clique.+ #[test]+ fn fixture_folder_chains_not_cliques() {+ let out = chain(&["alpha.md", "beta.md", "delta.md", "epsilon.md", "gamma.md"]);+ assert_eq!(out.len(), 4);+ let pairs: Vec<(&str, &str)> = out+ .iter()+ .map(|e| (e.src_file.as_str(), e.quote.as_str()))+ .collect();+ assert_eq!(+ pairs,+ vec![+ ("alpha.md", "alpha.md and beta.md share directory ."),+ ("beta.md", "beta.md and delta.md share directory ."),+ ("delta.md", "delta.md and epsilon.md share directory ."),+ ("epsilon.md", "epsilon.md and gamma.md share directory ."),+ ]+ );+ }++ /// The cap in numbers: n files emit exactly n-1 edges, so a 500-file+ /// directory emits 499 — not 124,750.+ #[test]+ fn fan_out_is_linear_in_directory_size() {+ let rels: Vec<String> = (0..500).map(|i| format!("dir/f{i:03}.md")).collect();+ let refs: Vec<&str> = rels.iter().map(String::as_str).collect();+ assert_eq!(chain(&refs).len(), 499);+ }++ #[test]+ fn directories_never_cross_and_singletons_emit_nothing() {+ let out = chain(&["a/x.md", "a/y.md", "b/z.md", "lone.md"]);+ assert_eq!(out.len(), 1);+ assert_eq!(out[0].quote, "a/x.md and a/y.md share directory a");+ }++ /// Numeric-prefix conventions (01-, 02-, …) sort into their intended+ /// reading order, so the chain is also the file-level sequence signal.+ #[test]+ fn numeric_prefixes_chain_in_reading_order() {+ let out = chain(&["c/02-setup.md", "c/01-intro.md", "c/03-usage.md"]);+ assert_eq!(out.len(), 2);+ assert!(out[0].quote.starts_with("c/01-intro.md and c/02-setup.md"));+ assert!(out[1].quote.starts_with("c/02-setup.md and c/03-usage.md"));+ }+}diff --git a/engine/crates/xerj-autoindex/src/detect/sequence.rs b/engine/crates/xerj-autoindex/src/detect/sequence.rsnew file mode 100644index 000000000..d6dd1b451--- /dev/null+++ b/engine/crates/xerj-autoindex/src/detect/sequence.rs@@ -0,0 +1,89 @@+//! sequence@1 — section s_{i-1} → s_i within one file.+//!+//! The natural ordering edge: autoindex splits long documents into+//! retrieval-sized sections, and without these edges the split would DESTROY+//! the document's own narrative adjacency (a hop from "the answer" to "the+//! paragraph after it" is exactly what an agent wants when a section ends+//! mid-thought). Weight 0.8 — structural but almost certainly meaningful;+//! confidence 0.99 because the extractor's own ordering cannot be wrong about+//! itself.+//!+//! Emitted from `detect_text` on section i>0 (the contract's trick to avoid a+//! second pass): every section after the first knows its predecessor's doc id+//! deterministically from (slug, file_key, "s{i-1}").++use super::{EdgeDetector, EdgeDraft, SectionCtx};++pub const TAG: &str = "sequence@1";+pub const EDGE_TYPE: &str = "sequence";+pub const WEIGHT: f32 = 0.8;+pub const CONFIDENCE: f32 = 0.99;++pub struct Sequence;++impl EdgeDetector for Sequence {+ fn tag(&self) -> &'static str {+ TAG+ }++ fn detect_text(&self, ctx: &SectionCtx<'_>, out: &mut Vec<EdgeDraft>) {+ if ctx.section_ordinal == 0 {+ return;+ }+ let prev_ordinal = ctx.section_ordinal - 1;+ let prev_doc_id = crate::ids::doc_id(+ &ctx.file.dataset_slug,+ &ctx.file.file_key,+ &format!("s{prev_ordinal}"),+ );+ out.push(EdgeDraft {+ src: prev_doc_id,+ dst: ctx.section_doc_id.to_string(),+ edge_type: EDGE_TYPE,+ weight: WEIGHT,+ confidence: CONFIDENCE,+ valid_at_ms: ctx.file.mtime_ms,+ src_file: ctx.file.rel.clone(),+ quote: format!(+ "section {prev_ordinal} precedes section {} of {}",+ ctx.section_ordinal, ctx.file.rel+ ),+ offset: 0,+ });+ }+}++#[cfg(test)]+mod tests {+ use super::super::{corpus_file, CorpusIndex, SectionCtx};+ use super::*;++ /// A 3-section file yields exactly the two adjacency edges, chained+ /// through the deterministic section doc ids.+ #[test]+ fn three_sections_chain_two_edges() {+ let corpus = CorpusIndex::build(vec![corpus_file("big.md", "kb", "docs", "txt-prose", 7)]);+ let file = &corpus.files["big.md"];+ let det = Sequence;+ let mut out = Vec::new();+ for ordinal in 0..3u32 {+ let doc_id = crate::ids::doc_id("docs", "kb", &format!("s{ordinal}"));+ let ctx = SectionCtx {+ corpus: &corpus,+ file,+ section_ordinal: ordinal,+ section_doc_id: &doc_id,+ text: "…",+ };+ det.detect_text(&ctx, &mut out);+ }+ assert_eq!(out.len(), 2);+ assert_eq!(out[0].src, crate::ids::doc_id("docs", "kb", "s0"));+ assert_eq!(out[0].dst, crate::ids::doc_id("docs", "kb", "s1"));+ assert_eq!(out[1].src, crate::ids::doc_id("docs", "kb", "s1"));+ assert_eq!(out[1].dst, crate::ids::doc_id("docs", "kb", "s2"));+ assert_eq!(out[0].quote, "section 0 precedes section 1 of big.md");+ assert_eq!(out[0].offset, 0);+ assert_eq!(out[0].valid_at_ms, 7);+ }+}diff --git a/engine/crates/xerj-autoindex/src/detect/wikilink.rs b/engine/crates/xerj-autoindex/src/detect/wikilink.rsnew file mode 100644index 000000000..34e1d4aca--- /dev/null+++ b/engine/crates/xerj-autoindex/src/detect/wikilink.rs@@ -0,0 +1,191 @@+//! wikilink@1 — `[[Target]]` / `[[Target|alias]]` in section text.+//!+//! The strongest signal the detectors have: a human deliberately named+//! another note. Hence weight 1.0; confidence 0.95 rather than 1.0 because+//! resolution is by name, and a corpus can hold two files answering to the+//! same stem (the ambiguity is counted, never hidden).++use super::{+ line_at, CorpusFile, CorpusIndex, DetectorCounters, EdgeDetector, EdgeDraft, SectionCtx,+};+use std::sync::atomic::{AtomicU64, Ordering};++pub const TAG: &str = "wikilink@1";+pub const EDGE_TYPE: &str = "wikilink";+pub const WEIGHT: f32 = 1.0;+pub const CONFIDENCE: f32 = 0.95;++/// Ignore absurd "targets": an unmatched `[[` in a code block would otherwise+/// swallow the rest of the section into one bogus link.+const MAX_TARGET_LEN: usize = 512;++#[derive(Default)]+pub struct Wikilink {+ unresolved: AtomicU64,+ ambiguous: AtomicU64,+}++enum Resolved<'a> {+ Hit(&'a CorpusFile),+ Ambiguous(&'a CorpusFile),+ Miss,+}++/// §6.5 resolution order: exact rel path (with, then without, extension),+/// else the lowercase stem table. Multiple candidates pick the+/// lexicographically smallest rel — deterministic, and counted as ambiguous.+fn resolve<'a>(corpus: &'a CorpusIndex, target: &str) -> Resolved<'a> {+ if let Some(f) = corpus.files.get(target) {+ return Resolved::Hit(f);+ }+ // Whole-path match minus extension: "notes/beta" → "notes/beta.md".+ // BTreeMap iteration is ascending, so the first match IS the smallest.+ let mut without_ext = corpus+ .files+ .iter()+ .filter(|(rel, _)| super::rel_without_ext(rel) == target);+ if let Some((_, first)) = without_ext.next() {+ return if without_ext.next().is_some() {+ Resolved::Ambiguous(first)+ } else {+ Resolved::Hit(first)+ };+ }+ match corpus.by_stem.get(&target.to_ascii_lowercase()) {+ Some(rels) if rels.len() == 1 => Resolved::Hit(&corpus.files[&rels[0]]),+ Some(rels) => Resolved::Ambiguous(&corpus.files[&rels[0]]),+ None => Resolved::Miss,+ }+}++impl EdgeDetector for Wikilink {+ fn tag(&self) -> &'static str {+ TAG+ }++ fn detect_text(&self, ctx: &SectionCtx<'_>, out: &mut Vec<EdgeDraft>) {+ let text = ctx.text;+ let mut i = 0usize;+ while let Some(open) = text[i..].find("[[") {+ let open = i + open;+ let Some(close) = text[open + 2..].find("]]") else {+ break;+ };+ let close = open + 2 + close;+ i = close + 2;+ let inner = &text[open + 2..close];+ if inner.len() > MAX_TARGET_LEN || inner.contains("[[") {+ continue;+ }+ let target = inner.split('|').next().unwrap_or("").trim();+ if target.is_empty() {+ continue;+ }+ let file = match resolve(ctx.corpus, target) {+ Resolved::Hit(f) => f,+ Resolved::Ambiguous(f) => {+ self.ambiguous.fetch_add(1, Ordering::Relaxed);+ f+ }+ Resolved::Miss => {+ // Dangling link: recorded in the run summary+ // (edges_unresolved), never an invented edge.+ self.unresolved.fetch_add(1, Ordering::Relaxed);+ continue;+ }+ };+ out.push(EdgeDraft {+ src: ctx.section_doc_id.to_string(),+ dst: file.anchor_doc_id.clone(),+ edge_type: EDGE_TYPE,+ weight: WEIGHT,+ confidence: CONFIDENCE,+ valid_at_ms: ctx.file.mtime_ms,+ src_file: ctx.file.rel.clone(),+ quote: line_at(text, open),+ offset: open as u64,+ });+ }+ }++ fn counters(&self) -> DetectorCounters {+ DetectorCounters {+ unresolved: self.unresolved.load(Ordering::Relaxed),+ ambiguous: self.ambiguous.load(Ordering::Relaxed),+ }+ }+}++#[cfg(test)]+mod tests {+ use super::super::{corpus_file, CorpusIndex, SectionCtx};+ use super::*;++ fn corpus() -> CorpusIndex {+ CorpusIndex::build(vec![+ corpus_file("alpha.md", "ka", "notes", "txt-prose", 1753600000000),+ corpus_file("beta.md", "kb", "notes", "txt-prose", 1753600000000),+ corpus_file("sub/beta.txt", "kc", "notes", "txt-prose", 1753600000000),+ ])+ }++ fn run(corpus: &CorpusIndex, text: &str) -> (Vec<EdgeDraft>, DetectorCounters) {+ let det = Wikilink::default();+ let file = &corpus.files["alpha.md"];+ let ctx = SectionCtx {+ corpus,+ file,+ section_ordinal: 0,+ section_doc_id: "sec0",+ text,+ };+ let mut out = Vec::new();+ det.detect_text(&ctx, &mut out);+ (out, det.counters())+ }++ #[test]+ fn fixture_offsets_and_targets() {+ let corpus = corpus();+ let (out, c) = run(&corpus, "Alpha is the hub note. It links to [[beta]] and [[gamma]].");+ // beta resolves ambiguously (beta.md vs sub/beta.txt) to the smallest+ // rel; gamma is dangling and must be counted, not emitted.+ assert_eq!(out.len(), 1);+ assert_eq!(out[0].dst, corpus.files["beta.md"].anchor_doc_id);+ assert_eq!(out[0].offset, 35);+ assert_eq!(+ out[0].quote,+ "Alpha is the hub note. It links to [[beta]] and [[gamma]]."+ );+ assert_eq!(c.unresolved, 1);+ assert_eq!(c.ambiguous, 1);+ }++ #[test]+ fn alias_exact_rel_and_case_insensitive_stem() {+ let corpus = corpus();+ let (out, c) = run(&corpus, "see [[beta.md|the beta note]] and [[BETA]]");+ assert_eq!(out.len(), 2);+ // exact rel match is unambiguous even though the stem table is not+ assert_eq!(out[0].dst, corpus.files["beta.md"].anchor_doc_id);+ assert_eq!(c.ambiguous, 1, "only the stem lookup was ambiguous");+ assert_eq!(c.unresolved, 0);+ }++ #[test]+ fn ambiguity_is_deterministic_smallest_rel() {+ let corpus = corpus();+ let (a, _) = run(&corpus, "[[beta]]");+ let (b, _) = run(&corpus, "[[beta]]");+ assert_eq!(a[0].dst, b[0].dst);+ assert_eq!(a[0].dst, corpus.files["beta.md"].anchor_doc_id);+ }++ #[test]+ fn unterminated_and_empty_links_emit_nothing() {+ let corpus = corpus();+ let (out, c) = run(&corpus, "broken [[beta and empty [[]] end");+ assert!(out.is_empty());+ assert_eq!(c.unresolved, 0);+ }+}diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rsindex 765ff8d41..1bffda1ca 100644--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs@@ -190,6 +190,11 @@ fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {max_file_gb: 1,sample: 50,no_semantic: true,+ // These tests count data bulks to place injected failures; graph+ // edge bulks would shift that numbering, so keep detection off here+ // (the graph pipeline has its own e2e suite in detect::e2e).+ brain: None,+ no_graph: true,dry_run: false,json: false,quiet: true,diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 0aa3f3380..a441146f4 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -9,6 +9,7 @@ pub mod coerce;mod content;pub mod correlate;pub mod dataset;+pub mod detect;pub mod esclient;pub mod extract;pub mod ids;@@ -124,6 +125,43 @@ fn record_bulk_outcome(}}+// ─── second-brain graph runtime ──────────────────────────────────────────++/// Per-run graph state, shared read-only with the Phase B workers+/// (SECOND_BRAIN_SPEC §6.6). Built after the plan is final because the+/// detectors resolve links against the FULL corpus — a per-file view could+/// not tell "dangling" from "not walked yet".+struct GraphRt {+ corpus: detect::CorpusIndex,+ detectors: Vec<Box<dyn detect::EdgeDetector>>,+ /// Raw-source href pass handle. Lives outside the registry because the+ /// HTML extractor strips markup before sectioning, so anchors only exist+ /// in the raw bytes — a source the `EdgeDetector` trait deliberately+ /// never sees (see `detect::href` module docs).+ href_raw: detect::href::Href,+ edges_index: String,+ brain: String,+ /// ONE wall-clock stamp per run: `created_at` is the single+ /// non-deterministic edge field (§6.4); per-worker clocks would make two+ /// halves of one run disagree about when it happened.+ created_at_ms: i64,+ /// detector tag → edges written this run (run-summary honesty §6.6.4).+ written: Mutex<std::collections::BTreeMap<&'static str, u64>>,+ self_dropped: AtomicU64,+ /// Prior-generation edges soft-invalidated before this run's writes.+ invalidated: u64,+}++/// `emit_document` section locators are `s{i}`; everything else (row/line/+/// table locators) is not a text section and must not reach `detect_text`.+fn section_ordinal(locator: &str) -> Option<u32> {+ let rest = locator.strip_prefix('s')?;+ if rest.is_empty() || !rest.bytes().all(|b| b.is_ascii_digit()) {+ return None;+ }+ rest.parse().ok()+}+// ─── Phase A: per-file scan (sniff + bounded sampling) ───────────────────struct FileScan {@@ -824,6 +862,162 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {journal.file_replace_start(key, generation)?;}}+ // ── second-brain graph: corpus table, structural edges, invalidation ──+ // (SECOND_BRAIN_SPEC §6.6.1/§6.6.3.) Runs after plan finalization so the+ // detectors see the whole corpus, and BEFORE any Phase B publication so+ // replacement invalidation can only ever see prior-generation edges —+ // running it later would invalidate this run's own fresh edges.+ let bulk_cut = cfg.bulk_mb << 20;+ let junk_records = AtomicU64::new(0);+ let bulk_errors = Mutex::new(Vec::<String>::new());+ let graph: Option<GraphRt> = if cfg.no_graph {+ None+ } else {+ let brain = match &cfg.brain {+ Some(b) => b.clone(),+ None => {+ let base = cfg+ .root+ .canonicalize()+ .unwrap_or_else(|_| cfg.root.clone());+ let name = base+ .file_name()+ .map(|s| s.to_string_lossy().to_string())+ .unwrap_or_default();+ let slug = dataset::sanitize_slug(&name);+ if slug.is_empty() {+ "brain".into()+ } else {+ slug+ }+ }+ };+ if let Err(reason) = detect::validate_brain(&brain) {+ anyhow::bail!(+ "brain name '{brain}' is invalid: {reason}. Pass an explicit --brain <name> \+ or disable relationship detection with --no-graph"+ );+ }+ // Corpus resolution table: every planned file's rel → identity ++ // anchor node. valid_at comes from the file mtime (§6.4) so an+ // unchanged corpus re-emits byte-identical edge_ids and re-runs+ // converge by overwrite, exactly like ids::doc_id does for nodes.+ let mut corpus_files = Vec::new();+ for (i, f) in files.iter().enumerate() {+ let key = &keys[i];+ if key.is_empty() {+ continue;+ }+ let Some(fa) = plan.files.get(key) else {+ continue; // junk or post-freeze files carry no node docs+ };+ let slug = fa+ .assignments+ .iter()+ .find(|(g, _)| g.is_none())+ .map(|(_, s)| s.clone())+ .or_else(|| fa.assignments.iter().map(|(_, s)| s.clone()).min());+ let Some(slug) = slug else { continue };+ let mtime_ms = std::fs::metadata(&f.path)+ .ok()+ .and_then(|m| m.modified().ok())+ .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())+ .map(|d| d.as_millis() as i64)+ .unwrap_or(0);+ corpus_files.push(detect::corpus_file(&f.rel, key, &slug, &fa.family, mtime_ms));+ }+ let corpus = detect::CorpusIndex::build(corpus_files);+ let detectors = detect::default_detectors();+ let edges_index = detect::edges_index_name(&brain);+ let created_at_ms = chrono::Utc::now().timestamp_millis();++ es.ensure_index(&edges_index, &detect::edge_index_mapping())+ .with_context(|| format!("create edges index {edges_index}"))?;+ let mut nodes_indices: Vec<&str> = plan.datasets.iter().map(|d| d.index.as_str()).collect();+ nodes_indices.sort_unstable();+ nodes_indices.dedup();+ detect::ensure_brain_meta(+ &es,+ &edges_index,+ &brain,+ &nodes_indices.join(","),+ created_at_ms,+ )?;++ // Replacement invalidation FIRST: soft-invalidate every live edge a+ // replaced file taught in earlier runs. The bi-temporal record stays+ // queryable (`as_of` time travel); nothing is deleted.+ let mut invalidated = 0u64;+ {+ let mut replaced_rels: Vec<&str> = todo+ .iter()+ .filter(|&&i| cleanup_required.contains(&keys[i]))+ .map(|&i| files[i].rel.as_str())+ .collect();+ replaced_rels.sort_unstable();+ replaced_rels.dedup();+ for rel in replaced_rels {+ invalidated += detect::invalidate_prior_edges(&es, &edges_index, rel, created_at_ms)+ .with_context(|| format!("invalidate prior edges taught by {rel}"))?;+ }+ }++ // Structural detection (samedir chains) + bulk write, cut at the same+ // --bulk-mb threshold as node bulks.+ let mut structural = Vec::new();+ for det in &detectors {+ det.detect_structure(&corpus, &mut structural);+ }+ let assembled = detect::assemble(&structural, &edges_index, created_at_ms);+ let mut written: std::collections::BTreeMap<&'static str, u64> =+ std::collections::BTreeMap::new();+ {+ let mut send_err: Option<String> = None;+ let mut buf: Vec<u8> = Vec::new();+ for edge in &assembled.edges {+ buf.extend_from_slice(&edge.ndjson);+ *written.entry(edge.detector).or_default() += 1;+ if buf.len() >= bulk_cut+ && record_bulk_outcome(+ &es,+ std::mem::take(&mut buf),+ &junk_records,+ &bulk_errors,+ &mut send_err,+ )+ {+ break;+ }+ }+ if send_err.is_none() && !buf.is_empty() {+ record_bulk_outcome(&es, buf, &junk_records, &bulk_errors, &mut send_err);+ }+ if let Some(e) = send_err {+ anyhow::bail!("write structural graph edges to {edges_index}: {e}");+ }+ }+ if !cfg.quiet {+ eprintln!(+ "graph: brain '{brain}' → {edges_index}; {} structural edges, {} prior edges \+ invalidated ({} detectors live)",+ assembled.edges.len(),+ invalidated,+ detectors.len()+ );+ }+ Some(GraphRt {+ corpus,+ detectors,+ href_raw: detect::href::Href::default(),+ edges_index,+ brain,+ created_at_ms,+ written: Mutex::new(written),+ self_dropped: AtomicU64::new(assembled.self_dropped),+ invalidated,+ })+ };+// ascending by size — workers pop() from the tail, so the BIGGEST files// start first and can't serialize the end of the run.todo.sort_by_key(|&i| files[i].size);@@ -854,10 +1048,7 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {let journal_mx = Mutex::new(&mut journal);let files_done = AtomicU64::new(0);let records_total = AtomicU64::new(0);- let junk_records = AtomicU64::new(0);- let bulk_errors = Mutex::new(Vec::<String>::new());let extra_junk = Mutex::new(Vec::<JunkFile>::new());- let bulk_cut = cfg.bulk_mb << 20;std::thread::scope(|scope| {for _ in 0..cfg.workers.min(n_todo.max(1)) {@@ -890,6 +1081,10 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {let mut file_records = 0u64;let mut file_junk = 0u64;let mut send_err: Option<String> = None;+ // Edges this file teaches — buffered apart from the node+ // staging file (different target index) and sent only+ // after the node bulks are accepted (§6.7).+ let mut edge_drafts: Vec<detect::EdgeDraft> = Vec::new();let mut staged = match tempfile::Builder::new().prefix(".autoindex-stage-").tempfile_in(&state_dir)@@ -948,12 +1143,12 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {fields.insert("ax_format".into(), Value::String(format_str(Some(&sn))));let id = ids::doc_id(slug, key, &rec.locator);let action = json!({"index": {"_index": rt.index, "_id": id}});+ let doc = Value::Object(fields);if let Err(error) = writeln!(staged.as_file_mut(),"{}\n{}",action,- serde_json::to_string(&Value::Object(fields))- .unwrap_or_else(|_| "{}".into())+ serde_json::to_string(&doc).unwrap_or_else(|_| "{}".into())) {send_err =Some(format!("stage extracted records for {}: {error}", f.rel));@@ -961,6 +1156,29 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {}rt.records.fetch_add(1, Ordering::Relaxed);file_records += 1;+ // Textual edge detection (§6.6.2), after the node+ // action is staged: `body` is the exact section+ // string the node doc carries, and `id` is the+ // section node the evidence lives in.+ if let Some(gr) = graph.as_ref() {+ if let Some(ordinal) = section_ordinal(&rec.locator) {+ if let (Some(cf), Some(body)) = (+ gr.corpus.files.get(&f.rel),+ doc.get("body").and_then(Value::as_str),+ ) {+ let ctx = detect::SectionCtx {+ corpus: &gr.corpus,+ file: cf,+ section_ordinal: ordinal,+ section_doc_id: &id,+ text: body,+ };+ for det in &gr.detectors {+ det.detect_text(&ctx, &mut edge_drafts);+ }+ }+ }+ }true};let res = extract::extract(&f.path, &sn, None, &mut sink);@@ -981,6 +1199,32 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {}}}+ // Raw-source href pass: the HTML extractor strips markup+ // before sectioning, so `<a href>` evidence exists only in+ // the raw bytes (detect::href module docs). The second+ // content::verify below still covers this re-read.+ if send_err.is_none() {+ if let Some(gr) = graph.as_ref() {+ if let Some(cf) = gr+ .corpus+ .files+ .get(&f.rel)+ .filter(|cf| cf.family == "html")+ {+ if let Ok(Some(bytes)) =+ extract::read_whole(&f.path, sn.gzip, extract::MAX_WHOLE_FILE)+ {+ let (raw, _) = sniff::decode_text(&bytes);+ gr.href_raw.detect_raw_html(+ &gr.corpus,+ cf,+ &raw,+ &mut edge_drafts,+ );+ }+ }+ }+ }if send_err.is_none() {if let Err(error) = content::verify(&f.path, f.size, expected_digest) {send_err = Some(format!(@@ -1084,6 +1328,49 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {);}}+ // Second-brain edges for this file (§6.7): only after the+ // node bulks were accepted, so an edge never precedes its+ // own src doc. A failed edge send leaves the file+ // un-journaled — the whole file (nodes AND edges) is+ // republished on the next run, which converges because+ // both sides overwrite by deterministic _id.+ if send_err.is_none() && !edge_drafts.is_empty() {+ if let Some(gr) = graph.as_ref() {+ let out =+ detect::assemble(&edge_drafts, &gr.edges_index, gr.created_at_ms);+ gr.self_dropped.fetch_add(out.self_dropped, Ordering::Relaxed);+ let mut ebuf: Vec<u8> = Vec::new();+ for edge in &out.edges {+ ebuf.extend_from_slice(&edge.ndjson);+ if ebuf.len() >= bulk_cut+ && record_bulk_outcome(+ &es,+ std::mem::take(&mut ebuf),+ &junk_records,+ &bulk_errors,+ &mut send_err,+ )+ {+ break;+ }+ }+ if send_err.is_none() && !ebuf.is_empty() {+ record_bulk_outcome(+ &es,+ ebuf,+ &junk_records,+ &bulk_errors,+ &mut send_err,+ );+ }+ if send_err.is_none() {+ let mut written = gr.written.lock().unwrap();+ for edge in &out.edges {+ *written.entry(edge.detector).or_default() += 1;+ }+ }+ }+ }if let Some(e) = send_err {// endpoint trouble: record, do NOT journal file_donelet mut be = bulk_errors.lock().unwrap();@@ -1159,6 +1446,10 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {// ── finalize: refresh, verify, correlate, catalog ────────────────────es.refresh(&format!("{}-*", cfg.prefix)).ok();+ // The dot-prefixed edges index is outside the {prefix}-* pattern.+ if let Some(gr) = &graph {+ es.refresh(&gr.edges_index).ok();+ }// live per-dataset counts + time ranges (every claim traces to a run)let mut ds_counts: HashMap<String, u64> = HashMap::new();@@ -1404,7 +1695,36 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {let wall = t0.elapsed().as_secs_f64();let total_records: u64 = ds_counts.values().sum();- let run_doc = json!({+ // Run-summary honesty (§6.6.4): what the detectors wrote AND what they+ // could not resolve — a dangling [[link]] is a fact about the corpus, not+ // something to swallow.+ let graph_summary = graph.as_ref().map(|gr| {+ let written = gr.written.lock().unwrap();+ let mut counters = detect::DetectorCounters::default();+ for det in &gr.detectors {+ let c = det.counters();+ counters.unresolved += c.unresolved;+ counters.ambiguous += c.ambiguous;+ }+ let raw = gr.href_raw.counters();+ counters.unresolved += raw.unresolved;+ counters.ambiguous += raw.ambiguous;+ let by_detector: Map<String, Value> = written+ .iter()+ .map(|(tag, n)| ((*tag).to_string(), json!(n)))+ .collect();+ json!({+ "brain": gr.brain,+ "edges_index": gr.edges_index,+ "edges_written": written.values().sum::<u64>(),+ "by_detector": by_detector,+ "edges_unresolved": counters.unresolved,+ "edges_ambiguous": counters.ambiguous,+ "edges_self_dropped": gr.self_dropped.load(Ordering::Relaxed),+ "edges_invalidated": gr.invalidated,+ })+ });+ let mut run_doc = json!({"doc_kind": "run","run_id": run_id,"root": root_str,@@ -1422,6 +1742,9 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {"workers": cfg.workers,"semantic": !cfg.no_semantic,});+ if let Some(g) = &graph_summary {+ run_doc["graph"] = g.clone();+ }push_doc(&format!("run:{run_id}"), &run_doc, &mut cat_buf);if !cat_buf.is_empty() {@@ -1446,6 +1769,26 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {for (idx, cnt) in rows {println!(" {idx:<40} {cnt:>10} docs");}+ if let Some(g) = &graph_summary {+ let by: Vec<String> = g["by_detector"]+ .as_object()+ .map(|m| {+ m.iter()+ .map(|(tag, n)| format!("{tag} {n}"))+ .collect()+ })+ .unwrap_or_default();+ println!(+ "graph: {} edges → {} ({}); {} unresolved, {} ambiguous, {} self-dropped, {} invalidated",+ g["edges_written"],+ g["edges_index"].as_str().unwrap_or(""),+ if by.is_empty() { "no detections".to_string() } else { by.join(", ") },+ g["edges_unresolved"],+ g["edges_ambiguous"],+ g["edges_self_dropped"],+ g["edges_invalidated"],+ );+ }println!("\nnext: `xerj autoindex map --url {}` for the data map; search via GET /{}-*/_search",cfg.url, cfg.prefix@@ -1583,6 +1926,33 @@ fn run_map(cfg: MapCfg) -> Result<i32> {junk_files.len() as u64));+ // Second-brain summary (§6.1): live edge count straight from the+ // edges index, scoped by `exists src` so the meta doc never counts.+ if let Some(g) = runs.first().and_then(|r| r.get("graph")) {+ if let (Some(brain), Some(edges_index)) = (+ g.get("brain").and_then(Value::as_str),+ g.get("edges_index").and_then(Value::as_str),+ ) {+ let live = es+ .search(+ edges_index,+ &json!({+ "size": 0,+ "track_total_hits": true,+ "query": {"bool": {+ "filter": [{"exists": {"field": "src"}}],+ "must_not": [{"exists": {"field": "invalid_at"}}]+ }}+ }),+ )+ .ok()+ .and_then(|v| v.pointer("/hits/total/value").and_then(Value::as_u64));+ match live {+ Some(n) => println!("\ngraph: {n} live edges in {edges_index} (brain {brain})"),+ None => println!("\ngraph: brain {brain} — edges index {edges_index} unreachable"),+ }+ }+ }}Ok(0)}@@ -1610,6 +1980,7 @@ fn run_status(cfg: StatusCfg) -> Result<i32> {let mut done = 0u64;let mut records = 0u64;let mut finished = false;+ let mut graph_line: Option<String> = None;if let Ok(f) = std::fs::File::open(&jp) {use std::io::BufRead;for line in std::io::BufReader::new(f).lines().map_while(|l| l.ok()) {@@ -1622,7 +1993,20 @@ fn run_status(cfg: StatusCfg) -> Result<i32> {done += 1;records += v.get("records").and_then(|r| r.as_u64()).unwrap_or(0);}- Some("finish") => finished = true,+ Some("finish") => {+ finished = true;+ // Latest finish wins — the summary embeds the run+ // doc, whose `graph` block is the edge count of+ // record for this journal.+ if let Some(g) = v.pointer("/summary/graph") {+ graph_line = Some(format!(+ "graph: {} edges written to {} (brain {})",+ g.get("edges_written").and_then(Value::as_u64).unwrap_or(0),+ g.get("edges_index").and_then(Value::as_str).unwrap_or("?"),+ g.get("brain").and_then(Value::as_str).unwrap_or("?"),+ ));+ }+ }_ => {}}}@@ -1636,6 +2020,9 @@ fn run_status(cfg: StatusCfg) -> Result<i32> {records,if finished { "FINISHED" } else { "in progress" });+ if let Some(line) = graph_line {+ println!(" {line}");+ }}// live indicesif let Ok(es) = Es::new(&cfg.url, cfg.api_key.clone()) {