Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

wip(second-brain/autoindex): autoindex link detectors

xerj-org3772201eab63
diff --git a/engine/crates/xerj-autoindex/src/cli.rs b/engine/crates/xerj-autoindex/src/cli.rs
index 09436f00c..ee81b6a7b 100644
--- a/engine/crates/xerj-autoindex/src/cli.rs
+++ b/engine/crates/xerj-autoindex/src/cli.rs
@@ -19,6 +19,10 @@ pub struct IndexCfg {
pub max_file_gb: u64,
pub sample: usize,
pub no_semantic: bool,
+ /// Second-brain name; None derives it from the root folder basename.
+ pub brain: Option<String>,
+ /// Disable edge detection entirely (no `.xerj-memory-*-edges` writes).
+ pub no_graph: bool,
pub dry_run: bool,
pub json: bool,
pub quiet: bool,
@@ -75,6 +79,10 @@ pub fn print_help() {
--max-file-gb <N> skip+record oversized non-streamable files (default 2)\n\
--sample <N> records sampled per file for inference (default 500)\n\
--no-semantic skip semantic_text on body fields (pure BM25+keyword)\n\
+ --brain <NAME> second-brain name; relationship edges land in\n\
+ .xerj-memory-<NAME>-edges (default: folder name slug)\n\
+ --no-graph skip relationship detection (wikilinks, local links,\n\
+ section order, directory chains) — no edges are written\n\
--dry-run walk+sniff+infer, print the plan, index nothing\n\
--json machine-readable output (map: raw catalog docs)\n\
--quiet errors only\n\
@@ -132,6 +140,8 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
let mut max_file_gb = 2u64;
let mut sample = 500usize;
let mut no_semantic = false;
+ let mut brain: Option<String> = None;
+ let mut no_graph = false;
let mut dry_run = false;
let mut json = false;
let mut quiet = false;
@@ -198,6 +208,15 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
.ok_or("--sample needs a number")?
}
"--no-semantic" => no_semantic = true,
+ "--brain" => {
+ let name = it.next().ok_or("--brain needs a value")?;
+ // Fail at parse time — a bad brain name after a long Phase A
+ // would be a rude place to learn about the '-edges' rule.
+ crate::detect::validate_brain(&name)
+ .map_err(|reason| format!("--brain {name}: {reason}"))?;
+ brain = Some(name);
+ }
+ "--no-graph" => no_graph = true,
"--dry-run" => dry_run = true,
"--json" => json = true,
"--md" => json = false,
@@ -254,6 +273,8 @@ pub fn parse(args: Vec<String>) -> Result<Cmd, String> {
max_file_gb,
sample: sample.max(50),
no_semantic,
+ brain,
+ no_graph,
dry_run,
json,
quiet,
@@ -278,6 +299,32 @@ mod tests {
assert_eq!(index(&["data"]).bulk_timeout_secs, 300);
}
+ #[test]
+ fn graph_is_on_by_default_with_derived_brain() {
+ let cfg = index(&["data"]);
+ assert!(!cfg.no_graph);
+ assert_eq!(cfg.brain, None, "brain derives from the folder at run time");
+ assert!(index(&["data", "--no-graph"]).no_graph);
+ assert_eq!(
+ index(&["data", "--brain", "notes"]).brain.as_deref(),
+ Some("notes")
+ );
+ }
+
+ #[test]
+ fn brain_names_are_validated_at_parse_time() {
+ for bad in ["kb-edges", "Notes", "-x", "a..b"] {
+ let err = parse(
+ ["data", "--brain", bad]
+ .into_iter()
+ .map(str::to_string)
+ .collect(),
+ )
+ .unwrap_err();
+ assert!(err.contains("--brain"), "{err}");
+ }
+ }
+
#[test]
fn bulk_timeout_accepts_custom_bounded_value() {
assert_eq!(
diff --git a/engine/crates/xerj-autoindex/src/detect/e2e.rs b/engine/crates/xerj-autoindex/src/detect/e2e.rs
new file mode 100644
index 000000000..4d8f199b7
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/detect/e2e.rs
@@ -0,0 +1,541 @@
+//! End-to-end detector tests over the contract's §8 fixture: a real
+//! `run_index` against a fake ES endpoint, asserting the EXACT edge set the
+//! five notes must produce — including what must NOT be produced (dangling
+//! links become counters, replaced files' prior edges become soft-invalidated
+//! documents, and a re-run converges byte-identically modulo `created_at`).
+
+use crate::cli::IndexCfg;
+use crate::detect;
+use serde_json::{json, Value};
+use std::collections::{BTreeMap, HashMap};
+use std::fs;
+use std::io::{BufRead, BufReader, Read, Write};
+use std::net::{TcpListener, TcpStream};
+use std::path::Path;
+use std::sync::{Arc, Mutex};
+use std::thread;
+
+/// index → (_id → _source). BTreeMaps keep every assertion order-stable.
+type Docs = BTreeMap<String, BTreeMap<String, Value>>;
+
+struct MockEs {
+ url: String,
+ docs: Arc<Mutex<Docs>>,
+ stop: Arc<Mutex<bool>>,
+ join: Option<thread::JoinHandle<()>>,
+}
+
+impl MockEs {
+ fn start() -> Self {
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ listener.set_nonblocking(true).unwrap();
+ let url = format!("http://{}", listener.local_addr().unwrap());
+ let docs: Arc<Mutex<Docs>> = Arc::default();
+ let stop = Arc::new(Mutex::new(false));
+ let (sd, ss) = (Arc::clone(&docs), Arc::clone(&stop));
+ let join = thread::spawn(move || loop {
+ match listener.accept() {
+ Ok((stream, _)) => handle(stream, &sd),
+ Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
+ if *ss.lock().unwrap() {
+ break;
+ }
+ thread::yield_now();
+ }
+ Err(e) => panic!("mock accept: {e}"),
+ }
+ });
+ Self {
+ url,
+ docs,
+ stop,
+ join: Some(join),
+ }
+ }
+
+ fn index(&self, name: &str) -> BTreeMap<String, Value> {
+ self.docs.lock().unwrap().get(name).cloned().unwrap_or_default()
+ }
+
+ /// rel path → anchor node `_id`, straight from the published node docs so
+ /// the expectation uses exactly the ids the pipeline derived.
+ fn anchors(&self) -> HashMap<String, String> {
+ let mut out = HashMap::new();
+ for (index, docs) in self.docs.lock().unwrap().iter() {
+ if index.starts_with('.') {
+ continue; // catalog + edges indices are not node datasets
+ }
+ for (id, source) in docs {
+ if source.get("ax_locator").and_then(Value::as_str) == Some("s0") {
+ if let Some(rel) = source.get("ax_path").and_then(Value::as_str) {
+ out.insert(rel.to_string(), id.clone());
+ }
+ }
+ }
+ }
+ out
+ }
+}
+
+impl Drop for MockEs {
+ fn drop(&mut self) {
+ *self.stop.lock().unwrap() = true;
+ let _ = TcpStream::connect(self.url.trim_start_matches("http://"));
+ self.join.take().unwrap().join().unwrap();
+ }
+}
+
+fn respond(mut stream: TcpStream, status: u16, body: &Value) {
+ let bytes = body.to_string();
+ let reason = if status == 200 { "OK" } else { "Not Found" };
+ write!(
+ stream,
+ "HTTP/1.1 {status} {reason}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
+ bytes.len(),
+ bytes
+ )
+ .unwrap();
+}
+
+fn handle(stream: TcpStream, docs: &Arc<Mutex<Docs>>) {
+ let clone = stream.try_clone().unwrap();
+ let mut reader = BufReader::new(clone);
+ let mut request_line = String::new();
+ if reader.read_line(&mut request_line).unwrap() == 0 {
+ return;
+ }
+ let mut content_length = 0usize;
+ loop {
+ let mut line = String::new();
+ reader.read_line(&mut line).unwrap();
+ if line == "\r\n" || line.is_empty() {
+ break;
+ }
+ if let Some(v) = line
+ .to_ascii_lowercase()
+ .strip_prefix("content-length:")
+ .map(str::trim)
+ {
+ content_length = v.parse().unwrap();
+ }
+ }
+ let mut body = vec![0; content_length];
+ reader.read_exact(&mut body).unwrap();
+ let mut parts = request_line.split_whitespace();
+ let method = parts.next().unwrap_or("");
+ let path = parts.next().unwrap_or("").to_string();
+
+ if method == "POST" && path == "/_bulk" {
+ return respond(stream, 200, &bulk(&body, docs));
+ }
+ if let Some((index, id)) = doc_route(&path) {
+ let locked = docs.lock().unwrap();
+ return match locked.get(&index).and_then(|d| d.get(&id)) {
+ Some(source) => respond(
+ stream,
+ 200,
+ &json!({"found": true, "_id": id, "_source": source}),
+ ),
+ None => respond(stream, 404, &json!({"found": false})),
+ };
+ }
+ if method == "POST" && path.contains("/_search") {
+ let index = path.trim_start_matches('/').split('/').next().unwrap_or("");
+ let query: Value = serde_json::from_slice(&body).unwrap_or(Value::Null);
+ return respond(stream, 200, &search(index, &query, docs));
+ }
+ if method == "POST" && path.contains("/_delete_by_query") {
+ let index = path.trim_start_matches('/').split('/').next().unwrap_or("");
+ let query: Value = serde_json::from_slice(&body).unwrap_or(Value::Null);
+ let key = query
+ .pointer("/query/term/ax_file")
+ .and_then(Value::as_str)
+ .map(str::to_owned);
+ let mut deleted = 0u64;
+ if let (Some(key), Some(store)) = (key, docs.lock().unwrap().get_mut(index)) {
+ let before = store.len();
+ store.retain(|_, d| d.get("ax_file").and_then(Value::as_str) != Some(key.as_str()));
+ deleted = (before - store.len()) as u64;
+ }
+ return respond(stream, 200, &json!({"deleted": deleted, "failures": []}));
+ }
+ respond(stream, 200, &json!({"acknowledged": true}));
+}
+
+fn doc_route(path: &str) -> Option<(String, String)> {
+ let mut segs = path.trim_start_matches('/').split('/');
+ let index = segs.next()?;
+ if segs.next()? != "_doc" {
+ return None;
+ }
+ let id = segs.next()?;
+ segs.next().is_none().then(|| (index.to_string(), id.to_string()))
+}
+
+fn bulk(body: &[u8], docs: &Arc<Mutex<Docs>>) -> Value {
+ let lines: Vec<&[u8]> = body
+ .split(|b| *b == b'\n')
+ .filter(|l| !l.is_empty())
+ .collect();
+ let mut locked = docs.lock().unwrap();
+ let mut items = Vec::new();
+ let mut errors = false;
+ for pair in lines.chunks_exact(2) {
+ let action: Value = serde_json::from_slice(pair[0]).unwrap();
+ let doc: Value = serde_json::from_slice(pair[1]).unwrap();
+ let (op, meta) = if let Some(m) = action.get("index") {
+ ("index", m)
+ } else if let Some(m) = action.get("create") {
+ ("create", m)
+ } else {
+ continue;
+ };
+ let index = meta["_index"].as_str().unwrap().to_string();
+ let id = meta["_id"].as_str().unwrap().to_string();
+ let store = locked.entry(index).or_default();
+ if op == "create" && store.contains_key(&id) {
+ errors = true;
+ items.push(json!({"create": {"status": 409,
+ "error": {"type": "version_conflict_engine_exception", "reason": "exists"}}}));
+ continue;
+ }
+ store.insert(id, doc);
+ items.push(json!({op: {"status": 200}}));
+ }
+ json!({"errors": errors, "items": items})
+}
+
+fn search(index: &str, body: &Value, docs: &Arc<Mutex<Docs>>) -> Value {
+ let locked = docs.lock().unwrap();
+ let store = locked.get(index).cloned().unwrap_or_default();
+ // The §6.6.3 invalidation query: live edges taught by one src_file.
+ if let Some(rel) = body
+ .pointer("/query/bool/filter/0/term/src_file")
+ .and_then(Value::as_str)
+ {
+ let size = body.get("size").and_then(Value::as_u64).unwrap_or(10) as usize;
+ let hits: Vec<Value> = store
+ .iter()
+ .filter(|(_, s)| {
+ s.get("src_file").and_then(Value::as_str) == Some(rel)
+ && s.get("invalid_at").is_none()
+ })
+ .take(size)
+ .map(|(id, s)| json!({"_id": id, "_source": s}))
+ .collect();
+ return json!({"hits": {"total": {"value": hits.len()}, "hits": hits}});
+ }
+ // size:0 totals (dataset counts, map summary): total = matching docs.
+ let live_only = body.pointer("/query/bool/must_not").is_some();
+ let total = store
+ .values()
+ .filter(|s| !live_only || s.get("invalid_at").is_none())
+ .count();
+ json!({"hits": {"total": {"value": total}, "hits": []}, "aggregations": {}})
+}
+
+// ─── scenarios ───────────────────────────────────────────────────────────
+
+const EDGES_INDEX: &str = ".xerj-memory-notes-edges";
+
+fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {
+ IndexCfg {
+ root: root.to_owned(),
+ url: url.to_owned(),
+ api_key: None,
+ workers: 1,
+ pdf_workers: 1,
+ pdf_timeout_secs: 30,
+ bulk_mb: 8,
+ bulk_timeout_secs: 300,
+ prefix: "ax".into(),
+ state_dir: Some(state_dir.to_owned()),
+ fresh: false,
+ follow_symlinks: false,
+ max_file_gb: 1,
+ sample: 50,
+ no_semantic: true,
+ brain: Some("notes".into()),
+ no_graph: false,
+ dry_run: false,
+ json: false,
+ quiet: true,
+ }
+}
+
+/// §8.1 — all five bodies verbatim; the wikilink byte offsets in §8.2 are
+/// offsets into these exact strings.
+fn write_fixture(dir: &Path) {
+ fs::write(
+ dir.join("alpha.md"),
+ "Alpha is the hub note. It links to [[beta]] and [[gamma]].",
+ )
+ .unwrap();
+ fs::write(
+ dir.join("beta.md"),
+ "Beta continues the thread and references [[gamma]].",
+ )
+ .unwrap();
+ fs::write(
+ dir.join("gamma.md"),
+ "Gamma is the sink note with no outgoing links.",
+ )
+ .unwrap();
+ fs::write(dir.join("delta.md"), "Delta cites [[alpha]] as its source.").unwrap();
+ fs::write(dir.join("epsilon.md"), "Epsilon stands alone.").unwrap();
+}
+
+fn journal_graph_summary(state_dir: &Path) -> Value {
+ let journal = fs::read_to_string(state_dir.join("journal.ndjson")).unwrap();
+ journal
+ .lines()
+ .filter_map(|l| serde_json::from_str::<Value>(l).ok())
+ .filter(|v| v.get("kind").and_then(Value::as_str) == Some("finish"))
+ .next_back()
+ .and_then(|v| v.pointer("/summary/graph").cloned())
+ .expect("finish event with graph summary")
+}
+
+/// Comparable edge shape: everything the contract §8.3 table pins for
+/// autoindex output (node-id derivation included via the live anchors).
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
+struct EdgeKey {
+ src: String,
+ dst: String,
+ edge_type: String,
+ detector: String,
+ weight: String,
+ confidence: String,
+ src_file: String,
+ quote: String,
+ offset: u64,
+}
+
+fn edge_keys(edges: &BTreeMap<String, Value>) -> Vec<EdgeKey> {
+ let mut out: Vec<EdgeKey> = edges
+ .iter()
+ .filter(|(id, _)| id.as_str() != detect::BRAIN_META_ID)
+ .map(|(id, s)| {
+ // Identity discipline: _id == edge_id field == the §2.3 hash.
+ assert_eq!(s["edge_id"].as_str().unwrap(), id);
+ assert_eq!(
+ detect::edge_id(
+ s["src"].as_str().unwrap(),
+ s["type"].as_str().unwrap(),
+ s["dst"].as_str().unwrap(),
+ s["valid_at"].as_i64().unwrap(),
+ ),
+ id.as_str()
+ );
+ assert_eq!(s["schema_version"], json!(1));
+ EdgeKey {
+ src: s["src"].as_str().unwrap().into(),
+ dst: s["dst"].as_str().unwrap().into(),
+ edge_type: s["type"].as_str().unwrap().into(),
+ detector: s["detector"].as_str().unwrap().into(),
+ weight: s["weight"].to_string(),
+ confidence: s["confidence"].to_string(),
+ src_file: s["src_file"].as_str().unwrap().into(),
+ quote: s["evidence"]["quote"].as_str().unwrap().into(),
+ offset: s["evidence"]["offset"].as_u64().unwrap(),
+ }
+ })
+ .collect();
+ out.sort();
+ out
+}
+
+#[allow(clippy::too_many_arguments)]
+fn expect(
+ anchors: &HashMap<String, String>,
+ src_rel: &str,
+ dst_rel: &str,
+ edge_type: &str,
+ detector: &str,
+ weight: &str,
+ confidence: &str,
+ quote: &str,
+ offset: u64,
+) -> EdgeKey {
+ EdgeKey {
+ src: anchors[src_rel].clone(),
+ dst: anchors[dst_rel].clone(),
+ edge_type: edge_type.into(),
+ detector: detector.into(),
+ weight: weight.into(),
+ confidence: confidence.into(),
+ src_file: src_rel.into(),
+ quote: quote.into(),
+ offset,
+ }
+}
+
+#[test]
+fn fixture_folder_end_to_end_matches_the_contract_edge_set() {
+ let corpus = tempfile::tempdir().unwrap();
+ let state = tempfile::tempdir().unwrap();
+ write_fixture(corpus.path());
+ let es = MockEs::start();
+
+ assert_eq!(
+ crate::run_index(cfg(corpus.path(), state.path(), &es.url)).unwrap(),
+ 0
+ );
+
+ let anchors = es.anchors();
+ assert_eq!(anchors.len(), 5, "five single-section note anchors");
+ let edges = es.index(EDGES_INDEX);
+ assert!(
+ edges.contains_key(detect::BRAIN_META_ID),
+ "brain meta doc must exist"
+ );
+ let meta = &edges[detect::BRAIN_META_ID];
+ assert_eq!(meta["brain"], json!("notes"));
+ assert_eq!(meta["meta_version"], json!(1));
+ assert!(meta["nodes_index"].as_str().unwrap().starts_with("ax-"));
+
+ let alpha_line = "Alpha is the hub note. It links to [[beta]] and [[gamma]].";
+ let mut expected = vec![
+ // §8.2/§8.3 wikilink edges — exact quotes and byte offsets.
+ expect(&anchors, "alpha.md", "beta.md", "wikilink", "wikilink@1", "1.0", "0.95", alpha_line, 35),
+ expect(&anchors, "alpha.md", "gamma.md", "wikilink", "wikilink@1", "1.0", "0.95", alpha_line, 48),
+ expect(&anchors, "beta.md", "gamma.md", "wikilink", "wikilink@1", "1.0", "0.95",
+ "Beta continues the thread and references [[gamma]].", 41),
+ expect(&anchors, "delta.md", "alpha.md", "wikilink", "wikilink@1", "1.0", "0.95",
+ "Delta cites [[alpha]] as its source.", 12),
+ // §8.3 samedir chain over rel-sorted files — 4 edges, not a clique.
+ expect(&anchors, "alpha.md", "beta.md", "same_dir", "samedir@1", "0.3", "0.4",
+ "alpha.md and beta.md share directory .", 0),
+ expect(&anchors, "beta.md", "delta.md", "same_dir", "samedir@1", "0.3", "0.4",
+ "beta.md and delta.md share directory .", 0),
+ expect(&anchors, "delta.md", "epsilon.md", "same_dir", "samedir@1", "0.3", "0.4",
+ "delta.md and epsilon.md share directory .", 0),
+ expect(&anchors, "epsilon.md", "gamma.md", "same_dir", "samedir@1", "0.3", "0.4",
+ "epsilon.md and gamma.md share directory .", 0),
+ ];
+ expected.sort();
+ assert_eq!(edge_keys(&edges), expected, "the exact §8.3 edge set");
+
+ let g = journal_graph_summary(state.path());
+ assert_eq!(g["edges_written"], json!(8));
+ assert_eq!(g["by_detector"]["wikilink@1"], json!(4));
+ assert_eq!(g["by_detector"]["samedir@1"], json!(4));
+ assert_eq!(g["edges_unresolved"], json!(0));
+ assert_eq!(g["edges_ambiguous"], json!(0));
+ assert_eq!(g["edges_self_dropped"], json!(0));
+ assert_eq!(g["edges_invalidated"], json!(0));
+
+ // Determinism (§6.4): a fresh re-run over the unchanged corpus overwrites
+ // every edge in place — byte-identical modulo created_at.
+ let strip_created = |m: &BTreeMap<String, Value>| -> BTreeMap<String, Value> {
+ m.iter()
+ .map(|(id, s)| {
+ let mut s = s.clone();
+ if let Some(o) = s.as_object_mut() {
+ o.remove("created_at");
+ }
+ (id.clone(), s)
+ })
+ .collect()
+ };
+ let first = strip_created(&edges);
+ let state2 = tempfile::tempdir().unwrap();
+ assert_eq!(
+ crate::run_index(cfg(corpus.path(), state2.path(), &es.url)).unwrap(),
+ 0
+ );
+ assert_eq!(
+ strip_created(&es.index(EDGES_INDEX)),
+ first,
+ "re-run must converge on the identical edge set"
+ );
+}
+
+#[test]
+fn dangling_wikilink_is_counted_not_silently_dropped() {
+ let corpus = tempfile::tempdir().unwrap();
+ let state = tempfile::tempdir().unwrap();
+ fs::write(
+ corpus.path().join("zeta.md"),
+ "Zeta links [[nowhere]] and [[eta]].",
+ )
+ .unwrap();
+ fs::write(corpus.path().join("eta.md"), "Eta stands alone.").unwrap();
+ let es = MockEs::start();
+
+ assert_eq!(
+ crate::run_index(cfg(corpus.path(), state.path(), &es.url)).unwrap(),
+ 0
+ );
+
+ let edges = es.index(EDGES_INDEX);
+ let keys = edge_keys(&edges);
+ let types: Vec<(&str, &str)> = keys
+ .iter()
+ .map(|k| (k.edge_type.as_str(), k.detector.as_str()))
+ .collect();
+ // One resolved wikilink + one samedir chain edge; [[nowhere]] must not
+ // fabricate an edge…
+ assert_eq!(
+ types,
+ vec![("same_dir", "samedir@1"), ("wikilink", "wikilink@1")]
+ );
+ // …but it must not vanish either: it is a dangling-link fact of record.
+ let g = journal_graph_summary(state.path());
+ assert_eq!(g["edges_unresolved"], json!(1));
+ assert_eq!(g["edges_written"], json!(2));
+}
+
+#[test]
+fn replaced_file_soft_invalidates_its_prior_edges() {
+ let corpus = tempfile::tempdir().unwrap();
+ let state = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("alpha.md"), "Alpha cites [[beta]] today.").unwrap();
+ fs::write(corpus.path().join("beta.md"), "Beta stands alone.").unwrap();
+ let es = MockEs::start();
+ assert_eq!(
+ crate::run_index(cfg(corpus.path(), state.path(), &es.url)).unwrap(),
+ 0
+ );
+ let before = es.index(EDGES_INDEX);
+ let old_wikilink: Vec<String> = before
+ .iter()
+ .filter(|(_, s)| s["type"] == json!("wikilink"))
+ .map(|(id, _)| id.clone())
+ .collect();
+ assert_eq!(old_wikilink.len(), 1);
+
+ // Rewrite alpha at a strictly later mtime-ms: same fact, new valid_at →
+ // a NEW edge_id; the old edge must survive as an invalidated document.
+ std::thread::sleep(std::time::Duration::from_millis(15));
+ fs::write(
+ corpus.path().join("alpha.md"),
+ "Alpha still cites [[beta]], reworded.",
+ )
+ .unwrap();
+ assert_eq!(
+ crate::run_index(cfg(corpus.path(), state.path(), &es.url)).unwrap(),
+ 0
+ );
+
+ let after = es.index(EDGES_INDEX);
+ let old = &after[&old_wikilink[0]];
+ assert!(
+ old.get("invalid_at").is_some() && old.get("expired_at").is_some(),
+ "prior edge is soft-invalidated, not deleted: {old}"
+ );
+ let live_wikilinks: Vec<&Value> = after
+ .values()
+ .filter(|s| s["type"] == json!("wikilink") && s.get("invalid_at").is_none())
+ .collect();
+ assert_eq!(live_wikilinks.len(), 1, "exactly one live replacement edge");
+ assert_eq!(
+ live_wikilinks[0]["evidence"]["quote"],
+ json!("Alpha still cites [[beta]], reworded.")
+ );
+ let g = journal_graph_summary(state.path());
+ assert!(
+ g["edges_invalidated"].as_u64().unwrap() >= 1,
+ "invalidation surfaced in the run summary: {g}"
+ );
+}
diff --git a/engine/crates/xerj-autoindex/src/detect/href.rs b/engine/crates/xerj-autoindex/src/detect/href.rs
new file mode 100644
index 000000000..8c561df4b
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/detect/href.rs
@@ -0,0 +1,228 @@
+//! href@1 — `<a href="…">` to a corpus file, html-extracted files only.
+//!
+//! Weight 0.7: an HTML anchor is authored intent, but exported HTML (wikis,
+//! doc generators) also mass-produces navigation links, so one anchor says
+//! less than one hand-written markdown link.
+//!
+//! ## Why there is a raw-source pass
+//! The HTML extractor strips markup before sectioning — an `<a>` tag never
+//! survives into section text, so `detect_text` alone would make this
+//! detector a no-op exactly where it matters. `detect_raw_html` therefore
+//! scans the RAW decoded source once per html file (the pipeline feeds it in,
+//! §6.6.2): edges anchor to the file's s0 node and carry offset 0, the
+//! contract's escape hatch for "the extractor lost byte positions". The two
+//! passes cannot double-fire: real tags exist only in the raw source, while
+//! entity-escaped tags (`&lt;a …&gt;`) exist only in the decoded section
+//! text.
+
+use super::{
+ line_at, resolve_local, CorpusFile, CorpusIndex, DetectorCounters, EdgeDetector, EdgeDraft,
+ LinkTarget, SectionCtx,
+};
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::OnceLock;
+
+pub const TAG: &str = "href@1";
+pub const EDGE_TYPE: &str = "href";
+pub const WEIGHT: f32 = 0.7;
+pub const CONFIDENCE: f32 = 0.85;
+
+/// Family string (`Family::as_str`) this detector is scoped to. Sniffing
+/// decides — a `.html` extension on a file that sniffed as prose does not
+/// qualify, and an extension-less html export does.
+const HTML_FAMILY: &str = "html";
+
+#[derive(Default)]
+pub struct Href {
+ unresolved: AtomicU64,
+}
+
+/// Case-insensitive `<a … href=VALUE>`; the capture groups locate the VALUE
+/// start byte so section-text hits (escaped markup) carry a real offset.
+fn anchor_re() -> &'static regex::Regex {
+ static RE: OnceLock<regex::Regex> = OnceLock::new();
+ RE.get_or_init(|| {
+ regex::Regex::new(r#"(?is)<a\s[^>]*?href\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))"#)
+ .expect("static anchor regex")
+ })
+}
+
+fn scan_anchors(text: &str, mut hit: impl FnMut(usize, &str)) {
+ for caps in anchor_re().captures_iter(text) {
+ let m = caps
+ .get(1)
+ .or_else(|| caps.get(2))
+ .or_else(|| caps.get(3));
+ if let Some(m) = m {
+ hit(m.start(), m.as_str());
+ }
+ }
+}
+
+fn draft(
+ file: &CorpusFile,
+ src: &str,
+ target: &CorpusFile,
+ quote: String,
+ offset: u64,
+) -> EdgeDraft {
+ EdgeDraft {
+ src: src.to_string(),
+ dst: target.anchor_doc_id.clone(),
+ edge_type: EDGE_TYPE,
+ weight: WEIGHT,
+ confidence: CONFIDENCE,
+ valid_at_ms: file.mtime_ms,
+ src_file: file.rel.clone(),
+ quote,
+ offset,
+ }
+}
+
+impl Href {
+ /// The raw-source pass (see module docs). `src` is the file's anchor node:
+ /// section attribution was lost with the markup, and honesty prefers a
+ /// coarse-but-true source node over a guessed section.
+ pub fn detect_raw_html(
+ &self,
+ corpus: &CorpusIndex,
+ file: &CorpusFile,
+ raw: &str,
+ out: &mut Vec<EdgeDraft>,
+ ) {
+ if file.family != HTML_FAMILY {
+ return;
+ }
+ scan_anchors(raw, |pos, url| {
+ match resolve_local(corpus, &file.dir, url) {
+ LinkTarget::Hit(target) => out.push(draft(
+ file,
+ &file.anchor_doc_id,
+ target,
+ line_at(raw, pos),
+ // Byte positions did not survive extraction — the §6.5
+ // "best effort" zero, NOT the raw-file offset, which no
+ // reader could map back onto a section.
+ 0,
+ )),
+ LinkTarget::External | LinkTarget::Empty => {}
+ LinkTarget::Miss => {
+ self.unresolved.fetch_add(1, Ordering::Relaxed);
+ }
+ }
+ });
+ }
+}
+
+impl EdgeDetector for Href {
+ fn tag(&self) -> &'static str {
+ TAG
+ }
+
+ /// Section-text pass: only fires when markup survived into the extracted
+ /// text (entity-escaped anchors) — then the byte offset is real.
+ fn detect_text(&self, ctx: &SectionCtx<'_>, out: &mut Vec<EdgeDraft>) {
+ if ctx.file.family != HTML_FAMILY {
+ return;
+ }
+ scan_anchors(ctx.text, |pos, url| {
+ match resolve_local(ctx.corpus, &ctx.file.dir, url) {
+ LinkTarget::Hit(target) => out.push(draft(
+ ctx.file,
+ ctx.section_doc_id,
+ target,
+ line_at(ctx.text, pos),
+ pos as u64,
+ )),
+ LinkTarget::External | LinkTarget::Empty => {}
+ LinkTarget::Miss => {
+ self.unresolved.fetch_add(1, Ordering::Relaxed);
+ }
+ }
+ });
+ }
+
+ fn counters(&self) -> DetectorCounters {
+ DetectorCounters {
+ unresolved: self.unresolved.load(Ordering::Relaxed),
+ ambiguous: 0,
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::super::{corpus_file, CorpusIndex, SectionCtx};
+ use super::*;
+
+ fn corpus() -> CorpusIndex {
+ CorpusIndex::build(vec![
+ corpus_file("site/index.html", "k1", "d", "html", 10),
+ corpus_file("site/about.html", "k2", "d", "html", 10),
+ corpus_file("notes.md", "k3", "d", "txt-prose", 10),
+ ])
+ }
+
+ #[test]
+ fn raw_pass_resolves_relative_and_counts_misses() {
+ let corpus = corpus();
+ let det = Href::default();
+ let mut out = Vec::new();
+ det.detect_raw_html(
+ &corpus,
+ &corpus.files["site/index.html"],
+ "<p><A HREF='about.html'>about</A> <a class=\"x\" href=\"https://e.co\">ext</a>\n\
+ <a href=gone.html>miss</a></p>",
+ &mut out,
+ );
+ assert_eq!(out.len(), 1);
+ assert_eq!(out[0].dst, corpus.files["site/about.html"].anchor_doc_id);
+ assert_eq!(out[0].src, corpus.files["site/index.html"].anchor_doc_id);
+ assert_eq!(out[0].offset, 0, "raw pass lost byte positions by contract");
+ assert!(out[0].quote.contains("about"));
+ assert_eq!(det.counters().unresolved, 1);
+ }
+
+ #[test]
+ fn non_html_families_are_out_of_scope() {
+ let corpus = corpus();
+ let det = Href::default();
+ let mut out = Vec::new();
+ det.detect_raw_html(
+ &corpus,
+ &corpus.files["notes.md"],
+ "<a href=\"site/about.html\">x</a>",
+ &mut out,
+ );
+ let ctx = SectionCtx {
+ corpus: &corpus,
+ file: &corpus.files["notes.md"],
+ section_ordinal: 0,
+ section_doc_id: "sec0",
+ text: "<a href=\"site/about.html\">x</a>",
+ };
+ det.detect_text(&ctx, &mut out);
+ assert!(out.is_empty());
+ assert_eq!(det.counters().unresolved, 0);
+ }
+
+ #[test]
+ fn section_text_pass_keeps_real_offsets() {
+ let corpus = corpus();
+ let det = Href::default();
+ let text = "escaped <a href=\"about.html\">about</a> anchor";
+ let ctx = SectionCtx {
+ corpus: &corpus,
+ file: &corpus.files["site/index.html"],
+ section_ordinal: 0,
+ section_doc_id: "sec0",
+ text,
+ };
+ let mut out = Vec::new();
+ det.detect_text(&ctx, &mut out);
+ assert_eq!(out.len(), 1);
+ assert_eq!(out[0].src, "sec0");
+ assert_eq!(out[0].offset, 17, "offset of the href VALUE start");
+ assert_eq!(&text[17..27], "about.html");
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/detect/mdlink.rs b/engine/crates/xerj-autoindex/src/detect/mdlink.rs
new file mode 100644
index 000000000..c8a524769
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/detect/mdlink.rs
@@ -0,0 +1,170 @@
+//! mdlink@1 — `[text](relative/path.md)` where the target is a corpus file.
+//!
+//! Slightly below wikilink (weight 0.9): an inline markdown link is still an
+//! explicit authorial act, but it is also how people cite external material,
+//! so a resolved local target carries marginally less "these notes belong
+//! together" intent than a `[[wiki link]]`.
+
+use super::{
+ line_at, resolve_local, DetectorCounters, EdgeDetector, EdgeDraft, LinkTarget, SectionCtx,
+};
+use std::sync::atomic::{AtomicU64, Ordering};
+
+pub const TAG: &str = "mdlink@1";
+pub const EDGE_TYPE: &str = "mdlink";
+pub const WEIGHT: f32 = 0.9;
+pub const CONFIDENCE: f32 = 0.9;
+
+#[derive(Default)]
+pub struct Mdlink {
+ unresolved: AtomicU64,
+}
+
+/// Find `[label](url)` occurrences. Hand scan, not regex: the grammar is two
+/// delimiter searches, and the scan must never backtrack pathologically on a
+/// section full of stray brackets. No nesting support — deterministic first
+/// `]` / first `)` wins, which matches how common renderers degrade.
+fn scan_links(text: &str, mut hit: impl FnMut(usize, &str)) {
+ let bytes = text.as_bytes();
+ let mut i = 0usize;
+ while i < bytes.len() {
+ let Some(open) = text[i..].find('[') else {
+ break;
+ };
+ let open = i + open;
+ let Some(close) = text[open + 1..].find(']') else {
+ break;
+ };
+ let close = open + 1 + close;
+ if bytes.get(close + 1) != Some(&b'(') {
+ i = open + 1;
+ continue;
+ }
+ let Some(paren) = text[close + 2..].find(')') else {
+ break;
+ };
+ let paren = close + 2 + paren;
+ // `[t](url "title")` — the optional title starts at the first
+ // whitespace inside the parens.
+ let raw = text[close + 2..paren].trim();
+ let url = raw.split_whitespace().next().unwrap_or("");
+ hit(open, url);
+ i = paren + 1;
+ }
+}
+
+impl EdgeDetector for Mdlink {
+ fn tag(&self) -> &'static str {
+ TAG
+ }
+
+ fn detect_text(&self, ctx: &SectionCtx<'_>, out: &mut Vec<EdgeDraft>) {
+ scan_links(ctx.text, |open, url| {
+ match resolve_local(ctx.corpus, &ctx.file.dir, url) {
+ LinkTarget::Hit(target) => out.push(EdgeDraft {
+ src: ctx.section_doc_id.to_string(),
+ dst: target.anchor_doc_id.clone(),
+ edge_type: EDGE_TYPE,
+ weight: WEIGHT,
+ confidence: CONFIDENCE,
+ valid_at_ms: ctx.file.mtime_ms,
+ src_file: ctx.file.rel.clone(),
+ quote: line_at(ctx.text, open),
+ offset: open as u64,
+ }),
+ // External links are what markdown links are FOR — skipping
+ // them silently is correct, only local misses are dangling.
+ LinkTarget::External | LinkTarget::Empty => {}
+ LinkTarget::Miss => {
+ self.unresolved.fetch_add(1, Ordering::Relaxed);
+ }
+ }
+ });
+ }
+
+ fn counters(&self) -> DetectorCounters {
+ DetectorCounters {
+ unresolved: self.unresolved.load(Ordering::Relaxed),
+ ambiguous: 0,
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::super::{corpus_file, CorpusIndex, SectionCtx};
+ use super::*;
+
+ fn corpus() -> CorpusIndex {
+ CorpusIndex::build(vec![
+ corpus_file("docs/guide.md", "k1", "d", "txt-prose", 10),
+ corpus_file("readme.md", "k2", "d", "txt-prose", 10),
+ ])
+ }
+
+ fn run(corpus: &CorpusIndex, from: &str, text: &str) -> (Vec<EdgeDraft>, DetectorCounters) {
+ let det = Mdlink::default();
+ let file = &corpus.files[from];
+ let ctx = SectionCtx {
+ corpus,
+ file,
+ section_ordinal: 0,
+ section_doc_id: "sec0",
+ text,
+ };
+ let mut out = Vec::new();
+ det.detect_text(&ctx, &mut out);
+ (out, det.counters())
+ }
+
+ #[test]
+ fn resolves_dir_relative_then_root_relative() {
+ let corpus = corpus();
+ let (out, c) = run(
+ &corpus,
+ "docs/guide.md",
+ "see [readme](../readme.md) and [self](guide.md) and [root](docs/guide.md)",
+ );
+ assert_eq!(out.len(), 3);
+ assert_eq!(out[0].dst, corpus.files["readme.md"].anchor_doc_id);
+ assert_eq!(out[1].dst, corpus.files["docs/guide.md"].anchor_doc_id);
+ // "docs/guide.md" fails dir-relative (docs/docs/guide.md), resolves
+ // root-relative.
+ assert_eq!(out[2].dst, corpus.files["docs/guide.md"].anchor_doc_id);
+ assert_eq!(c.unresolved, 0);
+ assert_eq!(out[0].offset, 4);
+ }
+
+ #[test]
+ fn external_links_skip_and_local_misses_count() {
+ let corpus = corpus();
+ let (out, c) = run(
+ &corpus,
+ "readme.md",
+ "[web](https://example.com/a.md) [mail](mailto:[email protected]) [gone](missing.md) [frag](#top)",
+ );
+ assert!(out.is_empty());
+ assert_eq!(c.unresolved, 1, "only the local miss is dangling");
+ }
+
+ #[test]
+ fn title_and_fragment_are_stripped() {
+ let corpus = corpus();
+ let (out, _) = run(
+ &corpus,
+ "readme.md",
+ "x [g](docs/guide.md#section \"The Guide\") y",
+ );
+ assert_eq!(out.len(), 1);
+ assert_eq!(out[0].dst, corpus.files["docs/guide.md"].anchor_doc_id);
+ assert_eq!(out[0].quote, "x [g](docs/guide.md#section \"The Guide\") y");
+ }
+
+ #[test]
+ fn wikilinks_do_not_false_positive() {
+ let corpus = corpus();
+ let (out, c) = run(&corpus, "readme.md", "a [[guide]] wikilink, no mdlink here");
+ assert!(out.is_empty());
+ assert_eq!(c.unresolved, 0);
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/detect/mod.rs b/engine/crates/xerj-autoindex/src/detect/mod.rs
new file mode 100644
index 000000000..201f8d35f
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/detect/mod.rs
@@ -0,0 +1,739 @@
+//! Second-brain edge detectors: deterministic, versioned relationship
+//! extraction over the corpus autoindex is already walking (contract:
+//! SECOND_BRAIN_SPEC §6).
+//!
+//! Why deterministic instead of an LLM pass: the same corpus must produce a
+//! byte-identical edge set on every run (given unchanged mtimes), because edge
+//! identity — and therefore idempotent re-runs, kill -9 safety, and the
+//! bi-temporal history — hangs off a content hash of (src, type, dst,
+//! valid_at). A model in that loop would turn every re-run into a new belief
+//! set. Every emitted edge instead carries the exact quote that taught it and
+//! a `detector` tag like `wikilink@1`; bump the `@N` on ANY behavior change so
+//! old edges remain attributable to the rules that produced them.
+//!
+//! Edges are ordinary documents in one reserved `.xerj-memory-{brain}-edges`
+//! index per brain — no storage-format change, they ride the normal
+//! WAL → memtable → segment path and are soft-invalidated (never deleted) so
+//! "what did it believe last Tuesday" stays answerable.
+
+pub mod href;
+pub mod mdlink;
+pub mod samedir;
+pub mod sequence;
+pub mod wikilink;
+
+#[cfg(test)]
+mod e2e;
+
+use crate::esclient::Es;
+use anyhow::{anyhow, Context, Result};
+use serde_json::{json, Value};
+use std::collections::BTreeMap;
+
+/// Edge-schema version stamped into every emitted edge (`schema_version`).
+pub const EDGE_SCHEMA_VERSION: u32 = 1;
+
+/// Reserved `_id` of the per-brain meta document. It carries no `src`/`dst`,
+/// so the hop path and every edge count (all filter on `exists src`) never
+/// see it.
+pub const BRAIN_META_ID: &str = "__xerj-brain-meta";
+
+/// edge_id = xxh3_128("xg1\0" src "\0" type "\0" dst "\0" decimal(valid_at_ms)),
+/// rendered as 32 lowercase hex chars ({:032x}).
+pub fn edge_id(src: &str, edge_type: &str, dst: &str, valid_at_ms: i64) -> String {
+ use xxhash_rust::xxh3::xxh3_128;
+ let mut input = Vec::with_capacity(16 + src.len() + edge_type.len() + dst.len());
+ input.extend_from_slice(b"xg1\x00");
+ input.extend_from_slice(src.as_bytes());
+ input.push(0);
+ input.extend_from_slice(edge_type.as_bytes());
+ input.push(0);
+ input.extend_from_slice(dst.as_bytes());
+ input.push(0);
+ input.extend_from_slice(valid_at_ms.to_string().as_bytes());
+ format!("{:032x}", xxh3_128(&input))
+}
+
+/// Edges index name for a brain (SECOND_BRAIN_SPEC §1). The leading dot keeps
+/// it out of user index APIs and `_cat` listings.
+pub fn edges_index_name(brain: &str) -> String {
+ format!(".xerj-memory-{brain}-edges")
+}
+
+/// Brain-name validation: identical rules to the agent-memory namespace
+/// (lowercase/digit start, `[a-z0-9._-]`, ≤200 chars, no `..`) plus the
+/// `-edges` suffix rejection — without it a brain named `kb-edges` would
+/// collide with brain `kb`'s edge index.
+pub fn validate_brain(brain: &str) -> std::result::Result<(), String> {
+ if brain.is_empty() {
+ return Err("brain name must not be empty".into());
+ }
+ if brain.len() > 200 {
+ return Err("brain name too long (max 200 chars)".into());
+ }
+ let first = brain.chars().next().unwrap();
+ if !(first.is_ascii_lowercase() || first.is_ascii_digit()) {
+ return Err("brain name must start with a lowercase letter or digit".into());
+ }
+ for c in brain.chars() {
+ let ok = c.is_ascii_lowercase() || c.is_ascii_digit() || matches!(c, '_' | '-' | '.');
+ if !ok {
+ return Err(format!(
+ "brain name contains illegal character '{c}' (allowed: a-z 0-9 _ - .)"
+ ));
+ }
+ }
+ if brain.contains("..") {
+ return Err("brain name must not contain '..'".into());
+ }
+ if brain.ends_with("-edges") {
+ return Err("namespace suffix '-edges' is reserved for graph edge indices".into());
+ }
+ Ok(())
+}
+
+/// The exact edges-index mapping body (SECOND_BRAIN_SPEC §2.1). The API
+/// stream's `graph_api::link` sends the byte-identical body — the two writers
+/// must agree or the first one to touch a brain decides the column types for
+/// everyone.
+pub fn edge_index_mapping() -> Value {
+ json!({
+ "mappings": {
+ "properties": {
+ "edge_id": { "type": "keyword" },
+ "src": { "type": "keyword" },
+ "dst": { "type": "keyword" },
+ "type": { "type": "keyword" },
+ "weight": { "type": "float" },
+ "valid_at": { "type": "date", "format": "epoch_millis" },
+ "invalid_at": { "type": "date", "format": "epoch_millis" },
+ "created_at": { "type": "date", "format": "epoch_millis" },
+ "expired_at": { "type": "date", "format": "epoch_millis" },
+ "detector": { "type": "keyword" },
+ "confidence": { "type": "float" },
+ "schema_version": { "type": "integer" },
+ "src_file": { "type": "keyword" },
+ "evidence": {
+ "properties": {
+ "quote": { "type": "text" },
+ "source": { "type": "keyword" },
+ "offset": { "type": "long" }
+ }
+ }
+ }
+ }
+ })
+}
+
+// ─── corpus resolution table ─────────────────────────────────────────────
+
+/// Corpus-wide resolution table, built once after Phase A (plan assignment):
+/// every indexed file's rel path → identity + the doc id of its first section.
+pub struct CorpusIndex {
+ /// key: root-relative path with forward slashes (`FileEntry.rel`).
+ pub files: BTreeMap<String, CorpusFile>,
+ /// lowercase file-stem → rel paths bearing it (wikilink resolution;
+ /// BTreeMap + sorted values so ambiguity resolution is deterministic).
+ pub by_stem: BTreeMap<String, Vec<String>>,
+}
+
+pub struct CorpusFile {
+ pub rel: String,
+ /// `ids::file_key` output.
+ pub file_key: String,
+ pub dataset_slug: String,
+ /// `ids::doc_id(dataset_slug, file_key, "s0")` — the file's anchor node.
+ pub anchor_doc_id: String,
+ pub mtime_ms: i64,
+ /// Parent dir rel path, "" for root.
+ pub dir: String,
+ /// Sniffed format family (`Family::as_str`), from the frozen plan — NOT
+ /// guessed from the extension. href@1 gates on this: the contract scopes
+ /// anchor detection to html-extracted files, and an `.html` extension is
+ /// not proof of that (sniffing decides the family).
+ pub family: String,
+}
+
+/// Build one [`CorpusFile`], deriving the anchor node id and parent dir from
+/// the identity fields so every caller (pipeline and tests) derives them the
+/// same way.
+pub fn corpus_file(
+ rel: &str,
+ file_key: &str,
+ dataset_slug: &str,
+ family: &str,
+ mtime_ms: i64,
+) -> CorpusFile {
+ CorpusFile {
+ rel: rel.to_string(),
+ file_key: file_key.to_string(),
+ dataset_slug: dataset_slug.to_string(),
+ anchor_doc_id: crate::ids::doc_id(dataset_slug, file_key, "s0"),
+ mtime_ms,
+ dir: rel.rsplit_once('/').map(|(d, _)| d.to_string()).unwrap_or_default(),
+ family: family.to_string(),
+ }
+}
+
+impl CorpusIndex {
+ pub fn build(files: Vec<CorpusFile>) -> Self {
+ let mut map: BTreeMap<String, CorpusFile> = BTreeMap::new();
+ let mut by_stem: BTreeMap<String, Vec<String>> = BTreeMap::new();
+ for f in files {
+ by_stem
+ .entry(stem_of(&f.rel).to_ascii_lowercase())
+ .or_default()
+ .push(f.rel.clone());
+ map.insert(f.rel.clone(), f);
+ }
+ for rels in by_stem.values_mut() {
+ rels.sort();
+ rels.dedup();
+ }
+ CorpusIndex {
+ files: map,
+ by_stem,
+ }
+ }
+}
+
+/// File stem of a rel path: final path segment minus its last extension.
+/// Dotfiles (".env") keep their full name — an empty stem would collide every
+/// dotfile into one wikilink bucket.
+pub(crate) fn stem_of(rel: &str) -> &str {
+ let name = rel.rsplit('/').next().unwrap_or(rel);
+ match name.rsplit_once('.') {
+ Some((stem, _)) if !stem.is_empty() => stem,
+ _ => name,
+ }
+}
+
+/// Rel path minus its last extension (whole-path form used by wikilink's
+/// "with or without extension" rule). "" stem is preserved as-is for dotfiles.
+pub(crate) fn rel_without_ext(rel: &str) -> String {
+ let (dir, name) = rel.rsplit_once('/').unwrap_or(("", rel));
+ let stem = match name.rsplit_once('.') {
+ Some((s, _)) if !s.is_empty() => s,
+ _ => name,
+ };
+ if dir.is_empty() {
+ stem.to_string()
+ } else {
+ format!("{dir}/{stem}")
+ }
+}
+
+// ─── detector trait ──────────────────────────────────────────────────────
+
+/// One detected edge before identity/envelope assembly.
+pub struct EdgeDraft {
+ /// Node doc id (usually the section containing the evidence).
+ pub src: String,
+ /// Node doc id (target file's anchor_doc_id).
+ pub dst: String,
+ pub edge_type: &'static str,
+ pub weight: f32,
+ pub confidence: f32,
+ /// §6.4 determinism rule: mtime of `src_file`, never the wall clock —
+ /// re-runs over an unchanged corpus reproduce identical edge_ids.
+ pub valid_at_ms: i64,
+ /// Rel path that taught this edge (top-level keyword — exists so
+ /// replacement invalidation is a doc-values term query, not an
+ /// `evidence.source` object scan).
+ pub src_file: String,
+ /// evidence.quote (≤240 chars, trimmed).
+ pub quote: String,
+ /// evidence.offset (byte offset in section text; 0 for structural).
+ pub offset: u64,
+}
+
+/// Per-section textual context. `text` is the exact section string that became
+/// the node doc's `body` (post `split_sections`).
+pub struct SectionCtx<'a> {
+ pub corpus: &'a CorpusIndex,
+ pub file: &'a CorpusFile,
+ pub section_ordinal: u32,
+ pub section_doc_id: &'a str,
+ pub text: &'a str,
+}
+
+/// What a resolving detector could not turn into edges. Surfaced in the run
+/// summary so a dangling `[[link]]` is recorded, never silently dropped.
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
+pub struct DetectorCounters {
+ /// Link targets that resolved to no corpus file (dangling).
+ pub unresolved: u64,
+ /// Link targets with >1 candidate; the lexicographically smallest rel won.
+ pub ambiguous: u64,
+}
+
+/// Deterministic, versioned edge detector. NO network, NO LLM, NO clock reads
+/// (all time comes from ctx mtimes) — same inputs must yield the same drafts.
+pub trait EdgeDetector: Sync {
+ /// Versioned tag stored in `detector`, e.g. "wikilink@1". Bump the @N on
+ /// ANY behavior change.
+ fn tag(&self) -> &'static str;
+ /// Per-section textual detection. Default: no-op.
+ fn detect_text(&self, _ctx: &SectionCtx<'_>, _out: &mut Vec<EdgeDraft>) {}
+ /// Corpus-structural detection, called once after Phase A. Default: no-op.
+ fn detect_structure(&self, _corpus: &CorpusIndex, _out: &mut Vec<EdgeDraft>) {}
+ /// Run-summary honesty counters (default: nothing to report). Detectors
+ /// that resolve link targets keep these in atomics so the counts survive
+ /// the shared `&self` the parallel Phase B workers hold.
+ fn counters(&self) -> DetectorCounters {
+ DetectorCounters::default()
+ }
+}
+
+/// The registry. Order is normative (fixture edge ordering depends on it only
+/// via edge sort, so this is cosmetic — but keep it).
+pub fn default_detectors() -> Vec<Box<dyn EdgeDetector>> {
+ vec![
+ Box::new(wikilink::Wikilink::default()),
+ Box::new(mdlink::Mdlink::default()),
+ Box::new(href::Href::default()),
+ Box::new(sequence::Sequence),
+ Box::new(samedir::SameDir),
+ ]
+}
+
+/// Detector tag for an edge type. The draft deliberately carries only the
+/// type (§6.3 shape); the 1:1 type→tag map lives here, next to the registry,
+/// so a version bump touches exactly one detector file plus nothing else.
+pub fn detector_tag_for(edge_type: &str) -> &'static str {
+ match edge_type {
+ wikilink::EDGE_TYPE => wikilink::TAG,
+ mdlink::EDGE_TYPE => mdlink::TAG,
+ href::EDGE_TYPE => href::TAG,
+ sequence::EDGE_TYPE => sequence::TAG,
+ samedir::EDGE_TYPE => samedir::TAG,
+ _ => "unknown@0",
+ }
+}
+
+// ─── shared text/link helpers ────────────────────────────────────────────
+
+/// Evidence quote discipline: the full trimmed line containing `offset`,
+/// clipped to 240 chars — enough to show a human (or agent) exactly what
+/// taught the edge without dragging whole sections into every edge doc.
+pub(crate) fn line_at(text: &str, offset: usize) -> String {
+ let offset = offset.min(text.len());
+ let start = text[..offset].rfind('\n').map(|i| i + 1).unwrap_or(0);
+ let end = text[offset..]
+ .find('\n')
+ .map(|i| offset + i)
+ .unwrap_or(text.len());
+ clip_quote(&text[start..end])
+}
+
+pub(crate) fn clip_quote(s: &str) -> String {
+ let t = s.trim();
+ if t.chars().count() <= 240 {
+ t.to_string()
+ } else {
+ t.chars().take(240).collect()
+ }
+}
+
+/// Outcome of scheme-less link resolution (mdlink/href rule, §6.5).
+pub(crate) enum LinkTarget<'a> {
+ /// Has a scheme or `//` prefix — external by design, not dangling.
+ External,
+ /// Nothing left after stripping fragment/query (same-page anchor).
+ Empty,
+ /// Scheme-less but no corpus file matches — dangling, counted.
+ Miss,
+ Hit(&'a CorpusFile),
+}
+
+/// Resolve a scheme-less url against the corpus: relative to the containing
+/// file's dir first, then as root-relative. Fragment (`#…`) and query (`?…`)
+/// are stripped; percent-encoding is NOT decoded (a deliberate v1 limit —
+/// decoding would need an escaping table and links in local corpora rarely
+/// use it; a miss is counted, not hidden).
+pub(crate) fn resolve_local<'a>(
+ corpus: &'a CorpusIndex,
+ containing_dir: &str,
+ url: &str,
+) -> LinkTarget<'a> {
+ let url = url.trim();
+ if url.is_empty() {
+ return LinkTarget::Empty;
+ }
+ if url.starts_with("//") || has_scheme(url) {
+ return LinkTarget::External;
+ }
+ let path = url.split(['#', '?']).next().unwrap_or("");
+ if path.is_empty() {
+ return LinkTarget::Empty;
+ }
+ let candidates: [Option<String>; 2] = if let Some(site_abs) = path.strip_prefix('/') {
+ // Site-absolute → root-relative only; joining it under the containing
+ // dir would invent a path the author never wrote.
+ [normalize_rel(site_abs), None]
+ } else {
+ let joined = if containing_dir.is_empty() {
+ path.to_string()
+ } else {
+ format!("{containing_dir}/{path}")
+ };
+ [normalize_rel(&joined), normalize_rel(path)]
+ };
+ for cand in candidates.into_iter().flatten() {
+ if let Some(f) = corpus.files.get(&cand) {
+ return LinkTarget::Hit(f);
+ }
+ }
+ LinkTarget::Miss
+}
+
+/// Any ':' before the first '/' marks the target external ("https:", "mailto:",
+/// "tel:", "host:8080"…). Corpus rel paths essentially never contain a colon,
+/// and a rule this blunt is deterministic where scheme-grammar parsing would
+/// accumulate special cases.
+fn has_scheme(url: &str) -> bool {
+ match (url.find(':'), url.find('/')) {
+ (Some(c), Some(s)) => c < s,
+ (Some(_), None) => true,
+ (None, _) => false,
+ }
+}
+
+/// Normalize `a/./b/../c` → `a/c`; `..` escaping the corpus root is a miss
+/// (None), not a guess.
+fn normalize_rel(path: &str) -> Option<String> {
+ let mut parts: Vec<&str> = Vec::new();
+ for seg in path.split('/') {
+ match seg {
+ "" | "." => {}
+ ".." => {
+ parts.pop()?;
+ }
+ s => parts.push(s),
+ }
+ }
+ if parts.is_empty() {
+ None
+ } else {
+ Some(parts.join("/"))
+ }
+}
+
+// ─── envelope assembly + write path ──────────────────────────────────────
+
+/// One edge ready for `_bulk`: the two ndjson lines plus the tag for
+/// per-detector accounting.
+pub struct AssembledEdge {
+ pub detector: &'static str,
+ pub edge_id: String,
+ pub ndjson: Vec<u8>,
+}
+
+pub struct AssembleOutcome {
+ pub edges: Vec<AssembledEdge>,
+ /// Self-edges (src == dst) dropped by the assembler — counted, per §6.4.
+ pub self_dropped: u64,
+}
+
+/// f32 → JSON number without widening artifacts: `0.3f32 as f64` is
+/// 0.30000001192092896, which would poison every stored weight and the
+/// doc-values compare downstream. The shortest f32 display ("0.3") parsed as
+/// f64 is the number the contract fixtures assert.
+fn clean_f64(v: f32) -> f64 {
+ format!("{v}").parse().unwrap_or(f64::from(v))
+}
+
+/// Turn drafts into `_bulk` actions (§2.2 stored-type discipline: every
+/// scalar a plain string/number, `invalid_at`/`expired_at` omitted — the
+/// null_bitmap IS the "still valid" signal). `created_at_ms` is the one
+/// non-deterministic field and never participates in `edge_id`.
+pub fn assemble(drafts: &[EdgeDraft], edges_index: &str, created_at_ms: i64) -> AssembleOutcome {
+ let mut edges = Vec::with_capacity(drafts.len());
+ let mut self_dropped = 0u64;
+ for d in drafts {
+ if d.src == d.dst {
+ self_dropped += 1;
+ continue;
+ }
+ let id = edge_id(&d.src, d.edge_type, &d.dst, d.valid_at_ms);
+ let tag = detector_tag_for(d.edge_type);
+ let action = json!({"index": {"_index": edges_index, "_id": id}});
+ let doc = json!({
+ "edge_id": id,
+ "src": d.src,
+ "dst": d.dst,
+ "type": d.edge_type,
+ "weight": clean_f64(d.weight),
+ "valid_at": d.valid_at_ms,
+ "created_at": created_at_ms,
+ "detector": tag,
+ "confidence": clean_f64(d.confidence),
+ "schema_version": EDGE_SCHEMA_VERSION,
+ "src_file": d.src_file,
+ "evidence": {
+ "quote": d.quote,
+ "source": d.src_file,
+ "offset": d.offset,
+ }
+ });
+ let mut ndjson = action.to_string().into_bytes();
+ ndjson.push(b'\n');
+ ndjson.extend_from_slice(doc.to_string().as_bytes());
+ ndjson.push(b'\n');
+ edges.push(AssembledEdge {
+ detector: tag,
+ edge_id: id,
+ ndjson,
+ });
+ }
+ AssembleOutcome {
+ edges,
+ self_dropped,
+ }
+}
+
+/// Create-if-absent write of the §2.5 brain meta doc. `create` semantics via
+/// a probing GET: a racing writer's meta doc must never be clobbered — it may
+/// carry a different `nodes_index` some reader already resolved through.
+pub fn ensure_brain_meta(
+ es: &Es,
+ edges_index: &str,
+ brain: &str,
+ nodes_index: &str,
+ created_at_ms: i64,
+) -> Result<()> {
+ if es.get_doc(edges_index, BRAIN_META_ID)?.is_some() {
+ return Ok(());
+ }
+ let action = json!({"create": {"_index": edges_index, "_id": BRAIN_META_ID}});
+ let doc = json!({
+ "meta_version": 1,
+ "brain": brain,
+ "nodes_index": nodes_index,
+ "created_at": created_at_ms,
+ });
+ let mut body = action.to_string().into_bytes();
+ body.push(b'\n');
+ body.extend_from_slice(doc.to_string().as_bytes());
+ body.push(b'\n');
+ let outcome = es.bulk(body).context("write brain meta doc")?;
+ if outcome.server_errors > 0 {
+ return Err(anyhow!(
+ "brain meta write failed: {}",
+ outcome
+ .first_server_error
+ .as_deref()
+ .unwrap_or("unknown server error")
+ ));
+ }
+ // item_errors here can only be the create-conflict race — the doc exists,
+ // which is exactly the state we want.
+ Ok(())
+}
+
+/// Soft-invalidate every live edge taught by `rel` (replacement hook, §6.6.3).
+/// Runs BEFORE any of this run's edges for that file are written, so the
+/// `src_file` + no-`invalid_at` query can only match prior-generation edges.
+/// Each pass re-indexes its hits with `invalid_at = expired_at = now` and
+/// refreshes, so repeat-until-empty converges; the pass bound turns a broken
+/// endpoint into an error instead of an infinite loop.
+pub fn invalidate_prior_edges(
+ es: &Es,
+ edges_index: &str,
+ rel: &str,
+ now_ms: i64,
+) -> Result<u64> {
+ const MAX_PASSES: usize = 1_000;
+ let query = json!({
+ "query": {"bool": {
+ "filter": [{"term": {"src_file": rel}}],
+ "must_not": [{"exists": {"field": "invalid_at"}}]
+ }},
+ "size": 1000,
+ "_source": true
+ });
+ let mut total = 0u64;
+ for _ in 0..MAX_PASSES {
+ let resp = es.search(edges_index, &query)?;
+ let hits = resp
+ .pointer("/hits/hits")
+ .and_then(Value::as_array)
+ .cloned()
+ .unwrap_or_default();
+ if hits.is_empty() {
+ return Ok(total);
+ }
+ let mut body = Vec::new();
+ for hit in &hits {
+ let id = hit
+ .get("_id")
+ .and_then(Value::as_str)
+ .ok_or_else(|| anyhow!("edge hit without _id in {edges_index}"))?;
+ let mut source = hit
+ .get("_source")
+ .and_then(Value::as_object)
+ .cloned()
+ .ok_or_else(|| anyhow!("edge {id} returned without _source"))?;
+ source.insert("invalid_at".into(), json!(now_ms));
+ source.insert("expired_at".into(), json!(now_ms));
+ body.extend_from_slice(
+ json!({"index": {"_index": edges_index, "_id": id}})
+ .to_string()
+ .as_bytes(),
+ );
+ body.push(b'\n');
+ body.extend_from_slice(Value::Object(source).to_string().as_bytes());
+ body.push(b'\n');
+ }
+ let outcome = es.bulk(body).context("invalidate prior edges")?;
+ if outcome.server_errors > 0 || outcome.item_errors > 0 {
+ return Err(anyhow!(
+ "edge invalidation for {} was partial: {}",
+ rel,
+ outcome
+ .first_server_error
+ .or(outcome.first_error)
+ .unwrap_or_else(|| "unknown bulk error".into())
+ ));
+ }
+ total += hits.len() as u64;
+ es.refresh(edges_index)?;
+ }
+ Err(anyhow!(
+ "edge invalidation for {rel} still found live edges after {MAX_PASSES} passes"
+ ))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// §2.3 pin vector — xerj-api carries a byte-identical copy of `edge_id`;
+ /// this shared vector is what keeps the two in lockstep.
+ #[test]
+ fn edge_id_pin_vector() {
+ assert_eq!(
+ edge_id("note-alpha", "wikilink", "note-beta", 1753600000000),
+ "bef814a75bd3d914c3e561f610154304"
+ );
+ }
+
+ #[test]
+ fn edge_id_is_valid_at_sensitive() {
+ let a = edge_id("a", "t", "b", 1);
+ assert_eq!(a, edge_id("a", "t", "b", 1));
+ assert_ne!(a, edge_id("a", "t", "b", 2));
+ assert_ne!(a, edge_id("a", "u", "b", 1));
+ }
+
+ #[test]
+ fn brain_names_reject_the_edges_suffix_and_bad_shapes() {
+ assert!(validate_brain("notes").is_ok());
+ assert!(validate_brain("a.b-c_9").is_ok());
+ assert!(validate_brain("kb-edges").is_err());
+ assert!(validate_brain("").is_err());
+ assert!(validate_brain("-notes").is_err());
+ assert!(validate_brain("Notes").is_err());
+ assert!(validate_brain("a..b").is_err());
+ }
+
+ #[test]
+ fn assembler_drops_self_edges_and_stamps_the_envelope() {
+ let drafts = vec![
+ EdgeDraft {
+ src: "n1".into(),
+ dst: "n1".into(),
+ edge_type: wikilink::EDGE_TYPE,
+ weight: 1.0,
+ confidence: 0.95,
+ valid_at_ms: 5,
+ src_file: "a.md".into(),
+ quote: "self".into(),
+ offset: 0,
+ },
+ EdgeDraft {
+ src: "n1".into(),
+ dst: "n2".into(),
+ edge_type: samedir::EDGE_TYPE,
+ weight: samedir::WEIGHT,
+ confidence: samedir::CONFIDENCE,
+ valid_at_ms: 5,
+ src_file: "a.md".into(),
+ quote: "a.md and b.md share directory .".into(),
+ offset: 0,
+ },
+ ];
+ let out = assemble(&drafts, ".xerj-memory-t-edges", 99);
+ assert_eq!(out.self_dropped, 1);
+ assert_eq!(out.edges.len(), 1);
+ let lines: Vec<&str> = std::str::from_utf8(&out.edges[0].ndjson)
+ .unwrap()
+ .lines()
+ .collect();
+ let doc: Value = serde_json::from_str(lines[1]).unwrap();
+ // f32 weights must land as their short decimal, not the widened f64.
+ assert_eq!(doc["weight"], json!(0.3));
+ assert_eq!(doc["confidence"], json!(0.4));
+ assert_eq!(doc["detector"], json!(samedir::TAG));
+ assert_eq!(doc["schema_version"], json!(1));
+ assert_eq!(doc["created_at"], json!(99));
+ assert_eq!(doc["evidence"]["source"], json!("a.md"));
+ assert!(doc.get("invalid_at").is_none(), "unset keys must be omitted");
+ assert_eq!(doc["edge_id"], json!(out.edges[0].edge_id));
+ }
+
+ #[test]
+ fn assembly_is_deterministic_for_a_fixed_created_at() {
+ let draft = EdgeDraft {
+ src: "n1".into(),
+ dst: "n2".into(),
+ edge_type: mdlink::EDGE_TYPE,
+ weight: mdlink::WEIGHT,
+ confidence: mdlink::CONFIDENCE,
+ valid_at_ms: 1753600000000,
+ src_file: "a.md".into(),
+ quote: "q".into(),
+ offset: 3,
+ };
+ let a = assemble(std::slice::from_ref(&draft), "i", 7);
+ let b = assemble(std::slice::from_ref(&draft), "i", 7);
+ assert_eq!(a.edges[0].ndjson, b.edges[0].ndjson);
+ }
+
+ #[test]
+ fn local_resolution_handles_relative_root_and_external_targets() {
+ let corpus = CorpusIndex::build(vec![
+ corpus_file("docs/a.md", "k1", "d", "txt-prose", 1),
+ corpus_file("b.md", "k2", "d", "txt-prose", 1),
+ ]);
+ assert!(matches!(
+ resolve_local(&corpus, "docs", "../b.md"),
+ LinkTarget::Hit(f) if f.rel == "b.md"
+ ));
+ assert!(matches!(
+ resolve_local(&corpus, "", "docs/a.md#frag?q=1"),
+ LinkTarget::Hit(f) if f.rel == "docs/a.md"
+ ));
+ assert!(matches!(
+ resolve_local(&corpus, "docs", "/b.md"),
+ LinkTarget::Hit(f) if f.rel == "b.md"
+ ));
+ assert!(matches!(
+ resolve_local(&corpus, "docs", "missing.md"),
+ LinkTarget::Miss
+ ));
+ assert!(matches!(
+ resolve_local(&corpus, "", "https://example.com/b.md"),
+ LinkTarget::External
+ ));
+ assert!(matches!(
+ resolve_local(&corpus, "", "//cdn.example.com/x"),
+ LinkTarget::External
+ ));
+ assert!(matches!(resolve_local(&corpus, "", "#anchor"), LinkTarget::Empty));
+ assert!(matches!(
+ resolve_local(&corpus, "", "../../escape.md"),
+ LinkTarget::Miss
+ ));
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/detect/samedir.rs b/engine/crates/xerj-autoindex/src/detect/samedir.rs
new file mode 100644
index 000000000..6cc00624e
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/detect/samedir.rs
@@ -0,0 +1,126 @@
+//! samedir@1 — files sharing a directory, chained, never a clique.
+//!
+//! Directory co-location is weak evidence (weight 0.3, confidence 0.4 — the
+//! floor of the detector set): it often means "same topic", but it is the
+//! only detector that fires on a corpus with no links at all, so it is what
+//! keeps an unlinked folder from being a graph of isolated points.
+//!
+//! ## The fan-out cap: a CHAIN over rel-sorted files
+//! A clique is O(n²): one 500-file directory would emit 500·499/2 = 124,750
+//! edges and drown every explicit-link signal under same_dir noise. The chain
+//! emits exactly n-1 edges (499 for that directory) — O(n) — while directory
+//! cohesion stays reachable inside the engine's 2-hop expansion cap for
+//! small dirs (±2 rel-sorted neighbors per hop pair). Sorting by rel makes
+//! the chain deterministic AND naturally follows name-ordered conventions
+//! (01-intro, 02-setup, …), which is why this detector doubles as the
+//! adjacent-file "sequence by naming" signal at the file level.
+
+use super::{CorpusIndex, EdgeDetector, EdgeDraft};
+
+pub const TAG: &str = "samedir@1";
+pub const EDGE_TYPE: &str = "same_dir";
+pub const WEIGHT: f32 = 0.3;
+pub const CONFIDENCE: f32 = 0.4;
+
+pub struct SameDir;
+
+impl EdgeDetector for SameDir {
+ fn tag(&self) -> &'static str {
+ TAG
+ }
+
+ fn detect_structure(&self, corpus: &CorpusIndex, out: &mut Vec<EdgeDraft>) {
+ // corpus.files is a BTreeMap keyed by rel: iteration is globally
+ // rel-sorted, so per-directory order needs no extra sort.
+ let mut by_dir: std::collections::BTreeMap<&str, Vec<&super::CorpusFile>> =
+ std::collections::BTreeMap::new();
+ for f in corpus.files.values() {
+ by_dir.entry(f.dir.as_str()).or_default().push(f);
+ }
+ for (dir, files) in by_dir {
+ let shown_dir = if dir.is_empty() { "." } else { dir };
+ for pair in files.windows(2) {
+ let (left, right) = (pair[0], pair[1]);
+ out.push(EdgeDraft {
+ src: left.anchor_doc_id.clone(),
+ dst: right.anchor_doc_id.clone(),
+ edge_type: EDGE_TYPE,
+ weight: WEIGHT,
+ confidence: CONFIDENCE,
+ valid_at_ms: left.mtime_ms,
+ src_file: left.rel.clone(),
+ quote: format!(
+ "{} and {} share directory {shown_dir}",
+ left.rel, right.rel
+ ),
+ offset: 0,
+ });
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::super::{corpus_file, CorpusIndex};
+ use super::*;
+
+ fn chain(rels: &[&str]) -> Vec<EdgeDraft> {
+ let corpus = CorpusIndex::build(
+ rels.iter()
+ .enumerate()
+ .map(|(i, rel)| corpus_file(rel, &format!("k{i}"), "notes", "txt-prose", 100 + i as i64))
+ .collect(),
+ );
+ let mut out = Vec::new();
+ SameDir.detect_structure(&corpus, &mut out);
+ out
+ }
+
+ /// The §8 fixture directory: 5 files chain into exactly 4 edges in
+ /// rel-sorted order — never the 10-edge clique.
+ #[test]
+ fn fixture_folder_chains_not_cliques() {
+ let out = chain(&["alpha.md", "beta.md", "delta.md", "epsilon.md", "gamma.md"]);
+ assert_eq!(out.len(), 4);
+ let pairs: Vec<(&str, &str)> = out
+ .iter()
+ .map(|e| (e.src_file.as_str(), e.quote.as_str()))
+ .collect();
+ assert_eq!(
+ pairs,
+ vec![
+ ("alpha.md", "alpha.md and beta.md share directory ."),
+ ("beta.md", "beta.md and delta.md share directory ."),
+ ("delta.md", "delta.md and epsilon.md share directory ."),
+ ("epsilon.md", "epsilon.md and gamma.md share directory ."),
+ ]
+ );
+ }
+
+ /// The cap in numbers: n files emit exactly n-1 edges, so a 500-file
+ /// directory emits 499 — not 124,750.
+ #[test]
+ fn fan_out_is_linear_in_directory_size() {
+ let rels: Vec<String> = (0..500).map(|i| format!("dir/f{i:03}.md")).collect();
+ let refs: Vec<&str> = rels.iter().map(String::as_str).collect();
+ assert_eq!(chain(&refs).len(), 499);
+ }
+
+ #[test]
+ fn directories_never_cross_and_singletons_emit_nothing() {
+ let out = chain(&["a/x.md", "a/y.md", "b/z.md", "lone.md"]);
+ assert_eq!(out.len(), 1);
+ assert_eq!(out[0].quote, "a/x.md and a/y.md share directory a");
+ }
+
+ /// Numeric-prefix conventions (01-, 02-, …) sort into their intended
+ /// reading order, so the chain is also the file-level sequence signal.
+ #[test]
+ fn numeric_prefixes_chain_in_reading_order() {
+ let out = chain(&["c/02-setup.md", "c/01-intro.md", "c/03-usage.md"]);
+ assert_eq!(out.len(), 2);
+ assert!(out[0].quote.starts_with("c/01-intro.md and c/02-setup.md"));
+ assert!(out[1].quote.starts_with("c/02-setup.md and c/03-usage.md"));
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/detect/sequence.rs b/engine/crates/xerj-autoindex/src/detect/sequence.rs
new file mode 100644
index 000000000..d6dd1b451
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/detect/sequence.rs
@@ -0,0 +1,89 @@
+//! sequence@1 — section s_{i-1} → s_i within one file.
+//!
+//! The natural ordering edge: autoindex splits long documents into
+//! retrieval-sized sections, and without these edges the split would DESTROY
+//! the document's own narrative adjacency (a hop from "the answer" to "the
+//! paragraph after it" is exactly what an agent wants when a section ends
+//! mid-thought). Weight 0.8 — structural but almost certainly meaningful;
+//! confidence 0.99 because the extractor's own ordering cannot be wrong about
+//! itself.
+//!
+//! Emitted from `detect_text` on section i>0 (the contract's trick to avoid a
+//! second pass): every section after the first knows its predecessor's doc id
+//! deterministically from (slug, file_key, "s{i-1}").
+
+use super::{EdgeDetector, EdgeDraft, SectionCtx};
+
+pub const TAG: &str = "sequence@1";
+pub const EDGE_TYPE: &str = "sequence";
+pub const WEIGHT: f32 = 0.8;
+pub const CONFIDENCE: f32 = 0.99;
+
+pub struct Sequence;
+
+impl EdgeDetector for Sequence {
+ fn tag(&self) -> &'static str {
+ TAG
+ }
+
+ fn detect_text(&self, ctx: &SectionCtx<'_>, out: &mut Vec<EdgeDraft>) {
+ if ctx.section_ordinal == 0 {
+ return;
+ }
+ let prev_ordinal = ctx.section_ordinal - 1;
+ let prev_doc_id = crate::ids::doc_id(
+ &ctx.file.dataset_slug,
+ &ctx.file.file_key,
+ &format!("s{prev_ordinal}"),
+ );
+ out.push(EdgeDraft {
+ src: prev_doc_id,
+ dst: ctx.section_doc_id.to_string(),
+ edge_type: EDGE_TYPE,
+ weight: WEIGHT,
+ confidence: CONFIDENCE,
+ valid_at_ms: ctx.file.mtime_ms,
+ src_file: ctx.file.rel.clone(),
+ quote: format!(
+ "section {prev_ordinal} precedes section {} of {}",
+ ctx.section_ordinal, ctx.file.rel
+ ),
+ offset: 0,
+ });
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::super::{corpus_file, CorpusIndex, SectionCtx};
+ use super::*;
+
+ /// A 3-section file yields exactly the two adjacency edges, chained
+ /// through the deterministic section doc ids.
+ #[test]
+ fn three_sections_chain_two_edges() {
+ let corpus = CorpusIndex::build(vec![corpus_file("big.md", "kb", "docs", "txt-prose", 7)]);
+ let file = &corpus.files["big.md"];
+ let det = Sequence;
+ let mut out = Vec::new();
+ for ordinal in 0..3u32 {
+ let doc_id = crate::ids::doc_id("docs", "kb", &format!("s{ordinal}"));
+ let ctx = SectionCtx {
+ corpus: &corpus,
+ file,
+ section_ordinal: ordinal,
+ section_doc_id: &doc_id,
+ text: "…",
+ };
+ det.detect_text(&ctx, &mut out);
+ }
+ assert_eq!(out.len(), 2);
+ assert_eq!(out[0].src, crate::ids::doc_id("docs", "kb", "s0"));
+ assert_eq!(out[0].dst, crate::ids::doc_id("docs", "kb", "s1"));
+ assert_eq!(out[1].src, crate::ids::doc_id("docs", "kb", "s1"));
+ assert_eq!(out[1].dst, crate::ids::doc_id("docs", "kb", "s2"));
+ assert_eq!(out[0].quote, "section 0 precedes section 1 of big.md");
+ assert_eq!(out[0].offset, 0);
+ assert_eq!(out[0].valid_at_ms, 7);
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/detect/wikilink.rs b/engine/crates/xerj-autoindex/src/detect/wikilink.rs
new file mode 100644
index 000000000..34e1d4aca
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/detect/wikilink.rs
@@ -0,0 +1,191 @@
+//! wikilink@1 — `[[Target]]` / `[[Target|alias]]` in section text.
+//!
+//! The strongest signal the detectors have: a human deliberately named
+//! another note. Hence weight 1.0; confidence 0.95 rather than 1.0 because
+//! resolution is by name, and a corpus can hold two files answering to the
+//! same stem (the ambiguity is counted, never hidden).
+
+use super::{
+ line_at, CorpusFile, CorpusIndex, DetectorCounters, EdgeDetector, EdgeDraft, SectionCtx,
+};
+use std::sync::atomic::{AtomicU64, Ordering};
+
+pub const TAG: &str = "wikilink@1";
+pub const EDGE_TYPE: &str = "wikilink";
+pub const WEIGHT: f32 = 1.0;
+pub const CONFIDENCE: f32 = 0.95;
+
+/// Ignore absurd "targets": an unmatched `[[` in a code block would otherwise
+/// swallow the rest of the section into one bogus link.
+const MAX_TARGET_LEN: usize = 512;
+
+#[derive(Default)]
+pub struct Wikilink {
+ unresolved: AtomicU64,
+ ambiguous: AtomicU64,
+}
+
+enum Resolved<'a> {
+ Hit(&'a CorpusFile),
+ Ambiguous(&'a CorpusFile),
+ Miss,
+}
+
+/// §6.5 resolution order: exact rel path (with, then without, extension),
+/// else the lowercase stem table. Multiple candidates pick the
+/// lexicographically smallest rel — deterministic, and counted as ambiguous.
+fn resolve<'a>(corpus: &'a CorpusIndex, target: &str) -> Resolved<'a> {
+ if let Some(f) = corpus.files.get(target) {
+ return Resolved::Hit(f);
+ }
+ // Whole-path match minus extension: "notes/beta" → "notes/beta.md".
+ // BTreeMap iteration is ascending, so the first match IS the smallest.
+ let mut without_ext = corpus
+ .files
+ .iter()
+ .filter(|(rel, _)| super::rel_without_ext(rel) == target);
+ if let Some((_, first)) = without_ext.next() {
+ return if without_ext.next().is_some() {
+ Resolved::Ambiguous(first)
+ } else {
+ Resolved::Hit(first)
+ };
+ }
+ match corpus.by_stem.get(&target.to_ascii_lowercase()) {
+ Some(rels) if rels.len() == 1 => Resolved::Hit(&corpus.files[&rels[0]]),
+ Some(rels) => Resolved::Ambiguous(&corpus.files[&rels[0]]),
+ None => Resolved::Miss,
+ }
+}
+
+impl EdgeDetector for Wikilink {
+ fn tag(&self) -> &'static str {
+ TAG
+ }
+
+ fn detect_text(&self, ctx: &SectionCtx<'_>, out: &mut Vec<EdgeDraft>) {
+ let text = ctx.text;
+ let mut i = 0usize;
+ while let Some(open) = text[i..].find("[[") {
+ let open = i + open;
+ let Some(close) = text[open + 2..].find("]]") else {
+ break;
+ };
+ let close = open + 2 + close;
+ i = close + 2;
+ let inner = &text[open + 2..close];
+ if inner.len() > MAX_TARGET_LEN || inner.contains("[[") {
+ continue;
+ }
+ let target = inner.split('|').next().unwrap_or("").trim();
+ if target.is_empty() {
+ continue;
+ }
+ let file = match resolve(ctx.corpus, target) {
+ Resolved::Hit(f) => f,
+ Resolved::Ambiguous(f) => {
+ self.ambiguous.fetch_add(1, Ordering::Relaxed);
+ f
+ }
+ Resolved::Miss => {
+ // Dangling link: recorded in the run summary
+ // (edges_unresolved), never an invented edge.
+ self.unresolved.fetch_add(1, Ordering::Relaxed);
+ continue;
+ }
+ };
+ out.push(EdgeDraft {
+ src: ctx.section_doc_id.to_string(),
+ dst: file.anchor_doc_id.clone(),
+ edge_type: EDGE_TYPE,
+ weight: WEIGHT,
+ confidence: CONFIDENCE,
+ valid_at_ms: ctx.file.mtime_ms,
+ src_file: ctx.file.rel.clone(),
+ quote: line_at(text, open),
+ offset: open as u64,
+ });
+ }
+ }
+
+ fn counters(&self) -> DetectorCounters {
+ DetectorCounters {
+ unresolved: self.unresolved.load(Ordering::Relaxed),
+ ambiguous: self.ambiguous.load(Ordering::Relaxed),
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::super::{corpus_file, CorpusIndex, SectionCtx};
+ use super::*;
+
+ fn corpus() -> CorpusIndex {
+ CorpusIndex::build(vec![
+ corpus_file("alpha.md", "ka", "notes", "txt-prose", 1753600000000),
+ corpus_file("beta.md", "kb", "notes", "txt-prose", 1753600000000),
+ corpus_file("sub/beta.txt", "kc", "notes", "txt-prose", 1753600000000),
+ ])
+ }
+
+ fn run(corpus: &CorpusIndex, text: &str) -> (Vec<EdgeDraft>, DetectorCounters) {
+ let det = Wikilink::default();
+ let file = &corpus.files["alpha.md"];
+ let ctx = SectionCtx {
+ corpus,
+ file,
+ section_ordinal: 0,
+ section_doc_id: "sec0",
+ text,
+ };
+ let mut out = Vec::new();
+ det.detect_text(&ctx, &mut out);
+ (out, det.counters())
+ }
+
+ #[test]
+ fn fixture_offsets_and_targets() {
+ let corpus = corpus();
+ let (out, c) = run(&corpus, "Alpha is the hub note. It links to [[beta]] and [[gamma]].");
+ // beta resolves ambiguously (beta.md vs sub/beta.txt) to the smallest
+ // rel; gamma is dangling and must be counted, not emitted.
+ assert_eq!(out.len(), 1);
+ assert_eq!(out[0].dst, corpus.files["beta.md"].anchor_doc_id);
+ assert_eq!(out[0].offset, 35);
+ assert_eq!(
+ out[0].quote,
+ "Alpha is the hub note. It links to [[beta]] and [[gamma]]."
+ );
+ assert_eq!(c.unresolved, 1);
+ assert_eq!(c.ambiguous, 1);
+ }
+
+ #[test]
+ fn alias_exact_rel_and_case_insensitive_stem() {
+ let corpus = corpus();
+ let (out, c) = run(&corpus, "see [[beta.md|the beta note]] and [[BETA]]");
+ assert_eq!(out.len(), 2);
+ // exact rel match is unambiguous even though the stem table is not
+ assert_eq!(out[0].dst, corpus.files["beta.md"].anchor_doc_id);
+ assert_eq!(c.ambiguous, 1, "only the stem lookup was ambiguous");
+ assert_eq!(c.unresolved, 0);
+ }
+
+ #[test]
+ fn ambiguity_is_deterministic_smallest_rel() {
+ let corpus = corpus();
+ let (a, _) = run(&corpus, "[[beta]]");
+ let (b, _) = run(&corpus, "[[beta]]");
+ assert_eq!(a[0].dst, b[0].dst);
+ assert_eq!(a[0].dst, corpus.files["beta.md"].anchor_doc_id);
+ }
+
+ #[test]
+ fn unterminated_and_empty_links_emit_nothing() {
+ let corpus = corpus();
+ let (out, c) = run(&corpus, "broken [[beta and empty [[]] end");
+ assert!(out.is_empty());
+ assert_eq!(c.unresolved, 0);
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index 765ff8d41..1bffda1ca 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -190,6 +190,11 @@ fn cfg(root: &Path, state_dir: &Path, url: &str) -> IndexCfg {
max_file_gb: 1,
sample: 50,
no_semantic: true,
+ // These tests count data bulks to place injected failures; graph
+ // edge bulks would shift that numbering, so keep detection off here
+ // (the graph pipeline has its own e2e suite in detect::e2e).
+ brain: None,
+ no_graph: true,
dry_run: false,
json: false,
quiet: true,
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 0aa3f3380..a441146f4 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -9,6 +9,7 @@ pub mod coerce;
mod content;
pub mod correlate;
pub mod dataset;
+pub mod detect;
pub mod esclient;
pub mod extract;
pub mod ids;
@@ -124,6 +125,43 @@ fn record_bulk_outcome(
}
}
+// ─── second-brain graph runtime ──────────────────────────────────────────
+
+/// Per-run graph state, shared read-only with the Phase B workers
+/// (SECOND_BRAIN_SPEC §6.6). Built after the plan is final because the
+/// detectors resolve links against the FULL corpus — a per-file view could
+/// not tell "dangling" from "not walked yet".
+struct GraphRt {
+ corpus: detect::CorpusIndex,
+ detectors: Vec<Box<dyn detect::EdgeDetector>>,
+ /// Raw-source href pass handle. Lives outside the registry because the
+ /// HTML extractor strips markup before sectioning, so anchors only exist
+ /// in the raw bytes — a source the `EdgeDetector` trait deliberately
+ /// never sees (see `detect::href` module docs).
+ href_raw: detect::href::Href,
+ edges_index: String,
+ brain: String,
+ /// ONE wall-clock stamp per run: `created_at` is the single
+ /// non-deterministic edge field (§6.4); per-worker clocks would make two
+ /// halves of one run disagree about when it happened.
+ created_at_ms: i64,
+ /// detector tag → edges written this run (run-summary honesty §6.6.4).
+ written: Mutex<std::collections::BTreeMap<&'static str, u64>>,
+ self_dropped: AtomicU64,
+ /// Prior-generation edges soft-invalidated before this run's writes.
+ invalidated: u64,
+}
+
+/// `emit_document` section locators are `s{i}`; everything else (row/line/
+/// table locators) is not a text section and must not reach `detect_text`.
+fn section_ordinal(locator: &str) -> Option<u32> {
+ let rest = locator.strip_prefix('s')?;
+ if rest.is_empty() || !rest.bytes().all(|b| b.is_ascii_digit()) {
+ return None;
+ }
+ rest.parse().ok()
+}
+
// ─── Phase A: per-file scan (sniff + bounded sampling) ───────────────────
struct FileScan {
@@ -824,6 +862,162 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {
journal.file_replace_start(key, generation)?;
}
}
+ // ── second-brain graph: corpus table, structural edges, invalidation ──
+ // (SECOND_BRAIN_SPEC §6.6.1/§6.6.3.) Runs after plan finalization so the
+ // detectors see the whole corpus, and BEFORE any Phase B publication so
+ // replacement invalidation can only ever see prior-generation edges —
+ // running it later would invalidate this run's own fresh edges.
+ let bulk_cut = cfg.bulk_mb << 20;
+ let junk_records = AtomicU64::new(0);
+ let bulk_errors = Mutex::new(Vec::<String>::new());
+ let graph: Option<GraphRt> = if cfg.no_graph {
+ None
+ } else {
+ let brain = match &cfg.brain {
+ Some(b) => b.clone(),
+ None => {
+ let base = cfg
+ .root
+ .canonicalize()
+ .unwrap_or_else(|_| cfg.root.clone());
+ let name = base
+ .file_name()
+ .map(|s| s.to_string_lossy().to_string())
+ .unwrap_or_default();
+ let slug = dataset::sanitize_slug(&name);
+ if slug.is_empty() {
+ "brain".into()
+ } else {
+ slug
+ }
+ }
+ };
+ if let Err(reason) = detect::validate_brain(&brain) {
+ anyhow::bail!(
+ "brain name '{brain}' is invalid: {reason}. Pass an explicit --brain <name> \
+ or disable relationship detection with --no-graph"
+ );
+ }
+ // Corpus resolution table: every planned file's rel → identity +
+ // anchor node. valid_at comes from the file mtime (§6.4) so an
+ // unchanged corpus re-emits byte-identical edge_ids and re-runs
+ // converge by overwrite, exactly like ids::doc_id does for nodes.
+ let mut corpus_files = Vec::new();
+ for (i, f) in files.iter().enumerate() {
+ let key = &keys[i];
+ if key.is_empty() {
+ continue;
+ }
+ let Some(fa) = plan.files.get(key) else {
+ continue; // junk or post-freeze files carry no node docs
+ };
+ let slug = fa
+ .assignments
+ .iter()
+ .find(|(g, _)| g.is_none())
+ .map(|(_, s)| s.clone())
+ .or_else(|| fa.assignments.iter().map(|(_, s)| s.clone()).min());
+ let Some(slug) = slug else { continue };
+ let mtime_ms = std::fs::metadata(&f.path)
+ .ok()
+ .and_then(|m| m.modified().ok())
+ .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
+ .map(|d| d.as_millis() as i64)
+ .unwrap_or(0);
+ corpus_files.push(detect::corpus_file(&f.rel, key, &slug, &fa.family, mtime_ms));
+ }
+ let corpus = detect::CorpusIndex::build(corpus_files);
+ let detectors = detect::default_detectors();
+ let edges_index = detect::edges_index_name(&brain);
+ let created_at_ms = chrono::Utc::now().timestamp_millis();
+
+ es.ensure_index(&edges_index, &detect::edge_index_mapping())
+ .with_context(|| format!("create edges index {edges_index}"))?;
+ let mut nodes_indices: Vec<&str> = plan.datasets.iter().map(|d| d.index.as_str()).collect();
+ nodes_indices.sort_unstable();
+ nodes_indices.dedup();
+ detect::ensure_brain_meta(
+ &es,
+ &edges_index,
+ &brain,
+ &nodes_indices.join(","),
+ created_at_ms,
+ )?;
+
+ // Replacement invalidation FIRST: soft-invalidate every live edge a
+ // replaced file taught in earlier runs. The bi-temporal record stays
+ // queryable (`as_of` time travel); nothing is deleted.
+ let mut invalidated = 0u64;
+ {
+ let mut replaced_rels: Vec<&str> = todo
+ .iter()
+ .filter(|&&i| cleanup_required.contains(&keys[i]))
+ .map(|&i| files[i].rel.as_str())
+ .collect();
+ replaced_rels.sort_unstable();
+ replaced_rels.dedup();
+ for rel in replaced_rels {
+ invalidated += detect::invalidate_prior_edges(&es, &edges_index, rel, created_at_ms)
+ .with_context(|| format!("invalidate prior edges taught by {rel}"))?;
+ }
+ }
+
+ // Structural detection (samedir chains) + bulk write, cut at the same
+ // --bulk-mb threshold as node bulks.
+ let mut structural = Vec::new();
+ for det in &detectors {
+ det.detect_structure(&corpus, &mut structural);
+ }
+ let assembled = detect::assemble(&structural, &edges_index, created_at_ms);
+ let mut written: std::collections::BTreeMap<&'static str, u64> =
+ std::collections::BTreeMap::new();
+ {
+ let mut send_err: Option<String> = None;
+ let mut buf: Vec<u8> = Vec::new();
+ for edge in &assembled.edges {
+ buf.extend_from_slice(&edge.ndjson);
+ *written.entry(edge.detector).or_default() += 1;
+ if buf.len() >= bulk_cut
+ && record_bulk_outcome(
+ &es,
+ std::mem::take(&mut buf),
+ &junk_records,
+ &bulk_errors,
+ &mut send_err,
+ )
+ {
+ break;
+ }
+ }
+ if send_err.is_none() && !buf.is_empty() {
+ record_bulk_outcome(&es, buf, &junk_records, &bulk_errors, &mut send_err);
+ }
+ if let Some(e) = send_err {
+ anyhow::bail!("write structural graph edges to {edges_index}: {e}");
+ }
+ }
+ if !cfg.quiet {
+ eprintln!(
+ "graph: brain '{brain}' → {edges_index}; {} structural edges, {} prior edges \
+ invalidated ({} detectors live)",
+ assembled.edges.len(),
+ invalidated,
+ detectors.len()
+ );
+ }
+ Some(GraphRt {
+ corpus,
+ detectors,
+ href_raw: detect::href::Href::default(),
+ edges_index,
+ brain,
+ created_at_ms,
+ written: Mutex::new(written),
+ self_dropped: AtomicU64::new(assembled.self_dropped),
+ invalidated,
+ })
+ };
+
// ascending by size — workers pop() from the tail, so the BIGGEST files
// start first and can't serialize the end of the run.
todo.sort_by_key(|&i| files[i].size);
@@ -854,10 +1048,7 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {
let journal_mx = Mutex::new(&mut journal);
let files_done = AtomicU64::new(0);
let records_total = AtomicU64::new(0);
- let junk_records = AtomicU64::new(0);
- let bulk_errors = Mutex::new(Vec::<String>::new());
let extra_junk = Mutex::new(Vec::<JunkFile>::new());
- let bulk_cut = cfg.bulk_mb << 20;
std::thread::scope(|scope| {
for _ in 0..cfg.workers.min(n_todo.max(1)) {
@@ -890,6 +1081,10 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {
let mut file_records = 0u64;
let mut file_junk = 0u64;
let mut send_err: Option<String> = None;
+ // Edges this file teaches — buffered apart from the node
+ // staging file (different target index) and sent only
+ // after the node bulks are accepted (§6.7).
+ let mut edge_drafts: Vec<detect::EdgeDraft> = Vec::new();
let mut staged = match tempfile::Builder::new()
.prefix(".autoindex-stage-")
.tempfile_in(&state_dir)
@@ -948,12 +1143,12 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {
fields.insert("ax_format".into(), Value::String(format_str(Some(&sn))));
let id = ids::doc_id(slug, key, &rec.locator);
let action = json!({"index": {"_index": rt.index, "_id": id}});
+ let doc = Value::Object(fields);
if let Err(error) = writeln!(
staged.as_file_mut(),
"{}\n{}",
action,
- serde_json::to_string(&Value::Object(fields))
- .unwrap_or_else(|_| "{}".into())
+ serde_json::to_string(&doc).unwrap_or_else(|_| "{}".into())
) {
send_err =
Some(format!("stage extracted records for {}: {error}", f.rel));
@@ -961,6 +1156,29 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {
}
rt.records.fetch_add(1, Ordering::Relaxed);
file_records += 1;
+ // Textual edge detection (§6.6.2), after the node
+ // action is staged: `body` is the exact section
+ // string the node doc carries, and `id` is the
+ // section node the evidence lives in.
+ if let Some(gr) = graph.as_ref() {
+ if let Some(ordinal) = section_ordinal(&rec.locator) {
+ if let (Some(cf), Some(body)) = (
+ gr.corpus.files.get(&f.rel),
+ doc.get("body").and_then(Value::as_str),
+ ) {
+ let ctx = detect::SectionCtx {
+ corpus: &gr.corpus,
+ file: cf,
+ section_ordinal: ordinal,
+ section_doc_id: &id,
+ text: body,
+ };
+ for det in &gr.detectors {
+ det.detect_text(&ctx, &mut edge_drafts);
+ }
+ }
+ }
+ }
true
};
let res = extract::extract(&f.path, &sn, None, &mut sink);
@@ -981,6 +1199,32 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {
}
}
}
+ // Raw-source href pass: the HTML extractor strips markup
+ // before sectioning, so `<a href>` evidence exists only in
+ // the raw bytes (detect::href module docs). The second
+ // content::verify below still covers this re-read.
+ if send_err.is_none() {
+ if let Some(gr) = graph.as_ref() {
+ if let Some(cf) = gr
+ .corpus
+ .files
+ .get(&f.rel)
+ .filter(|cf| cf.family == "html")
+ {
+ if let Ok(Some(bytes)) =
+ extract::read_whole(&f.path, sn.gzip, extract::MAX_WHOLE_FILE)
+ {
+ let (raw, _) = sniff::decode_text(&bytes);
+ gr.href_raw.detect_raw_html(
+ &gr.corpus,
+ cf,
+ &raw,
+ &mut edge_drafts,
+ );
+ }
+ }
+ }
+ }
if send_err.is_none() {
if let Err(error) = content::verify(&f.path, f.size, expected_digest) {
send_err = Some(format!(
@@ -1084,6 +1328,49 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {
);
}
}
+ // Second-brain edges for this file (§6.7): only after the
+ // node bulks were accepted, so an edge never precedes its
+ // own src doc. A failed edge send leaves the file
+ // un-journaled — the whole file (nodes AND edges) is
+ // republished on the next run, which converges because
+ // both sides overwrite by deterministic _id.
+ if send_err.is_none() && !edge_drafts.is_empty() {
+ if let Some(gr) = graph.as_ref() {
+ let out =
+ detect::assemble(&edge_drafts, &gr.edges_index, gr.created_at_ms);
+ gr.self_dropped.fetch_add(out.self_dropped, Ordering::Relaxed);
+ let mut ebuf: Vec<u8> = Vec::new();
+ for edge in &out.edges {
+ ebuf.extend_from_slice(&edge.ndjson);
+ if ebuf.len() >= bulk_cut
+ && record_bulk_outcome(
+ &es,
+ std::mem::take(&mut ebuf),
+ &junk_records,
+ &bulk_errors,
+ &mut send_err,
+ )
+ {
+ break;
+ }
+ }
+ if send_err.is_none() && !ebuf.is_empty() {
+ record_bulk_outcome(
+ &es,
+ ebuf,
+ &junk_records,
+ &bulk_errors,
+ &mut send_err,
+ );
+ }
+ if send_err.is_none() {
+ let mut written = gr.written.lock().unwrap();
+ for edge in &out.edges {
+ *written.entry(edge.detector).or_default() += 1;
+ }
+ }
+ }
+ }
if let Some(e) = send_err {
// endpoint trouble: record, do NOT journal file_done
let mut be = bulk_errors.lock().unwrap();
@@ -1159,6 +1446,10 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {
// ── finalize: refresh, verify, correlate, catalog ────────────────────
es.refresh(&format!("{}-*", cfg.prefix)).ok();
+ // The dot-prefixed edges index is outside the {prefix}-* pattern.
+ if let Some(gr) = &graph {
+ es.refresh(&gr.edges_index).ok();
+ }
// live per-dataset counts + time ranges (every claim traces to a run)
let mut ds_counts: HashMap<String, u64> = HashMap::new();
@@ -1404,7 +1695,36 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {
let wall = t0.elapsed().as_secs_f64();
let total_records: u64 = ds_counts.values().sum();
- let run_doc = json!({
+ // Run-summary honesty (§6.6.4): what the detectors wrote AND what they
+ // could not resolve — a dangling [[link]] is a fact about the corpus, not
+ // something to swallow.
+ let graph_summary = graph.as_ref().map(|gr| {
+ let written = gr.written.lock().unwrap();
+ let mut counters = detect::DetectorCounters::default();
+ for det in &gr.detectors {
+ let c = det.counters();
+ counters.unresolved += c.unresolved;
+ counters.ambiguous += c.ambiguous;
+ }
+ let raw = gr.href_raw.counters();
+ counters.unresolved += raw.unresolved;
+ counters.ambiguous += raw.ambiguous;
+ let by_detector: Map<String, Value> = written
+ .iter()
+ .map(|(tag, n)| ((*tag).to_string(), json!(n)))
+ .collect();
+ json!({
+ "brain": gr.brain,
+ "edges_index": gr.edges_index,
+ "edges_written": written.values().sum::<u64>(),
+ "by_detector": by_detector,
+ "edges_unresolved": counters.unresolved,
+ "edges_ambiguous": counters.ambiguous,
+ "edges_self_dropped": gr.self_dropped.load(Ordering::Relaxed),
+ "edges_invalidated": gr.invalidated,
+ })
+ });
+ let mut run_doc = json!({
"doc_kind": "run",
"run_id": run_id,
"root": root_str,
@@ -1422,6 +1742,9 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {
"workers": cfg.workers,
"semantic": !cfg.no_semantic,
});
+ if let Some(g) = &graph_summary {
+ run_doc["graph"] = g.clone();
+ }
push_doc(&format!("run:{run_id}"), &run_doc, &mut cat_buf);
if !cat_buf.is_empty() {
@@ -1446,6 +1769,26 @@ fn run_index(cfg: IndexCfg) -> Result<i32> {
for (idx, cnt) in rows {
println!(" {idx:<40} {cnt:>10} docs");
}
+ if let Some(g) = &graph_summary {
+ let by: Vec<String> = g["by_detector"]
+ .as_object()
+ .map(|m| {
+ m.iter()
+ .map(|(tag, n)| format!("{tag} {n}"))
+ .collect()
+ })
+ .unwrap_or_default();
+ println!(
+ "graph: {} edges → {} ({}); {} unresolved, {} ambiguous, {} self-dropped, {} invalidated",
+ g["edges_written"],
+ g["edges_index"].as_str().unwrap_or(""),
+ if by.is_empty() { "no detections".to_string() } else { by.join(", ") },
+ g["edges_unresolved"],
+ g["edges_ambiguous"],
+ g["edges_self_dropped"],
+ g["edges_invalidated"],
+ );
+ }
println!(
"\nnext: `xerj autoindex map --url {}` for the data map; search via GET /{}-*/_search",
cfg.url, cfg.prefix
@@ -1583,6 +1926,33 @@ fn run_map(cfg: MapCfg) -> Result<i32> {
junk_files.len() as u64
)
);
+ // Second-brain summary (§6.1): live edge count straight from the
+ // edges index, scoped by `exists src` so the meta doc never counts.
+ if let Some(g) = runs.first().and_then(|r| r.get("graph")) {
+ if let (Some(brain), Some(edges_index)) = (
+ g.get("brain").and_then(Value::as_str),
+ g.get("edges_index").and_then(Value::as_str),
+ ) {
+ let live = es
+ .search(
+ edges_index,
+ &json!({
+ "size": 0,
+ "track_total_hits": true,
+ "query": {"bool": {
+ "filter": [{"exists": {"field": "src"}}],
+ "must_not": [{"exists": {"field": "invalid_at"}}]
+ }}
+ }),
+ )
+ .ok()
+ .and_then(|v| v.pointer("/hits/total/value").and_then(Value::as_u64));
+ match live {
+ Some(n) => println!("\ngraph: {n} live edges in {edges_index} (brain {brain})"),
+ None => println!("\ngraph: brain {brain} — edges index {edges_index} unreachable"),
+ }
+ }
+ }
}
Ok(0)
}
@@ -1610,6 +1980,7 @@ fn run_status(cfg: StatusCfg) -> Result<i32> {
let mut done = 0u64;
let mut records = 0u64;
let mut finished = false;
+ let mut graph_line: Option<String> = None;
if let Ok(f) = std::fs::File::open(&jp) {
use std::io::BufRead;
for line in std::io::BufReader::new(f).lines().map_while(|l| l.ok()) {
@@ -1622,7 +1993,20 @@ fn run_status(cfg: StatusCfg) -> Result<i32> {
done += 1;
records += v.get("records").and_then(|r| r.as_u64()).unwrap_or(0);
}
- Some("finish") => finished = true,
+ Some("finish") => {
+ finished = true;
+ // Latest finish wins — the summary embeds the run
+ // doc, whose `graph` block is the edge count of
+ // record for this journal.
+ if let Some(g) = v.pointer("/summary/graph") {
+ graph_line = Some(format!(
+ "graph: {} edges written to {} (brain {})",
+ g.get("edges_written").and_then(Value::as_u64).unwrap_or(0),
+ g.get("edges_index").and_then(Value::as_str).unwrap_or("?"),
+ g.get("brain").and_then(Value::as_str).unwrap_or("?"),
+ ));
+ }
+ }
_ => {}
}
}
@@ -1636,6 +2020,9 @@ fn run_status(cfg: StatusCfg) -> Result<i32> {
records,
if finished { "FINISHED" } else { "in progress" }
);
+ if let Some(line) = graph_line {
+ println!(" {line}");
+ }
}
// live indices
if let Ok(es) = Es::new(&cfg.url, cfg.api_key.clone()) {