AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(autoindex): xerj corpus build — recipe-driven harvested-record packs (M1)
xerj-orgbe1d4841b810
diff --git a/engine/Cargo.lock b/engine/Cargo.lockindex 1c97eeb57..c54bca2bf 100644--- a/engine/Cargo.lock+++ b/engine/Cargo.lock@@ -7322,6 +7322,7 @@ dependencies = ["streaming-iterator","tempfile","tokio",+ "toml","tree-sitter","tree-sitter-bash","tree-sitter-c",diff --git a/engine/crates/xerj-autoindex/Cargo.toml b/engine/crates/xerj-autoindex/Cargo.tomlindex 1385a5483..cf9de8042 100644--- a/engine/crates/xerj-autoindex/Cargo.toml+++ b/engine/crates/xerj-autoindex/Cargo.toml@@ -18,6 +18,10 @@ serde_json.workspace = truesimd-json.workspace = truechrono.workspace = trueregex = "1"+# Corpus recipes (`xerj corpus build`) are authored TOML: human-readable and+# commented, so licence reasoning sits next to the source it justifies and+# ships verbatim inside the pack as provenance.+toml.workspace = truewalkdir = "2"# gitignore/.xerjignore matching (#276). ripgrep's matcher, used for its# semantics only — traversal stays on walkdir so the existing symlink,diff --git a/engine/crates/xerj-autoindex/src/harvest/identity.rs b/engine/crates/xerj-autoindex/src/harvest/identity.rsnew file mode 100644index 000000000..2b2e07fce--- /dev/null+++ b/engine/crates/xerj-autoindex/src/harvest/identity.rs@@ -0,0 +1,686 @@+//! Identity resolution: turn N source records into unique logical records.+//!+//! Union-find over identity-edge VALUES: every value a recipe declares as+//! an edge (`id`, each element of `aliases`, …) is a node, and a record+//! unions with each of its own values. Two records sharing ANY value merge,+//! transitively and cyclically — exactly the alias closure the sources+//! themselves do not provide (OSV ships one record per source ID, linked+//! only by `aliases`; it does not merge them for you).+//!+//! Derived relations (CVE↔commit, say) are deliberately NOT edges: a single+//! commit fixing two CVEs would fuse two advisories that are distinct+//! findings. Identity is what the sources SAY is the same thing, never what+//! we infer from shared references.+//!+//! Merge is first-non-empty-wins under the recipe's precedence, with arrays+//! unioned — deterministic given the member order, which is itself+//! (precedence rank, store key), never filesystem accident.++use std::collections::HashMap;++use serde_json::{Map, Value};++use super::recipe::{+ slug_rank, Derived, DerivedOp, Envelope, Identity, Merge, LICENCE, SRC, SRC_PATH, SRC_URL,+};++/// The store-key sentinel `store::read_all` injects. Sorting by it keeps+/// member order deterministic when two members share a source rank.+pub(crate) const KEY: &str = "\u{0}key";++/// One cluster of envelopes that are the same logical record.+pub struct Cluster {+ /// Indices into the envelope slice, in canonical order.+ pub members: Vec<usize>,+}++/// Group envelopes into clusters. Deterministic: iteration follows the+/// envelope order, and cluster output order follows each cluster's minimum+/// member index.+pub fn cluster(envelopes: &[Map<String, Value>], identity: &Identity) -> Vec<Cluster> {+ // DSU over [0..envelopes) ∪ value nodes (mapped to >= envelopes). Value+ // nodes always lose unions (records have smaller indices), so every+ // root stays below envelopes.len().+ let mut parent: Vec<usize> = (0..envelopes.len()).collect();+ let mut value_nodes: HashMap<String, usize> = HashMap::new();++ fn find(parent: &mut [usize], mut x: usize) -> usize {+ while parent[x] != x {+ parent[x] = parent[parent[x]]; // path halving+ x = parent[x];+ }+ x+ }++ for (i, env) in envelopes.iter().enumerate() {+ for value in identity_values(env, &identity.edges) {+ let node = *value_nodes.entry(value).or_insert_with(|| {+ parent.push(parent.len());+ parent.len() - 1+ });+ let (ra, rb) = (find(&mut parent, i), find(&mut parent, node));+ if ra != rb {+ let (lo, hi) = if ra < rb { (ra, rb) } else { (rb, ra) };+ parent[hi] = lo;+ }+ }+ }++ let mut by_root: Vec<Vec<usize>> = vec![Vec::new(); envelopes.len()];+ for i in 0..envelopes.len() {+ by_root[find(&mut parent, i)].push(i);+ }+ by_root+ .into_iter()+ .filter(|m| !m.is_empty())+ .map(|mut members| {+ members.sort_by_key(|&i| order_key(&envelopes[i], &identity.canonical_source_order));+ Cluster { members }+ })+ .collect()+}++/// The identity-edge values of one envelope. `each = true` explodes string+/// arrays element-wise; a plain string field is one value; empty strings+/// and other types contribute nothing.+fn identity_values(env: &Map<String, Value>, edges: &[super::recipe::IdentityEdge]) -> Vec<String> {+ let mut out = Vec::new();+ for e in edges {+ match env.get(&e.field) {+ Some(Value::String(s)) if !s.is_empty() => out.push(s.clone()),+ Some(Value::Array(a)) if e.each => {+ for v in a {+ if let Some(s) = v.as_str() {+ if !s.is_empty() {+ out.push(s.to_string());+ }+ }+ }+ }+ // each=false on an array: the array as a whole is not an+ // identity — arrays vary by source even for the same record.+ _ => {}+ }+ }+ out+}++/// Sort key for members: the order list's rank first, then the envelope's+/// store key — stability that does not depend on directory iteration order.+fn order_key(env: &Map<String, Value>, order: &[String]) -> (usize, String) {+ let slug = env.get(SRC).and_then(Value::as_str).unwrap_or("");+ let key = env.get(KEY).and_then(Value::as_str).unwrap_or("");+ (slug_rank(order, slug), key.to_string())+}++/// Which member "wins" a scalar: the first (by merge order) carrying a+/// non-empty value.+fn first_str<'a>(members: &[&'a Map<String, Value>], key: &str) -> Option<&'a str> {+ members+ .iter()+ .filter_map(|m| m.get(key).and_then(Value::as_str))+ .find(|s| !s.is_empty())+}++/// Merge one passthrough key: an ARRAY in any member makes the result an+/// array (union across members, deduped, scalars folded in as elements —+/// sources disagree on cardinality more often than they agree); scalars+/// everywhere keep it a scalar, first non-empty under precedence.+fn merge_field(ranked: &[&Map<String, Value>], key: &str) -> Option<Value> {+ let any_array = ranked+ .iter()+ .any(|m| matches!(m.get(key), Some(Value::Array(_))));+ if any_array {+ return union_arrays(ranked, key).map(Value::Array);+ }+ if let Some(s) = first_str(ranked, key) {+ return Some(Value::String(s.to_string()));+ }+ ranked+ .iter()+ .find_map(|m| m.get(key).filter(|v| !v.is_null() && !v.is_string()))+ .cloned()+}++/// Union arrays across members preserving member order, deduped. A lone+/// string folds in as a single element.+fn union_arrays(members: &[&Map<String, Value>], key: &str) -> Option<Vec<Value>> {+ let mut out: Vec<Value> = Vec::new();+ for m in members {+ match m.get(key) {+ Some(Value::Array(a)) => {+ for v in a {+ if !out.contains(v) {+ out.push(v.clone());+ }+ }+ }+ Some(Value::String(s)) if !s.is_empty() => {+ let v = Value::String(s.clone());+ if !out.contains(&v) {+ out.push(v);+ }+ }+ _ => {}+ }+ }+ (!out.is_empty()).then_some(out)+}++/// The most-restrictive licence across members. A merged record+/// redistributes text from EVERY member, so a restricted member must be+/// able to veto the permissive ones — `is_restricted` is binary (the+/// licence tuple has no total order), so: any restricted member's licence+/// wins (precedence-first among them); otherwise precedence-first+/// non-empty. Attribution-grade obligations the binary predicate cannot+/// rank (CC-BY-4.0 vs CC0) stay visible per record through `sources[]`+/// and per pack through the manifest's source list — the `licence` field+/// is the redistribution-warning signal, not a legal settlement.+fn most_restrictive(members: &[&Map<String, Value>]) -> String {+ let mut first_non_empty = String::new();+ for m in members {+ let lic = m.get(LICENCE).and_then(Value::as_str).unwrap_or("");+ if lic.is_empty() {+ continue;+ }+ if xerj_common::xccode::licence::is_restricted(lic) {+ return lic.to_string();+ }+ if first_non_empty.is_empty() {+ first_non_empty = lic.to_string();+ }+ }+ first_non_empty+}++/// Merge one cluster into a single output record (envelope fields, then+/// passthrough, then derived). `order` is the merge precedence.+pub fn merge_cluster(+ cluster: &[&Map<String, Value>],+ order: &[String],+ envelope: &Envelope,+ merge: &Merge,+ derived: &[Derived],+) -> Map<String, Value> {+ let mut members: Vec<&Map<String, Value>> = cluster.to_vec();+ members.sort_by_key(|m| order_key(m, order));+ let winner = members.first().copied();++ let mut out = Map::new();++ // — envelope fields ────────────────────────────────────────────────────+ let id = first_of(&members, &envelope.id_from);+ out.insert("id".into(), Value::String(id.unwrap_or_default()));+ let title = first_of(&members, &envelope.title_from);+ out.insert("title".into(), Value::String(title.unwrap_or_default()));++ // body: per member, join its body_join fields; the first non-empty+ // joined body wins (joining ACROSS sources would duplicate the same+ // advisory prose written twice).+ let mut body = String::new();+ for m in &members {+ let parts: Vec<&str> = envelope+ .body_join+ .iter()+ .filter_map(|k| m.get(k).and_then(Value::as_str).filter(|s| !s.is_empty()))+ .collect();+ if !parts.is_empty() {+ body = parts.join("\n\n");+ break;+ }+ }+ out.insert("body".into(), Value::String(body));++ // defs: union of the defs_from arrays across ALL members, space-joined —+ // this is the field `xerj code` matches definition queries against.+ let mut defs: Vec<String> = Vec::new();+ for m in &members {+ for k in &envelope.defs_from {+ if let Some(Value::Array(a)) = m.get(k) {+ for v in a {+ if let Some(s) = v.as_str() {+ if !s.is_empty() && !defs.iter().any(|d| d == s) {+ defs.push(s.to_string());+ }+ }+ }+ }+ }+ }+ out.insert("defs".into(), Value::String(defs.join(" ")));++ out.insert("licence".into(), Value::String(most_restrictive(&members)));++ // sources: contributing slugs in precedence order; `source` names the+ // winner, `source_url`/`origin` point back into it.+ let mut slugs: Vec<Value> = Vec::new();+ for m in &members {+ if let Some(s) = m.get(SRC).and_then(Value::as_str) {+ let v = Value::String(s.to_string());+ if !slugs.contains(&v) {+ slugs.push(v);+ }+ }+ }+ out.insert("sources".into(), Value::Array(slugs));+ let get = |k: &str| {+ winner+ .and_then(|m| m.get(k))+ .and_then(Value::as_str)+ .unwrap_or_default()+ .to_string()+ };+ out.insert("source".into(), Value::String(get(SRC)));+ out.insert("source_url".into(), Value::String(get(SRC_URL)));+ out.insert("origin".into(), Value::String(get(SRC_PATH)));++ // modified: max member value (RFC3339 UTC strings compare correctly).+ let modified = members+ .iter()+ .filter_map(|m| m.get("modified").and_then(Value::as_str))+ .filter(|s| !s.is_empty())+ .max()+ .unwrap_or("")+ .to_string();+ out.insert("modified".into(), Value::String(modified));++ // — passthrough ───────────────────────────────────────────────────────+ // Every data key any member carries, minus provenance sentinels and+ // what the envelope already fixed. Scalars are first-non-empty under+ // the field's precedence (global, or per-field when overridden); arrays+ // union across members regardless of precedence — evidence accumulates.+ if envelope.passthrough {+ let mut keys: Vec<&String> = members+ .iter()+ .flat_map(|m| m.keys())+ .filter(|k| {+ !k.starts_with('\u{0}') && !k.starts_with('_') && !out.contains_key(k.as_str())+ })+ .collect();+ keys.sort();+ keys.dedup();+ for k in keys {+ let order = merge+ .field_precedence+ .get(k)+ .map(Vec::as_slice)+ .unwrap_or(order);+ let mut ranked = members.clone();+ ranked.sort_by_key(|m| order_key(m, order));+ if let Some(v) = merge_field(&ranked, k) {+ out.insert(k.clone(), v);+ }+ }+ }++ // — derived fields ────────────────────────────────────────────────────+ for d in derived {+ let v = apply_derived(&out, &d.op);+ out.insert(d.name.clone(), v);+ }++ out+}++/// First non-empty string among any of the listed keys, scanning members in+/// order. The key list wins over member order: `title_from = ["summary",+/// "id"]` prefers a summary from ANY source over an id from the top source+/// — the human-readable name wherever it exists.+fn first_of(members: &[&Map<String, Value>], keys: &[String]) -> Option<String> {+ for k in keys {+ if let Some(s) = first_str(members, k) {+ return Some(s.to_string());+ }+ }+ None+}++fn apply_derived(rec: &Map<String, Value>, op: &DerivedOp) -> Value {+ match op {+ DerivedOp::RegexExtract { from, re, unique } => {+ let mut out: Vec<Value> = Vec::new();+ for k in from {+ let texts: Vec<String> = match rec.get(k) {+ Some(Value::String(s)) => vec![s.clone()],+ Some(Value::Array(a)) => a+ .iter()+ .filter_map(|v| v.as_str().map(str::to_string))+ .collect(),+ _ => Vec::new(),+ };+ for t in texts {+ for m in re.find_iter(&t) {+ let v = Value::String(m.as_str().to_string());+ if !*unique || !out.contains(&v) {+ out.push(v);+ }+ }+ }+ }+ Value::Array(out)+ }+ DerivedOp::Map {+ from,+ rules,+ default,+ } => {+ let text = rec.get(from).and_then(Value::as_str).unwrap_or("");+ for (re, value) in rules {+ if re.is_match(text) {+ return Value::String(value.clone());+ }+ }+ Value::String(default.clone())+ }+ DerivedOp::Present { from } => {+ let present = match rec.get(from) {+ None | Some(Value::Null) => false,+ Some(Value::String(s)) => !s.is_empty(),+ Some(_) => true,+ };+ Value::Bool(present)+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::harvest::recipe::IdentityEdge;+ use regex::Regex;+ use serde_json::json;++ fn env(src: &str, fields: &[(&str, Value)]) -> Map<String, Value> {+ let mut m = Map::new();+ m.insert(SRC.into(), json!(src));+ m.insert(SRC_URL.into(), json!(format!("https://example/{src}")));+ m.insert(SRC_PATH.into(), json!(format!("{src}/record.json")));+ m.insert(LICENCE.into(), json!("CC0-1.0"));+ m.insert(KEY.into(), json!(format!("{src}-key")));+ for (k, v) in fields {+ m.insert(k.to_string(), v.clone());+ }+ m+ }++ fn edges(spec: &[(&str, bool)]) -> Vec<IdentityEdge> {+ spec.iter()+ .map(|(f, e)| IdentityEdge {+ field: f.to_string(),+ each: *e,+ })+ .collect()+ }++ fn identity_of(edges: Vec<IdentityEdge>) -> Identity {+ Identity {+ edges,+ canonical_source_order: vec![],+ }+ }++ #[test]+ fn alias_chain_and_cycle_collapse_to_one_cluster() {+ // rustsec --CVE--> ghsa and a direct alias edge, plus a cycle+ let a = env(+ "rustsec",+ &[("id", json!("R1")), ("aliases", json!(["C1"]))],+ );+ let b = env(+ "ghsa",+ &[("id", json!("G1")), ("aliases", json!(["C1", "R1"]))],+ );+ let c = env(+ "osv",+ &[("id", json!("C1")), ("aliases", json!(["R1", "G1"]))],+ );+ let identity = identity_of(edges(&[("id", false), ("aliases", true)]));+ let clusters = cluster(&[a, b, c], &identity);+ assert_eq!(clusters.len(), 1, "all three are one logical record");+ assert_eq!(clusters[0].members.len(), 3);+ }++ #[test]+ fn disjoint_records_stay_apart() {+ let a = env("a", &[("id", json!("X1"))]);+ let b = env("a", &[("id", json!("X2"))]);+ let clusters = cluster(&[a, b], &identity_of(edges(&[("id", false)])));+ assert_eq!(clusters.len(), 2);+ }++ #[test]+ fn shared_commit_is_not_an_identity() {+ // two CVEs referencing the same fixing commit must NOT merge+ let a = env(+ "a",+ &[+ ("id", json!("CVE-1")),+ ("commit_urls", json!(["https://x/commit/deadbeef"])),+ ],+ );+ let b = env(+ "a",+ &[+ ("id", json!("CVE-2")),+ ("commit_urls", json!(["https://x/commit/deadbeef"])),+ ],+ );+ let clusters = cluster(&[a, b], &identity_of(edges(&[("id", false)])));+ assert_eq!(+ clusters.len(),+ 2,+ "commits are derived edges, never identity"+ );+ }++ fn envelope() -> Envelope {+ Envelope {+ id_from: vec!["id".into()],+ title_from: vec!["summary".into(), "id".into()],+ body_join: vec!["summary".into(), "details".into()],+ defs_from: vec!["affected_functions".into()],+ passthrough: true,+ }+ }++ #[test]+ fn merge_prefers_precedence_and_unions_arrays() {+ let r = env(+ "rustsec",+ &[+ ("id", json!("R1")),+ ("summary", json!("unsound insert_many")),+ ("details", json!("root cause prose")),+ ("aliases", json!(["CVE-1"])),+ (+ "affected_functions",+ json!(["smallvec::SmallVec::insert_many"]),+ ),+ ("fixed_versions", json!(["0.6.14"])),+ ],+ );+ let g = env(+ "ghsa",+ &[+ ("id", json!("G1")),+ ("aliases", json!(["CVE-1", "R1"])),+ ("fixed_versions", json!(["1.6.1", "0.6.14"])),+ ("severity", json!("HIGH")),+ ],+ );+ // g is CC-BY: non-restricted, so precedence-first CC0 wins the FIELD+ // (attribution still rides via sources[]/manifest)…+ let mut g = g;+ g.insert(LICENCE.into(), json!("CC-BY-4.0"));+ let merged = merge_cluster(+ &[&r, &g],+ &["rustsec".to_string(), "ghsa".to_string()],+ &envelope(),+ &Merge::default(),+ &[],+ );+ assert_eq!(+ merged["id"],+ json!("R1"),+ "precedence-first member names the id"+ );+ assert_eq!(merged["title"], json!("unsound insert_many"));+ assert!(merged["body"]+ .as_str()+ .unwrap()+ .contains("root cause prose"));+ assert_eq!(+ merged["defs"],+ json!("smallvec::SmallVec::insert_many"),+ "defs feeds xerj code's definition matching"+ );+ assert_eq!(merged["licence"], json!("CC0-1.0"));+ // …but one restricted member vetoes every permissive one+ let mut g2 = env(+ "cisa",+ &[("id", json!("C1")), ("aliases", json!(["CVE-1", "R1"]))],+ );+ g2.insert(LICENCE.into(), json!("GPL-3.0"));+ let merged = merge_cluster(+ &[&r, &g, &g2],+ &[+ "rustsec".to_string(),+ "ghsa".to_string(),+ "cisa".to_string(),+ ],+ &envelope(),+ &Merge::default(),+ &[],+ );+ assert_eq!(+ merged["licence"],+ json!("GPL-3.0"),+ "restricted wins outright"+ );+ assert_eq!(merged["sources"], json!(["rustsec", "ghsa", "cisa"]));+ assert_eq!(merged["source"], json!("rustsec"));+ assert_eq!(merged["source_url"], json!("https://example/rustsec"));+ assert_eq!(merged["origin"], json!("rustsec/record.json"));+ assert_eq!(+ merged["fixed_versions"],+ json!(["0.6.14", "1.6.1"]),+ "arrays union in member order, deduped"+ );+ assert_eq!(merged["aliases"], json!(["CVE-1", "R1"]));+ assert_eq!(merged["severity"], json!("HIGH"), "only g has it");+ }++ #[test]+ fn title_from_spans_sources_before_falling_back() {+ // a summary in a LOWER precedence source beats the id of a higher one+ let r = env("rustsec", &[("id", json!("R1"))]);+ let g = env(+ "ghsa",+ &[("id", json!("G1")), ("summary", json!("the real title"))],+ );+ let merged = merge_cluster(+ &[&r, &g],+ &["rustsec".to_string(), "ghsa".to_string()],+ &envelope(),+ &Merge::default(),+ &[],+ );+ assert_eq!(merged["title"], json!("the real title"));+ }++ #[test]+ fn modified_takes_the_latest_member_value() {+ let r = env(+ "rustsec",+ &[+ ("id", json!("R1")),+ ("modified", json!("2024-01-01T00:00:00Z")),+ ],+ );+ let g = env(+ "ghsa",+ &[+ ("id", json!("G1")),+ ("modified", json!("2025-06-01T00:00:00Z")),+ ],+ );+ let merged = merge_cluster(+ &[&r, &g],+ &["rustsec".to_string(), "ghsa".to_string()],+ &envelope(),+ &Merge::default(),+ &[],+ );+ assert_eq!(merged["modified"], json!("2025-06-01T00:00:00Z"));+ }++ #[test]+ fn derived_ops_run_over_the_merged_record() {+ let r = env(+ "rustsec",+ &[+ ("id", json!("RUSTSEC-2021-0003")),+ ("aliases", json!(["CVE-2021-25900"])),+ ("summary", json!("Unsound: insert_many")),+ ("withdrawn", Value::Null),+ (+ "reference_urls",+ json!([+ "https://github.com/servo/rust-smallvec/commit/abc1234",+ "https://github.com/servo/rust-smallvec/pull/9"+ ]),+ ),+ ],+ );+ let cve = Regex::new(r"(?i)\bCVE-\d{4}-\d{4,}\b").unwrap();+ let commit =+ Regex::new(r"https://github\.com/[^/\s]+/[^/\s]+/commit/[0-9a-f]{7,40}").unwrap();+ let unsound = Regex::new(r"(?i)\bunsound\b").unwrap();+ let merged = merge_cluster(+ &[&r],+ &["rustsec".to_string()],+ &envelope(),+ &Merge::default(),+ &[+ Derived {+ name: "cve_ids".into(),+ op: DerivedOp::RegexExtract {+ from: vec!["id".into(), "aliases".into()],+ re: cve,+ unique: true,+ },+ },+ Derived {+ name: "commit_urls".into(),+ op: DerivedOp::RegexExtract {+ from: vec!["reference_urls".into()],+ re: commit,+ unique: true,+ },+ },+ Derived {+ name: "record_kind".into(),+ op: DerivedOp::Map {+ from: "title".into(),+ rules: vec![(unsound, "informational".into())],+ default: "vulnerability".into(),+ },+ },+ Derived {+ name: "is_withdrawn".into(),+ op: DerivedOp::Present {+ from: "withdrawn".into(),+ },+ },+ ],+ );+ assert_eq!(merged["cve_ids"], json!(["CVE-2021-25900"]));+ assert_eq!(+ merged["commit_urls"],+ json!(["https://github.com/servo/rust-smallvec/commit/abc1234"]),+ "the PR link is not a commit"+ );+ assert_eq!(merged["record_kind"], json!("informational"));+ assert_eq!(merged["is_withdrawn"], json!(false), "null is not present");+ }+}diff --git a/engine/crates/xerj-autoindex/src/harvest/mod.rs b/engine/crates/xerj-autoindex/src/harvest/mod.rsnew file mode 100644index 000000000..b00936c52--- /dev/null+++ b/engine/crates/xerj-autoindex/src/harvest/mod.rs@@ -0,0 +1,600 @@+//! `xerj corpus build` — build a portable corpus pack from harvested records.+//!+//! The pipeline, end to end:+//!+//! ```text+//! recipe.toml ─→ harvest (dir walk / M2: http-zip, git)+//! ─→ normalize (format adapter: osv | flat)+//! ─→ store (content-addressed, presence = dedup)+//! ─→ identity (union-find over declared edge values)+//! ─→ merge (precedence + array union + strictest licence)+//! ─→ pack (deterministic sharded JSONL + manifest + SUMS)+//! ```+//!+//! Incrementality is not a mode, it is the storage layout: every normalized+//! record lands at `store/<slug>.<xxh3-of-canonical-json>.json`, so a re-run+//! recomputes the key, sees the file, and skips. Deleting the source tree's+//! unchanged files and re-running is a no-op beyond the walk; the pack is+//! then rebuilt from the store deterministically.+//!+//! The tool is domain-agnostic by directive: what a "record" is, which+//! fields are identity, and how fields merge all live in the recipe, never+//! here. Licence terms ride along as record data (`licence`, per-source+//! `sources[]`); the tool does not and must not enforce them — the pack+//! author owns what goes in the pack.++pub mod identity;+pub mod normalize;+pub mod pack;+pub mod recipe;+pub mod store;++pub(crate) use store::MAX_FIELDS;++use std::collections::HashSet;+use std::path::{Path, PathBuf};++use anyhow::{bail, Context, Result};+use serde_json::{Map, Value};++use crate::xc::code_root;+use pack::{RunStats, SourceStat};+use recipe::{SourceKind, LICENCE, SRC, SRC_PATH, SRC_URL};++const BUILD_USAGE: &str = "usage: xerj corpus build <name> [--recipe <path>] [--fresh]++build (or incrementally refresh) a harvested corpus pack:+ 1. load the recipe (default $XERJ_CODE_HOME/recipes/<name>.toml)+ 2. harvest every declared source into the content store+ (already-stored records are skipped — that is the point)+ 3. resolve identity, merge duplicates, emit a deterministic pack+ under $XERJ_CODE_HOME/builds/<name>/pack/<name>/++--fresh discards the build state first (full re-harvest).";++/// Entry point from the corpus dispatcher. 0 = pack built, 1 = failure,+/// 2 = usage error.+pub fn run_build(args: &[String]) -> i32 {+ match run_build_inner(args, &code_root()) {+ Ok(code) => code,+ Err(e) => {+ eprintln!("xerj corpus build: {e:#}");+ 1+ }+ }+}++fn run_build_inner(args: &[String], home: &Path) -> Result<i32> {+ let mut name: Option<String> = None;+ let mut recipe_path: Option<PathBuf> = None;+ let mut fresh = false;+ let mut it = args.iter();+ while let Some(a) = it.next() {+ match a.as_str() {+ "--recipe" => {+ recipe_path = Some(PathBuf::from(it.next().context("--recipe needs a path")?));+ }+ "--fresh" => fresh = true,+ "-h" | "--help" | "help" => {+ println!("{BUILD_USAGE}");+ return Ok(0);+ }+ s if s.starts_with('-') => bail!("unknown flag '{s}'\n\n{BUILD_USAGE}"),+ s => {+ if name.replace(s.to_string()).is_some() {+ bail!("unexpected second name '{s}'\n\n{BUILD_USAGE}");+ }+ }+ }+ }+ let Some(name) = name else {+ eprintln!("{BUILD_USAGE}");+ return Ok(2);+ };++ let recipe_file = match recipe_path {+ Some(p) => p,+ None => home.join("recipes").join(format!("{name}.toml")),+ };+ if !recipe_file.is_file() {+ bail!(+ "no recipe for '{name}' at {} (pass --recipe <path>, or create it there)",+ recipe_file.display()+ );+ }+ let recipe = recipe::load(&recipe_file)?;+ if recipe.name != name {+ bail!(+ "recipe at {} builds corpus '{}', not '{}' — the name must match",+ recipe_file.display(),+ recipe.name,+ name+ );+ }+ let recipe_dir = recipe_file+ .parent()+ .map(Path::to_path_buf)+ .unwrap_or_else(|| PathBuf::from("."));+ let recipe_toml = std::fs::read_to_string(&recipe_file)?;++ let build_dir = home.join("builds").join(&recipe.name);+ if fresh && build_dir.exists() {+ std::fs::remove_dir_all(&build_dir)+ .with_context(|| format!("--fresh: cannot clear {}", build_dir.display()))?;+ }+ let store_dir = build_dir.join("store");++ // ── harvest into the content store ───────────────────────────────────+ let mut per_source = Vec::new();+ for src in &recipe.sources {+ let SourceKind::Dir = src.kind;+ let root = recipe_dir.join(src.path.as_deref().unwrap_or("."));+ let files = walk(&root, &src.glob)?;+ let mut stat = SourceStat {+ slug: src.slug.clone(),+ kind: "dir".to_string(),+ path: src.path.clone(),+ url: src.url.clone(),+ licence: src.licence.clone(),+ records: 0,+ new: 0,+ skipped: 0,+ pruned: 0,+ };+ let mut keys = HashSet::new();+ for (rel, path) in files {+ for value in parse_file(&path)? {+ let mut env = normalize::normalize(src.format, &value);+ env.insert(SRC.into(), Value::String(src.slug.clone()));+ env.insert(+ SRC_URL.into(),+ Value::String(src.url.clone().unwrap_or_default()),+ );+ env.insert(SRC_PATH.into(), Value::String(rel.clone()));+ env.insert(LICENCE.into(), Value::String(src.licence.clone()));+ let (key, was_new) = store::put(&store_dir, &src.slug, &env)?;+ keys.insert(key);+ stat.records += 1;+ if was_new {+ stat.new += 1;+ } else {+ stat.skipped += 1;+ }+ }+ }+ stat.pruned = store::prune(&store_dir, &src.slug, &keys);+ per_source.push(stat);+ }++ // ── identity + merge over the FULL store (not just this run's files) ──+ let slugs: HashSet<String> = recipe.sources.iter().map(|s| s.slug.clone()).collect();+ let envelopes = store::read_all(&store_dir, &slugs);+ let clusters = identity::cluster(&envelopes, &recipe.identity);+ let merge_order = if recipe.merge.precedence.is_empty() {+ recipe.identity.canonical_source_order.clone()+ } else {+ recipe.merge.precedence.clone()+ };+ let mut records = Vec::with_capacity(clusters.len());+ for c in &clusters {+ let members: Vec<&Map<String, Value>> = c.members.iter().map(|&i| &envelopes[i]).collect();+ records.push(identity::merge_cluster(+ &members,+ &merge_order,+ &recipe.envelope,+ &recipe.merge,+ &recipe.derived,+ ));+ }++ // ── emit ──────────────────────────────────────────────────────────────+ let stats = RunStats {+ envelopes: envelopes.len(),+ per_source,+ };+ let result = pack::emit(&build_dir, &recipe, &recipe_toml, records, &stats)?;++ println!(+ "built '{}' generation {}: {} envelopes → {} records, {} shard file(s)",+ recipe.name, result.generation, stats.envelopes, result.records, result.shards+ );+ for s in &stats.per_source {+ println!(+ " {}: {} record(s) — {} new, {} skipped, {} pruned ({})",+ s.slug, s.records, s.new, s.skipped, s.pruned, s.licence+ );+ }+ println!("pack: {}", result.dir.display());+ Ok(0)+}++// ── dir source ──────────────────────────────────────────────────────────────++/// Recursively list files under `root` matching `glob`, as+/// (posix-relative-path, absolute-path), sorted — directory iteration order+/// must never reach the output.+fn walk(root: &Path, glob: &str) -> Result<Vec<(String, PathBuf)>> {+ let re = glob_regex(glob);+ let mut out = Vec::new();+ fn recurse(dir: &Path, root: &Path, re: ®ex::Regex, out: &mut Vec<(String, PathBuf)>) {+ let Ok(entries) = std::fs::read_dir(dir) else {+ return;+ };+ for e in entries.flatten() {+ let p = e.path();+ if p.is_dir() {+ recurse(&p, root, re, out);+ } else {+ let rel = p.strip_prefix(root).unwrap_or(&p);+ let rel = rel.to_string_lossy().replace('\\', "/");+ if re.is_match(&rel) {+ out.push((rel, p));+ }+ }+ }+ }+ recurse(root, root, &re, &mut out);+ out.sort();+ Ok(out)+}++/// Minimal glob → regex: `**` spans directories (`**/` also matches zero+/// directories), `*` stays inside one segment, `?` is one char. Recipes+/// need path matching, not the full glob grammar; if a recipe ever needs+/// character classes we take a real glob crate rather than growing this.+fn glob_regex(glob: &str) -> regex::Regex {+ let mut re = String::from("^");+ let mut chars = glob.chars().peekable();+ while let Some(c) = chars.next() {+ match c {+ '*' => {+ if chars.peek() == Some(&'*') {+ chars.next();+ if chars.peek() == Some(&'/') {+ chars.next();+ re.push_str("(.*/)?");+ } else {+ re.push_str(".*");+ }+ } else {+ re.push_str("[^/]*");+ }+ }+ '?' => re.push_str("[^/]"),+ other => re.push_str(®ex::escape(&other.to_string())),+ }+ }+ re.push('$');+ regex::Regex::new(&re).expect("glob translation always yields a valid regex")+}++/// Parse one harvested file: `.json`/`.jsonl`/`.ndjson`. A `.json` array+/// yields its elements; `.jsonl` yields one per non-empty line. Other+/// extensions are skipped by the glob, not here — reaching this with a+/// non-JSON file is a hard error, not a silent drop.+fn parse_file(path: &Path) -> Result<Vec<Value>> {+ let raw =+ std::fs::read_to_string(path).with_context(|| format!("cannot read {}", path.display()))?;+ let name = path+ .file_name()+ .and_then(|n| n.to_str())+ .unwrap_or_default()+ .to_lowercase();+ let jsonl = name.ends_with(".jsonl") || name.ends_with(".ndjson");+ if jsonl {+ let mut out = Vec::new();+ for (n, line) in raw.lines().enumerate() {+ let line = line.trim();+ if line.is_empty() {+ continue;+ }+ let v: Value = serde_json::from_str(line)+ .with_context(|| format!("{}:{}: not valid JSON", path.display(), n + 1))?;+ out.push(v);+ }+ Ok(out)+ } else {+ let v: Value = serde_json::from_str(&raw)+ .with_context(|| format!("{}: not valid JSON", path.display()))?;+ Ok(match v {+ Value::Array(a) => a,+ other => vec![other],+ })+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[test]+ fn glob_matches_paths_like_a_recipe_expects() {+ let re = glob_regex("**/*.json");+ assert!(re.is_match("a.json"));+ assert!(re.is_match("deep/nested/a.json"));+ assert!(!re.is_match("a.jsonl"));+ assert!(!re.is_match("dir/sub")); // '.json' must be a suffix++ let re = glob_regex("osv/*.json");+ assert!(re.is_match("osv/x.json"));+ assert!(!re.is_match("osv/nested/x.json"), "* does not cross '/'");+ assert!(!re.is_match("other/x.json"));++ let re = glob_regex("**/GHSA-????.json");+ assert!(re.is_match("GHSA-1234.json"));+ assert!(!re.is_match("GHSA-12345.json"));+ }++ #[test]+ fn walk_is_sorted_and_relative() {+ let tmp = tempfile::tempdir().unwrap();+ let root = tmp.path().join("src");+ std::fs::create_dir_all(root.join("b")).unwrap();+ std::fs::create_dir_all(root.join("a")).unwrap();+ std::fs::write(root.join("b/second.json"), "{}").unwrap();+ std::fs::write(root.join("a/first.json"), "{}").unwrap();+ std::fs::write(root.join("a/note.txt"), "x").unwrap();+ let files = walk(&root, "**/*.json").unwrap();+ let rels: Vec<&str> = files.iter().map(|(r, _)| r.as_str()).collect();+ assert_eq!(rels, vec!["a/first.json", "b/second.json"]);+ }++ #[test]+ fn parse_file_understands_json_and_jsonl() {+ let tmp = tempfile::tempdir().unwrap();+ let a = tmp.path().join("a.json");+ std::fs::write(&a, r#"[{"id":"x"},{"id":"y"}]"#).unwrap();+ assert_eq!(parse_file(&a).unwrap().len(), 2);+ let b = tmp.path().join("b.jsonl");+ std::fs::write(&b, "{\"id\":\"z\"}\n\n{\"id\":\"w\"}\n").unwrap();+ assert_eq!(parse_file(&b).unwrap().len(), 2, "blank lines skipped");+ let c = tmp.path().join("c.json");+ std::fs::write(&c, r#"{"id":"one"}"#).unwrap();+ assert_eq!(parse_file(&c).unwrap().len(), 1);+ }++ // ── end to end: recipe → harvest → identity → pack, twice ─────────────++ const E2E_RECIPE: &str = r#"+[recipe]+format = 1+name = "e2e"+description = "two sources, one alias-linked record"++[[sources]]+slug = "rustsec"+kind = "dir"+path = "data/rustsec"+format = "osv"+licence = "CC0-1.0"++[[sources]]+slug = "ghsa"+kind = "dir"+path = "data/ghsa"+glob = "**/*.jsonl"+format = "osv"+licence = "CC-BY-4.0"++[envelope]+id_from = ["id"]+title_from = ["summary", "id"]+body_join = ["summary", "details"]+defs_from = ["affected_functions"]++[identity]+edges = [{ field = "id" }, { field = "aliases", each = true }]+canonical_source_order = ["rustsec", "ghsa"]++[merge]+precedence = ["rustsec", "ghsa"]++[[derived]]+name = "cve_ids"+op = "regex_extract"+from = ["id", "aliases"]+pattern = '(?i)\bCVE-\d{4}-\d{4,}\b'+unique = true++[emit]+shards = 4+"#;++ fn e2e_env(home: &Path) {+ let root = home.join("recipes");+ std::fs::create_dir_all(root.join("data/rustsec")).unwrap();+ std::fs::create_dir_all(root.join("data/ghsa")).unwrap();+ std::fs::write(root.join("e2e.toml"), E2E_RECIPE).unwrap();+ std::fs::write(+ root.join("data/rustsec/RUSTSEC-2021-0003.json"),+ r#"{+ "id": "RUSTSEC-2021-0003",+ "aliases": ["CVE-2021-25900"],+ "summary": "Out of bounds write in SmallVec::insert_many",+ "details": "An issue was discovered that allows attackers...",+ "modified": "2023-06-20T00:00:00Z",+ "affected": [{+ "package": {"ecosystem": "crates.io", "name": "smallvec"},+ "ranges": [{"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "0.6.14"}]}]+ }],+ "ecosystem_specific": {"affects": {"functions": {"smallvec::SmallVec::insert_many": ["<0.6.14"]}}}+ }"#,+ )+ .unwrap();+ // second rustsec record with NO alias link: its own cluster+ std::fs::write(+ root.join("data/rustsec/RUSTSEC-2020-0001.json"),+ r#"{"id": "RUSTSEC-2020-0001", "summary": "other issue"}"#,+ )+ .unwrap();+ // same logical record, GHSA shape: split ranges, no functions+ std::fs::write(+ root.join("data/ghsa/reviewed.jsonl"),+ concat!(+ r#"{"id": "GHSA-43w2-9j62-hq99", "aliases": ["CVE-2021-25900"], "summary": "Out of bounds write in insert_many", "modified": "2025-01-02T00:00:00Z", "affected": [{"package": {"ecosystem": "crates.io", "name": "smallvec"}, "ranges": [{"type": "SEMVER", "events": [{"introduced": "1.0.0"}, {"fixed": "1.6.1"}]}]}]}"#,+ "\n",+ ),+ )+ .unwrap();+ }++ fn read_records(pack: &Path) -> Vec<Map<String, Value>> {+ let manifest: Value =+ serde_json::from_str(&std::fs::read_to_string(pack.join("manifest.json")).unwrap())+ .unwrap();+ let mut out = Vec::new();+ for name in manifest["files"].as_object().unwrap().keys() {+ for line in std::fs::read_to_string(pack.join(name)).unwrap().lines() {+ out.push(serde_json::from_str(line).unwrap());+ }+ }+ out.sort_by_key(|r: &Map<String, Value>| {+ r.get("id")+ .and_then(Value::as_str)+ .unwrap_or("")+ .to_string()+ });+ out+ }++ #[test]+ fn build_end_to_end_and_incrementally() {+ let home = tempfile::tempdir().unwrap();+ e2e_env(home.path());++ let args: Vec<String> = vec!["e2e".to_string()];+ assert_eq!(run_build_inner(&args, home.path()).unwrap(), 0);++ let pack = home.path().join("builds/e2e/pack/e2e");+ let records = read_records(&pack);+ assert_eq!(+ records.len(),+ 2,+ "GHSA+RUSTSEC merge via CVE alias; RUSTSEC-2020-0001 alone"+ );++ let merged = records+ .iter()+ .find(|r| r["id"] == json!("RUSTSEC-2021-0003"))+ .expect("precedence-first source names the id");+ assert_eq!(merged["sources"], json!(["rustsec", "ghsa"]));+ assert_eq!(merged["source"], json!("rustsec"));+ assert_eq!(+ merged["title"],+ json!("Out of bounds write in SmallVec::insert_many")+ );+ assert_eq!(+ merged["defs"],+ json!("smallvec::SmallVec::insert_many"),+ "rustsec-only field still feeds xerj code"+ );+ assert_eq!(+ merged["fixed_versions"],+ json!(["0.6.14", "1.6.1"]),+ "split GHSA range unions with the packed RustSec one"+ );+ assert_eq!(+ merged["modified"],+ json!("2025-01-02T00:00:00Z"),+ "max across members"+ );+ assert_eq!(+ merged["licence"],+ json!("CC0-1.0"),+ "neither is restricted; precedence first"+ );+ assert_eq!(+ merged["cve_ids"],+ json!(["CVE-2021-25900"]),+ "derived over the merged record"+ );++ // uniform keys: the GHSA-less record still carries every key+ let lone = records+ .iter()+ .find(|r| r["id"] == json!("RUSTSEC-2020-0001"))+ .unwrap();+ assert_eq!(lone["sources"], json!(["rustsec"]));+ assert_eq!(lone["fixed_versions"], json!([]), "array key fills empty");+ assert_eq!(lone["cve_ids"], json!([]));+ assert_eq!(lone["affected_functions"], json!([]));+ assert_eq!(lone["modified"], json!(""));++ // manifest structure + checksums+ let manifest: Value =+ serde_json::from_str(&std::fs::read_to_string(pack.join("manifest.json")).unwrap())+ .unwrap();+ assert_eq!(manifest["format_version"], json!(1));+ assert_eq!(manifest["kind"], json!("harvested"));+ assert_eq!(manifest["generation"], json!(1));+ assert_eq!(manifest["counts"]["records"], json!(2));+ assert_eq!(+ manifest["description"],+ json!("two sources, one alias-linked record")+ );+ let rs = &manifest["sources"][0];+ assert_eq!(rs["slug"], json!("rustsec"));+ assert_eq!(rs["new"], json!(2));+ assert_eq!(rs["skipped"], json!(0));+ assert!(manifest["recipe_sha256"]+ .as_str()+ .is_some_and(|s| s.len() == 64));+ assert!(pack.join("recipe.toml").is_file(), "recipe ships verbatim");++ // ── second run: nothing new, shards byte-identical, generation up ──+ let before: Vec<(String, Vec<u8>)> = std::fs::read_dir(&pack)+ .unwrap()+ .flatten()+ .map(|e| {+ (+ e.file_name().to_string_lossy().to_string(),+ std::fs::read(e.path()).unwrap(),+ )+ })+ .filter(|(n, _)| n.starts_with("records-"))+ .collect();+ assert_eq!(run_build_inner(&args, home.path()).unwrap(), 0);+ let manifest: Value =+ serde_json::from_str(&std::fs::read_to_string(pack.join("manifest.json")).unwrap())+ .unwrap();+ assert_eq!(+ manifest["generation"],+ json!(2),+ "rebuild advances generation"+ );+ for s in manifest["sources"].as_array().unwrap() {+ assert_eq!(+ s["new"],+ json!(0),+ "content-addressed store skipped everything"+ );+ assert_eq!(s["skipped"], json!(s["records"]));+ assert_eq!(s["pruned"], json!(0));+ }+ for (name, bytes) in before {+ assert_eq!(+ std::fs::read(pack.join(&name)).unwrap(),+ bytes,+ "{name} byte-identical"+ );+ }++ // ── third run after deleting an upstream file: prune removes it ────+ std::fs::remove_file(+ home.path()+ .join("recipes/data/rustsec/RUSTSEC-2020-0001.json"),+ )+ .unwrap();+ assert_eq!(run_build_inner(&args, home.path()).unwrap(), 0);+ let manifest: Value =+ serde_json::from_str(&std::fs::read_to_string(pack.join("manifest.json")).unwrap())+ .unwrap();+ assert_eq!(+ manifest["sources"][0]["pruned"],+ json!(1),+ "dead envelope pruned"+ );+ assert_eq!(read_records(&pack).len(), 1);+ }+}diff --git a/engine/crates/xerj-autoindex/src/harvest/normalize.rs b/engine/crates/xerj-autoindex/src/harvest/normalize.rsnew file mode 100644index 000000000..ae8b10b76--- /dev/null+++ b/engine/crates/xerj-autoindex/src/harvest/normalize.rs@@ -0,0 +1,381 @@+//! Input-format adapters: one function per known record FORMAT, each turning+//! a source JSON value into flat key paths.+//!+//! An adapter knows the INPUT format, never our domain: `osv` understands+//! the ossf/osv-schema shape (shared by osv.dev dumps, the RustSec `osv`+//! branch and github-reviewed GHSA files); `flat` does generic flattening+//! for everything else. Which keys of the result feed the query envelope is+//! the recipe's job, not the adapter's — the same `osv` adapter would serve+//! a Go vulnerability corpus unchanged.+//!+//! Flattening mirrors `extract::flatten_object` semantics (two levels of+//! nesting become `a_b` keys; deeper structure and arrays-of-objects are+//! stored as JSON strings; arrays of scalars stay arrays) so a record+//! normalized here does not transform a second time at index time.+//!+//! One deliberate omission: OSV `database_specific` is NEVER read. The+//! schema marks it "intended for internal use only" and "subject to change+//! without notice" — building on it means building on a field its owner+//! says may vanish. Ecosystem-specific fields ARE read (they are the+//! documented extension point upstreams actually publish to, and RustSec's+//! affected-functions list lives there).++use serde_json::{Map, Value};++use super::recipe::Format;++/// Normalize one record under `format` into flat key paths.+pub fn normalize(format: Format, v: &Value) -> Map<String, Value> {+ match format {+ Format::Flat => flatten(v),+ Format::Osv => normalize_osv(v),+ }+}++// ── generic flattening ──────────────────────────────────────────────────────++/// Flatten any JSON value into flat keys, `extract::flatten_object`+/// semantics. A non-object top level becomes `{"value": v}` so envelope+/// mapping still has something to point at.+pub fn flatten(v: &Value) -> Map<String, Value> {+ let mut out = Map::new();+ match v {+ Value::Object(m) => {+ for (k, vv) in m {+ flatten_into(k, vv.clone(), 0, &mut out);+ }+ }+ other => {+ out.insert("value".to_string(), other.clone());+ }+ }+ out+}++fn flatten_into(key: &str, v: Value, depth: usize, out: &mut Map<String, Value>) {+ if out.len() >= super::MAX_FIELDS {+ return;+ }+ match v {+ Value::Object(m) => {+ if depth < 2 {+ for (k, vv) in m {+ flatten_into(&format!("{key}_{k}"), vv, depth + 1, out);+ }+ } else {+ out.insert(+ key.to_string(),+ Value::String(serde_json::to_string(&m).unwrap_or_default()),+ );+ }+ }+ Value::Array(a) => {+ if a.iter().all(|e| !e.is_object() && !e.is_array()) {+ out.insert(key.to_string(), Value::Array(a));+ } else {+ out.insert(+ key.to_string(),+ Value::String(serde_json::to_string(&a).unwrap_or_default()),+ );+ }+ }+ other => {+ out.insert(key.to_string(), other);+ }+ }+}++// ── OSV ─────────────────────────────────────────────────────────────────────++/// Curated extraction of the OSV v1.x fields that carry meaning across+/// sources, plus generic flattening of anything else. See the module docs+/// for why `database_specific` is absent.+fn normalize_osv(v: &Value) -> Map<String, Value> {+ let obj: Map<String, Value> = match v {+ Value::Object(m) => m.clone(),+ other => return flatten(other),+ };+ let mut out = Map::new();++ // Direct scalars that exist in every OSV record class.+ for k in [+ "id",+ "modified",+ "published",+ "withdrawn",+ "summary",+ "details",+ ] {+ if let Some(s) = obj.get(k).and_then(Value::as_str) {+ if !s.is_empty() {+ out.insert(k.to_string(), Value::String(s.to_string()));+ }+ }+ }++ // `aliases` / `related`: scalar string arrays — they survive flattening+ // as arrays, and identity unions on every element.+ for k in ["aliases", "related"] {+ if let Some(arr) = string_array(obj.get(k)) {+ if !arr.is_empty() {+ out.insert(k.to_string(), Value::Array(arr));+ }+ }+ }++ // `severity`: [{type, score, source}] — first entry wins for the scalar,+ // all scores kept for evidence.+ if let Some(sev) = obj.get("severity").and_then(Value::as_array) {+ let scores: Vec<Value> = sev+ .iter()+ .filter_map(|s| s.get("score").and_then(Value::as_str))+ .map(|s| Value::String(s.to_string()))+ .collect();+ if let Some(first) = scores.first() {+ out.insert("severity".to_string(), first.clone());+ }+ if !scores.is_empty() {+ out.insert("severity_scores".to_string(), Value::Array(scores));+ }+ }++ // `affected[]`: version knowledge. OSV packs multiple introduced/fixed+ // pairs into one SEMVER range (RustSec shape) while GHSA splits them+ // across entries; both land in the same flat lists here so the merge+ // unions intervals instead of comparing shapes.+ let mut ecosystems: Vec<Value> = Vec::new();+ let mut packages: Vec<Value> = Vec::new();+ let mut versions: Vec<Value> = Vec::new();+ let mut introduced: Vec<Value> = Vec::new();+ let mut fixed: Vec<Value> = Vec::new();+ let mut last_affected: Vec<Value> = Vec::new();+ for aff in obj+ .get("affected")+ .and_then(Value::as_array)+ .unwrap_or(&Vec::new())+ {+ if let Some(name) = aff+ .get("package")+ .and_then(|p| p.get("name"))+ .and_then(Value::as_str)+ {+ packages.push(Value::String(name.to_string()));+ }+ if let Some(eco) = aff+ .get("package")+ .and_then(|p| p.get("ecosystem"))+ .and_then(Value::as_str)+ {+ ecosystems.push(Value::String(eco.to_string()));+ }+ if let Some(vs) = string_array(aff.get("versions")) {+ versions.extend(vs);+ }+ for range in aff+ .get("ranges")+ .and_then(Value::as_array)+ .unwrap_or(&Vec::new())+ {+ for ev in range+ .get("events")+ .and_then(Value::as_array)+ .unwrap_or(&Vec::new())+ {+ for (key, sink) in [+ ("introduced", &mut introduced),+ ("fixed", &mut fixed),+ ("last_affected", &mut last_affected),+ ] {+ if let Some(s) = ev.get(key).and_then(Value::as_str) {+ sink.push(Value::String(s.to_string()));+ }+ }+ }+ }+ }+ dedup_into(&mut out, "ecosystems", ecosystems);+ dedup_into(&mut out, "packages", packages);+ dedup_into(&mut out, "affected_versions", versions);+ dedup_into(&mut out, "introduced_versions", introduced);+ dedup_into(&mut out, "fixed_versions", fixed);+ dedup_into(&mut out, "last_affected_versions", last_affected);++ // `references[]`: {type, url} — flattened into parallel scalar arrays.+ let mut urls: Vec<Value> = Vec::new();+ let mut types: Vec<Value> = Vec::new();+ for r in obj+ .get("references")+ .and_then(Value::as_array)+ .unwrap_or(&Vec::new())+ {+ if let Some(u) = r.get("url").and_then(Value::as_str) {+ urls.push(Value::String(u.to_string()));+ }+ if let Some(t) = r.get("type").and_then(Value::as_str) {+ types.push(Value::String(t.to_string()));+ }+ }+ dedup_into(&mut out, "reference_urls", urls);+ dedup_into(&mut out, "reference_types", types);++ // `ecosystem_specific`: the documented upstream extension point. RustSec+ // publishes affected FUNCTIONS here (`affects.functions` — canonical+ // paths keyed to version ranges), which no other source in the stack+ // carries; that is the one structured extraction worth doing.+ if let Some(eco) = obj.get("ecosystem_specific") {+ if let Some(functions) = eco+ .pointer("/affects/functions")+ .and_then(Value::as_object)+ .map(|m| m.keys().cloned().collect::<Vec<_>>())+ {+ if !functions.is_empty() {+ out.insert(+ "affected_functions".to_string(),+ Value::Array(functions.into_iter().map(Value::String).collect::<Vec<_>>()),+ );+ }+ }+ if let Some(info) = eco.get("informational").and_then(Value::as_str) {+ out.insert("informational".to_string(), Value::String(info.to_string()));+ }+ }++ // Everything not already curated flows through the generic flattener so+ // unknown-but-real OSV fields survive into the pack (curated keys win on+ // collision — they are the ones with merge semantics).+ for (k, vv) in &obj {+ if matches!(k.as_str(), "database_specific") {+ continue;+ }+ if out.contains_key(k) {+ continue;+ }+ flatten_into(k, vv.clone(), 0, &mut out);+ }+ out+}++fn string_array(v: Option<&Value>) -> Option<Vec<Value>> {+ match v {+ Some(Value::Array(a)) if a.iter().all(|e| e.is_string()) => Some(a.to_vec()),+ Some(Value::String(s)) => Some(vec![Value::String(s.clone())]),+ _ => None,+ }+}++fn dedup_into(out: &mut Map<String, Value>, key: &str, vals: Vec<Value>) {+ if vals.is_empty() {+ return;+ }+ let mut seen: Vec<String> = Vec::new();+ let mut kept: Vec<Value> = Vec::new();+ for v in vals {+ if let Some(s) = v.as_str() {+ if s.is_empty() || seen.iter().any(|x| x == s) {+ continue;+ }+ seen.push(s.to_string());+ kept.push(v);+ }+ }+ if !kept.is_empty() {+ out.insert(key.to_string(), Value::Array(kept));+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ #[test]+ fn flat_mirrors_extract_semantics() {+ let v = json!({+ "id": "x",+ "meta": {"a": 1, "b": {"deep": true}},+ "deeper": {"x": {"y": {"z": 1}}},+ "tags": ["a", "b"],+ "rows": [{"k": 1}]+ });+ let m = normalize(Format::Flat, &v);+ assert_eq!(m["id"], json!("x"));+ assert_eq!(m["meta_a"], json!(1));+ // three key segments survive (b's object is at depth 1 < 2)…+ assert_eq!(m["meta_b_deep"], json!(true));+ // …the object at depth 2 is stringified, exactly like flatten_object+ assert_eq!(m["deeper_x_y"], json!("{\"z\":1}"));+ assert_eq!(m["tags"], json!(["a", "b"]));+ assert!(m["rows"].is_string(), "arrays of objects stringify");+ }++ #[test]+ fn osv_extracts_identity_and_versions() {+ let v = json!({+ "id": "RUSTSEC-2021-0003",+ "aliases": ["CVE-2021-25900", "GHSA-43w2-9j62-hq99"],+ "summary": "Out of bounds write in SmallVec::insert_many",+ "modified": "2023-06-20T00:00:00Z",+ "affected": [{+ "package": {"ecosystem": "crates.io", "name": "smallvec"},+ "ranges": [{+ "type": "SEMVER",+ "events": [+ {"introduced": "0.6.3"}, {"fixed": "0.6.14"},+ {"introduced": "1.0.0"}, {"fixed": "1.6.1"}+ ]+ }],+ "versions": ["0.6.5"]+ }],+ "references": [+ {"type": "WEB", "url": "https://github.com/servo/rust-smallvec/commit/abc"},+ {"type": "PACKAGE", "url": "https://crates.io/crates/smallvec"}+ ],+ "ecosystem_specific": {"affects": {"functions": {"smallvec::SmallVec::insert_many": ["<1.6.1"]}}},+ "database_specific": {"severity": "HIGH", "cwe_ids": ["CWE-787"]}+ });+ let m = normalize(Format::Osv, &v);+ assert_eq!(m["id"], json!("RUSTSEC-2021-0003"));+ assert_eq!(+ m["aliases"],+ json!(["CVE-2021-25900", "GHSA-43w2-9j62-hq99"])+ );+ assert_eq!(m["packages"], json!(["smallvec"]));+ assert_eq!(m["ecosystems"], json!(["crates.io"]));+ // both introduced/fixed pairs of the packed SEMVER range survive+ assert_eq!(m["introduced_versions"], json!(["0.6.3", "1.0.0"]));+ assert_eq!(m["fixed_versions"], json!(["0.6.14", "1.6.1"]));+ assert_eq!(m["affected_versions"], json!(["0.6.5"]));+ assert_eq!(+ m["affected_functions"],+ json!(["smallvec::SmallVec::insert_many"])+ );+ assert_eq!(m["reference_urls"].as_array().unwrap().len(), 2);+ assert!(!m.contains_key("database_specific"));+ assert!(!m.contains_key("database_specific_severity"));+ }++ #[test]+ fn osv_ghsa_split_ranges_land_in_the_same_lists() {+ // GHSA splits one advisory's two fixed ranges across two `affected`+ // entries; the union must equal the RustSec packed shape.+ let v = json!({+ "id": "GHSA-43w2-9j62-hq99",+ "affected": [+ {"package": {"name": "smallvec"}, "ranges": [{"events": [{"introduced": "0.6.3"}, {"fixed": "0.6.14"}]}]},+ {"package": {"name": "smallvec"}, "ranges": [{"events": [{"introduced": "1.0.0"}, {"fixed": "1.6.1"}]}]}+ ]+ });+ let m = normalize(Format::Osv, &v);+ assert_eq!(m["introduced_versions"], json!(["0.6.3", "1.0.0"]));+ assert_eq!(m["fixed_versions"], json!(["0.6.14", "1.6.1"]));+ assert_eq!(m["packages"], json!(["smallvec"]), "deduped across entries");+ }++ #[test]+ fn osv_unknown_fields_passthrough() {+ let v = json!({"id": "x", "schema_version": "1.9.0", "upstream": {"of": "y"}});+ let m = normalize(Format::Osv, &v);+ assert_eq!(m["schema_version"], json!("1.9.0"));+ assert_eq!(m["upstream_of"], json!("y"));+ }+}diff --git a/engine/crates/xerj-autoindex/src/harvest/pack.rs b/engine/crates/xerj-autoindex/src/harvest/pack.rsnew file mode 100644index 000000000..580c8f3c7--- /dev/null+++ b/engine/crates/xerj-autoindex/src/harvest/pack.rs@@ -0,0 +1,367 @@+//! Deterministic pack emission: merged records → the portable artifact.+//!+//! A pack is a directory (a zip of it later) that ANY tool can consume:+//! sharded `records-*.jsonl`, a `manifest.json` with per-file sha256s, the+//! recipe that built it, and a `SHA256SUMS` over everything. Determinism is+//! the load-bearing property: shard assignment is `xxh3_64(id) % shards`,+//! records are id-ordered, and every record carries the SAME key set (a+//! key absent from one record is `[]` for array keys, `null` otherwise) —+//! so an unchanged input rebuilds byte-identical shard files, and the+//! `_id`s the engine derives stay stable across rebuilds (autoindex+//! clusters datasets by field-name set; an omitted optional field would+//! silently move a record between datasets).+//!+//! `manifest.json` and `SHA256SUMS` necessarily vary (timestamp,+//! generation) — determinism is claimed for the RECORDS, never the wrapper.++use std::collections::BTreeMap;+use std::path::{Path, PathBuf};++use anyhow::{bail, Context, Result};+use serde_json::{Map, Value};+use sha2::{Digest, Sha256};+use xxhash_rust::xxh3::xxh3_64;++use super::recipe::Recipe;++/// Per-source counts for the manifest: what this run actually did.+pub struct SourceStat {+ pub slug: String,+ pub kind: String,+ pub path: Option<String>,+ pub url: Option<String>,+ pub licence: String,+ pub records: usize,+ pub new: usize,+ pub skipped: usize,+ pub pruned: usize,+}++pub struct RunStats {+ pub envelopes: usize,+ pub per_source: Vec<SourceStat>,+}++#[derive(Debug)]+pub struct PackResult {+ pub dir: PathBuf,+ pub generation: u64,+ pub records: usize,+ pub shards: usize,+}++/// Build (or rebuild) the pack under `<build_dir>/pack/<name>`.+pub fn emit(+ build_dir: &Path,+ recipe: &Recipe,+ recipe_toml: &str,+ mut records: Vec<Map<String, Value>>,+ stats: &RunStats,+) -> Result<PackResult> {+ let pack_dir = build_dir.join("pack").join(&recipe.name);+ let generation = previous_generation(&pack_dir)? + 1;+ std::fs::create_dir_all(&pack_dir)+ .with_context(|| format!("cannot create {}", pack_dir.display()))?;++ // Order first, then refuse duplicate ids: a pack with two records of+ // the same id breaks the engine-side _id determinism promise, and a+ // duplicate here means identity resolution failed silently.+ records.sort_by(|a, b| {+ a.get("id")+ .and_then(Value::as_str)+ .unwrap_or("")+ .cmp(b.get("id").and_then(Value::as_str).unwrap_or(""))+ });+ let mut seen_id = String::new();+ for r in &records {+ let id = r.get("id").and_then(Value::as_str).unwrap_or("");+ if !seen_id.is_empty() && id == seen_id {+ bail!("duplicate emitted id '{id}': identity edges failed to merge two records that claim the same id");+ }+ seen_id = id.to_string();+ }++ let shards = recipe.emit.shards.min(records.len()).max(1);+ let uniform = uniform_keys(&records);++ // shard → line buffer; records are id-sorted, so appending preserves+ // in-shard order without a second sort.+ let mut shard_lines: Vec<String> = vec![String::new(); shards];+ for r in &records {+ let id = r.get("id").and_then(Value::as_str).unwrap_or("");+ let bucket = (xxh3_64(id.as_bytes()) % shards as u64) as usize;+ let mut full = r.clone();+ for (k, is_array) in &uniform {+ full.entry(k.clone()).or_insert_with(|| {+ if *is_array {+ Value::Array(Vec::new())+ } else {+ Value::Null+ }+ });+ }+ shard_lines[bucket].push_str(&super::store::canonical_json(&full));+ shard_lines[bucket].push('\n');+ }++ let mut files: BTreeMap<String, Value> = BTreeMap::new();+ for (i, lines) in shard_lines.iter().enumerate() {+ let name = format!("records-{:05}.jsonl", i);+ if lines.is_empty() {+ continue; // small corpora need fewer files than shards+ }+ let path = pack_dir.join(&name);+ std::fs::write(&path, lines.as_bytes())+ .with_context(|| format!("cannot write {}", path.display()))?;+ let n = lines.lines().count();+ files.insert(+ name.clone(),+ serde_json::json!({ "sha256": hex(&Sha256::digest(lines.as_bytes())), "records": n }),+ );+ }++ // the recipe ships verbatim: it is the pack's provenance, licence+ // reasoning included+ std::fs::write(pack_dir.join("recipe.toml"), recipe_toml)?;+ let recipe_sha = hex(&Sha256::digest(recipe_toml.as_bytes()));++ let manifest = serde_json::json!({+ "format_version": 1,+ "kind": "harvested",+ "pack": recipe.name,+ "description": recipe.description,+ "created_at": chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true),+ "generation": generation,+ "recipe_sha256": recipe_sha,+ "counts": {+ "envelopes": stats.envelopes,+ "records": records.len(),+ "shards": shards,+ },+ "sources": stats.per_source.iter().map(|s| serde_json::json!({+ "slug": s.slug, "kind": s.kind, "path": s.path, "url": s.url,+ "licence": s.licence, "records": s.records,+ "new": s.new, "skipped": s.skipped, "pruned": s.pruned,+ })).collect::<Vec<_>>(),+ "files": Value::Object(files.clone().into_iter().collect()),+ "tool": { "name": "xerj", "version": env!("CARGO_PKG_VERSION") },+ });+ let manifest_bytes = serde_json::to_vec_pretty(&manifest)?;+ std::fs::write(pack_dir.join("manifest.json"), &manifest_bytes)?;++ let mut sums = String::new();+ let mut names: Vec<String> = files.keys().cloned().collect();+ names.push("recipe.toml".to_string());+ names.push("manifest.json".to_string());+ names.sort();+ for n in &names {+ let bytes = std::fs::read(pack_dir.join(n)).with_context(|| format!("checksumming {n}"))?;+ sums.push_str(&hex(&Sha256::digest(&bytes)));+ sums.push_str(" ");+ sums.push_str(n);+ sums.push('\n');+ }+ std::fs::write(pack_dir.join("SHA256SUMS"), sums)?;++ Ok(PackResult {+ dir: pack_dir,+ generation,+ records: records.len(),+ shards,+ })+}++/// The uniform key set: every key any record carries, and whether missing+/// values should fill as `[]` (array wherever it appears) or `null`.+fn uniform_keys(records: &[Map<String, Value>]) -> BTreeMap<String, bool> {+ let mut out: BTreeMap<String, (bool /*any array*/, bool /*all non-null array*/)> =+ BTreeMap::new();+ for r in records {+ for (k, v) in r {+ let e = out.entry(k.clone()).or_insert((false, true));+ if v.is_array() {+ e.0 = true;+ } else if !v.is_null() {+ e.1 = false;+ }+ }+ }+ out.into_iter()+ .map(|(k, (any, all))| (k, any && all))+ .collect()+}++fn previous_generation(pack_dir: &Path) -> Result<u64> {+ let Ok(bytes) = std::fs::read(pack_dir.join("manifest.json")) else {+ return Ok(0);+ };+ let v: Value = serde_json::from_slice(&bytes).unwrap_or(Value::Null);+ Ok(v.get("generation").and_then(Value::as_u64).unwrap_or(0))+}++fn hex(bytes: &[u8]) -> String {+ bytes.iter().map(|b| format!("{b:02x}")).collect()+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::harvest::recipe::{Emit, Envelope, Identity, Merge};+ use serde_json::json;++ fn recipe(name: &str, shards: usize) -> Recipe {+ Recipe {+ name: name.to_string(),+ description: String::new(),+ envelope: Envelope {+ id_from: vec!["id".into()],+ title_from: vec!["id".into()],+ body_join: vec![],+ defs_from: vec![],+ passthrough: false,+ },+ sources: vec![],+ identity: Identity {+ edges: vec![],+ canonical_source_order: vec![],+ },+ merge: Merge::default(),+ derived: vec![],+ emit: Emit { shards },+ }+ }++ fn rec(id: &str, extra: &[(&str, Value)]) -> Map<String, Value> {+ let mut m = Map::new();+ m.insert("id".into(), json!(id));+ for (k, v) in extra {+ m.insert(k.to_string(), v.clone());+ }+ m+ }++ fn stats() -> RunStats {+ RunStats {+ envelopes: 0,+ per_source: vec![],+ }+ }++ #[test]+ fn uniform_keys_fill_and_shard_determinism() {+ let tmp = tempfile::tempdir().unwrap();+ let records = vec![+ rec("a", &[("aliases", json!(["X1"]))]),+ rec("b", &[("severity", json!("HIGH"))]),+ ];+ let r1 = emit(+ tmp.path(),+ &recipe("t", 4),+ "# recipe\n",+ records.clone(),+ &stats(),+ )+ .unwrap();+ assert_eq!(r1.records, 2);+ assert_eq!(r1.generation, 1);++ // every shard file's records carry BOTH keys+ for entry in std::fs::read_dir(&r1.dir).unwrap().flatten() {+ let name = entry.file_name().to_string_lossy().to_string();+ if !name.starts_with("records-") {+ continue;+ }+ for line in std::fs::read_to_string(entry.path()).unwrap().lines() {+ let v: Value = serde_json::from_str(line).unwrap();+ assert!(v.get("aliases").is_some(), "array key filled");+ assert!(v.get("severity").is_some(), "scalar key filled");+ let id = v["id"].as_str().unwrap();+ if id == "a" {+ assert_eq!(v["aliases"], json!(["X1"]));+ assert_eq!(v["severity"], json!(null));+ } else {+ assert_eq!(v["aliases"], json!([]), "array key fills empty");+ assert_eq!(v["severity"], json!("HIGH"));+ }+ }+ }++ // rebuild from a DIFFERENT input order → byte-identical records+ let mut shuffled = records.clone();+ shuffled.reverse();+ let before: Vec<(String, Vec<u8>)> = std::fs::read_dir(&r1.dir)+ .unwrap()+ .flatten()+ .map(|e| {+ (+ e.file_name().to_string_lossy().to_string(),+ std::fs::read(e.path()).unwrap(),+ )+ })+ .filter(|(n, _)| n.starts_with("records-"))+ .collect();+ let r2 = emit(+ tmp.path(),+ &recipe("t", 4),+ "# recipe\n",+ shuffled,+ &stats(),+ )+ .unwrap();+ assert_eq!(r2.generation, 2, "generation advances");+ for (name, bytes) in before {+ assert_eq!(+ std::fs::read(r2.dir.join(&name)).unwrap(),+ bytes,+ "{name} identical"+ );+ }+ }++ #[test]+ fn duplicate_ids_are_refused() {+ let tmp = tempfile::tempdir().unwrap();+ let records = vec![rec("a", &[]), rec("a", &[])];+ let err = emit(tmp.path(), &recipe("t", 2), "", records, &stats())+ .unwrap_err()+ .to_string();+ assert!(err.contains("duplicate emitted id"), "{err}");+ }++ #[test]+ fn sums_cover_every_artifact() {+ let tmp = tempfile::tempdir().unwrap();+ let records = vec![rec("a", &[]), rec("b", &[]), rec("c", &[])];+ let r = emit(tmp.path(), &recipe("t", 2), "# r\n", records, &stats()).unwrap();+ let sums = std::fs::read_to_string(r.dir.join("SHA256SUMS")).unwrap();+ assert!(sums.contains(" manifest.json\n"));+ assert!(sums.contains(" recipe.toml\n"));+ for line in sums.lines() {+ let (sum, name) = line.split_once(" ").unwrap();+ let bytes = std::fs::read(r.dir.join(name)).unwrap();+ assert_eq!(sum, hex(&Sha256::digest(&bytes)), "{name} hash matches");+ }+ let m: Value =+ serde_json::from_str(&std::fs::read_to_string(r.dir.join("manifest.json")).unwrap())+ .unwrap();+ assert_eq!(m["format_version"], json!(1));+ assert_eq!(m["kind"], json!("harvested"));+ assert_eq!(m["counts"]["records"], json!(3));+ // shard assignment is a hash — do not assert WHICH files exist, only+ // that the ones that do cover every record and none is empty+ let files = m["files"].as_object().unwrap();+ assert!(!files.is_empty());+ assert!(files.len() <= 2);+ let total: usize = files+ .values()+ .map(|f| f["records"].as_u64().unwrap() as usize)+ .sum();+ assert_eq!(total, 3);+ for f in files.values() {+ assert!(+ f["records"].as_u64().unwrap() > 0,+ "empty shard files are never written"+ );+ }+ }+}diff --git a/engine/crates/xerj-autoindex/src/harvest/recipe.rs b/engine/crates/xerj-autoindex/src/harvest/recipe.rsnew file mode 100644index 000000000..4229dc5a5--- /dev/null+++ b/engine/crates/xerj-autoindex/src/harvest/recipe.rs@@ -0,0 +1,692 @@+//! The corpus recipe — the declarative half of `xerj corpus build`.+//!+//! A recipe says WHERE records come from, which of their fields carry+//! IDENTITY (so the same logical record arriving from three sources becomes+//! one), how fields merge across sources, and which derived fields to+//! compute. It is TOML on purpose: it is human-authored, commented — licence+//! reasoning belongs next to the source it justifies — and it ships verbatim+//! inside the built pack as provenance.+//!+//! The tool is deliberately schema-agnostic: the recipe maps whatever the+//! sources actually contain onto a small query envelope (`title`, `body`,+//! `defs`, …). Nothing here knows what a vulnerability is. The parse is+//! STRICT — unknown keys and bad values are loud errors, not silently+//! ignored defaults — because a typo'd `aliases` silently turns one unique+//! record into three, which is the exact failure this subsystem exists to+//! prevent. (Hub manifests are tolerant because they are untrusted input+//! validated elsewhere; a recipe is trusted author input and must round-trip+//! faithfully.)++use std::collections::HashMap;+use std::path::Path;++use anyhow::{bail, Context, Result};+use regex::Regex;+use serde::Deserialize;++/// Reserved envelope key names. Every emitted record carries these; derived+/// fields and passthrough data may not clobber them (validation refuses a+/// recipe that tries — a silent clobber here would make `xerj code` and the+/// ES-compat query surface disagree about what a record is).+pub const RESERVED_KEYS: &[&str] = &[+ "id",+ "title",+ "body",+ "defs",+ "licence",+ "source",+ "sources",+ "source_url",+ "modified",+ "origin",+];++/// The envelope provenance keys used INSIDE the content store. Underscored+/// so they can never collide with a data field at emit time, where they are+/// mapped onto `source`/`sources`/`source_url`/`origin` and dropped.+pub(crate) const SRC: &str = "_src";+pub(crate) const SRC_URL: &str = "_src_url";+pub(crate) const SRC_PATH: &str = "_src_path";+pub(crate) const LICENCE: &str = "_licence";++/// Names every emitted record carries (the uniform-key invariant): the+/// RESERVED_KEYS above plus every passthrough and derived key the recipe+/// declares. Absent values are `null`/`[]`, never omitted — autoindex+/// clusters datasets by field-name set, so an omitted optional field would+/// move a record between datasets (and change its `_id`) across rebuilds.+#[derive(Debug, Clone)]+pub struct Recipe {+ pub name: String,+ /// Free-text, from `[recipe] description`; lands in the pack manifest so+ /// a consumer browsing packs sees what the corpus IS without reading the+ /// recipe.+ pub description: String,+ pub envelope: Envelope,+ pub sources: Vec<SourceSpec>,+ pub identity: Identity,+ pub merge: Merge,+ pub derived: Vec<Derived>,+ pub emit: Emit,+}++/// Which normalized keys feed the query envelope. All are "first non-empty+/// wins" lists evaluated per merged record, in merge-precedence order.+#[derive(Debug, Clone)]+pub struct Envelope {+ pub id_from: Vec<String>,+ pub title_from: Vec<String>,+ pub body_join: Vec<String>,+ pub defs_from: Vec<String>,+ pub passthrough: bool,+}++#[derive(Debug, Clone)]+pub struct SourceSpec {+ pub slug: String,+ pub kind: SourceKind,+ /// `dir` sources: a path relative to the recipe file.+ pub path: Option<String>,+ pub url: Option<String>,+ pub glob: String,+ pub format: Format,+ /// Carried onto every record from this source as record DATA — the tool+ /// never enforces it; the pack author owns what goes in the pack.+ pub licence: String,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum SourceKind {+ Dir,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum Format {+ /// OSV v1.x (the `ossf/osv-schema` shape shared by osv.dev dumps, the+ /// RustSec `osv` branch and github-reviewed GHSA files). Curated+ /// extraction of the standard fields plus generic flattening of the rest.+ Osv,+ /// Already-flat JSON records: generic two-level flattening only. This is+ /// the "we have not written a format adapter for this yet" path.+ Flat,+}++#[derive(Debug, Clone)]+pub struct Identity {+ pub edges: Vec<IdentityEdge>,+ /// Source precedence for choosing a cluster's canonical id; empty means+ /// source declaration order.+ pub canonical_source_order: Vec<String>,+}++#[derive(Debug, Clone)]+pub struct IdentityEdge {+ pub field: String,+ /// Array fields: union on EVERY element (`aliases`), not on the array+ /// as a whole.+ pub each: bool,+}++#[derive(Debug, Clone, Default)]+pub struct Merge {+ pub precedence: Vec<String>,+ pub field_precedence: HashMap<String, Vec<String>>,+}++#[derive(Debug, Clone)]+pub struct Derived {+ pub name: String,+ pub op: DerivedOp,+}++#[derive(Debug, Clone)]+pub enum DerivedOp {+ /// Collect every regex match over the listed fields (strings, and each+ /// element of string arrays) into a string array.+ RegexExtract {+ from: Vec<String>,+ re: Regex,+ unique: bool,+ },+ /// First matching rule's value for the field; `default` when nothing+ /// matches (or the field is absent).+ Map {+ from: String,+ rules: Vec<(Regex, String)>,+ default: String,+ },+ /// True when the field is present and non-empty.+ Present { from: String },+}++#[derive(Debug, Clone)]+pub struct Emit {+ pub shards: usize,+}++// ── raw (serde) layer ───────────────────────────────────────────────────────++#[derive(Deserialize)]+#[serde(deny_unknown_fields)]+struct RawRecipe {+ recipe: RawHeader,+ #[serde(default)]+ envelope: RawEnvelope,+ sources: Vec<RawSource>,+ identity: RawIdentity,+ #[serde(default)]+ merge: RawMerge,+ #[serde(default)]+ derived: Vec<RawDerived>,+ #[serde(default)]+ emit: RawEmit,+}++#[derive(Deserialize)]+#[serde(deny_unknown_fields)]+struct RawHeader {+ format: u32,+ name: String,+ #[serde(default)]+ description: String,+}++#[derive(Deserialize)]+#[serde(deny_unknown_fields)]+struct RawEnvelope {+ #[serde(default = "d_id_from")]+ id_from: Vec<String>,+ #[serde(default = "d_title_from")]+ title_from: Vec<String>,+ #[serde(default)]+ body_join: Vec<String>,+ #[serde(default)]+ defs_from: Vec<String>,+ #[serde(default = "d_true")]+ passthrough: bool,+}++// `#[serde(default)]` on the whole-table fields in RawRecipe falls back to+// `Default::default()` when the table is absent — which would silently zero+// these. The manual impls keep absent-table defaults equal to field defaults.+impl Default for RawEnvelope {+ fn default() -> Self {+ RawEnvelope {+ id_from: d_id_from(),+ title_from: d_title_from(),+ body_join: Vec::new(),+ defs_from: Vec::new(),+ passthrough: d_true(),+ }+ }+}++fn d_id_from() -> Vec<String> {+ vec!["id".to_string()]+}+fn d_title_from() -> Vec<String> {+ vec!["id".to_string()]+}+fn d_true() -> bool {+ true+}++#[derive(Deserialize)]+#[serde(deny_unknown_fields)]+struct RawSource {+ slug: String,+ kind: String,+ #[serde(default)]+ path: Option<String>,+ #[serde(default)]+ url: Option<String>,+ #[serde(default = "d_glob")]+ glob: String,+ format: String,+ #[serde(default)]+ licence: String,+}++fn d_glob() -> String {+ "**/*.json".to_string()+}++#[derive(Deserialize)]+#[serde(deny_unknown_fields)]+struct RawIdentity {+ edges: Vec<RawEdge>,+ #[serde(default)]+ canonical_source_order: Vec<String>,+}++#[derive(Deserialize)]+#[serde(deny_unknown_fields)]+struct RawEdge {+ field: String,+ #[serde(default)]+ each: bool,+}++#[derive(Deserialize, Default)]+#[serde(deny_unknown_fields)]+struct RawMerge {+ #[serde(default)]+ precedence: Vec<String>,+ #[serde(default)]+ field_precedence: HashMap<String, Vec<String>>,+}++#[derive(Deserialize)]+#[serde(deny_unknown_fields)]+struct RawDerived {+ name: String,+ op: String,+ #[serde(default)]+ from: Vec<String>,+ #[serde(default)]+ pattern: Option<String>,+ #[serde(default)]+ rules: Vec<RawMapRule>,+ #[serde(default)]+ default: Option<String>,+ #[serde(default)]+ unique: bool,+}++#[derive(Deserialize)]+#[serde(deny_unknown_fields)]+struct RawMapRule {+ r#match: String,+ value: String,+}++#[derive(Deserialize)]+#[serde(deny_unknown_fields)]+struct RawEmit {+ #[serde(default = "d_shards")]+ shards: usize,+}++impl Default for RawEmit {+ fn default() -> Self {+ RawEmit { shards: d_shards() }+ }+}++fn d_shards() -> usize {+ 16+}++// ── load + validate + compile ───────────────────────────────────────────────++/// A slug becomes a filename component in the content store+/// (`<slug>.<key>.json`), so it is gated to a safe charset the way corpus+/// and repo names are — but locally, because slugs are a builder concept.+fn valid_slug(slug: &str) -> Result<()> {+ let ok = !slug.is_empty()+ && slug.len() <= 64+ && slug+ .chars()+ .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.')+ && !slug.starts_with('.');+ if !ok {+ bail!("source slug '{slug}' must be 1-64 chars of [A-Za-z0-9._-], not starting with '.'");+ }+ Ok(())+}++/// Read, validate and compile a recipe. Every error names the file and the+/// offending key — a recipe that fails here must never half-run.+pub fn load(path: &Path) -> Result<Recipe> {+ let raw = std::fs::read_to_string(path)+ .with_context(|| format!("cannot read recipe {}", path.display()))?;+ let file: RawRecipe = toml::from_str(&raw).map_err(|e| {+ // the toml error names the unknown key/line — keep it in the message+ anyhow::anyhow!("{}: recipe parse error: {e}", path.display())+ })?;+ compile(path, file)+}++fn compile(path: &Path, file: RawRecipe) -> Result<Recipe> {+ let at = |what: &str| format!("{}: {}", path.display(), what);++ if file.recipe.format != 1 {+ bail!(at(&format!(+ "recipe format {} is not supported by this build (supported: 1)",+ file.recipe.format+ )));+ }+ let name = file.recipe.name.trim().to_string();+ if let Err(e) = xerj_common::xccode::pathgate::valid_corpus_name(&name) {+ bail!(at(&format!("recipe name '{name}': {e}")));+ }++ if file.sources.is_empty() {+ bail!(at("recipe declares no [[sources]]"));+ }+ let mut slugs: Vec<&str> = Vec::new();+ let mut sources = Vec::new();+ for s in &file.sources {+ valid_slug(&s.slug).with_context(|| at(&format!("source '{}'", s.slug)))?;+ if slugs.contains(&s.slug.as_str()) {+ bail!(at(&format!("duplicate source slug '{}'", s.slug)));+ }+ let kind = match s.kind.as_str() {+ "dir" => SourceKind::Dir,+ other @ ("http-zip" | "git") => bail!(at(&format!(+ "source '{}' kind '{other}' is not supported yet (planned: incremental \+ network sources)",+ s.slug+ ))),+ other => bail!(at(&format!("source '{}': unknown kind '{other}'", s.slug))),+ };+ let format = match s.format.as_str() {+ "osv" => Format::Osv,+ "flat" => Format::Flat,+ other => bail!(at(&format!(+ "source '{}': unknown format '{other}' (supported: osv, flat)",+ s.slug+ ))),+ };+ if kind == SourceKind::Dir {+ if s.path.as_deref().unwrap_or("").is_empty() {+ bail!(at(&format!(+ "dir source '{}' needs a 'path' (relative to the recipe file)",+ s.slug+ )));+ }+ if let Some(p) = &s.path {+ if Path::new(p).is_absolute() {+ bail!(at(&format!(+ "dir source '{}' path must be recipe-relative, got absolute '{p}'",+ s.slug+ )));+ }+ }+ }+ slugs.push(&s.slug);+ sources.push(SourceSpec {+ slug: s.slug.clone(),+ kind,+ path: s.path.clone(),+ url: s.url.clone(),+ glob: s.glob.clone(),+ format,+ licence: s.licence.clone(),+ });+ }++ if file.identity.edges.is_empty() {+ bail!(at(+ "[identity] declares no edges: without identity edges every record is its \+ own cluster and nothing deduplicates"+ ));+ }+ let edges = file+ .identity+ .edges+ .iter()+ .map(|e| IdentityEdge {+ field: e.field.clone(),+ each: e.each,+ })+ .collect();++ let known = |s: &str| slugs.contains(&s);+ for s in &file.identity.canonical_source_order {+ if !known(s) {+ bail!(at(&format!(+ "identity.canonical_source_order names source '{s}' which the recipe \+ does not declare"+ )));+ }+ }+ for s in &file.merge.precedence {+ if !known(s) {+ bail!(at(&format!(+ "merge.precedence names source '{s}' which the recipe does not declare"+ )));+ }+ }+ for (field, order) in &file.merge.field_precedence {+ for s in order {+ if !known(s) {+ bail!(at(&format!(+ "merge.field_precedence.{field} names source '{s}' which the recipe \+ does not declare"+ )));+ }+ }+ }++ let mut derived = Vec::new();+ for d in &file.derived {+ if RESERVED_KEYS.contains(&d.name.as_str()) {+ bail!(at(&format!(+ "derived field '{}' collides with a reserved envelope key",+ d.name+ )));+ }+ let op = match d.op.as_str() {+ "regex_extract" => {+ let pattern = d.pattern.as_deref().unwrap_or("");+ if pattern.is_empty() {+ bail!(at(&format!(+ "derived '{}': regex_extract needs a 'pattern'",+ d.name+ )));+ }+ if d.from.is_empty() {+ bail!(at(&format!(+ "derived '{}': regex_extract needs a 'from' list",+ d.name+ )));+ }+ let re = Regex::new(pattern)+ .with_context(|| at(&format!("derived '{}': bad regex /{pattern}/", d.name)))?;+ DerivedOp::RegexExtract {+ from: d.from.clone(),+ re,+ unique: d.unique,+ }+ }+ "map" => {+ if d.from.len() != 1 {+ bail!(at(&format!(+ "derived '{}': map needs exactly one 'from' field",+ d.name+ )));+ }+ if d.rules.is_empty() {+ bail!(at(&format!(+ "derived '{}': map needs at least one rule",+ d.name+ )));+ }+ let mut rules = Vec::new();+ for r in &d.rules {+ let re = Regex::new(&r.r#match).with_context(|| {+ at(&format!(+ "derived '{}': bad rule regex /{}/",+ d.name, r.r#match+ ))+ })?;+ rules.push((re, r.value.clone()));+ }+ DerivedOp::Map {+ from: d.from[0].clone(),+ rules,+ default: d.default.clone().unwrap_or_default(),+ }+ }+ "present" => {+ if d.from.len() != 1 {+ bail!(at(&format!(+ "derived '{}': present needs exactly one 'from' field",+ d.name+ )));+ }+ DerivedOp::Present {+ from: d.from[0].clone(),+ }+ }+ other => bail!(at(&format!(+ "derived '{}': unknown op '{other}' (supported: regex_extract, map, present)",+ d.name+ ))),+ };+ derived.push(Derived {+ name: d.name.clone(),+ op,+ });+ }++ if file.emit.shards == 0 || file.emit.shards > 4096 {+ bail!(at(&format!(+ "emit.shards must be 1..=4096, got {}",+ file.emit.shards+ )));+ }++ Ok(Recipe {+ name,+ description: file.recipe.description,+ envelope: Envelope {+ id_from: file.envelope.id_from,+ title_from: file.envelope.title_from,+ body_join: file.envelope.body_join,+ defs_from: file.envelope.defs_from,+ passthrough: file.envelope.passthrough,+ },+ sources,+ identity: Identity {+ edges,+ canonical_source_order: file.identity.canonical_source_order,+ },+ merge: file.merge.into_recipe(),+ derived,+ emit: Emit {+ shards: file.emit.shards,+ },+ })+}++impl RawMerge {+ fn into_recipe(self) -> Merge {+ Merge {+ precedence: self.precedence,+ field_precedence: self.field_precedence,+ }+ }+}++/// Rank of a source slug under an order list (absent = last). Members of a+/// cluster are sorted by this before any first-wins merge decision, so the+/// order list — not filesystem accident — decides whose value wins.+pub fn slug_rank(order: &[String], slug: &str) -> usize {+ order+ .iter()+ .position(|s| s == slug)+ .unwrap_or(order.len() + 1)+}++#[cfg(test)]+mod tests {+ use super::*;++ fn write(tmp: &std::path::Path, body: &str) -> std::path::PathBuf {+ let p = tmp.join("recipe.toml");+ std::fs::write(&p, body).unwrap();+ p+ }++ const MINIMAL: &str = r#"+[recipe]+format = 1+name = "demo"++[[sources]]+slug = "a"+kind = "dir"+path = "data/a"+format = "flat"++[identity]+edges = [{ field = "id" }]+"#;++ #[test]+ fn minimal_recipe_loads_with_defaults() {+ let tmp = tempfile::tempdir().unwrap();+ let r = load(&write(tmp.path(), MINIMAL)).unwrap();+ assert_eq!(r.name, "demo");+ assert_eq!(r.envelope.id_from, vec!["id".to_string()]);+ assert_eq!(r.envelope.passthrough, true);+ assert_eq!(r.emit.shards, 16);+ assert_eq!(r.sources[0].glob, "**/*.json");+ }++ #[test]+ fn unknown_keys_are_loud() {+ let tmp = tempfile::tempdir().unwrap();+ let bad = MINIMAL.replace("format = \"flat\"", "format = \"flat\"\nbogus_key = 1");+ let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();+ assert!(err.contains("bogus_key"), "{err}");+ }++ #[test]+ fn unknown_source_kind_names_the_planned_one() {+ let tmp = tempfile::tempdir().unwrap();+ let bad = MINIMAL.replace("kind = \"dir\"", "kind = \"git\"");+ let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();+ assert!(err.contains("not supported yet"), "{err}");+ assert!(err.contains("'git'"), "{err}");+ }++ #[test]+ fn identity_edges_are_mandatory() {+ let tmp = tempfile::tempdir().unwrap();+ let bad = MINIMAL.replace("edges = [{ field = \"id\" }]", "edges = []");+ let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();+ assert!(err.contains("declares no edges"), "{err}");+ }++ #[test]+ fn precedence_must_name_declared_sources() {+ let tmp = tempfile::tempdir().unwrap();+ let bad = MINIMAL.replace(+ "[identity]",+ "[merge]\nprecedence = [\"nope\"]\n\n[identity]",+ );+ let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();+ assert!(err.contains("'nope'"), "{err}");+ }++ #[test]+ fn derived_name_may_not_clobber_the_envelope() {+ let tmp = tempfile::tempdir().unwrap();+ let bad =+ format!("{MINIMAL}\n[[derived]]\nname = \"title\"\nop = \"present\"\nfrom = [\"x\"]\n");+ let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();+ assert!(err.contains("reserved envelope key"), "{err}");+ }++ #[test]+ fn bad_regex_is_rejected_at_load_not_at_build() {+ let tmp = tempfile::tempdir().unwrap();+ let bad = format!(+ "{MINIMAL}\n[[derived]]\nname = \"cves\"\nop = \"regex_extract\"\nfrom = \+ [\"id\"]\npattern = \"([unclosed\"\n"+ );+ let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();+ assert!(err.contains("bad regex"), "{err}");+ }++ #[test]+ fn slugs_are_path_safe() {+ assert!(valid_slug("rustsec").is_ok());+ assert!(valid_slug("osv-crates-io").is_ok());+ assert!(valid_slug("../escape").is_err());+ assert!(valid_slug(".hidden").is_err());+ assert!(valid_slug("").is_err());+ }+}diff --git a/engine/crates/xerj-autoindex/src/harvest/store.rs b/engine/crates/xerj-autoindex/src/harvest/store.rsnew file mode 100644index 000000000..c4eaee673--- /dev/null+++ b/engine/crates/xerj-autoindex/src/harvest/store.rs@@ -0,0 +1,202 @@+//! The content-addressed envelope store — "skip already stored" as a+//! filesystem fact, not a promise.+//!+//! Each normalized record plus its provenance becomes ONE immutable JSON+//! file named `<slug>.<xxh3_128-of-canonical-json>.json`. File presence IS+//! the dedup index: a re-run recomputes the key and skips if the file+//! exists, the same recomputed-never-trusted philosophy as `ids.rs` (same+//! content ⇒ same key ⇒ the write converges). No database, no journal —+//! crash at any point and the next run converges.+//!+//! The canonical form is key-sorted JSON with f64 floats in+//! shortest-round-trip form — deterministic for the same logical content+//! regardless of the input file's key order. (Sorting is explicit: the+//! workspace enables serde_json's `preserve_order` for the engine's+//! document store, which would otherwise bake insertion order in.)++use std::collections::HashSet;+use std::io;+use std::path::{Path, PathBuf};++use serde_json::{Map, Value};+use xxhash_rust::xxh3::xxh3_128;++/// Cap on flattened fields per envelope, matching the extractor's own cap+/// (`extract::MAX_FIELDS_PER_RECORD`): an envelope fatter than what the+/// index path would keep would silently disagree with the indexed record.+pub(crate) const MAX_FIELDS: usize = 512;++/// Canonical JSON for content hashing: sorted-key serialization. The+/// workspace enables serde_json's `preserve_order` (xerj-common), so `Map`+/// is an IndexMap and `to_string` would bake in INSERTION order — which is+/// exactly what canonical form must not do. Sorting through a `BTreeMap`+/// sidesteps the feature entirely.+pub fn canonical_json(env: &Map<String, Value>) -> String {+ let sorted: std::collections::BTreeMap<&String, &Value> = env.iter().collect();+ serde_json::to_string(&sorted).unwrap_or_default()+}++/// The store key for an envelope: xxh3_128 of the canonical JSON, hex.+/// 128-bit so an accidental collision across a multi-million-record store is+/// not a scenario to reason about.+pub fn key_of(env: &Map<String, Value>) -> String {+ format!("{:032x}", xxh3_128(canonical_json(env).as_bytes()))+}++/// Store one envelope. Returns `true` when it was NEW (written), `false`+/// when an identical envelope was already stored (skipped). The write is+/// atomic (tmp + rename): a partial file never exists under its final name.+pub fn put(store_dir: &Path, slug: &str, env: &Map<String, Value>) -> io::Result<(String, bool)> {+ let key = key_of(env);+ let path = file_for(store_dir, slug, &key);+ if path.exists() {+ return Ok((key, false));+ }+ std::fs::create_dir_all(store_dir)?;+ let tmp = path.with_extension("json.tmp");+ std::fs::write(&tmp, canonical_json(env))?;+ std::fs::rename(&tmp, &path)?;+ Ok((key, true))+}++fn file_for(store_dir: &Path, slug: &str, key: &str) -> PathBuf {+ store_dir.join(format!("{slug}.{key}.json"))+}++/// Every envelope whose `_src` is one of `slugs`, in filename order (the+/// deterministic iteration order every downstream stage relies on).+/// Envelopes from sources the current recipe no longer declares are left on+/// disk but not returned — a retired source must not haunt the cluster set.+/// Each returned map carries the file's store key under the `KEY` sentinel+/// so member ordering can fall back to it; `put` computes the key BEFORE+/// injection, so the sentinel is never part of any stored content.+pub fn read_all(store_dir: &Path, slugs: &HashSet<String>) -> Vec<Map<String, Value>> {+ let mut out = Vec::new();+ let Ok(entries) = std::fs::read_dir(store_dir) else {+ return out;+ };+ let mut names: Vec<String> = entries+ .flatten()+ .filter(|e| e.path().extension().and_then(|x| x.to_str()) == Some("json"))+ .filter_map(|e| e.file_name().to_str().map(str::to_string))+ .collect();+ names.sort();+ for name in names {+ // "<slug>.<32-hex>.json" — split from the RIGHT: slugs may contain+ // dots themselves, the key never does.+ let Some(rest) = name.strip_suffix(".json") else {+ continue;+ };+ let Some((slug, key)) = rest.rsplit_once('.') else {+ continue;+ };+ if !slugs.contains(slug) {+ continue;+ }+ if let Ok(raw) = std::fs::read_to_string(store_dir.join(&name)) {+ if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(&raw) {+ m.insert(super::identity::KEY.into(), Value::String(key.to_string()));+ out.push(m);+ }+ }+ }+ out+}++/// Drop envelopes of `slug` that this run did NOT re-put — a file deleted+/// upstream, or a record the normalizer no longer emits. Returns how many+/// files were removed. Scoped to one slug so a PARTIAL harvest of another+/// source (network watermark, M2) can never prune what it did not walk.+pub fn prune(store_dir: &Path, slug: &str, live_keys: &HashSet<String>) -> usize {+ let Ok(entries) = std::fs::read_dir(store_dir) else {+ return 0;+ };+ let mut removed = 0usize;+ for e in entries.flatten() {+ let name = e.file_name().to_string_lossy().to_string();+ let Some(rest) = name.strip_prefix(&format!("{slug}.")) else {+ continue;+ };+ let Some(key) = rest.strip_suffix(".json") else {+ continue;+ };+ if live_keys.contains(key) {+ continue;+ }+ if std::fs::remove_file(e.path()).is_ok() {+ removed += 1;+ }+ }+ removed+}++#[cfg(test)]+mod tests {+ use super::*;+ use serde_json::json;++ fn env(id: &str, src: &str) -> Map<String, Value> {+ let mut m = Map::new();+ m.insert("id".into(), json!(id));+ m.insert("_src".into(), json!(src));+ m+ }++ #[test]+ fn same_content_same_key_regardless_of_key_order() {+ let mut a = Map::new();+ a.insert("id".into(), json!("x"));+ a.insert("zz".into(), json!(1));+ let mut b = Map::new();+ b.insert("zz".into(), json!(1));+ b.insert("id".into(), json!("x"));+ assert_eq!(key_of(&a), key_of(&b));+ assert_eq!(canonical_json(&a), canonical_json(&b));+ }++ #[test]+ fn put_is_idempotent_and_read_round_trips() {+ let tmp = tempfile::tempdir().unwrap();+ let dir = tmp.path().join("store");+ let (k1, new1) = put(&dir, "a", &env("x", "a")).unwrap();+ let (_, new2) = put(&dir, "a", &env("x", "a")).unwrap();+ assert!(new1);+ assert!(!new2, "second put of identical content skips");+ let all = read_all(&dir, &HashSet::from(["a".to_string()]));+ assert_eq!(all.len(), 1);+ assert_eq!(all[0]["id"], json!("x"));+ assert_eq!(k1.len(), 32);+ }++ #[test]+ fn different_source_is_a_different_envelope() {+ // identical record text from two sources is two evidence rows, not one+ let tmp = tempfile::tempdir().unwrap();+ let dir = tmp.path().join("store");+ put(&dir, "a", &env("x", "a")).unwrap();+ put(&dir, "b", &env("x", "b")).unwrap();+ let all = read_all(&dir, &HashSet::from(["a".to_string(), "b".to_string()]));+ assert_eq!(all.len(), 2);+ // retired source stays on disk but is not returned+ let only_a = read_all(&dir, &HashSet::from(["a".to_string()]));+ assert_eq!(only_a.len(), 1);+ assert_eq!(only_a[0]["_src"], json!("a"));+ }++ #[test]+ fn prune_drops_only_dead_keys_of_its_slug() {+ let tmp = tempfile::tempdir().unwrap();+ let dir = tmp.path().join("store");+ let (ka, _) = put(&dir, "a", &env("x", "a")).unwrap();+ let (kb, _) = put(&dir, "a", &env("y", "a")).unwrap();+ let (_kc, _) = put(&dir, "b", &env("z", "b")).unwrap();+ let removed = prune(&dir, "a", &HashSet::from([ka.clone(), kb]));+ assert_eq!(removed, 0);+ let removed = prune(&dir, "a", &HashSet::from([ka]));+ assert_eq!(removed, 1, "y died, x lived, b untouched");+ assert_eq!(+ read_all(&dir, &HashSet::from(["a".into(), "b".into()])).len(),+ 2+ );+ }+}diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 81c890934..de5ff3e25 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -20,6 +20,9 @@ pub mod gate;mod generation_catalog;#[cfg(test)]mod generation_catalog_http_tests;+/// `xerj corpus build` — harvested-record corpus packs (recipe → store →+/// identity → merge → deterministic pack).+pub mod harvest;pub mod ids;pub mod ignore_rules;pub mod infer;diff --git a/engine/crates/xerj-autoindex/src/xc.rs b/engine/crates/xerj-autoindex/src/xc.rsindex c8eaf6fd0..17be15f4b 100644--- a/engine/crates/xerj-autoindex/src/xc.rs+++ b/engine/crates/xerj-autoindex/src/xc.rs@@ -225,13 +225,16 @@ pub fn run_code_cli(args: &[String]) -> i32 {const CORPUS_USAGE: &str ="usage: xerj corpus add <name> <git-url>... | --from <manifest.json> [--as <name>]+ xerj corpus build <name> [--recipe <path>] [--fresh]xerj corpus index <name> [--fresh] [--url URL]xerj corpus list [--url URL]clone the repos, detect licences, write corpora/<name>/corpus.json;--from takes the corpus name from the manifest's 'corpus' field unless-<name> or --as overrides it; index builds/verifies/switches the corpus-(exit 3 skips junk — normal); list shows what is loaded on this node.";+<name> or --as overrides it; build harvests recipe sources into a+deterministic pack under builds/<name>/ (see tools/xerj-code/);+index builds/verifies/switches the corpus (exit 3 skips junk — normal);+list shows what is loaded on this node.";fn git(dir: Option<&Path>, args: &[&str]) -> Result<(i32, String)> {let mut c = Command::new("git");@@ -1206,7 +1209,7 @@ fn run_corpus_list(args: &[String]) -> i32 {// ── dispatch ────────────────────────────────────────────────────────────────-/// `xerj corpus <add|index|list> …` — returns the process exit code.+/// `xerj corpus <add|build|index|list> …` — returns the process exit code.pub fn run_corpus_cli(args: &[String]) -> i32 {let Some(sub) = args.first() else {eprintln!("{CORPUS_USAGE}");@@ -1215,6 +1218,7 @@ pub fn run_corpus_cli(args: &[String]) -> i32 {let rest: Vec<String> = args[1..].to_vec();match sub.as_str() {"add" => run_corpus_add(&rest),+ "build" => crate::harvest::run_build(&rest),"index" => run_corpus_index(&rest),"list" => run_corpus_list(&rest),"-h" | "--help" | "help" => {