Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(autoindex): xerj corpus build — recipe-driven harvested-record packs (M1)

xerj-orgbe1d4841b810
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index 1c97eeb57..c54bca2bf 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -7322,6 +7322,7 @@ dependencies = [
"streaming-iterator",
"tempfile",
"tokio",
+ "toml",
"tree-sitter",
"tree-sitter-bash",
"tree-sitter-c",
diff --git a/engine/crates/xerj-autoindex/Cargo.toml b/engine/crates/xerj-autoindex/Cargo.toml
index 1385a5483..cf9de8042 100644
--- a/engine/crates/xerj-autoindex/Cargo.toml
+++ b/engine/crates/xerj-autoindex/Cargo.toml
@@ -18,6 +18,10 @@ serde_json.workspace = true
simd-json.workspace = true
chrono.workspace = true
regex = "1"
+# Corpus recipes (`xerj corpus build`) are authored TOML: human-readable and
+# commented, so licence reasoning sits next to the source it justifies and
+# ships verbatim inside the pack as provenance.
+toml.workspace = true
walkdir = "2"
# gitignore/.xerjignore matching (#276). ripgrep's matcher, used for its
# semantics only — traversal stays on walkdir so the existing symlink,
diff --git a/engine/crates/xerj-autoindex/src/harvest/identity.rs b/engine/crates/xerj-autoindex/src/harvest/identity.rs
new file mode 100644
index 000000000..2b2e07fce
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/harvest/identity.rs
@@ -0,0 +1,686 @@
+//! Identity resolution: turn N source records into unique logical records.
+//!
+//! Union-find over identity-edge VALUES: every value a recipe declares as
+//! an edge (`id`, each element of `aliases`, …) is a node, and a record
+//! unions with each of its own values. Two records sharing ANY value merge,
+//! transitively and cyclically — exactly the alias closure the sources
+//! themselves do not provide (OSV ships one record per source ID, linked
+//! only by `aliases`; it does not merge them for you).
+//!
+//! Derived relations (CVE↔commit, say) are deliberately NOT edges: a single
+//! commit fixing two CVEs would fuse two advisories that are distinct
+//! findings. Identity is what the sources SAY is the same thing, never what
+//! we infer from shared references.
+//!
+//! Merge is first-non-empty-wins under the recipe's precedence, with arrays
+//! unioned — deterministic given the member order, which is itself
+//! (precedence rank, store key), never filesystem accident.
+
+use std::collections::HashMap;
+
+use serde_json::{Map, Value};
+
+use super::recipe::{
+ slug_rank, Derived, DerivedOp, Envelope, Identity, Merge, LICENCE, SRC, SRC_PATH, SRC_URL,
+};
+
+/// The store-key sentinel `store::read_all` injects. Sorting by it keeps
+/// member order deterministic when two members share a source rank.
+pub(crate) const KEY: &str = "\u{0}key";
+
+/// One cluster of envelopes that are the same logical record.
+pub struct Cluster {
+ /// Indices into the envelope slice, in canonical order.
+ pub members: Vec<usize>,
+}
+
+/// Group envelopes into clusters. Deterministic: iteration follows the
+/// envelope order, and cluster output order follows each cluster's minimum
+/// member index.
+pub fn cluster(envelopes: &[Map<String, Value>], identity: &Identity) -> Vec<Cluster> {
+ // DSU over [0..envelopes) ∪ value nodes (mapped to >= envelopes). Value
+ // nodes always lose unions (records have smaller indices), so every
+ // root stays below envelopes.len().
+ let mut parent: Vec<usize> = (0..envelopes.len()).collect();
+ let mut value_nodes: HashMap<String, usize> = HashMap::new();
+
+ fn find(parent: &mut [usize], mut x: usize) -> usize {
+ while parent[x] != x {
+ parent[x] = parent[parent[x]]; // path halving
+ x = parent[x];
+ }
+ x
+ }
+
+ for (i, env) in envelopes.iter().enumerate() {
+ for value in identity_values(env, &identity.edges) {
+ let node = *value_nodes.entry(value).or_insert_with(|| {
+ parent.push(parent.len());
+ parent.len() - 1
+ });
+ let (ra, rb) = (find(&mut parent, i), find(&mut parent, node));
+ if ra != rb {
+ let (lo, hi) = if ra < rb { (ra, rb) } else { (rb, ra) };
+ parent[hi] = lo;
+ }
+ }
+ }
+
+ let mut by_root: Vec<Vec<usize>> = vec![Vec::new(); envelopes.len()];
+ for i in 0..envelopes.len() {
+ by_root[find(&mut parent, i)].push(i);
+ }
+ by_root
+ .into_iter()
+ .filter(|m| !m.is_empty())
+ .map(|mut members| {
+ members.sort_by_key(|&i| order_key(&envelopes[i], &identity.canonical_source_order));
+ Cluster { members }
+ })
+ .collect()
+}
+
+/// The identity-edge values of one envelope. `each = true` explodes string
+/// arrays element-wise; a plain string field is one value; empty strings
+/// and other types contribute nothing.
+fn identity_values(env: &Map<String, Value>, edges: &[super::recipe::IdentityEdge]) -> Vec<String> {
+ let mut out = Vec::new();
+ for e in edges {
+ match env.get(&e.field) {
+ Some(Value::String(s)) if !s.is_empty() => out.push(s.clone()),
+ Some(Value::Array(a)) if e.each => {
+ for v in a {
+ if let Some(s) = v.as_str() {
+ if !s.is_empty() {
+ out.push(s.to_string());
+ }
+ }
+ }
+ }
+ // each=false on an array: the array as a whole is not an
+ // identity — arrays vary by source even for the same record.
+ _ => {}
+ }
+ }
+ out
+}
+
+/// Sort key for members: the order list's rank first, then the envelope's
+/// store key — stability that does not depend on directory iteration order.
+fn order_key(env: &Map<String, Value>, order: &[String]) -> (usize, String) {
+ let slug = env.get(SRC).and_then(Value::as_str).unwrap_or("");
+ let key = env.get(KEY).and_then(Value::as_str).unwrap_or("");
+ (slug_rank(order, slug), key.to_string())
+}
+
+/// Which member "wins" a scalar: the first (by merge order) carrying a
+/// non-empty value.
+fn first_str<'a>(members: &[&'a Map<String, Value>], key: &str) -> Option<&'a str> {
+ members
+ .iter()
+ .filter_map(|m| m.get(key).and_then(Value::as_str))
+ .find(|s| !s.is_empty())
+}
+
+/// Merge one passthrough key: an ARRAY in any member makes the result an
+/// array (union across members, deduped, scalars folded in as elements —
+/// sources disagree on cardinality more often than they agree); scalars
+/// everywhere keep it a scalar, first non-empty under precedence.
+fn merge_field(ranked: &[&Map<String, Value>], key: &str) -> Option<Value> {
+ let any_array = ranked
+ .iter()
+ .any(|m| matches!(m.get(key), Some(Value::Array(_))));
+ if any_array {
+ return union_arrays(ranked, key).map(Value::Array);
+ }
+ if let Some(s) = first_str(ranked, key) {
+ return Some(Value::String(s.to_string()));
+ }
+ ranked
+ .iter()
+ .find_map(|m| m.get(key).filter(|v| !v.is_null() && !v.is_string()))
+ .cloned()
+}
+
+/// Union arrays across members preserving member order, deduped. A lone
+/// string folds in as a single element.
+fn union_arrays(members: &[&Map<String, Value>], key: &str) -> Option<Vec<Value>> {
+ let mut out: Vec<Value> = Vec::new();
+ for m in members {
+ match m.get(key) {
+ Some(Value::Array(a)) => {
+ for v in a {
+ if !out.contains(v) {
+ out.push(v.clone());
+ }
+ }
+ }
+ Some(Value::String(s)) if !s.is_empty() => {
+ let v = Value::String(s.clone());
+ if !out.contains(&v) {
+ out.push(v);
+ }
+ }
+ _ => {}
+ }
+ }
+ (!out.is_empty()).then_some(out)
+}
+
+/// The most-restrictive licence across members. A merged record
+/// redistributes text from EVERY member, so a restricted member must be
+/// able to veto the permissive ones — `is_restricted` is binary (the
+/// licence tuple has no total order), so: any restricted member's licence
+/// wins (precedence-first among them); otherwise precedence-first
+/// non-empty. Attribution-grade obligations the binary predicate cannot
+/// rank (CC-BY-4.0 vs CC0) stay visible per record through `sources[]`
+/// and per pack through the manifest's source list — the `licence` field
+/// is the redistribution-warning signal, not a legal settlement.
+fn most_restrictive(members: &[&Map<String, Value>]) -> String {
+ let mut first_non_empty = String::new();
+ for m in members {
+ let lic = m.get(LICENCE).and_then(Value::as_str).unwrap_or("");
+ if lic.is_empty() {
+ continue;
+ }
+ if xerj_common::xccode::licence::is_restricted(lic) {
+ return lic.to_string();
+ }
+ if first_non_empty.is_empty() {
+ first_non_empty = lic.to_string();
+ }
+ }
+ first_non_empty
+}
+
+/// Merge one cluster into a single output record (envelope fields, then
+/// passthrough, then derived). `order` is the merge precedence.
+pub fn merge_cluster(
+ cluster: &[&Map<String, Value>],
+ order: &[String],
+ envelope: &Envelope,
+ merge: &Merge,
+ derived: &[Derived],
+) -> Map<String, Value> {
+ let mut members: Vec<&Map<String, Value>> = cluster.to_vec();
+ members.sort_by_key(|m| order_key(m, order));
+ let winner = members.first().copied();
+
+ let mut out = Map::new();
+
+ // — envelope fields ────────────────────────────────────────────────────
+ let id = first_of(&members, &envelope.id_from);
+ out.insert("id".into(), Value::String(id.unwrap_or_default()));
+ let title = first_of(&members, &envelope.title_from);
+ out.insert("title".into(), Value::String(title.unwrap_or_default()));
+
+ // body: per member, join its body_join fields; the first non-empty
+ // joined body wins (joining ACROSS sources would duplicate the same
+ // advisory prose written twice).
+ let mut body = String::new();
+ for m in &members {
+ let parts: Vec<&str> = envelope
+ .body_join
+ .iter()
+ .filter_map(|k| m.get(k).and_then(Value::as_str).filter(|s| !s.is_empty()))
+ .collect();
+ if !parts.is_empty() {
+ body = parts.join("\n\n");
+ break;
+ }
+ }
+ out.insert("body".into(), Value::String(body));
+
+ // defs: union of the defs_from arrays across ALL members, space-joined —
+ // this is the field `xerj code` matches definition queries against.
+ let mut defs: Vec<String> = Vec::new();
+ for m in &members {
+ for k in &envelope.defs_from {
+ if let Some(Value::Array(a)) = m.get(k) {
+ for v in a {
+ if let Some(s) = v.as_str() {
+ if !s.is_empty() && !defs.iter().any(|d| d == s) {
+ defs.push(s.to_string());
+ }
+ }
+ }
+ }
+ }
+ }
+ out.insert("defs".into(), Value::String(defs.join(" ")));
+
+ out.insert("licence".into(), Value::String(most_restrictive(&members)));
+
+ // sources: contributing slugs in precedence order; `source` names the
+ // winner, `source_url`/`origin` point back into it.
+ let mut slugs: Vec<Value> = Vec::new();
+ for m in &members {
+ if let Some(s) = m.get(SRC).and_then(Value::as_str) {
+ let v = Value::String(s.to_string());
+ if !slugs.contains(&v) {
+ slugs.push(v);
+ }
+ }
+ }
+ out.insert("sources".into(), Value::Array(slugs));
+ let get = |k: &str| {
+ winner
+ .and_then(|m| m.get(k))
+ .and_then(Value::as_str)
+ .unwrap_or_default()
+ .to_string()
+ };
+ out.insert("source".into(), Value::String(get(SRC)));
+ out.insert("source_url".into(), Value::String(get(SRC_URL)));
+ out.insert("origin".into(), Value::String(get(SRC_PATH)));
+
+ // modified: max member value (RFC3339 UTC strings compare correctly).
+ let modified = members
+ .iter()
+ .filter_map(|m| m.get("modified").and_then(Value::as_str))
+ .filter(|s| !s.is_empty())
+ .max()
+ .unwrap_or("")
+ .to_string();
+ out.insert("modified".into(), Value::String(modified));
+
+ // — passthrough ───────────────────────────────────────────────────────
+ // Every data key any member carries, minus provenance sentinels and
+ // what the envelope already fixed. Scalars are first-non-empty under
+ // the field's precedence (global, or per-field when overridden); arrays
+ // union across members regardless of precedence — evidence accumulates.
+ if envelope.passthrough {
+ let mut keys: Vec<&String> = members
+ .iter()
+ .flat_map(|m| m.keys())
+ .filter(|k| {
+ !k.starts_with('\u{0}') && !k.starts_with('_') && !out.contains_key(k.as_str())
+ })
+ .collect();
+ keys.sort();
+ keys.dedup();
+ for k in keys {
+ let order = merge
+ .field_precedence
+ .get(k)
+ .map(Vec::as_slice)
+ .unwrap_or(order);
+ let mut ranked = members.clone();
+ ranked.sort_by_key(|m| order_key(m, order));
+ if let Some(v) = merge_field(&ranked, k) {
+ out.insert(k.clone(), v);
+ }
+ }
+ }
+
+ // — derived fields ────────────────────────────────────────────────────
+ for d in derived {
+ let v = apply_derived(&out, &d.op);
+ out.insert(d.name.clone(), v);
+ }
+
+ out
+}
+
+/// First non-empty string among any of the listed keys, scanning members in
+/// order. The key list wins over member order: `title_from = ["summary",
+/// "id"]` prefers a summary from ANY source over an id from the top source
+/// — the human-readable name wherever it exists.
+fn first_of(members: &[&Map<String, Value>], keys: &[String]) -> Option<String> {
+ for k in keys {
+ if let Some(s) = first_str(members, k) {
+ return Some(s.to_string());
+ }
+ }
+ None
+}
+
+fn apply_derived(rec: &Map<String, Value>, op: &DerivedOp) -> Value {
+ match op {
+ DerivedOp::RegexExtract { from, re, unique } => {
+ let mut out: Vec<Value> = Vec::new();
+ for k in from {
+ let texts: Vec<String> = match rec.get(k) {
+ Some(Value::String(s)) => vec![s.clone()],
+ Some(Value::Array(a)) => a
+ .iter()
+ .filter_map(|v| v.as_str().map(str::to_string))
+ .collect(),
+ _ => Vec::new(),
+ };
+ for t in texts {
+ for m in re.find_iter(&t) {
+ let v = Value::String(m.as_str().to_string());
+ if !*unique || !out.contains(&v) {
+ out.push(v);
+ }
+ }
+ }
+ }
+ Value::Array(out)
+ }
+ DerivedOp::Map {
+ from,
+ rules,
+ default,
+ } => {
+ let text = rec.get(from).and_then(Value::as_str).unwrap_or("");
+ for (re, value) in rules {
+ if re.is_match(text) {
+ return Value::String(value.clone());
+ }
+ }
+ Value::String(default.clone())
+ }
+ DerivedOp::Present { from } => {
+ let present = match rec.get(from) {
+ None | Some(Value::Null) => false,
+ Some(Value::String(s)) => !s.is_empty(),
+ Some(_) => true,
+ };
+ Value::Bool(present)
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::harvest::recipe::IdentityEdge;
+ use regex::Regex;
+ use serde_json::json;
+
+ fn env(src: &str, fields: &[(&str, Value)]) -> Map<String, Value> {
+ let mut m = Map::new();
+ m.insert(SRC.into(), json!(src));
+ m.insert(SRC_URL.into(), json!(format!("https://example/{src}")));
+ m.insert(SRC_PATH.into(), json!(format!("{src}/record.json")));
+ m.insert(LICENCE.into(), json!("CC0-1.0"));
+ m.insert(KEY.into(), json!(format!("{src}-key")));
+ for (k, v) in fields {
+ m.insert(k.to_string(), v.clone());
+ }
+ m
+ }
+
+ fn edges(spec: &[(&str, bool)]) -> Vec<IdentityEdge> {
+ spec.iter()
+ .map(|(f, e)| IdentityEdge {
+ field: f.to_string(),
+ each: *e,
+ })
+ .collect()
+ }
+
+ fn identity_of(edges: Vec<IdentityEdge>) -> Identity {
+ Identity {
+ edges,
+ canonical_source_order: vec![],
+ }
+ }
+
+ #[test]
+ fn alias_chain_and_cycle_collapse_to_one_cluster() {
+ // rustsec --CVE--> ghsa and a direct alias edge, plus a cycle
+ let a = env(
+ "rustsec",
+ &[("id", json!("R1")), ("aliases", json!(["C1"]))],
+ );
+ let b = env(
+ "ghsa",
+ &[("id", json!("G1")), ("aliases", json!(["C1", "R1"]))],
+ );
+ let c = env(
+ "osv",
+ &[("id", json!("C1")), ("aliases", json!(["R1", "G1"]))],
+ );
+ let identity = identity_of(edges(&[("id", false), ("aliases", true)]));
+ let clusters = cluster(&[a, b, c], &identity);
+ assert_eq!(clusters.len(), 1, "all three are one logical record");
+ assert_eq!(clusters[0].members.len(), 3);
+ }
+
+ #[test]
+ fn disjoint_records_stay_apart() {
+ let a = env("a", &[("id", json!("X1"))]);
+ let b = env("a", &[("id", json!("X2"))]);
+ let clusters = cluster(&[a, b], &identity_of(edges(&[("id", false)])));
+ assert_eq!(clusters.len(), 2);
+ }
+
+ #[test]
+ fn shared_commit_is_not_an_identity() {
+ // two CVEs referencing the same fixing commit must NOT merge
+ let a = env(
+ "a",
+ &[
+ ("id", json!("CVE-1")),
+ ("commit_urls", json!(["https://x/commit/deadbeef"])),
+ ],
+ );
+ let b = env(
+ "a",
+ &[
+ ("id", json!("CVE-2")),
+ ("commit_urls", json!(["https://x/commit/deadbeef"])),
+ ],
+ );
+ let clusters = cluster(&[a, b], &identity_of(edges(&[("id", false)])));
+ assert_eq!(
+ clusters.len(),
+ 2,
+ "commits are derived edges, never identity"
+ );
+ }
+
+ fn envelope() -> Envelope {
+ Envelope {
+ id_from: vec!["id".into()],
+ title_from: vec!["summary".into(), "id".into()],
+ body_join: vec!["summary".into(), "details".into()],
+ defs_from: vec!["affected_functions".into()],
+ passthrough: true,
+ }
+ }
+
+ #[test]
+ fn merge_prefers_precedence_and_unions_arrays() {
+ let r = env(
+ "rustsec",
+ &[
+ ("id", json!("R1")),
+ ("summary", json!("unsound insert_many")),
+ ("details", json!("root cause prose")),
+ ("aliases", json!(["CVE-1"])),
+ (
+ "affected_functions",
+ json!(["smallvec::SmallVec::insert_many"]),
+ ),
+ ("fixed_versions", json!(["0.6.14"])),
+ ],
+ );
+ let g = env(
+ "ghsa",
+ &[
+ ("id", json!("G1")),
+ ("aliases", json!(["CVE-1", "R1"])),
+ ("fixed_versions", json!(["1.6.1", "0.6.14"])),
+ ("severity", json!("HIGH")),
+ ],
+ );
+ // g is CC-BY: non-restricted, so precedence-first CC0 wins the FIELD
+ // (attribution still rides via sources[]/manifest)…
+ let mut g = g;
+ g.insert(LICENCE.into(), json!("CC-BY-4.0"));
+ let merged = merge_cluster(
+ &[&r, &g],
+ &["rustsec".to_string(), "ghsa".to_string()],
+ &envelope(),
+ &Merge::default(),
+ &[],
+ );
+ assert_eq!(
+ merged["id"],
+ json!("R1"),
+ "precedence-first member names the id"
+ );
+ assert_eq!(merged["title"], json!("unsound insert_many"));
+ assert!(merged["body"]
+ .as_str()
+ .unwrap()
+ .contains("root cause prose"));
+ assert_eq!(
+ merged["defs"],
+ json!("smallvec::SmallVec::insert_many"),
+ "defs feeds xerj code's definition matching"
+ );
+ assert_eq!(merged["licence"], json!("CC0-1.0"));
+ // …but one restricted member vetoes every permissive one
+ let mut g2 = env(
+ "cisa",
+ &[("id", json!("C1")), ("aliases", json!(["CVE-1", "R1"]))],
+ );
+ g2.insert(LICENCE.into(), json!("GPL-3.0"));
+ let merged = merge_cluster(
+ &[&r, &g, &g2],
+ &[
+ "rustsec".to_string(),
+ "ghsa".to_string(),
+ "cisa".to_string(),
+ ],
+ &envelope(),
+ &Merge::default(),
+ &[],
+ );
+ assert_eq!(
+ merged["licence"],
+ json!("GPL-3.0"),
+ "restricted wins outright"
+ );
+ assert_eq!(merged["sources"], json!(["rustsec", "ghsa", "cisa"]));
+ assert_eq!(merged["source"], json!("rustsec"));
+ assert_eq!(merged["source_url"], json!("https://example/rustsec"));
+ assert_eq!(merged["origin"], json!("rustsec/record.json"));
+ assert_eq!(
+ merged["fixed_versions"],
+ json!(["0.6.14", "1.6.1"]),
+ "arrays union in member order, deduped"
+ );
+ assert_eq!(merged["aliases"], json!(["CVE-1", "R1"]));
+ assert_eq!(merged["severity"], json!("HIGH"), "only g has it");
+ }
+
+ #[test]
+ fn title_from_spans_sources_before_falling_back() {
+ // a summary in a LOWER precedence source beats the id of a higher one
+ let r = env("rustsec", &[("id", json!("R1"))]);
+ let g = env(
+ "ghsa",
+ &[("id", json!("G1")), ("summary", json!("the real title"))],
+ );
+ let merged = merge_cluster(
+ &[&r, &g],
+ &["rustsec".to_string(), "ghsa".to_string()],
+ &envelope(),
+ &Merge::default(),
+ &[],
+ );
+ assert_eq!(merged["title"], json!("the real title"));
+ }
+
+ #[test]
+ fn modified_takes_the_latest_member_value() {
+ let r = env(
+ "rustsec",
+ &[
+ ("id", json!("R1")),
+ ("modified", json!("2024-01-01T00:00:00Z")),
+ ],
+ );
+ let g = env(
+ "ghsa",
+ &[
+ ("id", json!("G1")),
+ ("modified", json!("2025-06-01T00:00:00Z")),
+ ],
+ );
+ let merged = merge_cluster(
+ &[&r, &g],
+ &["rustsec".to_string(), "ghsa".to_string()],
+ &envelope(),
+ &Merge::default(),
+ &[],
+ );
+ assert_eq!(merged["modified"], json!("2025-06-01T00:00:00Z"));
+ }
+
+ #[test]
+ fn derived_ops_run_over_the_merged_record() {
+ let r = env(
+ "rustsec",
+ &[
+ ("id", json!("RUSTSEC-2021-0003")),
+ ("aliases", json!(["CVE-2021-25900"])),
+ ("summary", json!("Unsound: insert_many")),
+ ("withdrawn", Value::Null),
+ (
+ "reference_urls",
+ json!([
+ "https://github.com/servo/rust-smallvec/commit/abc1234",
+ "https://github.com/servo/rust-smallvec/pull/9"
+ ]),
+ ),
+ ],
+ );
+ let cve = Regex::new(r"(?i)\bCVE-\d{4}-\d{4,}\b").unwrap();
+ let commit =
+ Regex::new(r"https://github\.com/[^/\s]+/[^/\s]+/commit/[0-9a-f]{7,40}").unwrap();
+ let unsound = Regex::new(r"(?i)\bunsound\b").unwrap();
+ let merged = merge_cluster(
+ &[&r],
+ &["rustsec".to_string()],
+ &envelope(),
+ &Merge::default(),
+ &[
+ Derived {
+ name: "cve_ids".into(),
+ op: DerivedOp::RegexExtract {
+ from: vec!["id".into(), "aliases".into()],
+ re: cve,
+ unique: true,
+ },
+ },
+ Derived {
+ name: "commit_urls".into(),
+ op: DerivedOp::RegexExtract {
+ from: vec!["reference_urls".into()],
+ re: commit,
+ unique: true,
+ },
+ },
+ Derived {
+ name: "record_kind".into(),
+ op: DerivedOp::Map {
+ from: "title".into(),
+ rules: vec![(unsound, "informational".into())],
+ default: "vulnerability".into(),
+ },
+ },
+ Derived {
+ name: "is_withdrawn".into(),
+ op: DerivedOp::Present {
+ from: "withdrawn".into(),
+ },
+ },
+ ],
+ );
+ assert_eq!(merged["cve_ids"], json!(["CVE-2021-25900"]));
+ assert_eq!(
+ merged["commit_urls"],
+ json!(["https://github.com/servo/rust-smallvec/commit/abc1234"]),
+ "the PR link is not a commit"
+ );
+ assert_eq!(merged["record_kind"], json!("informational"));
+ assert_eq!(merged["is_withdrawn"], json!(false), "null is not present");
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/harvest/mod.rs b/engine/crates/xerj-autoindex/src/harvest/mod.rs
new file mode 100644
index 000000000..b00936c52
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/harvest/mod.rs
@@ -0,0 +1,600 @@
+//! `xerj corpus build` — build a portable corpus pack from harvested records.
+//!
+//! The pipeline, end to end:
+//!
+//! ```text
+//! recipe.toml ─→ harvest (dir walk / M2: http-zip, git)
+//! ─→ normalize (format adapter: osv | flat)
+//! ─→ store (content-addressed, presence = dedup)
+//! ─→ identity (union-find over declared edge values)
+//! ─→ merge (precedence + array union + strictest licence)
+//! ─→ pack (deterministic sharded JSONL + manifest + SUMS)
+//! ```
+//!
+//! Incrementality is not a mode, it is the storage layout: every normalized
+//! record lands at `store/<slug>.<xxh3-of-canonical-json>.json`, so a re-run
+//! recomputes the key, sees the file, and skips. Deleting the source tree's
+//! unchanged files and re-running is a no-op beyond the walk; the pack is
+//! then rebuilt from the store deterministically.
+//!
+//! The tool is domain-agnostic by directive: what a "record" is, which
+//! fields are identity, and how fields merge all live in the recipe, never
+//! here. Licence terms ride along as record data (`licence`, per-source
+//! `sources[]`); the tool does not and must not enforce them — the pack
+//! author owns what goes in the pack.
+
+pub mod identity;
+pub mod normalize;
+pub mod pack;
+pub mod recipe;
+pub mod store;
+
+pub(crate) use store::MAX_FIELDS;
+
+use std::collections::HashSet;
+use std::path::{Path, PathBuf};
+
+use anyhow::{bail, Context, Result};
+use serde_json::{Map, Value};
+
+use crate::xc::code_root;
+use pack::{RunStats, SourceStat};
+use recipe::{SourceKind, LICENCE, SRC, SRC_PATH, SRC_URL};
+
+const BUILD_USAGE: &str = "usage: xerj corpus build <name> [--recipe <path>] [--fresh]
+
+build (or incrementally refresh) a harvested corpus pack:
+ 1. load the recipe (default $XERJ_CODE_HOME/recipes/<name>.toml)
+ 2. harvest every declared source into the content store
+ (already-stored records are skipped — that is the point)
+ 3. resolve identity, merge duplicates, emit a deterministic pack
+ under $XERJ_CODE_HOME/builds/<name>/pack/<name>/
+
+--fresh discards the build state first (full re-harvest).";
+
+/// Entry point from the corpus dispatcher. 0 = pack built, 1 = failure,
+/// 2 = usage error.
+pub fn run_build(args: &[String]) -> i32 {
+ match run_build_inner(args, &code_root()) {
+ Ok(code) => code,
+ Err(e) => {
+ eprintln!("xerj corpus build: {e:#}");
+ 1
+ }
+ }
+}
+
+fn run_build_inner(args: &[String], home: &Path) -> Result<i32> {
+ let mut name: Option<String> = None;
+ let mut recipe_path: Option<PathBuf> = None;
+ let mut fresh = false;
+ let mut it = args.iter();
+ while let Some(a) = it.next() {
+ match a.as_str() {
+ "--recipe" => {
+ recipe_path = Some(PathBuf::from(it.next().context("--recipe needs a path")?));
+ }
+ "--fresh" => fresh = true,
+ "-h" | "--help" | "help" => {
+ println!("{BUILD_USAGE}");
+ return Ok(0);
+ }
+ s if s.starts_with('-') => bail!("unknown flag '{s}'\n\n{BUILD_USAGE}"),
+ s => {
+ if name.replace(s.to_string()).is_some() {
+ bail!("unexpected second name '{s}'\n\n{BUILD_USAGE}");
+ }
+ }
+ }
+ }
+ let Some(name) = name else {
+ eprintln!("{BUILD_USAGE}");
+ return Ok(2);
+ };
+
+ let recipe_file = match recipe_path {
+ Some(p) => p,
+ None => home.join("recipes").join(format!("{name}.toml")),
+ };
+ if !recipe_file.is_file() {
+ bail!(
+ "no recipe for '{name}' at {} (pass --recipe <path>, or create it there)",
+ recipe_file.display()
+ );
+ }
+ let recipe = recipe::load(&recipe_file)?;
+ if recipe.name != name {
+ bail!(
+ "recipe at {} builds corpus '{}', not '{}' — the name must match",
+ recipe_file.display(),
+ recipe.name,
+ name
+ );
+ }
+ let recipe_dir = recipe_file
+ .parent()
+ .map(Path::to_path_buf)
+ .unwrap_or_else(|| PathBuf::from("."));
+ let recipe_toml = std::fs::read_to_string(&recipe_file)?;
+
+ let build_dir = home.join("builds").join(&recipe.name);
+ if fresh && build_dir.exists() {
+ std::fs::remove_dir_all(&build_dir)
+ .with_context(|| format!("--fresh: cannot clear {}", build_dir.display()))?;
+ }
+ let store_dir = build_dir.join("store");
+
+ // ── harvest into the content store ───────────────────────────────────
+ let mut per_source = Vec::new();
+ for src in &recipe.sources {
+ let SourceKind::Dir = src.kind;
+ let root = recipe_dir.join(src.path.as_deref().unwrap_or("."));
+ let files = walk(&root, &src.glob)?;
+ let mut stat = SourceStat {
+ slug: src.slug.clone(),
+ kind: "dir".to_string(),
+ path: src.path.clone(),
+ url: src.url.clone(),
+ licence: src.licence.clone(),
+ records: 0,
+ new: 0,
+ skipped: 0,
+ pruned: 0,
+ };
+ let mut keys = HashSet::new();
+ for (rel, path) in files {
+ for value in parse_file(&path)? {
+ let mut env = normalize::normalize(src.format, &value);
+ env.insert(SRC.into(), Value::String(src.slug.clone()));
+ env.insert(
+ SRC_URL.into(),
+ Value::String(src.url.clone().unwrap_or_default()),
+ );
+ env.insert(SRC_PATH.into(), Value::String(rel.clone()));
+ env.insert(LICENCE.into(), Value::String(src.licence.clone()));
+ let (key, was_new) = store::put(&store_dir, &src.slug, &env)?;
+ keys.insert(key);
+ stat.records += 1;
+ if was_new {
+ stat.new += 1;
+ } else {
+ stat.skipped += 1;
+ }
+ }
+ }
+ stat.pruned = store::prune(&store_dir, &src.slug, &keys);
+ per_source.push(stat);
+ }
+
+ // ── identity + merge over the FULL store (not just this run's files) ──
+ let slugs: HashSet<String> = recipe.sources.iter().map(|s| s.slug.clone()).collect();
+ let envelopes = store::read_all(&store_dir, &slugs);
+ let clusters = identity::cluster(&envelopes, &recipe.identity);
+ let merge_order = if recipe.merge.precedence.is_empty() {
+ recipe.identity.canonical_source_order.clone()
+ } else {
+ recipe.merge.precedence.clone()
+ };
+ let mut records = Vec::with_capacity(clusters.len());
+ for c in &clusters {
+ let members: Vec<&Map<String, Value>> = c.members.iter().map(|&i| &envelopes[i]).collect();
+ records.push(identity::merge_cluster(
+ &members,
+ &merge_order,
+ &recipe.envelope,
+ &recipe.merge,
+ &recipe.derived,
+ ));
+ }
+
+ // ── emit ──────────────────────────────────────────────────────────────
+ let stats = RunStats {
+ envelopes: envelopes.len(),
+ per_source,
+ };
+ let result = pack::emit(&build_dir, &recipe, &recipe_toml, records, &stats)?;
+
+ println!(
+ "built '{}' generation {}: {} envelopes → {} records, {} shard file(s)",
+ recipe.name, result.generation, stats.envelopes, result.records, result.shards
+ );
+ for s in &stats.per_source {
+ println!(
+ " {}: {} record(s) — {} new, {} skipped, {} pruned ({})",
+ s.slug, s.records, s.new, s.skipped, s.pruned, s.licence
+ );
+ }
+ println!("pack: {}", result.dir.display());
+ Ok(0)
+}
+
+// ── dir source ──────────────────────────────────────────────────────────────
+
+/// Recursively list files under `root` matching `glob`, as
+/// (posix-relative-path, absolute-path), sorted — directory iteration order
+/// must never reach the output.
+fn walk(root: &Path, glob: &str) -> Result<Vec<(String, PathBuf)>> {
+ let re = glob_regex(glob);
+ let mut out = Vec::new();
+ fn recurse(dir: &Path, root: &Path, re: &regex::Regex, out: &mut Vec<(String, PathBuf)>) {
+ let Ok(entries) = std::fs::read_dir(dir) else {
+ return;
+ };
+ for e in entries.flatten() {
+ let p = e.path();
+ if p.is_dir() {
+ recurse(&p, root, re, out);
+ } else {
+ let rel = p.strip_prefix(root).unwrap_or(&p);
+ let rel = rel.to_string_lossy().replace('\\', "/");
+ if re.is_match(&rel) {
+ out.push((rel, p));
+ }
+ }
+ }
+ }
+ recurse(root, root, &re, &mut out);
+ out.sort();
+ Ok(out)
+}
+
+/// Minimal glob → regex: `**` spans directories (`**/` also matches zero
+/// directories), `*` stays inside one segment, `?` is one char. Recipes
+/// need path matching, not the full glob grammar; if a recipe ever needs
+/// character classes we take a real glob crate rather than growing this.
+fn glob_regex(glob: &str) -> regex::Regex {
+ let mut re = String::from("^");
+ let mut chars = glob.chars().peekable();
+ while let Some(c) = chars.next() {
+ match c {
+ '*' => {
+ if chars.peek() == Some(&'*') {
+ chars.next();
+ if chars.peek() == Some(&'/') {
+ chars.next();
+ re.push_str("(.*/)?");
+ } else {
+ re.push_str(".*");
+ }
+ } else {
+ re.push_str("[^/]*");
+ }
+ }
+ '?' => re.push_str("[^/]"),
+ other => re.push_str(&regex::escape(&other.to_string())),
+ }
+ }
+ re.push('$');
+ regex::Regex::new(&re).expect("glob translation always yields a valid regex")
+}
+
+/// Parse one harvested file: `.json`/`.jsonl`/`.ndjson`. A `.json` array
+/// yields its elements; `.jsonl` yields one per non-empty line. Other
+/// extensions are skipped by the glob, not here — reaching this with a
+/// non-JSON file is a hard error, not a silent drop.
+fn parse_file(path: &Path) -> Result<Vec<Value>> {
+ let raw =
+ std::fs::read_to_string(path).with_context(|| format!("cannot read {}", path.display()))?;
+ let name = path
+ .file_name()
+ .and_then(|n| n.to_str())
+ .unwrap_or_default()
+ .to_lowercase();
+ let jsonl = name.ends_with(".jsonl") || name.ends_with(".ndjson");
+ if jsonl {
+ let mut out = Vec::new();
+ for (n, line) in raw.lines().enumerate() {
+ let line = line.trim();
+ if line.is_empty() {
+ continue;
+ }
+ let v: Value = serde_json::from_str(line)
+ .with_context(|| format!("{}:{}: not valid JSON", path.display(), n + 1))?;
+ out.push(v);
+ }
+ Ok(out)
+ } else {
+ let v: Value = serde_json::from_str(&raw)
+ .with_context(|| format!("{}: not valid JSON", path.display()))?;
+ Ok(match v {
+ Value::Array(a) => a,
+ other => vec![other],
+ })
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[test]
+ fn glob_matches_paths_like_a_recipe_expects() {
+ let re = glob_regex("**/*.json");
+ assert!(re.is_match("a.json"));
+ assert!(re.is_match("deep/nested/a.json"));
+ assert!(!re.is_match("a.jsonl"));
+ assert!(!re.is_match("dir/sub")); // '.json' must be a suffix
+
+ let re = glob_regex("osv/*.json");
+ assert!(re.is_match("osv/x.json"));
+ assert!(!re.is_match("osv/nested/x.json"), "* does not cross '/'");
+ assert!(!re.is_match("other/x.json"));
+
+ let re = glob_regex("**/GHSA-????.json");
+ assert!(re.is_match("GHSA-1234.json"));
+ assert!(!re.is_match("GHSA-12345.json"));
+ }
+
+ #[test]
+ fn walk_is_sorted_and_relative() {
+ let tmp = tempfile::tempdir().unwrap();
+ let root = tmp.path().join("src");
+ std::fs::create_dir_all(root.join("b")).unwrap();
+ std::fs::create_dir_all(root.join("a")).unwrap();
+ std::fs::write(root.join("b/second.json"), "{}").unwrap();
+ std::fs::write(root.join("a/first.json"), "{}").unwrap();
+ std::fs::write(root.join("a/note.txt"), "x").unwrap();
+ let files = walk(&root, "**/*.json").unwrap();
+ let rels: Vec<&str> = files.iter().map(|(r, _)| r.as_str()).collect();
+ assert_eq!(rels, vec!["a/first.json", "b/second.json"]);
+ }
+
+ #[test]
+ fn parse_file_understands_json_and_jsonl() {
+ let tmp = tempfile::tempdir().unwrap();
+ let a = tmp.path().join("a.json");
+ std::fs::write(&a, r#"[{"id":"x"},{"id":"y"}]"#).unwrap();
+ assert_eq!(parse_file(&a).unwrap().len(), 2);
+ let b = tmp.path().join("b.jsonl");
+ std::fs::write(&b, "{\"id\":\"z\"}\n\n{\"id\":\"w\"}\n").unwrap();
+ assert_eq!(parse_file(&b).unwrap().len(), 2, "blank lines skipped");
+ let c = tmp.path().join("c.json");
+ std::fs::write(&c, r#"{"id":"one"}"#).unwrap();
+ assert_eq!(parse_file(&c).unwrap().len(), 1);
+ }
+
+ // ── end to end: recipe → harvest → identity → pack, twice ─────────────
+
+ const E2E_RECIPE: &str = r#"
+[recipe]
+format = 1
+name = "e2e"
+description = "two sources, one alias-linked record"
+
+[[sources]]
+slug = "rustsec"
+kind = "dir"
+path = "data/rustsec"
+format = "osv"
+licence = "CC0-1.0"
+
+[[sources]]
+slug = "ghsa"
+kind = "dir"
+path = "data/ghsa"
+glob = "**/*.jsonl"
+format = "osv"
+licence = "CC-BY-4.0"
+
+[envelope]
+id_from = ["id"]
+title_from = ["summary", "id"]
+body_join = ["summary", "details"]
+defs_from = ["affected_functions"]
+
+[identity]
+edges = [{ field = "id" }, { field = "aliases", each = true }]
+canonical_source_order = ["rustsec", "ghsa"]
+
+[merge]
+precedence = ["rustsec", "ghsa"]
+
+[[derived]]
+name = "cve_ids"
+op = "regex_extract"
+from = ["id", "aliases"]
+pattern = '(?i)\bCVE-\d{4}-\d{4,}\b'
+unique = true
+
+[emit]
+shards = 4
+"#;
+
+ fn e2e_env(home: &Path) {
+ let root = home.join("recipes");
+ std::fs::create_dir_all(root.join("data/rustsec")).unwrap();
+ std::fs::create_dir_all(root.join("data/ghsa")).unwrap();
+ std::fs::write(root.join("e2e.toml"), E2E_RECIPE).unwrap();
+ std::fs::write(
+ root.join("data/rustsec/RUSTSEC-2021-0003.json"),
+ r#"{
+ "id": "RUSTSEC-2021-0003",
+ "aliases": ["CVE-2021-25900"],
+ "summary": "Out of bounds write in SmallVec::insert_many",
+ "details": "An issue was discovered that allows attackers...",
+ "modified": "2023-06-20T00:00:00Z",
+ "affected": [{
+ "package": {"ecosystem": "crates.io", "name": "smallvec"},
+ "ranges": [{"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "0.6.14"}]}]
+ }],
+ "ecosystem_specific": {"affects": {"functions": {"smallvec::SmallVec::insert_many": ["<0.6.14"]}}}
+ }"#,
+ )
+ .unwrap();
+ // second rustsec record with NO alias link: its own cluster
+ std::fs::write(
+ root.join("data/rustsec/RUSTSEC-2020-0001.json"),
+ r#"{"id": "RUSTSEC-2020-0001", "summary": "other issue"}"#,
+ )
+ .unwrap();
+ // same logical record, GHSA shape: split ranges, no functions
+ std::fs::write(
+ root.join("data/ghsa/reviewed.jsonl"),
+ concat!(
+ r#"{"id": "GHSA-43w2-9j62-hq99", "aliases": ["CVE-2021-25900"], "summary": "Out of bounds write in insert_many", "modified": "2025-01-02T00:00:00Z", "affected": [{"package": {"ecosystem": "crates.io", "name": "smallvec"}, "ranges": [{"type": "SEMVER", "events": [{"introduced": "1.0.0"}, {"fixed": "1.6.1"}]}]}]}"#,
+ "\n",
+ ),
+ )
+ .unwrap();
+ }
+
+ fn read_records(pack: &Path) -> Vec<Map<String, Value>> {
+ let manifest: Value =
+ serde_json::from_str(&std::fs::read_to_string(pack.join("manifest.json")).unwrap())
+ .unwrap();
+ let mut out = Vec::new();
+ for name in manifest["files"].as_object().unwrap().keys() {
+ for line in std::fs::read_to_string(pack.join(name)).unwrap().lines() {
+ out.push(serde_json::from_str(line).unwrap());
+ }
+ }
+ out.sort_by_key(|r: &Map<String, Value>| {
+ r.get("id")
+ .and_then(Value::as_str)
+ .unwrap_or("")
+ .to_string()
+ });
+ out
+ }
+
+ #[test]
+ fn build_end_to_end_and_incrementally() {
+ let home = tempfile::tempdir().unwrap();
+ e2e_env(home.path());
+
+ let args: Vec<String> = vec!["e2e".to_string()];
+ assert_eq!(run_build_inner(&args, home.path()).unwrap(), 0);
+
+ let pack = home.path().join("builds/e2e/pack/e2e");
+ let records = read_records(&pack);
+ assert_eq!(
+ records.len(),
+ 2,
+ "GHSA+RUSTSEC merge via CVE alias; RUSTSEC-2020-0001 alone"
+ );
+
+ let merged = records
+ .iter()
+ .find(|r| r["id"] == json!("RUSTSEC-2021-0003"))
+ .expect("precedence-first source names the id");
+ assert_eq!(merged["sources"], json!(["rustsec", "ghsa"]));
+ assert_eq!(merged["source"], json!("rustsec"));
+ assert_eq!(
+ merged["title"],
+ json!("Out of bounds write in SmallVec::insert_many")
+ );
+ assert_eq!(
+ merged["defs"],
+ json!("smallvec::SmallVec::insert_many"),
+ "rustsec-only field still feeds xerj code"
+ );
+ assert_eq!(
+ merged["fixed_versions"],
+ json!(["0.6.14", "1.6.1"]),
+ "split GHSA range unions with the packed RustSec one"
+ );
+ assert_eq!(
+ merged["modified"],
+ json!("2025-01-02T00:00:00Z"),
+ "max across members"
+ );
+ assert_eq!(
+ merged["licence"],
+ json!("CC0-1.0"),
+ "neither is restricted; precedence first"
+ );
+ assert_eq!(
+ merged["cve_ids"],
+ json!(["CVE-2021-25900"]),
+ "derived over the merged record"
+ );
+
+ // uniform keys: the GHSA-less record still carries every key
+ let lone = records
+ .iter()
+ .find(|r| r["id"] == json!("RUSTSEC-2020-0001"))
+ .unwrap();
+ assert_eq!(lone["sources"], json!(["rustsec"]));
+ assert_eq!(lone["fixed_versions"], json!([]), "array key fills empty");
+ assert_eq!(lone["cve_ids"], json!([]));
+ assert_eq!(lone["affected_functions"], json!([]));
+ assert_eq!(lone["modified"], json!(""));
+
+ // manifest structure + checksums
+ let manifest: Value =
+ serde_json::from_str(&std::fs::read_to_string(pack.join("manifest.json")).unwrap())
+ .unwrap();
+ assert_eq!(manifest["format_version"], json!(1));
+ assert_eq!(manifest["kind"], json!("harvested"));
+ assert_eq!(manifest["generation"], json!(1));
+ assert_eq!(manifest["counts"]["records"], json!(2));
+ assert_eq!(
+ manifest["description"],
+ json!("two sources, one alias-linked record")
+ );
+ let rs = &manifest["sources"][0];
+ assert_eq!(rs["slug"], json!("rustsec"));
+ assert_eq!(rs["new"], json!(2));
+ assert_eq!(rs["skipped"], json!(0));
+ assert!(manifest["recipe_sha256"]
+ .as_str()
+ .is_some_and(|s| s.len() == 64));
+ assert!(pack.join("recipe.toml").is_file(), "recipe ships verbatim");
+
+ // ── second run: nothing new, shards byte-identical, generation up ──
+ let before: Vec<(String, Vec<u8>)> = std::fs::read_dir(&pack)
+ .unwrap()
+ .flatten()
+ .map(|e| {
+ (
+ e.file_name().to_string_lossy().to_string(),
+ std::fs::read(e.path()).unwrap(),
+ )
+ })
+ .filter(|(n, _)| n.starts_with("records-"))
+ .collect();
+ assert_eq!(run_build_inner(&args, home.path()).unwrap(), 0);
+ let manifest: Value =
+ serde_json::from_str(&std::fs::read_to_string(pack.join("manifest.json")).unwrap())
+ .unwrap();
+ assert_eq!(
+ manifest["generation"],
+ json!(2),
+ "rebuild advances generation"
+ );
+ for s in manifest["sources"].as_array().unwrap() {
+ assert_eq!(
+ s["new"],
+ json!(0),
+ "content-addressed store skipped everything"
+ );
+ assert_eq!(s["skipped"], json!(s["records"]));
+ assert_eq!(s["pruned"], json!(0));
+ }
+ for (name, bytes) in before {
+ assert_eq!(
+ std::fs::read(pack.join(&name)).unwrap(),
+ bytes,
+ "{name} byte-identical"
+ );
+ }
+
+ // ── third run after deleting an upstream file: prune removes it ────
+ std::fs::remove_file(
+ home.path()
+ .join("recipes/data/rustsec/RUSTSEC-2020-0001.json"),
+ )
+ .unwrap();
+ assert_eq!(run_build_inner(&args, home.path()).unwrap(), 0);
+ let manifest: Value =
+ serde_json::from_str(&std::fs::read_to_string(pack.join("manifest.json")).unwrap())
+ .unwrap();
+ assert_eq!(
+ manifest["sources"][0]["pruned"],
+ json!(1),
+ "dead envelope pruned"
+ );
+ assert_eq!(read_records(&pack).len(), 1);
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/harvest/normalize.rs b/engine/crates/xerj-autoindex/src/harvest/normalize.rs
new file mode 100644
index 000000000..ae8b10b76
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/harvest/normalize.rs
@@ -0,0 +1,381 @@
+//! Input-format adapters: one function per known record FORMAT, each turning
+//! a source JSON value into flat key paths.
+//!
+//! An adapter knows the INPUT format, never our domain: `osv` understands
+//! the ossf/osv-schema shape (shared by osv.dev dumps, the RustSec `osv`
+//! branch and github-reviewed GHSA files); `flat` does generic flattening
+//! for everything else. Which keys of the result feed the query envelope is
+//! the recipe's job, not the adapter's — the same `osv` adapter would serve
+//! a Go vulnerability corpus unchanged.
+//!
+//! Flattening mirrors `extract::flatten_object` semantics (two levels of
+//! nesting become `a_b` keys; deeper structure and arrays-of-objects are
+//! stored as JSON strings; arrays of scalars stay arrays) so a record
+//! normalized here does not transform a second time at index time.
+//!
+//! One deliberate omission: OSV `database_specific` is NEVER read. The
+//! schema marks it "intended for internal use only" and "subject to change
+//! without notice" — building on it means building on a field its owner
+//! says may vanish. Ecosystem-specific fields ARE read (they are the
+//! documented extension point upstreams actually publish to, and RustSec's
+//! affected-functions list lives there).
+
+use serde_json::{Map, Value};
+
+use super::recipe::Format;
+
+/// Normalize one record under `format` into flat key paths.
+pub fn normalize(format: Format, v: &Value) -> Map<String, Value> {
+ match format {
+ Format::Flat => flatten(v),
+ Format::Osv => normalize_osv(v),
+ }
+}
+
+// ── generic flattening ──────────────────────────────────────────────────────
+
+/// Flatten any JSON value into flat keys, `extract::flatten_object`
+/// semantics. A non-object top level becomes `{"value": v}` so envelope
+/// mapping still has something to point at.
+pub fn flatten(v: &Value) -> Map<String, Value> {
+ let mut out = Map::new();
+ match v {
+ Value::Object(m) => {
+ for (k, vv) in m {
+ flatten_into(k, vv.clone(), 0, &mut out);
+ }
+ }
+ other => {
+ out.insert("value".to_string(), other.clone());
+ }
+ }
+ out
+}
+
+fn flatten_into(key: &str, v: Value, depth: usize, out: &mut Map<String, Value>) {
+ if out.len() >= super::MAX_FIELDS {
+ return;
+ }
+ match v {
+ Value::Object(m) => {
+ if depth < 2 {
+ for (k, vv) in m {
+ flatten_into(&format!("{key}_{k}"), vv, depth + 1, out);
+ }
+ } else {
+ out.insert(
+ key.to_string(),
+ Value::String(serde_json::to_string(&m).unwrap_or_default()),
+ );
+ }
+ }
+ Value::Array(a) => {
+ if a.iter().all(|e| !e.is_object() && !e.is_array()) {
+ out.insert(key.to_string(), Value::Array(a));
+ } else {
+ out.insert(
+ key.to_string(),
+ Value::String(serde_json::to_string(&a).unwrap_or_default()),
+ );
+ }
+ }
+ other => {
+ out.insert(key.to_string(), other);
+ }
+ }
+}
+
+// ── OSV ─────────────────────────────────────────────────────────────────────
+
+/// Curated extraction of the OSV v1.x fields that carry meaning across
+/// sources, plus generic flattening of anything else. See the module docs
+/// for why `database_specific` is absent.
+fn normalize_osv(v: &Value) -> Map<String, Value> {
+ let obj: Map<String, Value> = match v {
+ Value::Object(m) => m.clone(),
+ other => return flatten(other),
+ };
+ let mut out = Map::new();
+
+ // Direct scalars that exist in every OSV record class.
+ for k in [
+ "id",
+ "modified",
+ "published",
+ "withdrawn",
+ "summary",
+ "details",
+ ] {
+ if let Some(s) = obj.get(k).and_then(Value::as_str) {
+ if !s.is_empty() {
+ out.insert(k.to_string(), Value::String(s.to_string()));
+ }
+ }
+ }
+
+ // `aliases` / `related`: scalar string arrays — they survive flattening
+ // as arrays, and identity unions on every element.
+ for k in ["aliases", "related"] {
+ if let Some(arr) = string_array(obj.get(k)) {
+ if !arr.is_empty() {
+ out.insert(k.to_string(), Value::Array(arr));
+ }
+ }
+ }
+
+ // `severity`: [{type, score, source}] — first entry wins for the scalar,
+ // all scores kept for evidence.
+ if let Some(sev) = obj.get("severity").and_then(Value::as_array) {
+ let scores: Vec<Value> = sev
+ .iter()
+ .filter_map(|s| s.get("score").and_then(Value::as_str))
+ .map(|s| Value::String(s.to_string()))
+ .collect();
+ if let Some(first) = scores.first() {
+ out.insert("severity".to_string(), first.clone());
+ }
+ if !scores.is_empty() {
+ out.insert("severity_scores".to_string(), Value::Array(scores));
+ }
+ }
+
+ // `affected[]`: version knowledge. OSV packs multiple introduced/fixed
+ // pairs into one SEMVER range (RustSec shape) while GHSA splits them
+ // across entries; both land in the same flat lists here so the merge
+ // unions intervals instead of comparing shapes.
+ let mut ecosystems: Vec<Value> = Vec::new();
+ let mut packages: Vec<Value> = Vec::new();
+ let mut versions: Vec<Value> = Vec::new();
+ let mut introduced: Vec<Value> = Vec::new();
+ let mut fixed: Vec<Value> = Vec::new();
+ let mut last_affected: Vec<Value> = Vec::new();
+ for aff in obj
+ .get("affected")
+ .and_then(Value::as_array)
+ .unwrap_or(&Vec::new())
+ {
+ if let Some(name) = aff
+ .get("package")
+ .and_then(|p| p.get("name"))
+ .and_then(Value::as_str)
+ {
+ packages.push(Value::String(name.to_string()));
+ }
+ if let Some(eco) = aff
+ .get("package")
+ .and_then(|p| p.get("ecosystem"))
+ .and_then(Value::as_str)
+ {
+ ecosystems.push(Value::String(eco.to_string()));
+ }
+ if let Some(vs) = string_array(aff.get("versions")) {
+ versions.extend(vs);
+ }
+ for range in aff
+ .get("ranges")
+ .and_then(Value::as_array)
+ .unwrap_or(&Vec::new())
+ {
+ for ev in range
+ .get("events")
+ .and_then(Value::as_array)
+ .unwrap_or(&Vec::new())
+ {
+ for (key, sink) in [
+ ("introduced", &mut introduced),
+ ("fixed", &mut fixed),
+ ("last_affected", &mut last_affected),
+ ] {
+ if let Some(s) = ev.get(key).and_then(Value::as_str) {
+ sink.push(Value::String(s.to_string()));
+ }
+ }
+ }
+ }
+ }
+ dedup_into(&mut out, "ecosystems", ecosystems);
+ dedup_into(&mut out, "packages", packages);
+ dedup_into(&mut out, "affected_versions", versions);
+ dedup_into(&mut out, "introduced_versions", introduced);
+ dedup_into(&mut out, "fixed_versions", fixed);
+ dedup_into(&mut out, "last_affected_versions", last_affected);
+
+ // `references[]`: {type, url} — flattened into parallel scalar arrays.
+ let mut urls: Vec<Value> = Vec::new();
+ let mut types: Vec<Value> = Vec::new();
+ for r in obj
+ .get("references")
+ .and_then(Value::as_array)
+ .unwrap_or(&Vec::new())
+ {
+ if let Some(u) = r.get("url").and_then(Value::as_str) {
+ urls.push(Value::String(u.to_string()));
+ }
+ if let Some(t) = r.get("type").and_then(Value::as_str) {
+ types.push(Value::String(t.to_string()));
+ }
+ }
+ dedup_into(&mut out, "reference_urls", urls);
+ dedup_into(&mut out, "reference_types", types);
+
+ // `ecosystem_specific`: the documented upstream extension point. RustSec
+ // publishes affected FUNCTIONS here (`affects.functions` — canonical
+ // paths keyed to version ranges), which no other source in the stack
+ // carries; that is the one structured extraction worth doing.
+ if let Some(eco) = obj.get("ecosystem_specific") {
+ if let Some(functions) = eco
+ .pointer("/affects/functions")
+ .and_then(Value::as_object)
+ .map(|m| m.keys().cloned().collect::<Vec<_>>())
+ {
+ if !functions.is_empty() {
+ out.insert(
+ "affected_functions".to_string(),
+ Value::Array(functions.into_iter().map(Value::String).collect::<Vec<_>>()),
+ );
+ }
+ }
+ if let Some(info) = eco.get("informational").and_then(Value::as_str) {
+ out.insert("informational".to_string(), Value::String(info.to_string()));
+ }
+ }
+
+ // Everything not already curated flows through the generic flattener so
+ // unknown-but-real OSV fields survive into the pack (curated keys win on
+ // collision — they are the ones with merge semantics).
+ for (k, vv) in &obj {
+ if matches!(k.as_str(), "database_specific") {
+ continue;
+ }
+ if out.contains_key(k) {
+ continue;
+ }
+ flatten_into(k, vv.clone(), 0, &mut out);
+ }
+ out
+}
+
+fn string_array(v: Option<&Value>) -> Option<Vec<Value>> {
+ match v {
+ Some(Value::Array(a)) if a.iter().all(|e| e.is_string()) => Some(a.to_vec()),
+ Some(Value::String(s)) => Some(vec![Value::String(s.clone())]),
+ _ => None,
+ }
+}
+
+fn dedup_into(out: &mut Map<String, Value>, key: &str, vals: Vec<Value>) {
+ if vals.is_empty() {
+ return;
+ }
+ let mut seen: Vec<String> = Vec::new();
+ let mut kept: Vec<Value> = Vec::new();
+ for v in vals {
+ if let Some(s) = v.as_str() {
+ if s.is_empty() || seen.iter().any(|x| x == s) {
+ continue;
+ }
+ seen.push(s.to_string());
+ kept.push(v);
+ }
+ }
+ if !kept.is_empty() {
+ out.insert(key.to_string(), Value::Array(kept));
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ #[test]
+ fn flat_mirrors_extract_semantics() {
+ let v = json!({
+ "id": "x",
+ "meta": {"a": 1, "b": {"deep": true}},
+ "deeper": {"x": {"y": {"z": 1}}},
+ "tags": ["a", "b"],
+ "rows": [{"k": 1}]
+ });
+ let m = normalize(Format::Flat, &v);
+ assert_eq!(m["id"], json!("x"));
+ assert_eq!(m["meta_a"], json!(1));
+ // three key segments survive (b's object is at depth 1 < 2)…
+ assert_eq!(m["meta_b_deep"], json!(true));
+ // …the object at depth 2 is stringified, exactly like flatten_object
+ assert_eq!(m["deeper_x_y"], json!("{\"z\":1}"));
+ assert_eq!(m["tags"], json!(["a", "b"]));
+ assert!(m["rows"].is_string(), "arrays of objects stringify");
+ }
+
+ #[test]
+ fn osv_extracts_identity_and_versions() {
+ let v = json!({
+ "id": "RUSTSEC-2021-0003",
+ "aliases": ["CVE-2021-25900", "GHSA-43w2-9j62-hq99"],
+ "summary": "Out of bounds write in SmallVec::insert_many",
+ "modified": "2023-06-20T00:00:00Z",
+ "affected": [{
+ "package": {"ecosystem": "crates.io", "name": "smallvec"},
+ "ranges": [{
+ "type": "SEMVER",
+ "events": [
+ {"introduced": "0.6.3"}, {"fixed": "0.6.14"},
+ {"introduced": "1.0.0"}, {"fixed": "1.6.1"}
+ ]
+ }],
+ "versions": ["0.6.5"]
+ }],
+ "references": [
+ {"type": "WEB", "url": "https://github.com/servo/rust-smallvec/commit/abc"},
+ {"type": "PACKAGE", "url": "https://crates.io/crates/smallvec"}
+ ],
+ "ecosystem_specific": {"affects": {"functions": {"smallvec::SmallVec::insert_many": ["<1.6.1"]}}},
+ "database_specific": {"severity": "HIGH", "cwe_ids": ["CWE-787"]}
+ });
+ let m = normalize(Format::Osv, &v);
+ assert_eq!(m["id"], json!("RUSTSEC-2021-0003"));
+ assert_eq!(
+ m["aliases"],
+ json!(["CVE-2021-25900", "GHSA-43w2-9j62-hq99"])
+ );
+ assert_eq!(m["packages"], json!(["smallvec"]));
+ assert_eq!(m["ecosystems"], json!(["crates.io"]));
+ // both introduced/fixed pairs of the packed SEMVER range survive
+ assert_eq!(m["introduced_versions"], json!(["0.6.3", "1.0.0"]));
+ assert_eq!(m["fixed_versions"], json!(["0.6.14", "1.6.1"]));
+ assert_eq!(m["affected_versions"], json!(["0.6.5"]));
+ assert_eq!(
+ m["affected_functions"],
+ json!(["smallvec::SmallVec::insert_many"])
+ );
+ assert_eq!(m["reference_urls"].as_array().unwrap().len(), 2);
+ assert!(!m.contains_key("database_specific"));
+ assert!(!m.contains_key("database_specific_severity"));
+ }
+
+ #[test]
+ fn osv_ghsa_split_ranges_land_in_the_same_lists() {
+ // GHSA splits one advisory's two fixed ranges across two `affected`
+ // entries; the union must equal the RustSec packed shape.
+ let v = json!({
+ "id": "GHSA-43w2-9j62-hq99",
+ "affected": [
+ {"package": {"name": "smallvec"}, "ranges": [{"events": [{"introduced": "0.6.3"}, {"fixed": "0.6.14"}]}]},
+ {"package": {"name": "smallvec"}, "ranges": [{"events": [{"introduced": "1.0.0"}, {"fixed": "1.6.1"}]}]}
+ ]
+ });
+ let m = normalize(Format::Osv, &v);
+ assert_eq!(m["introduced_versions"], json!(["0.6.3", "1.0.0"]));
+ assert_eq!(m["fixed_versions"], json!(["0.6.14", "1.6.1"]));
+ assert_eq!(m["packages"], json!(["smallvec"]), "deduped across entries");
+ }
+
+ #[test]
+ fn osv_unknown_fields_passthrough() {
+ let v = json!({"id": "x", "schema_version": "1.9.0", "upstream": {"of": "y"}});
+ let m = normalize(Format::Osv, &v);
+ assert_eq!(m["schema_version"], json!("1.9.0"));
+ assert_eq!(m["upstream_of"], json!("y"));
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/harvest/pack.rs b/engine/crates/xerj-autoindex/src/harvest/pack.rs
new file mode 100644
index 000000000..580c8f3c7
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/harvest/pack.rs
@@ -0,0 +1,367 @@
+//! Deterministic pack emission: merged records → the portable artifact.
+//!
+//! A pack is a directory (a zip of it later) that ANY tool can consume:
+//! sharded `records-*.jsonl`, a `manifest.json` with per-file sha256s, the
+//! recipe that built it, and a `SHA256SUMS` over everything. Determinism is
+//! the load-bearing property: shard assignment is `xxh3_64(id) % shards`,
+//! records are id-ordered, and every record carries the SAME key set (a
+//! key absent from one record is `[]` for array keys, `null` otherwise) —
+//! so an unchanged input rebuilds byte-identical shard files, and the
+//! `_id`s the engine derives stay stable across rebuilds (autoindex
+//! clusters datasets by field-name set; an omitted optional field would
+//! silently move a record between datasets).
+//!
+//! `manifest.json` and `SHA256SUMS` necessarily vary (timestamp,
+//! generation) — determinism is claimed for the RECORDS, never the wrapper.
+
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+
+use anyhow::{bail, Context, Result};
+use serde_json::{Map, Value};
+use sha2::{Digest, Sha256};
+use xxhash_rust::xxh3::xxh3_64;
+
+use super::recipe::Recipe;
+
+/// Per-source counts for the manifest: what this run actually did.
+pub struct SourceStat {
+ pub slug: String,
+ pub kind: String,
+ pub path: Option<String>,
+ pub url: Option<String>,
+ pub licence: String,
+ pub records: usize,
+ pub new: usize,
+ pub skipped: usize,
+ pub pruned: usize,
+}
+
+pub struct RunStats {
+ pub envelopes: usize,
+ pub per_source: Vec<SourceStat>,
+}
+
+#[derive(Debug)]
+pub struct PackResult {
+ pub dir: PathBuf,
+ pub generation: u64,
+ pub records: usize,
+ pub shards: usize,
+}
+
+/// Build (or rebuild) the pack under `<build_dir>/pack/<name>`.
+pub fn emit(
+ build_dir: &Path,
+ recipe: &Recipe,
+ recipe_toml: &str,
+ mut records: Vec<Map<String, Value>>,
+ stats: &RunStats,
+) -> Result<PackResult> {
+ let pack_dir = build_dir.join("pack").join(&recipe.name);
+ let generation = previous_generation(&pack_dir)? + 1;
+ std::fs::create_dir_all(&pack_dir)
+ .with_context(|| format!("cannot create {}", pack_dir.display()))?;
+
+ // Order first, then refuse duplicate ids: a pack with two records of
+ // the same id breaks the engine-side _id determinism promise, and a
+ // duplicate here means identity resolution failed silently.
+ records.sort_by(|a, b| {
+ a.get("id")
+ .and_then(Value::as_str)
+ .unwrap_or("")
+ .cmp(b.get("id").and_then(Value::as_str).unwrap_or(""))
+ });
+ let mut seen_id = String::new();
+ for r in &records {
+ let id = r.get("id").and_then(Value::as_str).unwrap_or("");
+ if !seen_id.is_empty() && id == seen_id {
+ bail!("duplicate emitted id '{id}': identity edges failed to merge two records that claim the same id");
+ }
+ seen_id = id.to_string();
+ }
+
+ let shards = recipe.emit.shards.min(records.len()).max(1);
+ let uniform = uniform_keys(&records);
+
+ // shard → line buffer; records are id-sorted, so appending preserves
+ // in-shard order without a second sort.
+ let mut shard_lines: Vec<String> = vec![String::new(); shards];
+ for r in &records {
+ let id = r.get("id").and_then(Value::as_str).unwrap_or("");
+ let bucket = (xxh3_64(id.as_bytes()) % shards as u64) as usize;
+ let mut full = r.clone();
+ for (k, is_array) in &uniform {
+ full.entry(k.clone()).or_insert_with(|| {
+ if *is_array {
+ Value::Array(Vec::new())
+ } else {
+ Value::Null
+ }
+ });
+ }
+ shard_lines[bucket].push_str(&super::store::canonical_json(&full));
+ shard_lines[bucket].push('\n');
+ }
+
+ let mut files: BTreeMap<String, Value> = BTreeMap::new();
+ for (i, lines) in shard_lines.iter().enumerate() {
+ let name = format!("records-{:05}.jsonl", i);
+ if lines.is_empty() {
+ continue; // small corpora need fewer files than shards
+ }
+ let path = pack_dir.join(&name);
+ std::fs::write(&path, lines.as_bytes())
+ .with_context(|| format!("cannot write {}", path.display()))?;
+ let n = lines.lines().count();
+ files.insert(
+ name.clone(),
+ serde_json::json!({ "sha256": hex(&Sha256::digest(lines.as_bytes())), "records": n }),
+ );
+ }
+
+ // the recipe ships verbatim: it is the pack's provenance, licence
+ // reasoning included
+ std::fs::write(pack_dir.join("recipe.toml"), recipe_toml)?;
+ let recipe_sha = hex(&Sha256::digest(recipe_toml.as_bytes()));
+
+ let manifest = serde_json::json!({
+ "format_version": 1,
+ "kind": "harvested",
+ "pack": recipe.name,
+ "description": recipe.description,
+ "created_at": chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true),
+ "generation": generation,
+ "recipe_sha256": recipe_sha,
+ "counts": {
+ "envelopes": stats.envelopes,
+ "records": records.len(),
+ "shards": shards,
+ },
+ "sources": stats.per_source.iter().map(|s| serde_json::json!({
+ "slug": s.slug, "kind": s.kind, "path": s.path, "url": s.url,
+ "licence": s.licence, "records": s.records,
+ "new": s.new, "skipped": s.skipped, "pruned": s.pruned,
+ })).collect::<Vec<_>>(),
+ "files": Value::Object(files.clone().into_iter().collect()),
+ "tool": { "name": "xerj", "version": env!("CARGO_PKG_VERSION") },
+ });
+ let manifest_bytes = serde_json::to_vec_pretty(&manifest)?;
+ std::fs::write(pack_dir.join("manifest.json"), &manifest_bytes)?;
+
+ let mut sums = String::new();
+ let mut names: Vec<String> = files.keys().cloned().collect();
+ names.push("recipe.toml".to_string());
+ names.push("manifest.json".to_string());
+ names.sort();
+ for n in &names {
+ let bytes = std::fs::read(pack_dir.join(n)).with_context(|| format!("checksumming {n}"))?;
+ sums.push_str(&hex(&Sha256::digest(&bytes)));
+ sums.push_str(" ");
+ sums.push_str(n);
+ sums.push('\n');
+ }
+ std::fs::write(pack_dir.join("SHA256SUMS"), sums)?;
+
+ Ok(PackResult {
+ dir: pack_dir,
+ generation,
+ records: records.len(),
+ shards,
+ })
+}
+
+/// The uniform key set: every key any record carries, and whether missing
+/// values should fill as `[]` (array wherever it appears) or `null`.
+fn uniform_keys(records: &[Map<String, Value>]) -> BTreeMap<String, bool> {
+ let mut out: BTreeMap<String, (bool /*any array*/, bool /*all non-null array*/)> =
+ BTreeMap::new();
+ for r in records {
+ for (k, v) in r {
+ let e = out.entry(k.clone()).or_insert((false, true));
+ if v.is_array() {
+ e.0 = true;
+ } else if !v.is_null() {
+ e.1 = false;
+ }
+ }
+ }
+ out.into_iter()
+ .map(|(k, (any, all))| (k, any && all))
+ .collect()
+}
+
+fn previous_generation(pack_dir: &Path) -> Result<u64> {
+ let Ok(bytes) = std::fs::read(pack_dir.join("manifest.json")) else {
+ return Ok(0);
+ };
+ let v: Value = serde_json::from_slice(&bytes).unwrap_or(Value::Null);
+ Ok(v.get("generation").and_then(Value::as_u64).unwrap_or(0))
+}
+
+fn hex(bytes: &[u8]) -> String {
+ bytes.iter().map(|b| format!("{b:02x}")).collect()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::harvest::recipe::{Emit, Envelope, Identity, Merge};
+ use serde_json::json;
+
+ fn recipe(name: &str, shards: usize) -> Recipe {
+ Recipe {
+ name: name.to_string(),
+ description: String::new(),
+ envelope: Envelope {
+ id_from: vec!["id".into()],
+ title_from: vec!["id".into()],
+ body_join: vec![],
+ defs_from: vec![],
+ passthrough: false,
+ },
+ sources: vec![],
+ identity: Identity {
+ edges: vec![],
+ canonical_source_order: vec![],
+ },
+ merge: Merge::default(),
+ derived: vec![],
+ emit: Emit { shards },
+ }
+ }
+
+ fn rec(id: &str, extra: &[(&str, Value)]) -> Map<String, Value> {
+ let mut m = Map::new();
+ m.insert("id".into(), json!(id));
+ for (k, v) in extra {
+ m.insert(k.to_string(), v.clone());
+ }
+ m
+ }
+
+ fn stats() -> RunStats {
+ RunStats {
+ envelopes: 0,
+ per_source: vec![],
+ }
+ }
+
+ #[test]
+ fn uniform_keys_fill_and_shard_determinism() {
+ let tmp = tempfile::tempdir().unwrap();
+ let records = vec![
+ rec("a", &[("aliases", json!(["X1"]))]),
+ rec("b", &[("severity", json!("HIGH"))]),
+ ];
+ let r1 = emit(
+ tmp.path(),
+ &recipe("t", 4),
+ "# recipe\n",
+ records.clone(),
+ &stats(),
+ )
+ .unwrap();
+ assert_eq!(r1.records, 2);
+ assert_eq!(r1.generation, 1);
+
+ // every shard file's records carry BOTH keys
+ for entry in std::fs::read_dir(&r1.dir).unwrap().flatten() {
+ let name = entry.file_name().to_string_lossy().to_string();
+ if !name.starts_with("records-") {
+ continue;
+ }
+ for line in std::fs::read_to_string(entry.path()).unwrap().lines() {
+ let v: Value = serde_json::from_str(line).unwrap();
+ assert!(v.get("aliases").is_some(), "array key filled");
+ assert!(v.get("severity").is_some(), "scalar key filled");
+ let id = v["id"].as_str().unwrap();
+ if id == "a" {
+ assert_eq!(v["aliases"], json!(["X1"]));
+ assert_eq!(v["severity"], json!(null));
+ } else {
+ assert_eq!(v["aliases"], json!([]), "array key fills empty");
+ assert_eq!(v["severity"], json!("HIGH"));
+ }
+ }
+ }
+
+ // rebuild from a DIFFERENT input order → byte-identical records
+ let mut shuffled = records.clone();
+ shuffled.reverse();
+ let before: Vec<(String, Vec<u8>)> = std::fs::read_dir(&r1.dir)
+ .unwrap()
+ .flatten()
+ .map(|e| {
+ (
+ e.file_name().to_string_lossy().to_string(),
+ std::fs::read(e.path()).unwrap(),
+ )
+ })
+ .filter(|(n, _)| n.starts_with("records-"))
+ .collect();
+ let r2 = emit(
+ tmp.path(),
+ &recipe("t", 4),
+ "# recipe\n",
+ shuffled,
+ &stats(),
+ )
+ .unwrap();
+ assert_eq!(r2.generation, 2, "generation advances");
+ for (name, bytes) in before {
+ assert_eq!(
+ std::fs::read(r2.dir.join(&name)).unwrap(),
+ bytes,
+ "{name} identical"
+ );
+ }
+ }
+
+ #[test]
+ fn duplicate_ids_are_refused() {
+ let tmp = tempfile::tempdir().unwrap();
+ let records = vec![rec("a", &[]), rec("a", &[])];
+ let err = emit(tmp.path(), &recipe("t", 2), "", records, &stats())
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("duplicate emitted id"), "{err}");
+ }
+
+ #[test]
+ fn sums_cover_every_artifact() {
+ let tmp = tempfile::tempdir().unwrap();
+ let records = vec![rec("a", &[]), rec("b", &[]), rec("c", &[])];
+ let r = emit(tmp.path(), &recipe("t", 2), "# r\n", records, &stats()).unwrap();
+ let sums = std::fs::read_to_string(r.dir.join("SHA256SUMS")).unwrap();
+ assert!(sums.contains(" manifest.json\n"));
+ assert!(sums.contains(" recipe.toml\n"));
+ for line in sums.lines() {
+ let (sum, name) = line.split_once(" ").unwrap();
+ let bytes = std::fs::read(r.dir.join(name)).unwrap();
+ assert_eq!(sum, hex(&Sha256::digest(&bytes)), "{name} hash matches");
+ }
+ let m: Value =
+ serde_json::from_str(&std::fs::read_to_string(r.dir.join("manifest.json")).unwrap())
+ .unwrap();
+ assert_eq!(m["format_version"], json!(1));
+ assert_eq!(m["kind"], json!("harvested"));
+ assert_eq!(m["counts"]["records"], json!(3));
+ // shard assignment is a hash — do not assert WHICH files exist, only
+ // that the ones that do cover every record and none is empty
+ let files = m["files"].as_object().unwrap();
+ assert!(!files.is_empty());
+ assert!(files.len() <= 2);
+ let total: usize = files
+ .values()
+ .map(|f| f["records"].as_u64().unwrap() as usize)
+ .sum();
+ assert_eq!(total, 3);
+ for f in files.values() {
+ assert!(
+ f["records"].as_u64().unwrap() > 0,
+ "empty shard files are never written"
+ );
+ }
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/harvest/recipe.rs b/engine/crates/xerj-autoindex/src/harvest/recipe.rs
new file mode 100644
index 000000000..4229dc5a5
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/harvest/recipe.rs
@@ -0,0 +1,692 @@
+//! The corpus recipe — the declarative half of `xerj corpus build`.
+//!
+//! A recipe says WHERE records come from, which of their fields carry
+//! IDENTITY (so the same logical record arriving from three sources becomes
+//! one), how fields merge across sources, and which derived fields to
+//! compute. It is TOML on purpose: it is human-authored, commented — licence
+//! reasoning belongs next to the source it justifies — and it ships verbatim
+//! inside the built pack as provenance.
+//!
+//! The tool is deliberately schema-agnostic: the recipe maps whatever the
+//! sources actually contain onto a small query envelope (`title`, `body`,
+//! `defs`, …). Nothing here knows what a vulnerability is. The parse is
+//! STRICT — unknown keys and bad values are loud errors, not silently
+//! ignored defaults — because a typo'd `aliases` silently turns one unique
+//! record into three, which is the exact failure this subsystem exists to
+//! prevent. (Hub manifests are tolerant because they are untrusted input
+//! validated elsewhere; a recipe is trusted author input and must round-trip
+//! faithfully.)
+
+use std::collections::HashMap;
+use std::path::Path;
+
+use anyhow::{bail, Context, Result};
+use regex::Regex;
+use serde::Deserialize;
+
+/// Reserved envelope key names. Every emitted record carries these; derived
+/// fields and passthrough data may not clobber them (validation refuses a
+/// recipe that tries — a silent clobber here would make `xerj code` and the
+/// ES-compat query surface disagree about what a record is).
+pub const RESERVED_KEYS: &[&str] = &[
+ "id",
+ "title",
+ "body",
+ "defs",
+ "licence",
+ "source",
+ "sources",
+ "source_url",
+ "modified",
+ "origin",
+];
+
+/// The envelope provenance keys used INSIDE the content store. Underscored
+/// so they can never collide with a data field at emit time, where they are
+/// mapped onto `source`/`sources`/`source_url`/`origin` and dropped.
+pub(crate) const SRC: &str = "_src";
+pub(crate) const SRC_URL: &str = "_src_url";
+pub(crate) const SRC_PATH: &str = "_src_path";
+pub(crate) const LICENCE: &str = "_licence";
+
+/// Names every emitted record carries (the uniform-key invariant): the
+/// RESERVED_KEYS above plus every passthrough and derived key the recipe
+/// declares. Absent values are `null`/`[]`, never omitted — autoindex
+/// clusters datasets by field-name set, so an omitted optional field would
+/// move a record between datasets (and change its `_id`) across rebuilds.
+#[derive(Debug, Clone)]
+pub struct Recipe {
+ pub name: String,
+ /// Free-text, from `[recipe] description`; lands in the pack manifest so
+ /// a consumer browsing packs sees what the corpus IS without reading the
+ /// recipe.
+ pub description: String,
+ pub envelope: Envelope,
+ pub sources: Vec<SourceSpec>,
+ pub identity: Identity,
+ pub merge: Merge,
+ pub derived: Vec<Derived>,
+ pub emit: Emit,
+}
+
+/// Which normalized keys feed the query envelope. All are "first non-empty
+/// wins" lists evaluated per merged record, in merge-precedence order.
+#[derive(Debug, Clone)]
+pub struct Envelope {
+ pub id_from: Vec<String>,
+ pub title_from: Vec<String>,
+ pub body_join: Vec<String>,
+ pub defs_from: Vec<String>,
+ pub passthrough: bool,
+}
+
+#[derive(Debug, Clone)]
+pub struct SourceSpec {
+ pub slug: String,
+ pub kind: SourceKind,
+ /// `dir` sources: a path relative to the recipe file.
+ pub path: Option<String>,
+ pub url: Option<String>,
+ pub glob: String,
+ pub format: Format,
+ /// Carried onto every record from this source as record DATA — the tool
+ /// never enforces it; the pack author owns what goes in the pack.
+ pub licence: String,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum SourceKind {
+ Dir,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Format {
+ /// OSV v1.x (the `ossf/osv-schema` shape shared by osv.dev dumps, the
+ /// RustSec `osv` branch and github-reviewed GHSA files). Curated
+ /// extraction of the standard fields plus generic flattening of the rest.
+ Osv,
+ /// Already-flat JSON records: generic two-level flattening only. This is
+ /// the "we have not written a format adapter for this yet" path.
+ Flat,
+}
+
+#[derive(Debug, Clone)]
+pub struct Identity {
+ pub edges: Vec<IdentityEdge>,
+ /// Source precedence for choosing a cluster's canonical id; empty means
+ /// source declaration order.
+ pub canonical_source_order: Vec<String>,
+}
+
+#[derive(Debug, Clone)]
+pub struct IdentityEdge {
+ pub field: String,
+ /// Array fields: union on EVERY element (`aliases`), not on the array
+ /// as a whole.
+ pub each: bool,
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct Merge {
+ pub precedence: Vec<String>,
+ pub field_precedence: HashMap<String, Vec<String>>,
+}
+
+#[derive(Debug, Clone)]
+pub struct Derived {
+ pub name: String,
+ pub op: DerivedOp,
+}
+
+#[derive(Debug, Clone)]
+pub enum DerivedOp {
+ /// Collect every regex match over the listed fields (strings, and each
+ /// element of string arrays) into a string array.
+ RegexExtract {
+ from: Vec<String>,
+ re: Regex,
+ unique: bool,
+ },
+ /// First matching rule's value for the field; `default` when nothing
+ /// matches (or the field is absent).
+ Map {
+ from: String,
+ rules: Vec<(Regex, String)>,
+ default: String,
+ },
+ /// True when the field is present and non-empty.
+ Present { from: String },
+}
+
+#[derive(Debug, Clone)]
+pub struct Emit {
+ pub shards: usize,
+}
+
+// ── raw (serde) layer ───────────────────────────────────────────────────────
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct RawRecipe {
+ recipe: RawHeader,
+ #[serde(default)]
+ envelope: RawEnvelope,
+ sources: Vec<RawSource>,
+ identity: RawIdentity,
+ #[serde(default)]
+ merge: RawMerge,
+ #[serde(default)]
+ derived: Vec<RawDerived>,
+ #[serde(default)]
+ emit: RawEmit,
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct RawHeader {
+ format: u32,
+ name: String,
+ #[serde(default)]
+ description: String,
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct RawEnvelope {
+ #[serde(default = "d_id_from")]
+ id_from: Vec<String>,
+ #[serde(default = "d_title_from")]
+ title_from: Vec<String>,
+ #[serde(default)]
+ body_join: Vec<String>,
+ #[serde(default)]
+ defs_from: Vec<String>,
+ #[serde(default = "d_true")]
+ passthrough: bool,
+}
+
+// `#[serde(default)]` on the whole-table fields in RawRecipe falls back to
+// `Default::default()` when the table is absent — which would silently zero
+// these. The manual impls keep absent-table defaults equal to field defaults.
+impl Default for RawEnvelope {
+ fn default() -> Self {
+ RawEnvelope {
+ id_from: d_id_from(),
+ title_from: d_title_from(),
+ body_join: Vec::new(),
+ defs_from: Vec::new(),
+ passthrough: d_true(),
+ }
+ }
+}
+
+fn d_id_from() -> Vec<String> {
+ vec!["id".to_string()]
+}
+fn d_title_from() -> Vec<String> {
+ vec!["id".to_string()]
+}
+fn d_true() -> bool {
+ true
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct RawSource {
+ slug: String,
+ kind: String,
+ #[serde(default)]
+ path: Option<String>,
+ #[serde(default)]
+ url: Option<String>,
+ #[serde(default = "d_glob")]
+ glob: String,
+ format: String,
+ #[serde(default)]
+ licence: String,
+}
+
+fn d_glob() -> String {
+ "**/*.json".to_string()
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct RawIdentity {
+ edges: Vec<RawEdge>,
+ #[serde(default)]
+ canonical_source_order: Vec<String>,
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct RawEdge {
+ field: String,
+ #[serde(default)]
+ each: bool,
+}
+
+#[derive(Deserialize, Default)]
+#[serde(deny_unknown_fields)]
+struct RawMerge {
+ #[serde(default)]
+ precedence: Vec<String>,
+ #[serde(default)]
+ field_precedence: HashMap<String, Vec<String>>,
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct RawDerived {
+ name: String,
+ op: String,
+ #[serde(default)]
+ from: Vec<String>,
+ #[serde(default)]
+ pattern: Option<String>,
+ #[serde(default)]
+ rules: Vec<RawMapRule>,
+ #[serde(default)]
+ default: Option<String>,
+ #[serde(default)]
+ unique: bool,
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct RawMapRule {
+ r#match: String,
+ value: String,
+}
+
+#[derive(Deserialize)]
+#[serde(deny_unknown_fields)]
+struct RawEmit {
+ #[serde(default = "d_shards")]
+ shards: usize,
+}
+
+impl Default for RawEmit {
+ fn default() -> Self {
+ RawEmit { shards: d_shards() }
+ }
+}
+
+fn d_shards() -> usize {
+ 16
+}
+
+// ── load + validate + compile ───────────────────────────────────────────────
+
+/// A slug becomes a filename component in the content store
+/// (`<slug>.<key>.json`), so it is gated to a safe charset the way corpus
+/// and repo names are — but locally, because slugs are a builder concept.
+fn valid_slug(slug: &str) -> Result<()> {
+ let ok = !slug.is_empty()
+ && slug.len() <= 64
+ && slug
+ .chars()
+ .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.')
+ && !slug.starts_with('.');
+ if !ok {
+ bail!("source slug '{slug}' must be 1-64 chars of [A-Za-z0-9._-], not starting with '.'");
+ }
+ Ok(())
+}
+
+/// Read, validate and compile a recipe. Every error names the file and the
+/// offending key — a recipe that fails here must never half-run.
+pub fn load(path: &Path) -> Result<Recipe> {
+ let raw = std::fs::read_to_string(path)
+ .with_context(|| format!("cannot read recipe {}", path.display()))?;
+ let file: RawRecipe = toml::from_str(&raw).map_err(|e| {
+ // the toml error names the unknown key/line — keep it in the message
+ anyhow::anyhow!("{}: recipe parse error: {e}", path.display())
+ })?;
+ compile(path, file)
+}
+
+fn compile(path: &Path, file: RawRecipe) -> Result<Recipe> {
+ let at = |what: &str| format!("{}: {}", path.display(), what);
+
+ if file.recipe.format != 1 {
+ bail!(at(&format!(
+ "recipe format {} is not supported by this build (supported: 1)",
+ file.recipe.format
+ )));
+ }
+ let name = file.recipe.name.trim().to_string();
+ if let Err(e) = xerj_common::xccode::pathgate::valid_corpus_name(&name) {
+ bail!(at(&format!("recipe name '{name}': {e}")));
+ }
+
+ if file.sources.is_empty() {
+ bail!(at("recipe declares no [[sources]]"));
+ }
+ let mut slugs: Vec<&str> = Vec::new();
+ let mut sources = Vec::new();
+ for s in &file.sources {
+ valid_slug(&s.slug).with_context(|| at(&format!("source '{}'", s.slug)))?;
+ if slugs.contains(&s.slug.as_str()) {
+ bail!(at(&format!("duplicate source slug '{}'", s.slug)));
+ }
+ let kind = match s.kind.as_str() {
+ "dir" => SourceKind::Dir,
+ other @ ("http-zip" | "git") => bail!(at(&format!(
+ "source '{}' kind '{other}' is not supported yet (planned: incremental \
+ network sources)",
+ s.slug
+ ))),
+ other => bail!(at(&format!("source '{}': unknown kind '{other}'", s.slug))),
+ };
+ let format = match s.format.as_str() {
+ "osv" => Format::Osv,
+ "flat" => Format::Flat,
+ other => bail!(at(&format!(
+ "source '{}': unknown format '{other}' (supported: osv, flat)",
+ s.slug
+ ))),
+ };
+ if kind == SourceKind::Dir {
+ if s.path.as_deref().unwrap_or("").is_empty() {
+ bail!(at(&format!(
+ "dir source '{}' needs a 'path' (relative to the recipe file)",
+ s.slug
+ )));
+ }
+ if let Some(p) = &s.path {
+ if Path::new(p).is_absolute() {
+ bail!(at(&format!(
+ "dir source '{}' path must be recipe-relative, got absolute '{p}'",
+ s.slug
+ )));
+ }
+ }
+ }
+ slugs.push(&s.slug);
+ sources.push(SourceSpec {
+ slug: s.slug.clone(),
+ kind,
+ path: s.path.clone(),
+ url: s.url.clone(),
+ glob: s.glob.clone(),
+ format,
+ licence: s.licence.clone(),
+ });
+ }
+
+ if file.identity.edges.is_empty() {
+ bail!(at(
+ "[identity] declares no edges: without identity edges every record is its \
+ own cluster and nothing deduplicates"
+ ));
+ }
+ let edges = file
+ .identity
+ .edges
+ .iter()
+ .map(|e| IdentityEdge {
+ field: e.field.clone(),
+ each: e.each,
+ })
+ .collect();
+
+ let known = |s: &str| slugs.contains(&s);
+ for s in &file.identity.canonical_source_order {
+ if !known(s) {
+ bail!(at(&format!(
+ "identity.canonical_source_order names source '{s}' which the recipe \
+ does not declare"
+ )));
+ }
+ }
+ for s in &file.merge.precedence {
+ if !known(s) {
+ bail!(at(&format!(
+ "merge.precedence names source '{s}' which the recipe does not declare"
+ )));
+ }
+ }
+ for (field, order) in &file.merge.field_precedence {
+ for s in order {
+ if !known(s) {
+ bail!(at(&format!(
+ "merge.field_precedence.{field} names source '{s}' which the recipe \
+ does not declare"
+ )));
+ }
+ }
+ }
+
+ let mut derived = Vec::new();
+ for d in &file.derived {
+ if RESERVED_KEYS.contains(&d.name.as_str()) {
+ bail!(at(&format!(
+ "derived field '{}' collides with a reserved envelope key",
+ d.name
+ )));
+ }
+ let op = match d.op.as_str() {
+ "regex_extract" => {
+ let pattern = d.pattern.as_deref().unwrap_or("");
+ if pattern.is_empty() {
+ bail!(at(&format!(
+ "derived '{}': regex_extract needs a 'pattern'",
+ d.name
+ )));
+ }
+ if d.from.is_empty() {
+ bail!(at(&format!(
+ "derived '{}': regex_extract needs a 'from' list",
+ d.name
+ )));
+ }
+ let re = Regex::new(pattern)
+ .with_context(|| at(&format!("derived '{}': bad regex /{pattern}/", d.name)))?;
+ DerivedOp::RegexExtract {
+ from: d.from.clone(),
+ re,
+ unique: d.unique,
+ }
+ }
+ "map" => {
+ if d.from.len() != 1 {
+ bail!(at(&format!(
+ "derived '{}': map needs exactly one 'from' field",
+ d.name
+ )));
+ }
+ if d.rules.is_empty() {
+ bail!(at(&format!(
+ "derived '{}': map needs at least one rule",
+ d.name
+ )));
+ }
+ let mut rules = Vec::new();
+ for r in &d.rules {
+ let re = Regex::new(&r.r#match).with_context(|| {
+ at(&format!(
+ "derived '{}': bad rule regex /{}/",
+ d.name, r.r#match
+ ))
+ })?;
+ rules.push((re, r.value.clone()));
+ }
+ DerivedOp::Map {
+ from: d.from[0].clone(),
+ rules,
+ default: d.default.clone().unwrap_or_default(),
+ }
+ }
+ "present" => {
+ if d.from.len() != 1 {
+ bail!(at(&format!(
+ "derived '{}': present needs exactly one 'from' field",
+ d.name
+ )));
+ }
+ DerivedOp::Present {
+ from: d.from[0].clone(),
+ }
+ }
+ other => bail!(at(&format!(
+ "derived '{}': unknown op '{other}' (supported: regex_extract, map, present)",
+ d.name
+ ))),
+ };
+ derived.push(Derived {
+ name: d.name.clone(),
+ op,
+ });
+ }
+
+ if file.emit.shards == 0 || file.emit.shards > 4096 {
+ bail!(at(&format!(
+ "emit.shards must be 1..=4096, got {}",
+ file.emit.shards
+ )));
+ }
+
+ Ok(Recipe {
+ name,
+ description: file.recipe.description,
+ envelope: Envelope {
+ id_from: file.envelope.id_from,
+ title_from: file.envelope.title_from,
+ body_join: file.envelope.body_join,
+ defs_from: file.envelope.defs_from,
+ passthrough: file.envelope.passthrough,
+ },
+ sources,
+ identity: Identity {
+ edges,
+ canonical_source_order: file.identity.canonical_source_order,
+ },
+ merge: file.merge.into_recipe(),
+ derived,
+ emit: Emit {
+ shards: file.emit.shards,
+ },
+ })
+}
+
+impl RawMerge {
+ fn into_recipe(self) -> Merge {
+ Merge {
+ precedence: self.precedence,
+ field_precedence: self.field_precedence,
+ }
+ }
+}
+
+/// Rank of a source slug under an order list (absent = last). Members of a
+/// cluster are sorted by this before any first-wins merge decision, so the
+/// order list — not filesystem accident — decides whose value wins.
+pub fn slug_rank(order: &[String], slug: &str) -> usize {
+ order
+ .iter()
+ .position(|s| s == slug)
+ .unwrap_or(order.len() + 1)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn write(tmp: &std::path::Path, body: &str) -> std::path::PathBuf {
+ let p = tmp.join("recipe.toml");
+ std::fs::write(&p, body).unwrap();
+ p
+ }
+
+ const MINIMAL: &str = r#"
+[recipe]
+format = 1
+name = "demo"
+
+[[sources]]
+slug = "a"
+kind = "dir"
+path = "data/a"
+format = "flat"
+
+[identity]
+edges = [{ field = "id" }]
+"#;
+
+ #[test]
+ fn minimal_recipe_loads_with_defaults() {
+ let tmp = tempfile::tempdir().unwrap();
+ let r = load(&write(tmp.path(), MINIMAL)).unwrap();
+ assert_eq!(r.name, "demo");
+ assert_eq!(r.envelope.id_from, vec!["id".to_string()]);
+ assert_eq!(r.envelope.passthrough, true);
+ assert_eq!(r.emit.shards, 16);
+ assert_eq!(r.sources[0].glob, "**/*.json");
+ }
+
+ #[test]
+ fn unknown_keys_are_loud() {
+ let tmp = tempfile::tempdir().unwrap();
+ let bad = MINIMAL.replace("format = \"flat\"", "format = \"flat\"\nbogus_key = 1");
+ let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();
+ assert!(err.contains("bogus_key"), "{err}");
+ }
+
+ #[test]
+ fn unknown_source_kind_names_the_planned_one() {
+ let tmp = tempfile::tempdir().unwrap();
+ let bad = MINIMAL.replace("kind = \"dir\"", "kind = \"git\"");
+ let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();
+ assert!(err.contains("not supported yet"), "{err}");
+ assert!(err.contains("'git'"), "{err}");
+ }
+
+ #[test]
+ fn identity_edges_are_mandatory() {
+ let tmp = tempfile::tempdir().unwrap();
+ let bad = MINIMAL.replace("edges = [{ field = \"id\" }]", "edges = []");
+ let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();
+ assert!(err.contains("declares no edges"), "{err}");
+ }
+
+ #[test]
+ fn precedence_must_name_declared_sources() {
+ let tmp = tempfile::tempdir().unwrap();
+ let bad = MINIMAL.replace(
+ "[identity]",
+ "[merge]\nprecedence = [\"nope\"]\n\n[identity]",
+ );
+ let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();
+ assert!(err.contains("'nope'"), "{err}");
+ }
+
+ #[test]
+ fn derived_name_may_not_clobber_the_envelope() {
+ let tmp = tempfile::tempdir().unwrap();
+ let bad =
+ format!("{MINIMAL}\n[[derived]]\nname = \"title\"\nop = \"present\"\nfrom = [\"x\"]\n");
+ let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();
+ assert!(err.contains("reserved envelope key"), "{err}");
+ }
+
+ #[test]
+ fn bad_regex_is_rejected_at_load_not_at_build() {
+ let tmp = tempfile::tempdir().unwrap();
+ let bad = format!(
+ "{MINIMAL}\n[[derived]]\nname = \"cves\"\nop = \"regex_extract\"\nfrom = \
+ [\"id\"]\npattern = \"([unclosed\"\n"
+ );
+ let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();
+ assert!(err.contains("bad regex"), "{err}");
+ }
+
+ #[test]
+ fn slugs_are_path_safe() {
+ assert!(valid_slug("rustsec").is_ok());
+ assert!(valid_slug("osv-crates-io").is_ok());
+ assert!(valid_slug("../escape").is_err());
+ assert!(valid_slug(".hidden").is_err());
+ assert!(valid_slug("").is_err());
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/harvest/store.rs b/engine/crates/xerj-autoindex/src/harvest/store.rs
new file mode 100644
index 000000000..c4eaee673
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/harvest/store.rs
@@ -0,0 +1,202 @@
+//! The content-addressed envelope store — "skip already stored" as a
+//! filesystem fact, not a promise.
+//!
+//! Each normalized record plus its provenance becomes ONE immutable JSON
+//! file named `<slug>.<xxh3_128-of-canonical-json>.json`. File presence IS
+//! the dedup index: a re-run recomputes the key and skips if the file
+//! exists, the same recomputed-never-trusted philosophy as `ids.rs` (same
+//! content ⇒ same key ⇒ the write converges). No database, no journal —
+//! crash at any point and the next run converges.
+//!
+//! The canonical form is key-sorted JSON with f64 floats in
+//! shortest-round-trip form — deterministic for the same logical content
+//! regardless of the input file's key order. (Sorting is explicit: the
+//! workspace enables serde_json's `preserve_order` for the engine's
+//! document store, which would otherwise bake insertion order in.)
+
+use std::collections::HashSet;
+use std::io;
+use std::path::{Path, PathBuf};
+
+use serde_json::{Map, Value};
+use xxhash_rust::xxh3::xxh3_128;
+
+/// Cap on flattened fields per envelope, matching the extractor's own cap
+/// (`extract::MAX_FIELDS_PER_RECORD`): an envelope fatter than what the
+/// index path would keep would silently disagree with the indexed record.
+pub(crate) const MAX_FIELDS: usize = 512;
+
+/// Canonical JSON for content hashing: sorted-key serialization. The
+/// workspace enables serde_json's `preserve_order` (xerj-common), so `Map`
+/// is an IndexMap and `to_string` would bake in INSERTION order — which is
+/// exactly what canonical form must not do. Sorting through a `BTreeMap`
+/// sidesteps the feature entirely.
+pub fn canonical_json(env: &Map<String, Value>) -> String {
+ let sorted: std::collections::BTreeMap<&String, &Value> = env.iter().collect();
+ serde_json::to_string(&sorted).unwrap_or_default()
+}
+
+/// The store key for an envelope: xxh3_128 of the canonical JSON, hex.
+/// 128-bit so an accidental collision across a multi-million-record store is
+/// not a scenario to reason about.
+pub fn key_of(env: &Map<String, Value>) -> String {
+ format!("{:032x}", xxh3_128(canonical_json(env).as_bytes()))
+}
+
+/// Store one envelope. Returns `true` when it was NEW (written), `false`
+/// when an identical envelope was already stored (skipped). The write is
+/// atomic (tmp + rename): a partial file never exists under its final name.
+pub fn put(store_dir: &Path, slug: &str, env: &Map<String, Value>) -> io::Result<(String, bool)> {
+ let key = key_of(env);
+ let path = file_for(store_dir, slug, &key);
+ if path.exists() {
+ return Ok((key, false));
+ }
+ std::fs::create_dir_all(store_dir)?;
+ let tmp = path.with_extension("json.tmp");
+ std::fs::write(&tmp, canonical_json(env))?;
+ std::fs::rename(&tmp, &path)?;
+ Ok((key, true))
+}
+
+fn file_for(store_dir: &Path, slug: &str, key: &str) -> PathBuf {
+ store_dir.join(format!("{slug}.{key}.json"))
+}
+
+/// Every envelope whose `_src` is one of `slugs`, in filename order (the
+/// deterministic iteration order every downstream stage relies on).
+/// Envelopes from sources the current recipe no longer declares are left on
+/// disk but not returned — a retired source must not haunt the cluster set.
+/// Each returned map carries the file's store key under the `KEY` sentinel
+/// so member ordering can fall back to it; `put` computes the key BEFORE
+/// injection, so the sentinel is never part of any stored content.
+pub fn read_all(store_dir: &Path, slugs: &HashSet<String>) -> Vec<Map<String, Value>> {
+ let mut out = Vec::new();
+ let Ok(entries) = std::fs::read_dir(store_dir) else {
+ return out;
+ };
+ let mut names: Vec<String> = entries
+ .flatten()
+ .filter(|e| e.path().extension().and_then(|x| x.to_str()) == Some("json"))
+ .filter_map(|e| e.file_name().to_str().map(str::to_string))
+ .collect();
+ names.sort();
+ for name in names {
+ // "<slug>.<32-hex>.json" — split from the RIGHT: slugs may contain
+ // dots themselves, the key never does.
+ let Some(rest) = name.strip_suffix(".json") else {
+ continue;
+ };
+ let Some((slug, key)) = rest.rsplit_once('.') else {
+ continue;
+ };
+ if !slugs.contains(slug) {
+ continue;
+ }
+ if let Ok(raw) = std::fs::read_to_string(store_dir.join(&name)) {
+ if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(&raw) {
+ m.insert(super::identity::KEY.into(), Value::String(key.to_string()));
+ out.push(m);
+ }
+ }
+ }
+ out
+}
+
+/// Drop envelopes of `slug` that this run did NOT re-put — a file deleted
+/// upstream, or a record the normalizer no longer emits. Returns how many
+/// files were removed. Scoped to one slug so a PARTIAL harvest of another
+/// source (network watermark, M2) can never prune what it did not walk.
+pub fn prune(store_dir: &Path, slug: &str, live_keys: &HashSet<String>) -> usize {
+ let Ok(entries) = std::fs::read_dir(store_dir) else {
+ return 0;
+ };
+ let mut removed = 0usize;
+ for e in entries.flatten() {
+ let name = e.file_name().to_string_lossy().to_string();
+ let Some(rest) = name.strip_prefix(&format!("{slug}.")) else {
+ continue;
+ };
+ let Some(key) = rest.strip_suffix(".json") else {
+ continue;
+ };
+ if live_keys.contains(key) {
+ continue;
+ }
+ if std::fs::remove_file(e.path()).is_ok() {
+ removed += 1;
+ }
+ }
+ removed
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ fn env(id: &str, src: &str) -> Map<String, Value> {
+ let mut m = Map::new();
+ m.insert("id".into(), json!(id));
+ m.insert("_src".into(), json!(src));
+ m
+ }
+
+ #[test]
+ fn same_content_same_key_regardless_of_key_order() {
+ let mut a = Map::new();
+ a.insert("id".into(), json!("x"));
+ a.insert("zz".into(), json!(1));
+ let mut b = Map::new();
+ b.insert("zz".into(), json!(1));
+ b.insert("id".into(), json!("x"));
+ assert_eq!(key_of(&a), key_of(&b));
+ assert_eq!(canonical_json(&a), canonical_json(&b));
+ }
+
+ #[test]
+ fn put_is_idempotent_and_read_round_trips() {
+ let tmp = tempfile::tempdir().unwrap();
+ let dir = tmp.path().join("store");
+ let (k1, new1) = put(&dir, "a", &env("x", "a")).unwrap();
+ let (_, new2) = put(&dir, "a", &env("x", "a")).unwrap();
+ assert!(new1);
+ assert!(!new2, "second put of identical content skips");
+ let all = read_all(&dir, &HashSet::from(["a".to_string()]));
+ assert_eq!(all.len(), 1);
+ assert_eq!(all[0]["id"], json!("x"));
+ assert_eq!(k1.len(), 32);
+ }
+
+ #[test]
+ fn different_source_is_a_different_envelope() {
+ // identical record text from two sources is two evidence rows, not one
+ let tmp = tempfile::tempdir().unwrap();
+ let dir = tmp.path().join("store");
+ put(&dir, "a", &env("x", "a")).unwrap();
+ put(&dir, "b", &env("x", "b")).unwrap();
+ let all = read_all(&dir, &HashSet::from(["a".to_string(), "b".to_string()]));
+ assert_eq!(all.len(), 2);
+ // retired source stays on disk but is not returned
+ let only_a = read_all(&dir, &HashSet::from(["a".to_string()]));
+ assert_eq!(only_a.len(), 1);
+ assert_eq!(only_a[0]["_src"], json!("a"));
+ }
+
+ #[test]
+ fn prune_drops_only_dead_keys_of_its_slug() {
+ let tmp = tempfile::tempdir().unwrap();
+ let dir = tmp.path().join("store");
+ let (ka, _) = put(&dir, "a", &env("x", "a")).unwrap();
+ let (kb, _) = put(&dir, "a", &env("y", "a")).unwrap();
+ let (_kc, _) = put(&dir, "b", &env("z", "b")).unwrap();
+ let removed = prune(&dir, "a", &HashSet::from([ka.clone(), kb]));
+ assert_eq!(removed, 0);
+ let removed = prune(&dir, "a", &HashSet::from([ka]));
+ assert_eq!(removed, 1, "y died, x lived, b untouched");
+ assert_eq!(
+ read_all(&dir, &HashSet::from(["a".into(), "b".into()])).len(),
+ 2
+ );
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 81c890934..de5ff3e25 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -20,6 +20,9 @@ pub mod gate;
mod generation_catalog;
#[cfg(test)]
mod generation_catalog_http_tests;
+/// `xerj corpus build` — harvested-record corpus packs (recipe → store →
+/// identity → merge → deterministic pack).
+pub mod harvest;
pub mod ids;
pub mod ignore_rules;
pub mod infer;
diff --git a/engine/crates/xerj-autoindex/src/xc.rs b/engine/crates/xerj-autoindex/src/xc.rs
index c8eaf6fd0..17be15f4b 100644
--- a/engine/crates/xerj-autoindex/src/xc.rs
+++ b/engine/crates/xerj-autoindex/src/xc.rs
@@ -225,13 +225,16 @@ pub fn run_code_cli(args: &[String]) -> i32 {
const CORPUS_USAGE: &str =
"usage: xerj corpus add <name> <git-url>... | --from <manifest.json> [--as <name>]
+ xerj corpus build <name> [--recipe <path>] [--fresh]
xerj corpus index <name> [--fresh] [--url URL]
xerj corpus list [--url URL]
clone the repos, detect licences, write corpora/<name>/corpus.json;
--from takes the corpus name from the manifest's 'corpus' field unless
-<name> or --as overrides it; index builds/verifies/switches the corpus
-(exit 3 skips junk — normal); list shows what is loaded on this node.";
+<name> or --as overrides it; build harvests recipe sources into a
+deterministic pack under builds/<name>/ (see tools/xerj-code/);
+index builds/verifies/switches the corpus (exit 3 skips junk — normal);
+list shows what is loaded on this node.";
fn git(dir: Option<&Path>, args: &[&str]) -> Result<(i32, String)> {
let mut c = Command::new("git");
@@ -1206,7 +1209,7 @@ fn run_corpus_list(args: &[String]) -> i32 {
// ── dispatch ────────────────────────────────────────────────────────────────
-/// `xerj corpus <add|index|list> …` — returns the process exit code.
+/// `xerj corpus <add|build|index|list> …` — returns the process exit code.
pub fn run_corpus_cli(args: &[String]) -> i32 {
let Some(sub) = args.first() else {
eprintln!("{CORPUS_USAGE}");
@@ -1215,6 +1218,7 @@ pub fn run_corpus_cli(args: &[String]) -> i32 {
let rest: Vec<String> = args[1..].to_vec();
match sub.as_str() {
"add" => run_corpus_add(&rest),
+ "build" => crate::harvest::run_build(&rest),
"index" => run_corpus_index(&rest),
"list" => run_corpus_list(&rest),
"-h" | "--help" | "help" => {