AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(autoindex): harden corpus harvest negative paths and add cisa-kev pack (#1110)
Christian M. Todiec4784f6f78f2
diff --git a/docs/CORPUS_PACKS.md b/docs/CORPUS_PACKS.mdindex 9dbf9c598..b451e523a 100644--- a/docs/CORPUS_PACKS.md+++ b/docs/CORPUS_PACKS.md@@ -23,7 +23,7 @@ Every statement below points at the code that makes it true:|---|---|| Pipeline orchestration, `corpus build` CLI | `engine/crates/xerj-autoindex/src/harvest/mod.rs` || Recipe parsing + strict validation | `engine/crates/xerj-autoindex/src/harvest/recipe.rs` |-| Source adapters (`dir`, `http-zip`, `git`) | `engine/crates/xerj-autoindex/src/harvest/source.rs` |+| Source adapters (`dir`, `http-zip`, `http-json`, `git`) | `engine/crates/xerj-autoindex/src/harvest/source.rs` || Input-format normalisation (`osv`, `rustsec-md`, `flat`) | `engine/crates/xerj-autoindex/src/harvest/normalize.rs` || Content-addressed store (presence = dedup) | `engine/crates/xerj-autoindex/src/harvest/store.rs` || Union-find identity resolution | `engine/crates/xerj-autoindex/src/harvest/identity.rs` |@@ -32,7 +32,7 @@ Every statement below points at the code that makes it true:| ed25519 keygen / sign / verify | `engine/crates/xerj-autoindex/src/harvest/sign.rs` || `corpus add --from <pack>` + `--verify-sig` | `engine/crates/xerj-autoindex/src/xc.rs` || Claims pinned to a measured build | `engine/crates/xerj-autoindex/tests/pack_claims.rs` |-| Showcase recipe + stats | `tools/packs/rust-vulns/` |+| Showcase recipes + stats | `tools/packs/rust-vulns/`, `tools/packs/cisa-kev/` || Scheduled build/sign/publish | `.github/workflows/pack-publish.yml` |## Quick start@@ -54,10 +54,11 @@ xerj corpus add rust-vulns --from ~/.xerj-code/builds/rust-vulns/pack/rust-vulnsA recipe is a commented TOML file — licence reasoning belongs in comments, andthe recipe ships verbatim inside the pack as provenance. Sections:-- `[[sources]]` — `slug`, `kind` (`dir` | `http-zip` | `git`), `url`/`path`,- `format` (the normalisation adapter), `licence`, optional `glob` and- watermark. Sources are advisory-listed upstream trees, HTTP zip exports, or- git repos fetched shallow at a pinned ref.+- `[[sources]]` — `slug`, `kind` (`dir` | `http-zip` | `http-json` | `git`),+ `url`/`path`, `format` (the normalisation adapter), `licence`, optional+ `glob`, `records_field` (to unwrap a top-level JSON object's array field), and+ watermark. Sources are advisory-listed upstream trees, HTTP zip exports,+ single-URL JSON feeds, or git repos fetched shallow at a pinned ref.- `[envelope]` — which normalised fields become the universal envelope:`id_from`, `title_from`, `body_join`, `defs_from`. `defs` (space-joinedsymbol paths) is what feeds `xerj code`'s definition search.diff --git a/engine/crates/xerj-autoindex/src/harvest/mod.rs b/engine/crates/xerj-autoindex/src/harvest/mod.rsindex 4dc49d147..21a5288ca 100644--- a/engine/crates/xerj-autoindex/src/harvest/mod.rs+++ b/engine/crates/xerj-autoindex/src/harvest/mod.rs@@ -162,7 +162,7 @@ fn run_build_inner(args: &[String], home: &Path) -> Result<i32> {let files = walk(&fetched.root, &src.glob)?;let mut keys = HashSet::new();for (rel, path) in files {- for value in parse_file(&path)? {+ for value in parse_file(&path, src.records_field.as_deref())? {let mut env = normalize::normalize(src.format, &value);env.insert(SRC.into(), Value::String(src.slug.clone()));env.insert(@@ -192,7 +192,7 @@ fn run_build_inner(args: &[String], home: &Path) -> Result<i32> {// ── identity + merge over the FULL store (not just this run's files) ──let slugs: HashSet<String> = recipe.sources.iter().map(|s| s.slug.clone()).collect();- let envelopes = store::read_all(&store_dir, &slugs);+ let envelopes = store::read_all(&store_dir, &slugs)?;let clusters = identity::cluster(&envelopes, &recipe.identity);let merge_order = if recipe.merge.precedence.is_empty() {recipe.identity.canonical_source_order.clone()@@ -278,6 +278,7 @@ fn kind_name(kind: SourceKind) -> &'static str {match kind {SourceKind::Dir => "dir",SourceKind::HttpZip => "http-zip",+ SourceKind::HttpJson => "http-json",SourceKind::Git => "git",}}@@ -367,7 +368,7 @@ fn glob_regex(glob: &str) -> regex::Regex {/// source is visible junk, not a silent parse error. Other extensions are/// skipped by the glob, not here — reaching this with an unlisted type is a/// hard error, not a silent drop.-fn parse_file(path: &Path) -> Result<Vec<Value>> {+fn parse_file(path: &Path, records_field: Option<&str>) -> Result<Vec<Value>> {let raw =std::fs::read_to_string(path).with_context(|| format!("cannot read {}", path.display()))?;let name = path@@ -394,6 +395,21 @@ fn parse_file(path: &Path) -> Result<Vec<Value>> {} else {let v: Value = serde_json::from_str(&raw).with_context(|| format!("{}: not valid JSON", path.display()))?;+ if let Some(field) = records_field {+ let Value::Object(mut m) = v else {+ bail!(+ "{}: records_field '{field}' requires a top-level JSON object",+ path.display()+ );+ };+ let Some(Value::Array(a)) = m.remove(field) else {+ bail!(+ "{}: records_field '{field}' missing or not a JSON array",+ path.display()+ );+ };+ return Ok(a);+ }Ok(match v {Value::Array(a) => a,other => vec![other],@@ -465,18 +481,36 @@ mod tests {let tmp = tempfile::tempdir().unwrap();let a = tmp.path().join("a.json");std::fs::write(&a, r#"[{"id":"x"},{"id":"y"}]"#).unwrap();- assert_eq!(parse_file(&a).unwrap().len(), 2);+ assert_eq!(parse_file(&a, None).unwrap().len(), 2);let b = tmp.path().join("b.jsonl");std::fs::write(&b, "{\"id\":\"z\"}\n\n{\"id\":\"w\"}\n").unwrap();- assert_eq!(parse_file(&b).unwrap().len(), 2, "blank lines skipped");+ assert_eq!(+ parse_file(&b, None).unwrap().len(),+ 2,+ "blank lines skipped"+ );let c = tmp.path().join("c.json");std::fs::write(&c, r#"{"id":"one"}"#).unwrap();- assert_eq!(parse_file(&c).unwrap().len(), 1);+ assert_eq!(parse_file(&c, None).unwrap().len(), 1);let d = tmp.path().join("d.md");std::fs::write(&d, "```toml\n[advisory]\nid = \"R\"\n```\n\n# T\n").unwrap();- let parsed = parse_file(&d).unwrap();+ let parsed = parse_file(&d, None).unwrap();assert_eq!(parsed.len(), 1, "one raw string value, not json-parsed");assert!(parsed[0].as_str().unwrap().starts_with("```toml"));+ let wrapped = tmp.path().join("wrapped.json");+ std::fs::write(+ &wrapped,+ r#"{"title":"KEV","vulnerabilities":[{"cveID":"CVE-1"},{"cveID":"CVE-2"}]}"#,+ )+ .unwrap();+ assert_eq!(+ parse_file(&wrapped, Some("vulnerabilities")).unwrap().len(),+ 2+ );+ let err = parse_file(&wrapped, Some("missing"))+ .unwrap_err()+ .to_string();+ assert!(err.contains("missing or not a JSON array"), "{err}");}// ── end to end: recipe → harvest → identity → pack, twice ─────────────@@ -765,6 +799,22 @@ shards = 4"dead envelope pruned");assert_eq!(read_records(&pack).len(), 1);++ // ── fourth run with --fresh: wipes build dir, resets generation ────+ let fresh_args: Vec<String> = vec!["e2e".to_string(), "--fresh".to_string()];+ assert_eq!(run_build_inner(&fresh_args, home.path()).unwrap(), 0);+ let manifest: Value =+ serde_json::from_str(&std::fs::read_to_string(pack.join("manifest.json")).unwrap())+ .unwrap();+ assert_eq!(+ manifest["generation"],+ json!(1),+ "--fresh wipes previous build dir and resets generation to 1"+ );+ for s in manifest["sources"].as_array().unwrap() {+ assert_eq!(s["skipped"], json!(0), "--fresh re-harvests from scratch");+ assert_eq!(s["new"], s["records"]);+ }}// ── end to end: a git source, watermark-skipped on the second run ─────diff --git a/engine/crates/xerj-autoindex/src/harvest/pack.rs b/engine/crates/xerj-autoindex/src/harvest/pack.rsindex 35fd1694d..839035dc1 100644--- a/engine/crates/xerj-autoindex/src/harvest/pack.rs+++ b/engine/crates/xerj-autoindex/src/harvest/pack.rs@@ -132,6 +132,21 @@ pub fn emit(shard_lines[bucket].push('\n');}+ // Remove any `records-*.jsonl` left in `pack_dir` by an earlier generation+ // that populated different (or more) shard buckets: otherwise orphan shard+ // files survive on disk and ship in a zip while invisible to `manifest.json`+ // and `SHA256SUMS` (#1110 gap 5a).+ for entry in std::fs::read_dir(&pack_dir)+ .with_context(|| format!("cannot read pack dir {}", pack_dir.display()))?+ {+ let e = entry.with_context(|| format!("cannot read entry in {}", pack_dir.display()))?;+ let name = e.file_name().to_string_lossy().to_string();+ if name.starts_with("records-") && name.ends_with(".jsonl") {+ std::fs::remove_file(e.path())+ .with_context(|| format!("cannot remove stale shard {}", e.path().display()))?;+ }+ }+let mut files: BTreeMap<String, Value> = BTreeMap::new();for (i, lines) in shard_lines.iter().enumerate() {let name = format!("records-{:05}.jsonl", i);@@ -236,11 +251,27 @@ fn uniform_keys(records: &[Map<String, Value>]) -> BTreeMap<String, bool> {}fn previous_generation(pack_dir: &Path) -> Result<u64> {- let Ok(bytes) = std::fs::read(pack_dir.join("manifest.json")) else {- return Ok(0);+ let manifest_path = pack_dir.join("manifest.json");+ let bytes = match std::fs::read(&manifest_path) {+ Ok(b) => b,+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(0),+ Err(e) => {+ return Err(e).with_context(|| format!("cannot read {}", manifest_path.display()))+ }};- let v: Value = serde_json::from_slice(&bytes).unwrap_or(Value::Null);- Ok(v.get("generation").and_then(Value::as_u64).unwrap_or(0))+ let v: Value = serde_json::from_slice(&bytes).with_context(|| {+ format!(+ "corrupt previous manifest {} (not valid JSON; pass --fresh to rebuild from scratch)",+ manifest_path.display()+ )+ })?;+ v.get("generation").and_then(Value::as_u64).ok_or_else(|| {+ anyhow::anyhow!(+ "corrupt previous manifest {}: missing or non-integer 'generation' \+ (pass --fresh to rebuild from scratch)",+ manifest_path.display()+ )+ })}fn hex(bytes: &[u8]) -> String {@@ -914,4 +945,93 @@ mod tests {);}}++ #[test]+ fn rebuild_with_fewer_shards_removes_orphan_records_files() {+ // #1110 gap 5a: when a rebuild populates fewer (or different) shard+ // buckets than the previous generation, stale `records-*.jsonl` files+ // must be swept before writing new shards so they never survive on+ // disk outside `manifest.json` and `SHA256SUMS`.+ let tmp = tempfile::tempdir().unwrap();+ let many: Vec<Map<String, Value>> = (0..32).map(|i| rec(&format!("id-{i}"), &[])).collect();+ let r1 = emit(+ tmp.path(),+ &recipe("t", 16),+ "# r\n",+ many.clone(),+ &stats_with_source(),+ &sugg(&many),+ )+ .unwrap();+ let gen1_shards: Vec<String> = std::fs::read_dir(&r1.dir)+ .unwrap()+ .flatten()+ .map(|e| e.file_name().to_string_lossy().to_string())+ .filter(|n| n.starts_with("records-") && n.ends_with(".jsonl"))+ .collect();+ assert!(gen1_shards.len() > 1, "multiple shard files populated");++ let one = vec![rec("id-0", &[])];+ let r2 = emit(+ tmp.path(),+ &recipe("t", 1),+ "# r\n",+ one.clone(),+ &stats_with_source(),+ &sugg(&one),+ )+ .unwrap();+ assert_eq!(r2.generation, 2);+ let gen2_shards: Vec<String> = std::fs::read_dir(&r2.dir)+ .unwrap()+ .flatten()+ .map(|e| e.file_name().to_string_lossy().to_string())+ .filter(|n| n.starts_with("records-") && n.ends_with(".jsonl"))+ .collect();+ assert_eq!(gen2_shards, vec!["records-00000.jsonl".to_string()]);+ }++ #[test]+ fn emit_refuses_corrupt_previous_manifest_instead_of_resetting_generation() {+ // #1110 gap 5b: a truncated or generation-less existing `manifest.json`+ // must fail the rebuild loudly, not silently reset `generation` to 1.+ let tmp = tempfile::tempdir().unwrap();+ let one = vec![rec("a", &[])];+ let r1 = emit(+ tmp.path(),+ &recipe("t", 1),+ "# r\n",+ one.clone(),+ &stats_with_source(),+ &sugg(&one),+ )+ .unwrap();+ assert_eq!(r1.generation, 1);++ std::fs::write(r1.dir.join("manifest.json"), "{truncated").unwrap();+ let err = emit(+ tmp.path(),+ &recipe("t", 1),+ "# r\n",+ one.clone(),+ &stats_with_source(),+ &sugg(&one),+ )+ .unwrap_err()+ .to_string();+ assert!(err.contains("corrupt previous manifest"), "{err}");++ std::fs::write(r1.dir.join("manifest.json"), r#"{"format_version":1}"#).unwrap();+ let err = emit(+ tmp.path(),+ &recipe("t", 1),+ "# r\n",+ one.clone(),+ &stats_with_source(),+ &sugg(&one),+ )+ .unwrap_err()+ .to_string();+ assert!(err.contains("missing or non-integer 'generation'"), "{err}");+ }}diff --git a/engine/crates/xerj-autoindex/src/harvest/recipe.rs b/engine/crates/xerj-autoindex/src/harvest/recipe.rsindex ca99e23ff..b7ef8462d 100644--- a/engine/crates/xerj-autoindex/src/harvest/recipe.rs+++ b/engine/crates/xerj-autoindex/src/harvest/recipe.rs@@ -87,13 +87,16 @@ pub struct SourceSpec {pub kind: SourceKind,/// `dir` sources: a path relative to the recipe file.pub path: Option<String>,- /// `http-zip`/`git` sources: where the bytes come from. A git url may- /// also be a local path (git itself accepts both — tests rely on it).+ /// `http-zip`/`http-json`/`git` sources: where the bytes come from. A git+ /// url may also be a local path (git itself accepts both — tests rely on it).pub url: Option<String>,/// `git` sources: a full commit sha (pinned), a branch or tag name, or/// absent to track the remote's default branch (HEAD).pub rev: Option<String>,pub glob: String,+ /// Optional top-level JSON object key holding the records array when a+ /// `.json` file is a wrapper catalog (e.g. CISA KEV's `"vulnerabilities"`).+ pub records_field: Option<String>,pub format: Format,/// Carried onto every record from this source as record DATA — the tool/// never enforces it; the pack author owns what goes in the pack.@@ -104,6 +107,7 @@ pub struct SourceSpec {pub enum SourceKind {Dir,HttpZip,+ HttpJson,Git,}@@ -267,6 +271,8 @@ struct RawSource {rev: Option<String>,#[serde(default = "d_glob")]glob: String,+ #[serde(default)]+ records_field: Option<String>,format: String,#[serde(default)]licence: String,@@ -418,9 +424,10 @@ fn compile(path: &Path, file: RawRecipe) -> Result<Recipe> {let kind = match s.kind.as_str() {"dir" => SourceKind::Dir,"http-zip" => SourceKind::HttpZip,+ "http-json" => SourceKind::HttpJson,"git" => SourceKind::Git,other => bail!(at(&format!(- "source '{}': unknown kind '{other}' (supported: dir, http-zip, git)",+ "source '{}': unknown kind '{other}' (supported: dir, http-zip, http-json, git)",s.slug))),};@@ -433,6 +440,14 @@ fn compile(path: &Path, file: RawRecipe) -> Result<Recipe> {s.slug))),};+ if let Some(rf) = &s.records_field {+ if rf.trim().is_empty() {+ bail!(at(&format!(+ "source '{}': 'records_field' cannot be empty",+ s.slug+ )));+ }+ }let url = s.url.as_deref().unwrap_or_default().to_string();let rev = s.rev.as_deref().unwrap_or_default().to_string();match kind {@@ -458,16 +473,21 @@ fn compile(path: &Path, file: RawRecipe) -> Result<Recipe> {)));}}- SourceKind::HttpZip => {+ SourceKind::HttpZip | SourceKind::HttpJson => {+ let kind_str = if kind == SourceKind::HttpZip {+ "http-zip"+ } else {+ "http-json"+ };if !url.starts_with("http://") && !url.starts_with("https://") {bail!(at(&format!(- "http-zip source '{}' needs an http(s) 'url', got '{url}'",+ "{kind_str} source '{}' needs an http(s) 'url', got '{url}'",s.slug)));}if s.path.is_some() || !rev.is_empty() {bail!(at(&format!(- "http-zip source '{}' takes 'url' only — no 'path'/'rev'",+ "{kind_str} source '{}' takes 'url' only — no 'path'/'rev'",s.slug)));}@@ -495,6 +515,7 @@ fn compile(path: &Path, file: RawRecipe) -> Result<Recipe> {url: s.url.clone(),rev: if rev.is_empty() { None } else { Some(rev) },glob: s.glob.clone(),+ records_field: s.records_field.clone(),format,licence: s.licence.clone(),});diff --git a/engine/crates/xerj-autoindex/src/harvest/source.rs b/engine/crates/xerj-autoindex/src/harvest/source.rsindex ad2a7a09c..76fece29f 100644--- a/engine/crates/xerj-autoindex/src/harvest/source.rs+++ b/engine/crates/xerj-autoindex/src/harvest/source.rs@@ -69,6 +69,24 @@ pub fn fetch(src: &SourceSpec, recipe_dir: &Path, cache_dir: &Path) -> Result<Feunchanged: false,})}+ super::recipe::SourceKind::HttpJson => {+ let url = src.url.as_deref().unwrap_or_default();+ let root = cache_dir.join(&src.slug);+ if root.exists() {+ std::fs::remove_dir_all(&root)+ .with_context(|| format!("source '{}': clear {}", src.slug, root.display()))?;+ }+ std::fs::create_dir_all(&root)+ .with_context(|| format!("source '{}': create {}", src.slug, root.display()))?;+ let json_path = root.join(format!("{}.json", src.slug));+ httpget::download(url, &json_path)+ .with_context(|| format!("source '{}': fetch", src.slug))?;+ Ok(Fetched {+ root,+ watermark: None,+ unchanged: false,+ })+ }super::recipe::SourceKind::Git => {let url = src.url.as_deref().unwrap_or_default();let root = cache_dir.join(&src.slug);@@ -329,6 +347,7 @@ mod tests {rev: None,glob: "**/*.json".into(),format: super::super::recipe::Format::Flat,+ records_field: None,licence: "CC0-1.0".into(),};let f1 = fetch(&spec, tmp.path(), &cache).unwrap();@@ -408,6 +427,7 @@ mod tests {rev: Some("export".into()),glob: "**/*.json".into(),format: super::super::recipe::Format::Flat,+ records_field: None,licence: "CC0-1.0".into(),};let f = fetch(&spec, tmp.path(), &tmp.path().join("cache")).unwrap();diff --git a/engine/crates/xerj-autoindex/src/harvest/store.rs b/engine/crates/xerj-autoindex/src/harvest/store.rsindex c4eaee673..0bc105321 100644--- a/engine/crates/xerj-autoindex/src/harvest/store.rs+++ b/engine/crates/xerj-autoindex/src/harvest/store.rs@@ -18,6 +18,7 @@ use std::collections::HashSet;use std::io;use std::path::{Path, PathBuf};+use anyhow::{bail, Context, Result};use serde_json::{Map, Value};use xxhash_rust::xxh3::xxh3_128;@@ -70,16 +71,28 @@ fn file_for(store_dir: &Path, slug: &str, key: &str) -> PathBuf {/// Each returned map carries the file's store key under the `KEY` sentinel/// so member ordering can fall back to it; `put` computes the key BEFORE/// injection, so the sentinel is never part of any stored content.-pub fn read_all(store_dir: &Path, slugs: &HashSet<String>) -> Vec<Map<String, Value>> {+///+/// Any unreadable or non-object `<slug>.<key>.json` for a live slug is a+/// hard error — silently skipping a corrupt store file would omit that+/// record from the emitted pack with a green exit (#1110 gap 4).+pub fn read_all(store_dir: &Path, slugs: &HashSet<String>) -> Result<Vec<Map<String, Value>>> {let mut out = Vec::new();- let Ok(entries) = std::fs::read_dir(store_dir) else {- return out;+ let entries = match std::fs::read_dir(store_dir) {+ Ok(entries) => entries,+ Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(out),+ Err(e) => {+ return Err(e).with_context(|| format!("cannot read store dir {}", store_dir.display()))+ }};- let mut names: Vec<String> = entries- .flatten()- .filter(|e| e.path().extension().and_then(|x| x.to_str()) == Some("json"))- .filter_map(|e| e.file_name().to_str().map(str::to_string))- .collect();+ let mut names: Vec<String> = Vec::new();+ for entry in entries {+ let e = entry.with_context(|| format!("cannot read entry in {}", store_dir.display()))?;+ if e.path().extension().and_then(|x| x.to_str()) == Some("json") {+ if let Some(name) = e.file_name().to_str() {+ names.push(name.to_string());+ }+ }+ }names.sort();for name in names {// "<slug>.<32-hex>.json" — split from the RIGHT: slugs may contain@@ -93,14 +106,22 @@ pub fn read_all(store_dir: &Path, slugs: &HashSet<String>) -> Vec<Map<String, Vaif !slugs.contains(slug) {continue;}- if let Ok(raw) = std::fs::read_to_string(store_dir.join(&name)) {- if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(&raw) {- m.insert(super::identity::KEY.into(), Value::String(key.to_string()));- out.push(m);- }- }+ let path = store_dir.join(&name);+ let raw = std::fs::read_to_string(&path)+ .with_context(|| format!("cannot read store envelope {}", path.display()))?;+ let v: Value = serde_json::from_str(&raw).with_context(|| {+ format!("corrupt store envelope {} (not valid JSON)", path.display())+ })?;+ let Value::Object(mut m) = v else {+ bail!(+ "corrupt store envelope {} (expected JSON object)",+ path.display()+ );+ };+ m.insert(super::identity::KEY.into(), Value::String(key.to_string()));+ out.push(m);}- out+ Ok(out)}/// Drop envelopes of `slug` that this run did NOT re-put — a file deleted@@ -162,7 +183,7 @@ mod tests {let (_, new2) = put(&dir, "a", &env("x", "a")).unwrap();assert!(new1);assert!(!new2, "second put of identical content skips");- let all = read_all(&dir, &HashSet::from(["a".to_string()]));+ let all = read_all(&dir, &HashSet::from(["a".to_string()])).unwrap();assert_eq!(all.len(), 1);assert_eq!(all[0]["id"], json!("x"));assert_eq!(k1.len(), 32);@@ -175,10 +196,10 @@ mod tests {let dir = tmp.path().join("store");put(&dir, "a", &env("x", "a")).unwrap();put(&dir, "b", &env("x", "b")).unwrap();- let all = read_all(&dir, &HashSet::from(["a".to_string(), "b".to_string()]));+ let all = read_all(&dir, &HashSet::from(["a".to_string(), "b".to_string()])).unwrap();assert_eq!(all.len(), 2);// retired source stays on disk but is not returned- let only_a = read_all(&dir, &HashSet::from(["a".to_string()]));+ let only_a = read_all(&dir, &HashSet::from(["a".to_string()])).unwrap();assert_eq!(only_a.len(), 1);assert_eq!(only_a[0]["_src"], json!("a"));}@@ -195,8 +216,38 @@ mod tests {let removed = prune(&dir, "a", &HashSet::from([ka]));assert_eq!(removed, 1, "y died, x lived, b untouched");assert_eq!(- read_all(&dir, &HashSet::from(["a".into(), "b".into()])).len(),+ read_all(&dir, &HashSet::from(["a".into(), "b".into()]))+ .unwrap()+ .len(),2);}++ #[test]+ fn read_all_refuses_corrupt_store_envelope_instead_of_dropping_it() {+ // #1110 gap 4: a truncated or non-object file in store/ for a live+ // slug must fail the build loudly, never silently omit the record.+ let tmp = tempfile::tempdir().unwrap();+ let dir = tmp.path().join("store");+ let (k, _) = put(&dir, "a", &env("x", "a")).unwrap();+ let path = dir.join(format!("a.{k}.json"));++ std::fs::write(&path, "{truncated").unwrap();+ let err = read_all(&dir, &HashSet::from(["a".to_string()]))+ .unwrap_err()+ .to_string();+ assert!(err.contains("corrupt store envelope"), "{err}");++ std::fs::write(&path, "[1, 2, 3]").unwrap();+ let err = read_all(&dir, &HashSet::from(["a".to_string()]))+ .unwrap_err()+ .to_string();+ assert!(err.contains("expected JSON object"), "{err}");++ // a corrupt file belonging to a RETIRED slug is ignored, matching the+ // retired-source contract above.+ assert!(read_all(&dir, &HashSet::from(["other".to_string()]))+ .unwrap()+ .is_empty());+ }}diff --git a/engine/crates/xerj-autoindex/tests/pack_claims.rs b/engine/crates/xerj-autoindex/tests/pack_claims.rsindex 945289525..587838663 100644--- a/engine/crates/xerj-autoindex/tests/pack_claims.rs+++ b/engine/crates/xerj-autoindex/tests/pack_claims.rs@@ -28,35 +28,35 @@ fn repo_root() -> PathBuf {}}-fn pack_dir(root: &Path) -> PathBuf {- root.join("tools/packs/rust-vulns")+fn pack_dir_named(root: &Path, name: &str) -> PathBuf {+ root.join("tools/packs").join(name)}fn read(path: &Path) -> String {std::fs::read_to_string(path).unwrap_or_else(|e| panic!("read {}: {e}", path.display()))}-fn stats(root: &Path) -> serde_json::Value {- let dir = pack_dir(root);+fn stats_named(root: &Path, name: &str) -> serde_json::Value {+ let dir = pack_dir_named(root, name);let text = read(&dir.join("pack-stats.json"));serde_json::from_str(&text).unwrap_or_else(|e| panic!("pack-stats.json is not valid JSON: {e}"))}/// The README's table row for `label` must carry exactly the number in/// pack-stats.json under `key`.-fn readme_row_pins(root: &Path, label: &str, key: &str) {- let readme = read(&pack_dir(root).join("README.md"));- let want = stats(root)+fn readme_row_pins_named(root: &Path, name: &str, label: &str, key: &str) {+ let readme = read(&pack_dir_named(root, name).join("README.md"));+ let want = stats_named(root, name).get(key).and_then(|v| v.as_u64())- .unwrap_or_else(|| panic!("pack-stats.json has no numeric '{key}'"));+ .unwrap_or_else(|| panic!("{name}/pack-stats.json has no numeric '{key}'"));// rows look like `| … label … | 1950 |` or `| … label … | 270 (707 …) |`,// so pin on maximal digit runs, not whole cellslet needle = label.to_string();let row = readme.lines().find(|l| l.contains(&needle))- .unwrap_or_else(|| panic!("README has no row containing '{needle}'"));+ .unwrap_or_else(|| panic!("{name}/README has no row containing '{needle}'"));let numbers: Vec<String> = row.split(|c: char| !c.is_ascii_digit()).filter(|s| !s.is_empty())@@ -64,7 +64,7 @@ fn readme_row_pins(root: &Path, label: &str, key: &str) {.collect();assert!(numbers.contains(&want.to_string()),- "README row '{needle}' does not carry {want} (pack-stats.{key}); rebuild \+ "{name}/README row '{needle}' does not carry {want} (pack-stats.{key}); rebuild \the pack and move the docs with the numbers",);}@@ -72,7 +72,7 @@ fn readme_row_pins(root: &Path, label: &str, key: &str) {#[test]fn pack_stats_are_measured_not_asserted() {let root = repo_root();- let s = stats(&root);+ let s = stats_named(&root, "rust-vulns");assert_eq!(s["provenance"], "measured","pack-stats.json must record provenance 'measured' — 'estimated' numbers \@@ -90,7 +90,7 @@ fn pack_stats_are_measured_not_asserted() {] {assert!(s.get(key).and_then(|v| v.as_u64()).is_some_and(|v| v > 0),- "pack-stats.json lacks a positive numeric '{key}'"+ "rust-vulns/pack-stats.json lacks a positive numeric '{key}'");}// internal consistency: the parts cannot exceed the whole@@ -99,67 +99,222 @@ fn pack_stats_are_measured_not_asserted() {let with_fn = s["records_with_functions"].as_u64().unwrap();assert!(merged <= records, "more merged records than records");assert!(with_fn <= records, "more function records than records");++ let kev = stats_named(&root, "cisa-kev");+ assert_eq!(kev["provenance"], "measured");+ for key in [+ "envelopes",+ "records",+ "vendors",+ "products",+ "records_with_cwe",+ "cwe_ids_distinct",+ "records_ransomware_known",+ "records_bytes",+ ] {+ assert!(+ kev.get(key).and_then(|v| v.as_u64()).is_some_and(|v| v > 0),+ "cisa-kev/pack-stats.json lacks a positive numeric '{key}'"+ );+ }+ let kev_records = kev["records"].as_u64().unwrap();+ assert!(kev["records_with_cwe"].as_u64().unwrap() <= kev_records);+ assert!(kev["records_ransomware_known"].as_u64().unwrap() <= kev_records);}#[test]fn readme_counts_match_the_measured_stats() {let root = repo_root();- readme_row_pins(&root, "Source records harvested", "envelopes");- readme_row_pins(&root, "Records after identity resolution", "records");- readme_row_pins(+ readme_row_pins_named(&root, "rust-vulns", "Source records harvested", "envelopes");+ readme_row_pins_named(+ &root,+ "rust-vulns",+ "Records after identity resolution",+ "records",+ );+ readme_row_pins_named(&root,+ "rust-vulns","…merged from both sources","merged_from_both_sources",);- readme_row_pins(+ readme_row_pins_named(&root,+ "rust-vulns","Records with affected-function paths","records_with_functions",);- readme_row_pins(+ readme_row_pins_named(&root,+ "rust-vulns","Records with affected-function paths","function_paths_distinct",);- readme_row_pins(&root, "Records withdrawn", "records_withdrawn");- readme_row_pins(&root, "Packages covered", "packages");+ readme_row_pins_named(+ &root,+ "rust-vulns",+ "Records withdrawn",+ "records_withdrawn",+ );+ readme_row_pins_named(&root, "rust-vulns", "Packages covered", "packages");++ readme_row_pins_named(&root, "cisa-kev", "Source records harvested", "envelopes");+ readme_row_pins_named(+ &root,+ "cisa-kev",+ "Records after identity resolution",+ "records",+ );+ readme_row_pins_named(&root, "cisa-kev", "Vendors covered", "vendors");+ readme_row_pins_named(&root, "cisa-kev", "Products covered", "products");+ readme_row_pins_named(+ &root,+ "cisa-kev",+ "Records with CWE classification",+ "records_with_cwe",+ );+ readme_row_pins_named(+ &root,+ "cisa-kev",+ "Records with CWE classification",+ "cwe_ids_distinct",+ );+ readme_row_pins_named(+ &root,+ "cisa-kev",+ "Records flagged known ransomware use",+ "records_ransomware_known",+ );+ readme_row_pins_named(&root, "cisa-kev", "Records JSONL bytes", "records_bytes");}#[test]fn the_recipe_is_the_one_the_stats_measured() {let root = repo_root();- let dir = pack_dir(&root);- let recipe = read(&dir.join("recipe.toml"));- let mut hasher = sha2::Sha256::new();- use sha2::Digest;- hasher.update(recipe.as_bytes());- let got = format!("{:x}", hasher.finalize());- let want = stats(&root)["recipe_sha256"]- .as_str()- .expect("pack-stats.json pins recipe_sha256")- .to_string();- assert_eq!(- got, want,- "recipe.toml changed since pack-stats.json was measured — rebuild the \- pack and refresh pack-stats.json (and the README numbers with it)"- );-- // and the stats' source table names exactly the recipe's sources- for slug in ["rustsec", "osv"] {- assert!(- recipe.contains(&format!("slug = \"{slug}\"")),- "recipe no longer declares source '{slug}'"+ for (pack, slugs) in [+ ("rust-vulns", &["rustsec", "osv"][..]),+ ("cisa-kev", &["kev"][..]),+ ] {+ let dir = pack_dir_named(&root, pack);+ let recipe = read(&dir.join("recipe.toml"));+ let mut hasher = sha2::Sha256::new();+ use sha2::Digest;+ hasher.update(recipe.as_bytes());+ let got = format!("{:x}", hasher.finalize());+ let want = stats_named(&root, pack)["recipe_sha256"]+ .as_str()+ .expect("pack-stats.json pins recipe_sha256")+ .to_string();+ assert_eq!(+ got, want,+ "{pack}/recipe.toml changed since pack-stats.json was measured — rebuild the \+ pack and refresh pack-stats.json (and the README numbers with it)");++ for slug in slugs {+ assert!(+ recipe.contains(&format!("slug = \"{slug}\"")),+ "{pack}/recipe no longer declares source '{slug}'"+ );+ assert!(+ stats_named(&root, pack)["sources"][slug].is_object(),+ "{pack}/pack-stats.json no longer records source '{slug}'"+ );+ }assert!(- stats(&root)["sources"][slug].is_object(),- "pack-stats.json no longer records source '{slug}'"+ !recipe.contains("CC-BY") && !recipe.contains("BUSL") && !recipe.contains("GPL"),+ "{pack} README says all record data is CC0; a new source with a \+ different licence must move that claim too");}- // every licence in the recipe is CC0; the README's licence claims must- // not silently grow a restricted term- assert!(- !recipe.contains("CC-BY") && !recipe.contains("BUSL") && !recipe.contains("GPL"),- "the pack README says all record data is CC0; a new source with a \- different licence must move that claim too"+}++#[test]+#[ignore = "fetches live CISA KEV feed over HTTPS to measure/refresh tools/packs/cisa-kev/pack-stats.json"]+fn measure_cisa_kev_pack_live() {+ use std::collections::BTreeSet;+ let root = repo_root();+ let recipe_path = root.join("tools/packs/cisa-kev/recipe.toml");+ let recipe_text = read(&recipe_path);+ use sha2::Digest;+ let mut hasher = sha2::Sha256::new();+ hasher.update(recipe_text.as_bytes());+ let recipe_sha256 = format!("{:x}", hasher.finalize());++ let home = tempfile::tempdir().unwrap();+ std::env::set_var("XERJ_CODE_HOME", home.path());+ let t0 = std::time::Instant::now();+ let code = xerj_autoindex::harvest::run_build(&[+ "cisa-kev".to_string(),+ "--recipe".to_string(),+ recipe_path.to_string_lossy().to_string(),+ ]);+ let elapsed = t0.elapsed().as_secs().max(1);+ assert_eq!(code, 0, "cisa-kev build failed");++ let pack = home.path().join("builds/cisa-kev/pack/cisa-kev");+ let manifest: serde_json::Value =+ serde_json::from_str(&read(&pack.join("manifest.json"))).unwrap();+ let envelopes = manifest["counts"]["envelopes"].as_u64().unwrap();+ let records_count = manifest["counts"]["records"].as_u64().unwrap();++ let mut vendors = BTreeSet::new();+ let mut products = BTreeSet::new();+ let mut cwes_distinct = BTreeSet::new();+ let mut records_with_cwe = 0u64;+ let mut records_ransomware_known = 0u64;+ let mut records_bytes = 0u64;++ for name in manifest["files"].as_object().unwrap().keys() {+ let p = pack.join(name);+ records_bytes += std::fs::metadata(&p).unwrap().len();+ for line in read(&p).lines() {+ if line.trim().is_empty() {+ continue;+ }+ let rec: serde_json::Value = serde_json::from_str(line).unwrap();+ if let Some(v) = rec["vendorProject"].as_str().filter(|s| !s.is_empty()) {+ vendors.insert(v.to_string());+ }+ if let Some(p) = rec["product"].as_str().filter(|s| !s.is_empty()) {+ products.insert(p.to_string());+ }+ if let Some(arr) = rec["cwe_ids"].as_array() {+ if !arr.is_empty() {+ records_with_cwe += 1;+ for c in arr {+ if let Some(s) = c.as_str() {+ cwes_distinct.insert(s.to_string());+ }+ }+ }+ }+ if rec["ransomware_status"] == serde_json::json!("known") {+ records_ransomware_known += 1;+ }+ }+ }++ let stats_json = serde_json::json!({+ "provenance": "measured",+ "measured_at": "2026-10-15",+ "pack": "cisa-kev",+ "recipe_sha256": recipe_sha256,+ "sources": {+ "kev": { "kind": "http-json", "envelopes": envelopes, "licence": "CC0-1.0" }+ },+ "envelopes": envelopes,+ "records": records_count,+ "vendors": vendors.len(),+ "products": products.len(),+ "records_with_cwe": records_with_cwe,+ "cwe_ids_distinct": cwes_distinct.len(),+ "records_ransomware_known": records_ransomware_known,+ "records_bytes": records_bytes,+ "build_seconds": elapsed,+ });+ println!(+ "MEASURED_CISA_KEV_STATS:\n{}",+ serde_json::to_string_pretty(&stats_json).unwrap());}diff --git a/tools/packs/cisa-kev/README.md b/tools/packs/cisa-kev/README.mdnew file mode 100644index 000000000..e372ba101--- /dev/null+++ b/tools/packs/cisa-kev/README.md@@ -0,0 +1,64 @@+# cisa-kev — CISA Known Exploited Vulnerabilities corpus pack++The US Cybersecurity and Infrastructure Security Agency (CISA) **Known Exploited+Vulnerabilities (KEV)** catalog as a portable, checksum-verified corpus pack —+built with `xerj corpus build` from this directory's+[recipe.toml](./recipe.toml) without a line of domain-specific Rust code.++Every number below is measured from a real build+([pack-stats.json](./pack-stats.json), `provenance: measured`), and+`tests/pack_claims.rs` pins this README against `pack-stats.json` and the recipe+SHA-256 so the three cannot drift apart.++| What | Count |+|---|---|+| Source records harvested (envelopes) | 1746 |+| Records after identity resolution | 1746 |+| Vendors covered (`vendorProject`) | 289 |+| Products covered (`product`) | 705 |+| Records with CWE classification | 1571 (189 distinct CWE IDs) |+| Records flagged known ransomware use | 361 |+| Records JSONL bytes | 2993632 |+| Cold build (network fetch + harvest + pack) | ~1 s |++## Why this pack exists++`rust-vulns` exercises multi-source OSV + RustSec markdown advisory merging.+`cisa-kev` exercises a completely different shape on the same pipeline (#1110):++- **Wrapped JSON feed (`kind = "http-json"`, `records_field = "vulnerabilities"`)**+ instead of a git repository or zip archive.+- **Flat record normalization (`format = "flat"`)** with domain-native field+ names (`cveID`, `vendorProject`, `product`, `vulnerabilityName`,+ `shortDescription`, `requiredAction`, `knownRansomwareCampaignUse`, `cwes`).+- **Derived fields** via `regex_extract` (`cwe_ids`) and `map`+ (`ransomware_status`: `"known"` vs `"unknown"`).+- **Public domain licence (`CC0-1.0`)**: US federal government work under+ 17 U.S.C. § 105.++## Build, consume, query++```sh+# 1. build (or incrementally refresh) from the recipe+xerj corpus build cisa-kev --recipe tools/packs/cisa-kev/recipe.toml++# 2. install the pack into your corpus store+xerj corpus add cisa-kev --from ~/.xerj-code/builds/cisa-kev/pack/cisa-kev++# 3. index and query+xerj corpus index cisa-kev+xerj code cisa-kev "exchange server SSRF proxylogon"+```++## Fields worth knowing++| Field | What it carries |+|---|---|+| `id` / `cveID` | canonical CVE identifier (`CVE-YYYY-NNNN+`) |+| `title` / `vulnerabilityName` | human-readable vulnerability title |+| `vendorProject` / `product` | vendor and product names (also routed into `defs`) |+| `shortDescription` / `requiredAction` | CISA summary and remediation directive |+| `dateAdded` / `dueDate` | catalog addition date and federal remediation deadline |+| `cwe_ids` | derived array of `CWE-NNN` identifiers |+| `ransomware_status` | derived `"known"` or `"unknown"` from `knownRansomwareCampaignUse` |+| `licence` / `sources` / `origin` | provenance per record (`CC0-1.0`, `["kev"]`) |diff --git a/tools/packs/cisa-kev/pack-stats.json b/tools/packs/cisa-kev/pack-stats.jsonnew file mode 100644index 000000000..7c51fc796--- /dev/null+++ b/tools/packs/cisa-kev/pack-stats.json@@ -0,0 +1,18 @@+{+ "provenance": "measured",+ "measured_at": "2026-10-15",+ "pack": "cisa-kev",+ "recipe_sha256": "b2f5b24f5fbcd0c076bdc19d844c962da0883783a121528f0e29c32fcdf9abf9",+ "sources": {+ "kev": { "kind": "http-json", "envelopes": 1746, "licence": "CC0-1.0" }+ },+ "envelopes": 1746,+ "records": 1746,+ "vendors": 289,+ "products": 705,+ "records_with_cwe": 1571,+ "cwe_ids_distinct": 189,+ "records_ransomware_known": 361,+ "records_bytes": 2993632,+ "build_seconds": 1+}diff --git a/tools/packs/cisa-kev/recipe.toml b/tools/packs/cisa-kev/recipe.tomlnew file mode 100644index 000000000..94ebfb6df--- /dev/null+++ b/tools/packs/cisa-kev/recipe.toml@@ -0,0 +1,56 @@+# cisa-kev — CISA Known Exploited Vulnerabilities (KEV) catalog.+#+# Second-domain corpus pack (#1110) proving the corpus builder generalizes+# without Rust/OSV assumptions:+# - Upstream is a single wrapped JSON document (`kind = "http-json"`,+# `records_field = "vulnerabilities"`) rather than an OSV zip or git tree.+# - Records use `format = "flat"` (no OSV or RustSec adapter) with domain-native+# keys (`cveID`, `vendorProject`, `product`, `vulnerabilityName`,+# `shortDescription`, `requiredAction`, `knownRansomwareCampaignUse`, `cwes`).+# - Licence: US federal government work (17 U.S.C. § 105), public domain /+# CC0-1.0 equivalent.++[recipe]+format = 1+name = "cisa-kev"+description = "CISA Known Exploited Vulnerabilities (KEV) catalog — actively exploited CVEs across all vendors and products"++[[sources]]+slug = "kev"+kind = "http-json"+url = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"+records_field = "vulnerabilities"+format = "flat"+licence = "CC0-1.0"++[envelope]+id_from = ["cveID"]+title_from = ["vulnerabilityName", "cveID"]+body_join = ["vendorProject", "product", "vulnerabilityName", "shortDescription", "requiredAction", "notes"]+defs_from = ["vendorProject", "product", "cwes"]++[identity]+edges = [{ field = "cveID" }]+canonical_source_order = ["kev"]++[merge]+precedence = ["kev"]++[[derived]]+name = "cwe_ids"+op = "regex_extract"+from = ["cwes", "notes"]+pattern = '\bCWE-\d+\b'+unique = true++[[derived]]+name = "ransomware_status"+op = "map"+from = ["knownRansomwareCampaignUse"]+rules = [+ { match = "(?i)^known$", value = "known" },+]+default = "unknown"++[emit]+shards = 8