Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(autoindex): harden corpus harvest negative paths and add cisa-kev pack (#1110)

Christian M. Todiec4784f6f78f2
diff --git a/docs/CORPUS_PACKS.md b/docs/CORPUS_PACKS.md
index 9dbf9c598..b451e523a 100644
--- a/docs/CORPUS_PACKS.md
+++ b/docs/CORPUS_PACKS.md
@@ -23,7 +23,7 @@ Every statement below points at the code that makes it true:
|---|---|
| Pipeline orchestration, `corpus build` CLI | `engine/crates/xerj-autoindex/src/harvest/mod.rs` |
| Recipe parsing + strict validation | `engine/crates/xerj-autoindex/src/harvest/recipe.rs` |
-| Source adapters (`dir`, `http-zip`, `git`) | `engine/crates/xerj-autoindex/src/harvest/source.rs` |
+| Source adapters (`dir`, `http-zip`, `http-json`, `git`) | `engine/crates/xerj-autoindex/src/harvest/source.rs` |
| Input-format normalisation (`osv`, `rustsec-md`, `flat`) | `engine/crates/xerj-autoindex/src/harvest/normalize.rs` |
| Content-addressed store (presence = dedup) | `engine/crates/xerj-autoindex/src/harvest/store.rs` |
| Union-find identity resolution | `engine/crates/xerj-autoindex/src/harvest/identity.rs` |
@@ -32,7 +32,7 @@ Every statement below points at the code that makes it true:
| ed25519 keygen / sign / verify | `engine/crates/xerj-autoindex/src/harvest/sign.rs` |
| `corpus add --from <pack>` + `--verify-sig` | `engine/crates/xerj-autoindex/src/xc.rs` |
| Claims pinned to a measured build | `engine/crates/xerj-autoindex/tests/pack_claims.rs` |
-| Showcase recipe + stats | `tools/packs/rust-vulns/` |
+| Showcase recipes + stats | `tools/packs/rust-vulns/`, `tools/packs/cisa-kev/` |
| Scheduled build/sign/publish | `.github/workflows/pack-publish.yml` |
## Quick start
@@ -54,10 +54,11 @@ xerj corpus add rust-vulns --from ~/.xerj-code/builds/rust-vulns/pack/rust-vulns
A recipe is a commented TOML file — licence reasoning belongs in comments, and
the recipe ships verbatim inside the pack as provenance. Sections:
-- `[[sources]]` — `slug`, `kind` (`dir` | `http-zip` | `git`), `url`/`path`,
- `format` (the normalisation adapter), `licence`, optional `glob` and
- watermark. Sources are advisory-listed upstream trees, HTTP zip exports, or
- git repos fetched shallow at a pinned ref.
+- `[[sources]]` — `slug`, `kind` (`dir` | `http-zip` | `http-json` | `git`),
+ `url`/`path`, `format` (the normalisation adapter), `licence`, optional
+ `glob`, `records_field` (to unwrap a top-level JSON object's array field), and
+ watermark. Sources are advisory-listed upstream trees, HTTP zip exports,
+ single-URL JSON feeds, or git repos fetched shallow at a pinned ref.
- `[envelope]` — which normalised fields become the universal envelope:
`id_from`, `title_from`, `body_join`, `defs_from`. `defs` (space-joined
symbol paths) is what feeds `xerj code`'s definition search.
diff --git a/engine/crates/xerj-autoindex/src/harvest/mod.rs b/engine/crates/xerj-autoindex/src/harvest/mod.rs
index 4dc49d147..21a5288ca 100644
--- a/engine/crates/xerj-autoindex/src/harvest/mod.rs
+++ b/engine/crates/xerj-autoindex/src/harvest/mod.rs
@@ -162,7 +162,7 @@ fn run_build_inner(args: &[String], home: &Path) -> Result<i32> {
let files = walk(&fetched.root, &src.glob)?;
let mut keys = HashSet::new();
for (rel, path) in files {
- for value in parse_file(&path)? {
+ for value in parse_file(&path, src.records_field.as_deref())? {
let mut env = normalize::normalize(src.format, &value);
env.insert(SRC.into(), Value::String(src.slug.clone()));
env.insert(
@@ -192,7 +192,7 @@ fn run_build_inner(args: &[String], home: &Path) -> Result<i32> {
// ── identity + merge over the FULL store (not just this run's files) ──
let slugs: HashSet<String> = recipe.sources.iter().map(|s| s.slug.clone()).collect();
- let envelopes = store::read_all(&store_dir, &slugs);
+ let envelopes = store::read_all(&store_dir, &slugs)?;
let clusters = identity::cluster(&envelopes, &recipe.identity);
let merge_order = if recipe.merge.precedence.is_empty() {
recipe.identity.canonical_source_order.clone()
@@ -278,6 +278,7 @@ fn kind_name(kind: SourceKind) -> &'static str {
match kind {
SourceKind::Dir => "dir",
SourceKind::HttpZip => "http-zip",
+ SourceKind::HttpJson => "http-json",
SourceKind::Git => "git",
}
}
@@ -367,7 +368,7 @@ fn glob_regex(glob: &str) -> regex::Regex {
/// source is visible junk, not a silent parse error. Other extensions are
/// skipped by the glob, not here — reaching this with an unlisted type is a
/// hard error, not a silent drop.
-fn parse_file(path: &Path) -> Result<Vec<Value>> {
+fn parse_file(path: &Path, records_field: Option<&str>) -> Result<Vec<Value>> {
let raw =
std::fs::read_to_string(path).with_context(|| format!("cannot read {}", path.display()))?;
let name = path
@@ -394,6 +395,21 @@ fn parse_file(path: &Path) -> Result<Vec<Value>> {
} else {
let v: Value = serde_json::from_str(&raw)
.with_context(|| format!("{}: not valid JSON", path.display()))?;
+ if let Some(field) = records_field {
+ let Value::Object(mut m) = v else {
+ bail!(
+ "{}: records_field '{field}' requires a top-level JSON object",
+ path.display()
+ );
+ };
+ let Some(Value::Array(a)) = m.remove(field) else {
+ bail!(
+ "{}: records_field '{field}' missing or not a JSON array",
+ path.display()
+ );
+ };
+ return Ok(a);
+ }
Ok(match v {
Value::Array(a) => a,
other => vec![other],
@@ -465,18 +481,36 @@ mod tests {
let tmp = tempfile::tempdir().unwrap();
let a = tmp.path().join("a.json");
std::fs::write(&a, r#"[{"id":"x"},{"id":"y"}]"#).unwrap();
- assert_eq!(parse_file(&a).unwrap().len(), 2);
+ assert_eq!(parse_file(&a, None).unwrap().len(), 2);
let b = tmp.path().join("b.jsonl");
std::fs::write(&b, "{\"id\":\"z\"}\n\n{\"id\":\"w\"}\n").unwrap();
- assert_eq!(parse_file(&b).unwrap().len(), 2, "blank lines skipped");
+ assert_eq!(
+ parse_file(&b, None).unwrap().len(),
+ 2,
+ "blank lines skipped"
+ );
let c = tmp.path().join("c.json");
std::fs::write(&c, r#"{"id":"one"}"#).unwrap();
- assert_eq!(parse_file(&c).unwrap().len(), 1);
+ assert_eq!(parse_file(&c, None).unwrap().len(), 1);
let d = tmp.path().join("d.md");
std::fs::write(&d, "```toml\n[advisory]\nid = \"R\"\n```\n\n# T\n").unwrap();
- let parsed = parse_file(&d).unwrap();
+ let parsed = parse_file(&d, None).unwrap();
assert_eq!(parsed.len(), 1, "one raw string value, not json-parsed");
assert!(parsed[0].as_str().unwrap().starts_with("```toml"));
+ let wrapped = tmp.path().join("wrapped.json");
+ std::fs::write(
+ &wrapped,
+ r#"{"title":"KEV","vulnerabilities":[{"cveID":"CVE-1"},{"cveID":"CVE-2"}]}"#,
+ )
+ .unwrap();
+ assert_eq!(
+ parse_file(&wrapped, Some("vulnerabilities")).unwrap().len(),
+ 2
+ );
+ let err = parse_file(&wrapped, Some("missing"))
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("missing or not a JSON array"), "{err}");
}
// ── end to end: recipe → harvest → identity → pack, twice ─────────────
@@ -765,6 +799,22 @@ shards = 4
"dead envelope pruned"
);
assert_eq!(read_records(&pack).len(), 1);
+
+ // ── fourth run with --fresh: wipes build dir, resets generation ────
+ let fresh_args: Vec<String> = vec!["e2e".to_string(), "--fresh".to_string()];
+ assert_eq!(run_build_inner(&fresh_args, home.path()).unwrap(), 0);
+ let manifest: Value =
+ serde_json::from_str(&std::fs::read_to_string(pack.join("manifest.json")).unwrap())
+ .unwrap();
+ assert_eq!(
+ manifest["generation"],
+ json!(1),
+ "--fresh wipes previous build dir and resets generation to 1"
+ );
+ for s in manifest["sources"].as_array().unwrap() {
+ assert_eq!(s["skipped"], json!(0), "--fresh re-harvests from scratch");
+ assert_eq!(s["new"], s["records"]);
+ }
}
// ── end to end: a git source, watermark-skipped on the second run ─────
diff --git a/engine/crates/xerj-autoindex/src/harvest/pack.rs b/engine/crates/xerj-autoindex/src/harvest/pack.rs
index 35fd1694d..839035dc1 100644
--- a/engine/crates/xerj-autoindex/src/harvest/pack.rs
+++ b/engine/crates/xerj-autoindex/src/harvest/pack.rs
@@ -132,6 +132,21 @@ pub fn emit(
shard_lines[bucket].push('\n');
}
+ // Remove any `records-*.jsonl` left in `pack_dir` by an earlier generation
+ // that populated different (or more) shard buckets: otherwise orphan shard
+ // files survive on disk and ship in a zip while invisible to `manifest.json`
+ // and `SHA256SUMS` (#1110 gap 5a).
+ for entry in std::fs::read_dir(&pack_dir)
+ .with_context(|| format!("cannot read pack dir {}", pack_dir.display()))?
+ {
+ let e = entry.with_context(|| format!("cannot read entry in {}", pack_dir.display()))?;
+ let name = e.file_name().to_string_lossy().to_string();
+ if name.starts_with("records-") && name.ends_with(".jsonl") {
+ std::fs::remove_file(e.path())
+ .with_context(|| format!("cannot remove stale shard {}", e.path().display()))?;
+ }
+ }
+
let mut files: BTreeMap<String, Value> = BTreeMap::new();
for (i, lines) in shard_lines.iter().enumerate() {
let name = format!("records-{:05}.jsonl", i);
@@ -236,11 +251,27 @@ fn uniform_keys(records: &[Map<String, Value>]) -> BTreeMap<String, bool> {
}
fn previous_generation(pack_dir: &Path) -> Result<u64> {
- let Ok(bytes) = std::fs::read(pack_dir.join("manifest.json")) else {
- return Ok(0);
+ let manifest_path = pack_dir.join("manifest.json");
+ let bytes = match std::fs::read(&manifest_path) {
+ Ok(b) => b,
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(0),
+ Err(e) => {
+ return Err(e).with_context(|| format!("cannot read {}", manifest_path.display()))
+ }
};
- let v: Value = serde_json::from_slice(&bytes).unwrap_or(Value::Null);
- Ok(v.get("generation").and_then(Value::as_u64).unwrap_or(0))
+ let v: Value = serde_json::from_slice(&bytes).with_context(|| {
+ format!(
+ "corrupt previous manifest {} (not valid JSON; pass --fresh to rebuild from scratch)",
+ manifest_path.display()
+ )
+ })?;
+ v.get("generation").and_then(Value::as_u64).ok_or_else(|| {
+ anyhow::anyhow!(
+ "corrupt previous manifest {}: missing or non-integer 'generation' \
+ (pass --fresh to rebuild from scratch)",
+ manifest_path.display()
+ )
+ })
}
fn hex(bytes: &[u8]) -> String {
@@ -914,4 +945,93 @@ mod tests {
);
}
}
+
+ #[test]
+ fn rebuild_with_fewer_shards_removes_orphan_records_files() {
+ // #1110 gap 5a: when a rebuild populates fewer (or different) shard
+ // buckets than the previous generation, stale `records-*.jsonl` files
+ // must be swept before writing new shards so they never survive on
+ // disk outside `manifest.json` and `SHA256SUMS`.
+ let tmp = tempfile::tempdir().unwrap();
+ let many: Vec<Map<String, Value>> = (0..32).map(|i| rec(&format!("id-{i}"), &[])).collect();
+ let r1 = emit(
+ tmp.path(),
+ &recipe("t", 16),
+ "# r\n",
+ many.clone(),
+ &stats_with_source(),
+ &sugg(&many),
+ )
+ .unwrap();
+ let gen1_shards: Vec<String> = std::fs::read_dir(&r1.dir)
+ .unwrap()
+ .flatten()
+ .map(|e| e.file_name().to_string_lossy().to_string())
+ .filter(|n| n.starts_with("records-") && n.ends_with(".jsonl"))
+ .collect();
+ assert!(gen1_shards.len() > 1, "multiple shard files populated");
+
+ let one = vec![rec("id-0", &[])];
+ let r2 = emit(
+ tmp.path(),
+ &recipe("t", 1),
+ "# r\n",
+ one.clone(),
+ &stats_with_source(),
+ &sugg(&one),
+ )
+ .unwrap();
+ assert_eq!(r2.generation, 2);
+ let gen2_shards: Vec<String> = std::fs::read_dir(&r2.dir)
+ .unwrap()
+ .flatten()
+ .map(|e| e.file_name().to_string_lossy().to_string())
+ .filter(|n| n.starts_with("records-") && n.ends_with(".jsonl"))
+ .collect();
+ assert_eq!(gen2_shards, vec!["records-00000.jsonl".to_string()]);
+ }
+
+ #[test]
+ fn emit_refuses_corrupt_previous_manifest_instead_of_resetting_generation() {
+ // #1110 gap 5b: a truncated or generation-less existing `manifest.json`
+ // must fail the rebuild loudly, not silently reset `generation` to 1.
+ let tmp = tempfile::tempdir().unwrap();
+ let one = vec![rec("a", &[])];
+ let r1 = emit(
+ tmp.path(),
+ &recipe("t", 1),
+ "# r\n",
+ one.clone(),
+ &stats_with_source(),
+ &sugg(&one),
+ )
+ .unwrap();
+ assert_eq!(r1.generation, 1);
+
+ std::fs::write(r1.dir.join("manifest.json"), "{truncated").unwrap();
+ let err = emit(
+ tmp.path(),
+ &recipe("t", 1),
+ "# r\n",
+ one.clone(),
+ &stats_with_source(),
+ &sugg(&one),
+ )
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("corrupt previous manifest"), "{err}");
+
+ std::fs::write(r1.dir.join("manifest.json"), r#"{"format_version":1}"#).unwrap();
+ let err = emit(
+ tmp.path(),
+ &recipe("t", 1),
+ "# r\n",
+ one.clone(),
+ &stats_with_source(),
+ &sugg(&one),
+ )
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("missing or non-integer 'generation'"), "{err}");
+ }
}
diff --git a/engine/crates/xerj-autoindex/src/harvest/recipe.rs b/engine/crates/xerj-autoindex/src/harvest/recipe.rs
index ca99e23ff..b7ef8462d 100644
--- a/engine/crates/xerj-autoindex/src/harvest/recipe.rs
+++ b/engine/crates/xerj-autoindex/src/harvest/recipe.rs
@@ -87,13 +87,16 @@ pub struct SourceSpec {
pub kind: SourceKind,
/// `dir` sources: a path relative to the recipe file.
pub path: Option<String>,
- /// `http-zip`/`git` sources: where the bytes come from. A git url may
- /// also be a local path (git itself accepts both — tests rely on it).
+ /// `http-zip`/`http-json`/`git` sources: where the bytes come from. A git
+ /// url may also be a local path (git itself accepts both — tests rely on it).
pub url: Option<String>,
/// `git` sources: a full commit sha (pinned), a branch or tag name, or
/// absent to track the remote's default branch (HEAD).
pub rev: Option<String>,
pub glob: String,
+ /// Optional top-level JSON object key holding the records array when a
+ /// `.json` file is a wrapper catalog (e.g. CISA KEV's `"vulnerabilities"`).
+ pub records_field: Option<String>,
pub format: Format,
/// Carried onto every record from this source as record DATA — the tool
/// never enforces it; the pack author owns what goes in the pack.
@@ -104,6 +107,7 @@ pub struct SourceSpec {
pub enum SourceKind {
Dir,
HttpZip,
+ HttpJson,
Git,
}
@@ -267,6 +271,8 @@ struct RawSource {
rev: Option<String>,
#[serde(default = "d_glob")]
glob: String,
+ #[serde(default)]
+ records_field: Option<String>,
format: String,
#[serde(default)]
licence: String,
@@ -418,9 +424,10 @@ fn compile(path: &Path, file: RawRecipe) -> Result<Recipe> {
let kind = match s.kind.as_str() {
"dir" => SourceKind::Dir,
"http-zip" => SourceKind::HttpZip,
+ "http-json" => SourceKind::HttpJson,
"git" => SourceKind::Git,
other => bail!(at(&format!(
- "source '{}': unknown kind '{other}' (supported: dir, http-zip, git)",
+ "source '{}': unknown kind '{other}' (supported: dir, http-zip, http-json, git)",
s.slug
))),
};
@@ -433,6 +440,14 @@ fn compile(path: &Path, file: RawRecipe) -> Result<Recipe> {
s.slug
))),
};
+ if let Some(rf) = &s.records_field {
+ if rf.trim().is_empty() {
+ bail!(at(&format!(
+ "source '{}': 'records_field' cannot be empty",
+ s.slug
+ )));
+ }
+ }
let url = s.url.as_deref().unwrap_or_default().to_string();
let rev = s.rev.as_deref().unwrap_or_default().to_string();
match kind {
@@ -458,16 +473,21 @@ fn compile(path: &Path, file: RawRecipe) -> Result<Recipe> {
)));
}
}
- SourceKind::HttpZip => {
+ SourceKind::HttpZip | SourceKind::HttpJson => {
+ let kind_str = if kind == SourceKind::HttpZip {
+ "http-zip"
+ } else {
+ "http-json"
+ };
if !url.starts_with("http://") && !url.starts_with("https://") {
bail!(at(&format!(
- "http-zip source '{}' needs an http(s) 'url', got '{url}'",
+ "{kind_str} source '{}' needs an http(s) 'url', got '{url}'",
s.slug
)));
}
if s.path.is_some() || !rev.is_empty() {
bail!(at(&format!(
- "http-zip source '{}' takes 'url' only — no 'path'/'rev'",
+ "{kind_str} source '{}' takes 'url' only — no 'path'/'rev'",
s.slug
)));
}
@@ -495,6 +515,7 @@ fn compile(path: &Path, file: RawRecipe) -> Result<Recipe> {
url: s.url.clone(),
rev: if rev.is_empty() { None } else { Some(rev) },
glob: s.glob.clone(),
+ records_field: s.records_field.clone(),
format,
licence: s.licence.clone(),
});
diff --git a/engine/crates/xerj-autoindex/src/harvest/source.rs b/engine/crates/xerj-autoindex/src/harvest/source.rs
index ad2a7a09c..76fece29f 100644
--- a/engine/crates/xerj-autoindex/src/harvest/source.rs
+++ b/engine/crates/xerj-autoindex/src/harvest/source.rs
@@ -69,6 +69,24 @@ pub fn fetch(src: &SourceSpec, recipe_dir: &Path, cache_dir: &Path) -> Result<Fe
unchanged: false,
})
}
+ super::recipe::SourceKind::HttpJson => {
+ let url = src.url.as_deref().unwrap_or_default();
+ let root = cache_dir.join(&src.slug);
+ if root.exists() {
+ std::fs::remove_dir_all(&root)
+ .with_context(|| format!("source '{}': clear {}", src.slug, root.display()))?;
+ }
+ std::fs::create_dir_all(&root)
+ .with_context(|| format!("source '{}': create {}", src.slug, root.display()))?;
+ let json_path = root.join(format!("{}.json", src.slug));
+ httpget::download(url, &json_path)
+ .with_context(|| format!("source '{}': fetch", src.slug))?;
+ Ok(Fetched {
+ root,
+ watermark: None,
+ unchanged: false,
+ })
+ }
super::recipe::SourceKind::Git => {
let url = src.url.as_deref().unwrap_or_default();
let root = cache_dir.join(&src.slug);
@@ -329,6 +347,7 @@ mod tests {
rev: None,
glob: "**/*.json".into(),
format: super::super::recipe::Format::Flat,
+ records_field: None,
licence: "CC0-1.0".into(),
};
let f1 = fetch(&spec, tmp.path(), &cache).unwrap();
@@ -408,6 +427,7 @@ mod tests {
rev: Some("export".into()),
glob: "**/*.json".into(),
format: super::super::recipe::Format::Flat,
+ records_field: None,
licence: "CC0-1.0".into(),
};
let f = fetch(&spec, tmp.path(), &tmp.path().join("cache")).unwrap();
diff --git a/engine/crates/xerj-autoindex/src/harvest/store.rs b/engine/crates/xerj-autoindex/src/harvest/store.rs
index c4eaee673..0bc105321 100644
--- a/engine/crates/xerj-autoindex/src/harvest/store.rs
+++ b/engine/crates/xerj-autoindex/src/harvest/store.rs
@@ -18,6 +18,7 @@ use std::collections::HashSet;
use std::io;
use std::path::{Path, PathBuf};
+use anyhow::{bail, Context, Result};
use serde_json::{Map, Value};
use xxhash_rust::xxh3::xxh3_128;
@@ -70,16 +71,28 @@ fn file_for(store_dir: &Path, slug: &str, key: &str) -> PathBuf {
/// Each returned map carries the file's store key under the `KEY` sentinel
/// so member ordering can fall back to it; `put` computes the key BEFORE
/// injection, so the sentinel is never part of any stored content.
-pub fn read_all(store_dir: &Path, slugs: &HashSet<String>) -> Vec<Map<String, Value>> {
+///
+/// Any unreadable or non-object `<slug>.<key>.json` for a live slug is a
+/// hard error — silently skipping a corrupt store file would omit that
+/// record from the emitted pack with a green exit (#1110 gap 4).
+pub fn read_all(store_dir: &Path, slugs: &HashSet<String>) -> Result<Vec<Map<String, Value>>> {
let mut out = Vec::new();
- let Ok(entries) = std::fs::read_dir(store_dir) else {
- return out;
+ let entries = match std::fs::read_dir(store_dir) {
+ Ok(entries) => entries,
+ Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(out),
+ Err(e) => {
+ return Err(e).with_context(|| format!("cannot read store dir {}", store_dir.display()))
+ }
};
- let mut names: Vec<String> = entries
- .flatten()
- .filter(|e| e.path().extension().and_then(|x| x.to_str()) == Some("json"))
- .filter_map(|e| e.file_name().to_str().map(str::to_string))
- .collect();
+ let mut names: Vec<String> = Vec::new();
+ for entry in entries {
+ let e = entry.with_context(|| format!("cannot read entry in {}", store_dir.display()))?;
+ if e.path().extension().and_then(|x| x.to_str()) == Some("json") {
+ if let Some(name) = e.file_name().to_str() {
+ names.push(name.to_string());
+ }
+ }
+ }
names.sort();
for name in names {
// "<slug>.<32-hex>.json" — split from the RIGHT: slugs may contain
@@ -93,14 +106,22 @@ pub fn read_all(store_dir: &Path, slugs: &HashSet<String>) -> Vec<Map<String, Va
if !slugs.contains(slug) {
continue;
}
- if let Ok(raw) = std::fs::read_to_string(store_dir.join(&name)) {
- if let Ok(Value::Object(mut m)) = serde_json::from_str::<Value>(&raw) {
- m.insert(super::identity::KEY.into(), Value::String(key.to_string()));
- out.push(m);
- }
- }
+ let path = store_dir.join(&name);
+ let raw = std::fs::read_to_string(&path)
+ .with_context(|| format!("cannot read store envelope {}", path.display()))?;
+ let v: Value = serde_json::from_str(&raw).with_context(|| {
+ format!("corrupt store envelope {} (not valid JSON)", path.display())
+ })?;
+ let Value::Object(mut m) = v else {
+ bail!(
+ "corrupt store envelope {} (expected JSON object)",
+ path.display()
+ );
+ };
+ m.insert(super::identity::KEY.into(), Value::String(key.to_string()));
+ out.push(m);
}
- out
+ Ok(out)
}
/// Drop envelopes of `slug` that this run did NOT re-put — a file deleted
@@ -162,7 +183,7 @@ mod tests {
let (_, new2) = put(&dir, "a", &env("x", "a")).unwrap();
assert!(new1);
assert!(!new2, "second put of identical content skips");
- let all = read_all(&dir, &HashSet::from(["a".to_string()]));
+ let all = read_all(&dir, &HashSet::from(["a".to_string()])).unwrap();
assert_eq!(all.len(), 1);
assert_eq!(all[0]["id"], json!("x"));
assert_eq!(k1.len(), 32);
@@ -175,10 +196,10 @@ mod tests {
let dir = tmp.path().join("store");
put(&dir, "a", &env("x", "a")).unwrap();
put(&dir, "b", &env("x", "b")).unwrap();
- let all = read_all(&dir, &HashSet::from(["a".to_string(), "b".to_string()]));
+ let all = read_all(&dir, &HashSet::from(["a".to_string(), "b".to_string()])).unwrap();
assert_eq!(all.len(), 2);
// retired source stays on disk but is not returned
- let only_a = read_all(&dir, &HashSet::from(["a".to_string()]));
+ let only_a = read_all(&dir, &HashSet::from(["a".to_string()])).unwrap();
assert_eq!(only_a.len(), 1);
assert_eq!(only_a[0]["_src"], json!("a"));
}
@@ -195,8 +216,38 @@ mod tests {
let removed = prune(&dir, "a", &HashSet::from([ka]));
assert_eq!(removed, 1, "y died, x lived, b untouched");
assert_eq!(
- read_all(&dir, &HashSet::from(["a".into(), "b".into()])).len(),
+ read_all(&dir, &HashSet::from(["a".into(), "b".into()]))
+ .unwrap()
+ .len(),
2
);
}
+
+ #[test]
+ fn read_all_refuses_corrupt_store_envelope_instead_of_dropping_it() {
+ // #1110 gap 4: a truncated or non-object file in store/ for a live
+ // slug must fail the build loudly, never silently omit the record.
+ let tmp = tempfile::tempdir().unwrap();
+ let dir = tmp.path().join("store");
+ let (k, _) = put(&dir, "a", &env("x", "a")).unwrap();
+ let path = dir.join(format!("a.{k}.json"));
+
+ std::fs::write(&path, "{truncated").unwrap();
+ let err = read_all(&dir, &HashSet::from(["a".to_string()]))
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("corrupt store envelope"), "{err}");
+
+ std::fs::write(&path, "[1, 2, 3]").unwrap();
+ let err = read_all(&dir, &HashSet::from(["a".to_string()]))
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("expected JSON object"), "{err}");
+
+ // a corrupt file belonging to a RETIRED slug is ignored, matching the
+ // retired-source contract above.
+ assert!(read_all(&dir, &HashSet::from(["other".to_string()]))
+ .unwrap()
+ .is_empty());
+ }
}
diff --git a/engine/crates/xerj-autoindex/tests/pack_claims.rs b/engine/crates/xerj-autoindex/tests/pack_claims.rs
index 945289525..587838663 100644
--- a/engine/crates/xerj-autoindex/tests/pack_claims.rs
+++ b/engine/crates/xerj-autoindex/tests/pack_claims.rs
@@ -28,35 +28,35 @@ fn repo_root() -> PathBuf {
}
}
-fn pack_dir(root: &Path) -> PathBuf {
- root.join("tools/packs/rust-vulns")
+fn pack_dir_named(root: &Path, name: &str) -> PathBuf {
+ root.join("tools/packs").join(name)
}
fn read(path: &Path) -> String {
std::fs::read_to_string(path).unwrap_or_else(|e| panic!("read {}: {e}", path.display()))
}
-fn stats(root: &Path) -> serde_json::Value {
- let dir = pack_dir(root);
+fn stats_named(root: &Path, name: &str) -> serde_json::Value {
+ let dir = pack_dir_named(root, name);
let text = read(&dir.join("pack-stats.json"));
serde_json::from_str(&text).unwrap_or_else(|e| panic!("pack-stats.json is not valid JSON: {e}"))
}
/// The README's table row for `label` must carry exactly the number in
/// pack-stats.json under `key`.
-fn readme_row_pins(root: &Path, label: &str, key: &str) {
- let readme = read(&pack_dir(root).join("README.md"));
- let want = stats(root)
+fn readme_row_pins_named(root: &Path, name: &str, label: &str, key: &str) {
+ let readme = read(&pack_dir_named(root, name).join("README.md"));
+ let want = stats_named(root, name)
.get(key)
.and_then(|v| v.as_u64())
- .unwrap_or_else(|| panic!("pack-stats.json has no numeric '{key}'"));
+ .unwrap_or_else(|| panic!("{name}/pack-stats.json has no numeric '{key}'"));
// rows look like `| … label … | 1950 |` or `| … label … | 270 (707 …) |`,
// so pin on maximal digit runs, not whole cells
let needle = label.to_string();
let row = readme
.lines()
.find(|l| l.contains(&needle))
- .unwrap_or_else(|| panic!("README has no row containing '{needle}'"));
+ .unwrap_or_else(|| panic!("{name}/README has no row containing '{needle}'"));
let numbers: Vec<String> = row
.split(|c: char| !c.is_ascii_digit())
.filter(|s| !s.is_empty())
@@ -64,7 +64,7 @@ fn readme_row_pins(root: &Path, label: &str, key: &str) {
.collect();
assert!(
numbers.contains(&want.to_string()),
- "README row '{needle}' does not carry {want} (pack-stats.{key}); rebuild \
+ "{name}/README row '{needle}' does not carry {want} (pack-stats.{key}); rebuild \
the pack and move the docs with the numbers",
);
}
@@ -72,7 +72,7 @@ fn readme_row_pins(root: &Path, label: &str, key: &str) {
#[test]
fn pack_stats_are_measured_not_asserted() {
let root = repo_root();
- let s = stats(&root);
+ let s = stats_named(&root, "rust-vulns");
assert_eq!(
s["provenance"], "measured",
"pack-stats.json must record provenance 'measured' — 'estimated' numbers \
@@ -90,7 +90,7 @@ fn pack_stats_are_measured_not_asserted() {
] {
assert!(
s.get(key).and_then(|v| v.as_u64()).is_some_and(|v| v > 0),
- "pack-stats.json lacks a positive numeric '{key}'"
+ "rust-vulns/pack-stats.json lacks a positive numeric '{key}'"
);
}
// internal consistency: the parts cannot exceed the whole
@@ -99,67 +99,222 @@ fn pack_stats_are_measured_not_asserted() {
let with_fn = s["records_with_functions"].as_u64().unwrap();
assert!(merged <= records, "more merged records than records");
assert!(with_fn <= records, "more function records than records");
+
+ let kev = stats_named(&root, "cisa-kev");
+ assert_eq!(kev["provenance"], "measured");
+ for key in [
+ "envelopes",
+ "records",
+ "vendors",
+ "products",
+ "records_with_cwe",
+ "cwe_ids_distinct",
+ "records_ransomware_known",
+ "records_bytes",
+ ] {
+ assert!(
+ kev.get(key).and_then(|v| v.as_u64()).is_some_and(|v| v > 0),
+ "cisa-kev/pack-stats.json lacks a positive numeric '{key}'"
+ );
+ }
+ let kev_records = kev["records"].as_u64().unwrap();
+ assert!(kev["records_with_cwe"].as_u64().unwrap() <= kev_records);
+ assert!(kev["records_ransomware_known"].as_u64().unwrap() <= kev_records);
}
#[test]
fn readme_counts_match_the_measured_stats() {
let root = repo_root();
- readme_row_pins(&root, "Source records harvested", "envelopes");
- readme_row_pins(&root, "Records after identity resolution", "records");
- readme_row_pins(
+ readme_row_pins_named(&root, "rust-vulns", "Source records harvested", "envelopes");
+ readme_row_pins_named(
+ &root,
+ "rust-vulns",
+ "Records after identity resolution",
+ "records",
+ );
+ readme_row_pins_named(
&root,
+ "rust-vulns",
"…merged from both sources",
"merged_from_both_sources",
);
- readme_row_pins(
+ readme_row_pins_named(
&root,
+ "rust-vulns",
"Records with affected-function paths",
"records_with_functions",
);
- readme_row_pins(
+ readme_row_pins_named(
&root,
+ "rust-vulns",
"Records with affected-function paths",
"function_paths_distinct",
);
- readme_row_pins(&root, "Records withdrawn", "records_withdrawn");
- readme_row_pins(&root, "Packages covered", "packages");
+ readme_row_pins_named(
+ &root,
+ "rust-vulns",
+ "Records withdrawn",
+ "records_withdrawn",
+ );
+ readme_row_pins_named(&root, "rust-vulns", "Packages covered", "packages");
+
+ readme_row_pins_named(&root, "cisa-kev", "Source records harvested", "envelopes");
+ readme_row_pins_named(
+ &root,
+ "cisa-kev",
+ "Records after identity resolution",
+ "records",
+ );
+ readme_row_pins_named(&root, "cisa-kev", "Vendors covered", "vendors");
+ readme_row_pins_named(&root, "cisa-kev", "Products covered", "products");
+ readme_row_pins_named(
+ &root,
+ "cisa-kev",
+ "Records with CWE classification",
+ "records_with_cwe",
+ );
+ readme_row_pins_named(
+ &root,
+ "cisa-kev",
+ "Records with CWE classification",
+ "cwe_ids_distinct",
+ );
+ readme_row_pins_named(
+ &root,
+ "cisa-kev",
+ "Records flagged known ransomware use",
+ "records_ransomware_known",
+ );
+ readme_row_pins_named(&root, "cisa-kev", "Records JSONL bytes", "records_bytes");
}
#[test]
fn the_recipe_is_the_one_the_stats_measured() {
let root = repo_root();
- let dir = pack_dir(&root);
- let recipe = read(&dir.join("recipe.toml"));
- let mut hasher = sha2::Sha256::new();
- use sha2::Digest;
- hasher.update(recipe.as_bytes());
- let got = format!("{:x}", hasher.finalize());
- let want = stats(&root)["recipe_sha256"]
- .as_str()
- .expect("pack-stats.json pins recipe_sha256")
- .to_string();
- assert_eq!(
- got, want,
- "recipe.toml changed since pack-stats.json was measured — rebuild the \
- pack and refresh pack-stats.json (and the README numbers with it)"
- );
-
- // and the stats' source table names exactly the recipe's sources
- for slug in ["rustsec", "osv"] {
- assert!(
- recipe.contains(&format!("slug = \"{slug}\"")),
- "recipe no longer declares source '{slug}'"
+ for (pack, slugs) in [
+ ("rust-vulns", &["rustsec", "osv"][..]),
+ ("cisa-kev", &["kev"][..]),
+ ] {
+ let dir = pack_dir_named(&root, pack);
+ let recipe = read(&dir.join("recipe.toml"));
+ let mut hasher = sha2::Sha256::new();
+ use sha2::Digest;
+ hasher.update(recipe.as_bytes());
+ let got = format!("{:x}", hasher.finalize());
+ let want = stats_named(&root, pack)["recipe_sha256"]
+ .as_str()
+ .expect("pack-stats.json pins recipe_sha256")
+ .to_string();
+ assert_eq!(
+ got, want,
+ "{pack}/recipe.toml changed since pack-stats.json was measured — rebuild the \
+ pack and refresh pack-stats.json (and the README numbers with it)"
);
+
+ for slug in slugs {
+ assert!(
+ recipe.contains(&format!("slug = \"{slug}\"")),
+ "{pack}/recipe no longer declares source '{slug}'"
+ );
+ assert!(
+ stats_named(&root, pack)["sources"][slug].is_object(),
+ "{pack}/pack-stats.json no longer records source '{slug}'"
+ );
+ }
assert!(
- stats(&root)["sources"][slug].is_object(),
- "pack-stats.json no longer records source '{slug}'"
+ !recipe.contains("CC-BY") && !recipe.contains("BUSL") && !recipe.contains("GPL"),
+ "{pack} README says all record data is CC0; a new source with a \
+ different licence must move that claim too"
);
}
- // every licence in the recipe is CC0; the README's licence claims must
- // not silently grow a restricted term
- assert!(
- !recipe.contains("CC-BY") && !recipe.contains("BUSL") && !recipe.contains("GPL"),
- "the pack README says all record data is CC0; a new source with a \
- different licence must move that claim too"
+}
+
+#[test]
+#[ignore = "fetches live CISA KEV feed over HTTPS to measure/refresh tools/packs/cisa-kev/pack-stats.json"]
+fn measure_cisa_kev_pack_live() {
+ use std::collections::BTreeSet;
+ let root = repo_root();
+ let recipe_path = root.join("tools/packs/cisa-kev/recipe.toml");
+ let recipe_text = read(&recipe_path);
+ use sha2::Digest;
+ let mut hasher = sha2::Sha256::new();
+ hasher.update(recipe_text.as_bytes());
+ let recipe_sha256 = format!("{:x}", hasher.finalize());
+
+ let home = tempfile::tempdir().unwrap();
+ std::env::set_var("XERJ_CODE_HOME", home.path());
+ let t0 = std::time::Instant::now();
+ let code = xerj_autoindex::harvest::run_build(&[
+ "cisa-kev".to_string(),
+ "--recipe".to_string(),
+ recipe_path.to_string_lossy().to_string(),
+ ]);
+ let elapsed = t0.elapsed().as_secs().max(1);
+ assert_eq!(code, 0, "cisa-kev build failed");
+
+ let pack = home.path().join("builds/cisa-kev/pack/cisa-kev");
+ let manifest: serde_json::Value =
+ serde_json::from_str(&read(&pack.join("manifest.json"))).unwrap();
+ let envelopes = manifest["counts"]["envelopes"].as_u64().unwrap();
+ let records_count = manifest["counts"]["records"].as_u64().unwrap();
+
+ let mut vendors = BTreeSet::new();
+ let mut products = BTreeSet::new();
+ let mut cwes_distinct = BTreeSet::new();
+ let mut records_with_cwe = 0u64;
+ let mut records_ransomware_known = 0u64;
+ let mut records_bytes = 0u64;
+
+ for name in manifest["files"].as_object().unwrap().keys() {
+ let p = pack.join(name);
+ records_bytes += std::fs::metadata(&p).unwrap().len();
+ for line in read(&p).lines() {
+ if line.trim().is_empty() {
+ continue;
+ }
+ let rec: serde_json::Value = serde_json::from_str(line).unwrap();
+ if let Some(v) = rec["vendorProject"].as_str().filter(|s| !s.is_empty()) {
+ vendors.insert(v.to_string());
+ }
+ if let Some(p) = rec["product"].as_str().filter(|s| !s.is_empty()) {
+ products.insert(p.to_string());
+ }
+ if let Some(arr) = rec["cwe_ids"].as_array() {
+ if !arr.is_empty() {
+ records_with_cwe += 1;
+ for c in arr {
+ if let Some(s) = c.as_str() {
+ cwes_distinct.insert(s.to_string());
+ }
+ }
+ }
+ }
+ if rec["ransomware_status"] == serde_json::json!("known") {
+ records_ransomware_known += 1;
+ }
+ }
+ }
+
+ let stats_json = serde_json::json!({
+ "provenance": "measured",
+ "measured_at": "2026-10-15",
+ "pack": "cisa-kev",
+ "recipe_sha256": recipe_sha256,
+ "sources": {
+ "kev": { "kind": "http-json", "envelopes": envelopes, "licence": "CC0-1.0" }
+ },
+ "envelopes": envelopes,
+ "records": records_count,
+ "vendors": vendors.len(),
+ "products": products.len(),
+ "records_with_cwe": records_with_cwe,
+ "cwe_ids_distinct": cwes_distinct.len(),
+ "records_ransomware_known": records_ransomware_known,
+ "records_bytes": records_bytes,
+ "build_seconds": elapsed,
+ });
+ println!(
+ "MEASURED_CISA_KEV_STATS:\n{}",
+ serde_json::to_string_pretty(&stats_json).unwrap()
);
}
diff --git a/tools/packs/cisa-kev/README.md b/tools/packs/cisa-kev/README.md
new file mode 100644
index 000000000..e372ba101
--- /dev/null
+++ b/tools/packs/cisa-kev/README.md
@@ -0,0 +1,64 @@
+# cisa-kev — CISA Known Exploited Vulnerabilities corpus pack
+
+The US Cybersecurity and Infrastructure Security Agency (CISA) **Known Exploited
+Vulnerabilities (KEV)** catalog as a portable, checksum-verified corpus pack —
+built with `xerj corpus build` from this directory's
+[recipe.toml](./recipe.toml) without a line of domain-specific Rust code.
+
+Every number below is measured from a real build
+([pack-stats.json](./pack-stats.json), `provenance: measured`), and
+`tests/pack_claims.rs` pins this README against `pack-stats.json` and the recipe
+SHA-256 so the three cannot drift apart.
+
+| What | Count |
+|---|---|
+| Source records harvested (envelopes) | 1746 |
+| Records after identity resolution | 1746 |
+| Vendors covered (`vendorProject`) | 289 |
+| Products covered (`product`) | 705 |
+| Records with CWE classification | 1571 (189 distinct CWE IDs) |
+| Records flagged known ransomware use | 361 |
+| Records JSONL bytes | 2993632 |
+| Cold build (network fetch + harvest + pack) | ~1 s |
+
+## Why this pack exists
+
+`rust-vulns` exercises multi-source OSV + RustSec markdown advisory merging.
+`cisa-kev` exercises a completely different shape on the same pipeline (#1110):
+
+- **Wrapped JSON feed (`kind = "http-json"`, `records_field = "vulnerabilities"`)**
+ instead of a git repository or zip archive.
+- **Flat record normalization (`format = "flat"`)** with domain-native field
+ names (`cveID`, `vendorProject`, `product`, `vulnerabilityName`,
+ `shortDescription`, `requiredAction`, `knownRansomwareCampaignUse`, `cwes`).
+- **Derived fields** via `regex_extract` (`cwe_ids`) and `map`
+ (`ransomware_status`: `"known"` vs `"unknown"`).
+- **Public domain licence (`CC0-1.0`)**: US federal government work under
+ 17 U.S.C. § 105.
+
+## Build, consume, query
+
+```sh
+# 1. build (or incrementally refresh) from the recipe
+xerj corpus build cisa-kev --recipe tools/packs/cisa-kev/recipe.toml
+
+# 2. install the pack into your corpus store
+xerj corpus add cisa-kev --from ~/.xerj-code/builds/cisa-kev/pack/cisa-kev
+
+# 3. index and query
+xerj corpus index cisa-kev
+xerj code cisa-kev "exchange server SSRF proxylogon"
+```
+
+## Fields worth knowing
+
+| Field | What it carries |
+|---|---|
+| `id` / `cveID` | canonical CVE identifier (`CVE-YYYY-NNNN+`) |
+| `title` / `vulnerabilityName` | human-readable vulnerability title |
+| `vendorProject` / `product` | vendor and product names (also routed into `defs`) |
+| `shortDescription` / `requiredAction` | CISA summary and remediation directive |
+| `dateAdded` / `dueDate` | catalog addition date and federal remediation deadline |
+| `cwe_ids` | derived array of `CWE-NNN` identifiers |
+| `ransomware_status` | derived `"known"` or `"unknown"` from `knownRansomwareCampaignUse` |
+| `licence` / `sources` / `origin` | provenance per record (`CC0-1.0`, `["kev"]`) |
diff --git a/tools/packs/cisa-kev/pack-stats.json b/tools/packs/cisa-kev/pack-stats.json
new file mode 100644
index 000000000..7c51fc796
--- /dev/null
+++ b/tools/packs/cisa-kev/pack-stats.json
@@ -0,0 +1,18 @@
+{
+ "provenance": "measured",
+ "measured_at": "2026-10-15",
+ "pack": "cisa-kev",
+ "recipe_sha256": "b2f5b24f5fbcd0c076bdc19d844c962da0883783a121528f0e29c32fcdf9abf9",
+ "sources": {
+ "kev": { "kind": "http-json", "envelopes": 1746, "licence": "CC0-1.0" }
+ },
+ "envelopes": 1746,
+ "records": 1746,
+ "vendors": 289,
+ "products": 705,
+ "records_with_cwe": 1571,
+ "cwe_ids_distinct": 189,
+ "records_ransomware_known": 361,
+ "records_bytes": 2993632,
+ "build_seconds": 1
+}
diff --git a/tools/packs/cisa-kev/recipe.toml b/tools/packs/cisa-kev/recipe.toml
new file mode 100644
index 000000000..94ebfb6df
--- /dev/null
+++ b/tools/packs/cisa-kev/recipe.toml
@@ -0,0 +1,56 @@
+# cisa-kev — CISA Known Exploited Vulnerabilities (KEV) catalog.
+#
+# Second-domain corpus pack (#1110) proving the corpus builder generalizes
+# without Rust/OSV assumptions:
+# - Upstream is a single wrapped JSON document (`kind = "http-json"`,
+# `records_field = "vulnerabilities"`) rather than an OSV zip or git tree.
+# - Records use `format = "flat"` (no OSV or RustSec adapter) with domain-native
+# keys (`cveID`, `vendorProject`, `product`, `vulnerabilityName`,
+# `shortDescription`, `requiredAction`, `knownRansomwareCampaignUse`, `cwes`).
+# - Licence: US federal government work (17 U.S.C. § 105), public domain /
+# CC0-1.0 equivalent.
+
+[recipe]
+format = 1
+name = "cisa-kev"
+description = "CISA Known Exploited Vulnerabilities (KEV) catalog — actively exploited CVEs across all vendors and products"
+
+[[sources]]
+slug = "kev"
+kind = "http-json"
+url = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
+records_field = "vulnerabilities"
+format = "flat"
+licence = "CC0-1.0"
+
+[envelope]
+id_from = ["cveID"]
+title_from = ["vulnerabilityName", "cveID"]
+body_join = ["vendorProject", "product", "vulnerabilityName", "shortDescription", "requiredAction", "notes"]
+defs_from = ["vendorProject", "product", "cwes"]
+
+[identity]
+edges = [{ field = "cveID" }]
+canonical_source_order = ["kev"]
+
+[merge]
+precedence = ["kev"]
+
+[[derived]]
+name = "cwe_ids"
+op = "regex_extract"
+from = ["cwes", "notes"]
+pattern = '\bCWE-\d+\b'
+unique = true
+
+[[derived]]
+name = "ransomware_status"
+op = "map"
+from = ["knownRansomwareCampaignUse"]
+rules = [
+ { match = "(?i)^known$", value = "known" },
+]
+default = "unknown"
+
+[emit]
+shards = 8