Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

feat(autoindex): network sources for corpus build — http-zip, git, watermarks (M2)

xerj-org5bd29f38a18d
diff --git a/engine/crates/xerj-autoindex/src/harvest/httpget.rs b/engine/crates/xerj-autoindex/src/harvest/httpget.rs
new file mode 100644
index 000000000..afe3bfff6
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/harvest/httpget.rs
@@ -0,0 +1,153 @@
+//! Blocking HTTP download for `http-zip` recipe sources.
+//!
+//! Deliberately small next to `esclient`'s retry loop: that one protects an
+//! interactive indexing session against a live server's 429/503 storms; this
+//! one fetches a handful of public dump URLs per run. The part worth copying
+//! is the SHAPE — bounded attempts, exponential backoff, one-line errors that
+//! name the URL and status, and an atomic tmp+rename write so a partial
+//! download never exists under the cache's final name.
+
+use std::path::Path;
+use std::time::Duration;
+
+use anyhow::{bail, Context, Result};
+
+/// Refuse to buffer more than this on disk. Public advisory dumps are
+/// megabytes-to-a-few-hundred-MB; a runaway URL should fail loudly, not fill
+/// the disk. Decompression safety is enforced separately at zip-extract time
+/// (a small zip can expand enormously — see `source::extract_zip`).
+const MAX_DOWNLOAD_BYTES: u64 = 4 << 30;
+
+const ATTEMPTS: u32 = 3;
+const INITIAL_BACKOFF: Duration = Duration::from_secs(1);
+
+/// Download `url` to `dest` (atomically). Returns the byte count.
+pub fn download(url: &str, dest: &Path) -> Result<u64> {
+ let client = reqwest::blocking::Client::builder()
+ .timeout(Duration::from_secs(600))
+ .connect_timeout(Duration::from_secs(30))
+ .redirect(reqwest::redirect::Policy::limited(10))
+ .build()
+ .context("build http client")?;
+
+ let mut last_err = String::new();
+ for attempt in 0..ATTEMPTS {
+ if attempt > 0 {
+ std::thread::sleep(INITIAL_BACKOFF * (1 << (attempt - 1)));
+ }
+ match once(&client, url, dest) {
+ Ok(n) => return Ok(n),
+ // keep the FIRST error: later attempts against a dead endpoint
+ // report less (bare connect failure vs "HTTP 404")
+ Err(e) => {
+ if last_err.is_empty() {
+ last_err = e.to_string();
+ }
+ }
+ }
+ }
+ bail!("download {url} failed after {ATTEMPTS} attempts: {last_err}")
+}
+
+fn once(client: &reqwest::blocking::Client, url: &str, dest: &Path) -> Result<u64> {
+ use std::io::{Read, Write};
+ let resp = client
+ .get(url)
+ .send()
+ .with_context(|| format!("GET {url}"))?;
+ let status = resp.status();
+ if !status.is_success() {
+ // not retry-worthy in the same way as a timeout, but cheap to retry
+ // once upstream hiccuped — the loop above bounds it
+ bail!("HTTP {status} for {url}");
+ }
+ if let Some(len) = resp.content_length() {
+ if len > MAX_DOWNLOAD_BYTES {
+ bail!("{url} declares {len} bytes, over the {MAX_DOWNLOAD_BYTES} cap");
+ }
+ }
+
+ let tmp = dest.with_extension("part");
+ if let Some(parent) = dest.parent() {
+ std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;
+ }
+ let mut file =
+ std::fs::File::create(&tmp).with_context(|| format!("create {}", tmp.display()))?;
+ let mut written: u64 = 0;
+ // stream: `resp.bytes()` would buffer the entire body in memory before
+ // any cap could fire
+ let mut reader = resp.take(MAX_DOWNLOAD_BYTES + 1);
+ let mut buf = vec![0u8; 64 * 1024];
+ loop {
+ let n = reader.read(&mut buf)?;
+ if n == 0 {
+ break;
+ }
+ written += n as u64;
+ if written > MAX_DOWNLOAD_BYTES {
+ let _ = std::fs::remove_file(&tmp);
+ bail!("{url} exceeded the {MAX_DOWNLOAD_BYTES} download cap");
+ }
+ file.write_all(&buf[..n])?;
+ }
+ file.flush()?;
+ drop(file);
+ std::fs::rename(&tmp, dest).with_context(|| format!("move {} into place", tmp.display()))?;
+ Ok(written)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ /// One canned HTTP response over a raw TcpListener — the fake-node
+ /// pattern from `failure_resume_http_tests.rs`. No reqwest server dep.
+ /// Owned arguments: the serving thread must be 'static.
+ fn serve_once(head: String, body: Vec<u8>) -> std::io::Result<std::net::SocketAddr> {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;
+ let addr = listener.local_addr()?;
+ std::thread::spawn(move || {
+ if let Ok((mut sock, _)) = listener.accept() {
+ use std::io::{Read, Write};
+ let mut buf = [0u8; 1024];
+ let _ = sock.read(&mut buf); // discard the request
+ let _ = sock.write_all(head.as_bytes());
+ let _ = sock.write_all(&body);
+ let _ = sock.flush();
+ }
+ });
+ Ok(addr)
+ }
+
+ #[test]
+ fn downloads_and_retries() {
+ let tmp = tempfile::tempdir().unwrap();
+ let dest = tmp.path().join("dump.zip");
+
+ // success path
+ let addr = serve_once(
+ "HTTP/1.1 200 OK\r\nContent-Length: 6\r\nConnection: close\r\n\r\n".to_string(),
+ b"hello!".to_vec(),
+ )
+ .unwrap();
+ let n = download(&format!("http://{addr}/x.zip"), &dest).unwrap();
+ assert_eq!(n, 6);
+ assert_eq!(std::fs::read(&dest).unwrap(), b"hello!");
+ assert!(
+ !tmp.path().join("dump.part").exists(),
+ "no partial left behind"
+ );
+
+ // failure path: 404 on every attempt → named error, no dest file
+ let addr = serve_once(
+ "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string(),
+ Vec::new(),
+ )
+ .unwrap();
+ let err = download(&format!("http://{addr}/gone.zip"), &dest)
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("404"), "{err}");
+ assert!(err.contains("3 attempts"), "{err}");
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/harvest/mod.rs b/engine/crates/xerj-autoindex/src/harvest/mod.rs
index b00936c52..696deb822 100644
--- a/engine/crates/xerj-autoindex/src/harvest/mod.rs
+++ b/engine/crates/xerj-autoindex/src/harvest/mod.rs
@@ -23,10 +23,12 @@
//! `sources[]`); the tool does not and must not enforce them — the pack
//! author owns what goes in the pack.
+pub mod httpget;
pub mod identity;
pub mod normalize;
pub mod pack;
pub mod recipe;
+pub mod source;
pub mod store;
pub(crate) use store::MAX_FIELDS;
@@ -123,24 +125,33 @@ fn run_build_inner(args: &[String], home: &Path) -> Result<i32> {
.with_context(|| format!("--fresh: cannot clear {}", build_dir.display()))?;
}
let store_dir = build_dir.join("store");
+ let cache_dir = build_dir.join("cache");
// ── harvest into the content store ───────────────────────────────────
let mut per_source = Vec::new();
for src in &recipe.sources {
- let SourceKind::Dir = src.kind;
- let root = recipe_dir.join(src.path.as_deref().unwrap_or("."));
- let files = walk(&root, &src.glob)?;
+ let fetched = source::fetch(src, &recipe_dir, &cache_dir)?;
let mut stat = SourceStat {
slug: src.slug.clone(),
- kind: "dir".to_string(),
+ kind: kind_name(src.kind).to_string(),
path: src.path.clone(),
url: src.url.clone(),
licence: src.licence.clone(),
+ watermark: fetched.watermark.clone(),
+ unchanged: fetched.unchanged,
records: 0,
new: 0,
skipped: 0,
pruned: 0,
};
+ if fetched.unchanged {
+ // The whole upstream tree is byte-identical to the previous run:
+ // skip the walk AND the prune — every envelope this source ever
+ // stored is still live, and pruning needs the full key set.
+ per_source.push(stat);
+ continue;
+ }
+ let files = walk(&fetched.root, &src.glob)?;
let mut keys = HashSet::new();
for (rel, path) in files {
for value in parse_file(&path)? {
@@ -163,6 +174,11 @@ fn run_build_inner(args: &[String], home: &Path) -> Result<i32> {
}
}
stat.pruned = store::prune(&store_dir, &src.slug, &keys);
+ // persist the watermark only after the source's harvest fully
+ // succeeded — a failed run must re-walk
+ if let Some(wm) = &fetched.watermark {
+ source::save_watermark(&cache_dir, &src.slug, wm)?;
+ }
per_source.push(stat);
}
@@ -194,11 +210,37 @@ fn run_build_inner(args: &[String], home: &Path) -> Result<i32> {
};
let result = pack::emit(&build_dir, &recipe, &recipe_toml, records, &stats)?;
+ // run report: the machine-readable twin of the stdout summary, with the
+ // watermarks a scheduled rebuild (M6) will diff against
+ let report = serde_json::json!({
+ "finished_at": chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true),
+ "pack": recipe.name,
+ "generation": result.generation,
+ "records": result.records,
+ "envelopes": stats.envelopes,
+ "sources": stats.per_source.iter()
+ .map(|s| serde_json::to_value(s).expect("SourceStat serializes"))
+ .collect::<Vec<_>>(),
+ });
+ std::fs::write(
+ build_dir.join("report.json"),
+ serde_json::to_vec_pretty(&report)?,
+ )
+ .with_context(|| format!("write {}", build_dir.join("report.json").display()))?;
+
println!(
"built '{}' generation {}: {} envelopes → {} records, {} shard file(s)",
recipe.name, result.generation, stats.envelopes, result.records, result.shards
);
for s in &stats.per_source {
+ if s.unchanged {
+ println!(
+ " {}: unchanged (watermark {})",
+ s.slug,
+ s.watermark.clone().unwrap_or_default()
+ );
+ continue;
+ }
println!(
" {}: {} record(s) — {} new, {} skipped, {} pruned ({})",
s.slug, s.records, s.new, s.skipped, s.pruned, s.licence
@@ -208,6 +250,14 @@ fn run_build_inner(args: &[String], home: &Path) -> Result<i32> {
Ok(0)
}
+fn kind_name(kind: SourceKind) -> &'static str {
+ match kind {
+ SourceKind::Dir => "dir",
+ SourceKind::HttpZip => "http-zip",
+ SourceKind::Git => "git",
+ }
+}
+
// ── dir source ──────────────────────────────────────────────────────────────
/// Recursively list files under `root` matching `glob`, as
@@ -597,4 +647,94 @@ shards = 4
);
assert_eq!(read_records(&pack).len(), 1);
}
+
+ // ── end to end: a git source, watermark-skipped on the second run ─────
+
+ #[test]
+ fn build_from_git_source_and_skip_when_unchanged() {
+ let home = tempfile::tempdir().unwrap();
+ let origin = home.path().join("origin");
+ std::fs::create_dir_all(&origin).unwrap();
+ crate::xc::git(Some(&origin), &["init", "--quiet", "-b", "main"]).ok();
+ std::fs::write(origin.join("a.json"), r#"{"id": "R1", "aliases": ["C1"]}"#).unwrap();
+ crate::xc::git(Some(&origin), &["add", "."]).ok();
+ crate::xc::git(
+ Some(&origin),
+ &[
+ "-c",
+ "user.email=t@t",
+ "-c",
+ "user.name=t",
+ "commit",
+ "--quiet",
+ "-m",
+ "one",
+ ],
+ )
+ .ok();
+
+ let recipe = format!(
+ "[recipe]\nformat = 1\nname = \"gitdemo\"\n\n\
+ [[sources]]\nslug = \"upstream\"\nkind = \"git\"\nurl = \"{}\"\nformat = \"flat\"\n\
+ licence = \"CC0-1.0\"\n\n\
+ [identity]\nedges = [{{ field = \"id\" }}]\n",
+ origin.to_string_lossy()
+ );
+ std::fs::create_dir_all(home.path().join("recipes")).unwrap();
+ std::fs::write(home.path().join("recipes/gitdemo.toml"), recipe).unwrap();
+
+ let args: Vec<String> = vec!["gitdemo".to_string()];
+ assert_eq!(run_build_inner(&args, home.path()).unwrap(), 0);
+ let pack = home.path().join("builds/gitdemo/pack/gitdemo");
+ assert_eq!(read_records(&pack).len(), 1);
+
+ let manifest: Value =
+ serde_json::from_str(&std::fs::read_to_string(pack.join("manifest.json")).unwrap())
+ .unwrap();
+ let s = &manifest["sources"][0];
+ assert_eq!(s["kind"], json!("git"));
+ assert_eq!(s["new"], json!(1));
+ assert!(s["watermark"].as_str().is_some_and(|w| w.len() == 40));
+
+ // second run: same HEAD → the source is not walked, the store is
+ // untouched, the pack is rebuilt identically from the store
+ assert_eq!(run_build_inner(&args, home.path()).unwrap(), 0);
+ let manifest: Value =
+ serde_json::from_str(&std::fs::read_to_string(pack.join("manifest.json")).unwrap())
+ .unwrap();
+ let s = &manifest["sources"][0];
+ assert_eq!(s["unchanged"], json!(true));
+ assert_eq!(s["records"], json!(0), "no walk happened");
+ assert_eq!(
+ read_records(&pack).len(),
+ 1,
+ "records survive via the store"
+ );
+ assert!(home.path().join("builds/gitdemo/report.json").is_file());
+
+ // a new commit upstream re-activates the source
+ std::fs::write(origin.join("b.json"), r#"{"id": "R2"}"#).unwrap();
+ crate::xc::git(Some(&origin), &["add", "."]).ok();
+ crate::xc::git(
+ Some(&origin),
+ &[
+ "-c",
+ "user.email=t@t",
+ "-c",
+ "user.name=t",
+ "commit",
+ "--quiet",
+ "-m",
+ "two",
+ ],
+ )
+ .ok();
+ assert_eq!(run_build_inner(&args, home.path()).unwrap(), 0);
+ assert_eq!(read_records(&pack).len(), 2);
+ let manifest: Value =
+ serde_json::from_str(&std::fs::read_to_string(pack.join("manifest.json")).unwrap())
+ .unwrap();
+ assert_eq!(manifest["sources"][0]["unchanged"], json!(false));
+ assert_eq!(manifest["sources"][0]["new"], json!(1));
+ }
}
diff --git a/engine/crates/xerj-autoindex/src/harvest/pack.rs b/engine/crates/xerj-autoindex/src/harvest/pack.rs
index 580c8f3c7..357614967 100644
--- a/engine/crates/xerj-autoindex/src/harvest/pack.rs
+++ b/engine/crates/xerj-autoindex/src/harvest/pack.rs
@@ -24,13 +24,18 @@ use xxhash_rust::xxh3::xxh3_64;
use super::recipe::Recipe;
-/// Per-source counts for the manifest: what this run actually did.
+/// Per-source counts for the manifest and report: what this run actually
+/// did. `unchanged` = watermark matched, so the source was not walked at
+/// all (and must not be pruned either — everything it stored is live).
+#[derive(serde::Serialize)]
pub struct SourceStat {
pub slug: String,
pub kind: String,
pub path: Option<String>,
pub url: Option<String>,
pub licence: String,
+ pub watermark: Option<String>,
+ pub unchanged: bool,
pub records: usize,
pub new: usize,
pub skipped: usize,
@@ -138,11 +143,11 @@ pub fn emit(
"records": records.len(),
"shards": shards,
},
- "sources": stats.per_source.iter().map(|s| serde_json::json!({
- "slug": s.slug, "kind": s.kind, "path": s.path, "url": s.url,
- "licence": s.licence, "records": s.records,
- "new": s.new, "skipped": s.skipped, "pruned": s.pruned,
- })).collect::<Vec<_>>(),
+ "sources": stats
+ .per_source
+ .iter()
+ .map(|s| serde_json::to_value(s).expect("SourceStat serializes"))
+ .collect::<Vec<_>>(),
"files": Value::Object(files.clone().into_iter().collect()),
"tool": { "name": "xerj", "version": env!("CARGO_PKG_VERSION") },
});
diff --git a/engine/crates/xerj-autoindex/src/harvest/recipe.rs b/engine/crates/xerj-autoindex/src/harvest/recipe.rs
index 4229dc5a5..0af8a9309 100644
--- a/engine/crates/xerj-autoindex/src/harvest/recipe.rs
+++ b/engine/crates/xerj-autoindex/src/harvest/recipe.rs
@@ -86,7 +86,11 @@ pub struct SourceSpec {
pub kind: SourceKind,
/// `dir` sources: a path relative to the recipe file.
pub path: Option<String>,
+ /// `http-zip`/`git` sources: where the bytes come from. A git url may
+ /// also be a local path (git itself accepts both — tests rely on it).
pub url: Option<String>,
+ /// `git` sources: a pinned commit sha, or absent to track remote HEAD.
+ pub rev: Option<String>,
pub glob: String,
pub format: Format,
/// Carried onto every record from this source as record DATA — the tool
@@ -97,6 +101,8 @@ pub struct SourceSpec {
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SourceKind {
Dir,
+ HttpZip,
+ Git,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -239,6 +245,8 @@ struct RawSource {
path: Option<String>,
#[serde(default)]
url: Option<String>,
+ #[serde(default)]
+ rev: Option<String>,
#[serde(default = "d_glob")]
glob: String,
format: String,
@@ -372,12 +380,12 @@ fn compile(path: &Path, file: RawRecipe) -> Result<Recipe> {
}
let kind = match s.kind.as_str() {
"dir" => SourceKind::Dir,
- other @ ("http-zip" | "git") => bail!(at(&format!(
- "source '{}' kind '{other}' is not supported yet (planned: incremental \
- network sources)",
+ "http-zip" => SourceKind::HttpZip,
+ "git" => SourceKind::Git,
+ other => bail!(at(&format!(
+ "source '{}': unknown kind '{other}' (supported: dir, http-zip, git)",
s.slug
))),
- other => bail!(at(&format!("source '{}': unknown kind '{other}'", s.slug))),
};
let format = match s.format.as_str() {
"osv" => Format::Osv,
@@ -387,17 +395,55 @@ fn compile(path: &Path, file: RawRecipe) -> Result<Recipe> {
s.slug
))),
};
- if kind == SourceKind::Dir {
- if s.path.as_deref().unwrap_or("").is_empty() {
- bail!(at(&format!(
- "dir source '{}' needs a 'path' (relative to the recipe file)",
- s.slug
- )));
+ let url = s.url.as_deref().unwrap_or_default().to_string();
+ let rev = s.rev.as_deref().unwrap_or_default().to_string();
+ match kind {
+ SourceKind::Dir => {
+ if s.path.as_deref().unwrap_or("").is_empty() {
+ bail!(at(&format!(
+ "dir source '{}' needs a 'path' (relative to the recipe file)",
+ s.slug
+ )));
+ }
+ if let Some(p) = &s.path {
+ if Path::new(p).is_absolute() {
+ bail!(at(&format!(
+ "dir source '{}' path must be recipe-relative, got absolute '{p}'",
+ s.slug
+ )));
+ }
+ }
+ if !url.is_empty() || !rev.is_empty() {
+ bail!(at(&format!(
+ "dir source '{}' takes 'path', not 'url'/'rev'",
+ s.slug
+ )));
+ }
}
- if let Some(p) = &s.path {
- if Path::new(p).is_absolute() {
+ SourceKind::HttpZip => {
+ if !url.starts_with("http://") && !url.starts_with("https://") {
bail!(at(&format!(
- "dir source '{}' path must be recipe-relative, got absolute '{p}'",
+ "http-zip source '{}' needs an http(s) 'url', got '{url}'",
+ s.slug
+ )));
+ }
+ if s.path.is_some() || !rev.is_empty() {
+ bail!(at(&format!(
+ "http-zip source '{}' takes 'url' only — no 'path'/'rev'",
+ s.slug
+ )));
+ }
+ }
+ SourceKind::Git => {
+ if url.is_empty() {
+ bail!(at(&format!(
+ "git source '{}' needs a 'url' (a git URL or local path)",
+ s.slug
+ )));
+ }
+ if s.path.is_some() {
+ bail!(at(&format!(
+ "git source '{}' takes 'url' (+optional 'rev'), not 'path'",
s.slug
)));
}
@@ -409,6 +455,7 @@ fn compile(path: &Path, file: RawRecipe) -> Result<Recipe> {
kind,
path: s.path.clone(),
url: s.url.clone(),
+ rev: if rev.is_empty() { None } else { Some(rev) },
glob: s.glob.clone(),
format,
licence: s.licence.clone(),
@@ -634,12 +681,34 @@ edges = [{ field = "id" }]
}
#[test]
- fn unknown_source_kind_names_the_planned_one() {
+ fn unknown_source_kind_is_loud() {
+ let tmp = tempfile::tempdir().unwrap();
+ let bad = MINIMAL.replace("kind = \"dir\"", "kind = \"svn\"");
+ let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();
+ assert!(err.contains("unknown kind 'svn'"), "{err}");
+ }
+
+ #[test]
+ fn network_kinds_validate_their_targets() {
let tmp = tempfile::tempdir().unwrap();
- let bad = MINIMAL.replace("kind = \"dir\"", "kind = \"git\"");
+ // http-zip without an http(s) url
+ let bad = MINIMAL.replace(
+ "kind = \"dir\"\npath = \"data/a\"",
+ "kind = \"http-zip\"\nurl = \"ftp://x/y.zip\"",
+ );
+ let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();
+ assert!(err.contains("http(s) 'url'"), "{err}");
+ // git with a stray path
+ let bad = MINIMAL.replace("kind = \"dir\"", "kind = \"git\"\nurl = \"https://x/y\"");
+ let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();
+ assert!(err.contains("not 'path'"), "{err}");
+ // dir with a url
+ let bad = MINIMAL.replace(
+ "path = \"data/a\"",
+ "path = \"data/a\"\nurl = \"https://x/y\"",
+ );
let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();
- assert!(err.contains("not supported yet"), "{err}");
- assert!(err.contains("'git'"), "{err}");
+ assert!(err.contains("not 'url'/'rev'"), "{err}");
}
#[test]
diff --git a/engine/crates/xerj-autoindex/src/harvest/source.rs b/engine/crates/xerj-autoindex/src/harvest/source.rs
new file mode 100644
index 000000000..dbda8910c
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/harvest/source.rs
@@ -0,0 +1,361 @@
+//! Source adapters: turn a recipe `[[sources]]` entry into a directory to
+//! walk, plus the upstream watermark that lets an unchanged source be
+//! skipped wholesale on the next run.
+//!
+//! Kinds:
+//! - `dir` — walk a recipe-relative directory (the fixture/local kind)
+//! - `http-zip`— download a zip, extract into the cache, walk the
+//! extraction. No watermark: the zip is re-downloaded every run and the
+//! content store dedups what did not change (correctness lives there,
+//! watermarks are only an optimization).
+//! - `git` — promisor shallow checkout at `rev` (pinned) or the remote
+//! HEAD (tracking). Watermark = HEAD sha: unchanged tree ⇒ the walk is
+//! skipped entirely this run.
+//!
+//! Zip extraction is defensive by default: entry names are normalized and
+//! any path escaping the extraction root (zip-slip), absolute path, or
+//! symlink entry is skipped and COUNTED, never trusted — dump URLs are
+//! fetched from the open internet.
+
+use std::io::Read;
+use std::path::{Component, Path, PathBuf};
+
+use anyhow::{bail, Context, Result};
+
+use super::httpget;
+use super::recipe::SourceSpec;
+
+/// Total decompressed cap for one zip. Real advisory dumps are well under
+/// this; the cap exists because a zip's declared sizes are untrusted and a
+/// 3 MiB file can claim to expand to terabytes.
+const MAX_EXTRACT_BYTES: u64 = 2 << 30;
+
+pub struct Fetched {
+ /// The directory the glob walk starts from.
+ pub root: PathBuf,
+ /// Current upstream version marker, if the kind has one.
+ pub watermark: Option<String>,
+ /// The watermark matches the previous run's: the caller should skip
+ /// walking AND pruning this source (everything it stored is still live).
+ pub unchanged: bool,
+}
+
+/// Where a source's watermark from the previous run is kept.
+fn watermark_path(cache_dir: &Path, slug: &str) -> PathBuf {
+ cache_dir.join(format!("{slug}.watermark"))
+}
+
+/// Materialize a source. `cache_dir` is `<build>/cache`; it only exists to
+/// make network kinds incremental.
+pub fn fetch(src: &SourceSpec, recipe_dir: &Path, cache_dir: &Path) -> Result<Fetched> {
+ match src.kind {
+ super::recipe::SourceKind::Dir => Ok(Fetched {
+ root: recipe_dir.join(src.path.as_deref().unwrap_or(".")),
+ watermark: None,
+ unchanged: false,
+ }),
+ super::recipe::SourceKind::HttpZip => {
+ let url = src.url.as_deref().unwrap_or_default();
+ let zip_path = cache_dir.join(format!("{}.zip", src.slug));
+ httpget::download(url, &zip_path)
+ .with_context(|| format!("source '{}': fetch", src.slug))?;
+ let root = cache_dir.join(&src.slug);
+ extract_zip(&zip_path, &root)
+ .with_context(|| format!("source '{}': extract", src.slug))?;
+ Ok(Fetched {
+ root,
+ watermark: None,
+ unchanged: false,
+ })
+ }
+ super::recipe::SourceKind::Git => {
+ let url = src.url.as_deref().unwrap_or_default();
+ let root = cache_dir.join(&src.slug);
+ let rev = src.rev.as_deref().unwrap_or("HEAD");
+ let watermark = git_checkout(&root, url, rev, &src.slug)?;
+ let prev = std::fs::read_to_string(watermark_path(cache_dir, &src.slug))
+ .map(|s| s.trim().to_string())
+ .unwrap_or_default();
+ Ok(Fetched {
+ root,
+ watermark: Some(watermark.clone()),
+ unchanged: !watermark.is_empty() && watermark == prev,
+ })
+ }
+ }
+}
+
+/// Persist a source's watermark after its harvest fully succeeded (never
+/// before — a failed run must re-walk).
+pub fn save_watermark(cache_dir: &Path, slug: &str, watermark: &str) -> Result<()> {
+ std::fs::create_dir_all(cache_dir)?;
+ std::fs::write(watermark_path(cache_dir, slug), watermark)
+ .with_context(|| format!("save watermark for '{slug}'"))
+}
+
+/// Shallow promisor checkout of `rev` (a sha or `HEAD`) into `target`,
+/// reusing the corpus-clone machinery: init + blob:none filter so a pin at
+/// an old sha costs one small fetch, never a full history. Returns HEAD.
+fn git_checkout(target: &Path, url: &str, rev: &str, slug: &str) -> Result<String> {
+ if !target.join(".git").exists() {
+ std::fs::create_dir_all(target)
+ .with_context(|| format!("source '{slug}': create {}", target.display()))?;
+ crate::xc::git(Some(target), &["init", "--quiet"])?;
+ crate::xc::git(Some(target), &["remote", "add", "origin", url]).ok();
+ crate::xc::git(Some(target), &["config", "remote.origin.promisor", "true"]).ok();
+ crate::xc::git(
+ Some(target),
+ &["config", "remote.origin.partialclonefilter", "blob:none"],
+ )
+ .ok();
+ }
+ // A pinned rev (full sha) can reuse checkout_at_sha; tracking HEAD goes
+ // through FETCH_HEAD, which checkout_at_sha cannot name.
+ let ok = |r: Result<(i32, String)>| r.map(|(rc, _)| rc == 0).unwrap_or(false);
+ let looks_like_sha = rev.len() == 40 && rev.chars().all(|c| c.is_ascii_hexdigit());
+ if looks_like_sha {
+ crate::xc::checkout_at_sha(target, url, rev)
+ .with_context(|| format!("source '{slug}': checkout {rev}"))?;
+ } else {
+ if !ok(crate::xc::git(
+ Some(target),
+ &["fetch", "--depth", "1", "origin", "HEAD"],
+ )) {
+ bail!("source '{slug}': git fetch HEAD from {url} failed");
+ }
+ if !ok(crate::xc::git(
+ Some(target),
+ &["checkout", "--quiet", "--force", "--detach", "FETCH_HEAD"],
+ )) {
+ bail!("source '{slug}': git checkout FETCH_HEAD failed");
+ }
+ crate::xc::git(Some(target), &["clean", "-qfd"]).ok();
+ }
+ let (_, head) = crate::xc::git(Some(target), &["rev-parse", "HEAD"])
+ .with_context(|| format!("source '{slug}': rev-parse"))?;
+ if head.is_empty() {
+ bail!("source '{slug}': empty HEAD after checkout");
+ }
+ Ok(head)
+}
+
+/// Extract `zip` into `dest`, replacing any previous extraction of the same
+/// source. Defensive: entry paths are sanitized, escapes are skipped and
+/// counted (returned), total decompressed bytes are capped.
+pub fn extract_zip(zip: &Path, dest: &Path) -> Result<usize> {
+ let f = std::fs::File::open(zip).with_context(|| format!("open {}", zip.display()))?;
+ let mut z = zip::ZipArchive::new(f).context("open zip")?;
+
+ // clear the previous extraction first: a file renamed upstream must not
+ // survive as a stale duplicate
+ if dest.exists() {
+ std::fs::remove_dir_all(dest).with_context(|| format!("clear {}", dest.display()))?;
+ }
+ std::fs::create_dir_all(dest)?;
+
+ let mut skipped = 0usize;
+ let mut total: u64 = 0;
+ for i in 0..z.len() {
+ let mut entry = z.by_index(i).context("zip entry")?;
+ if entry.is_dir() {
+ continue;
+ }
+ if entry.is_symlink() {
+ skipped += 1;
+ continue;
+ }
+ let raw = entry.name().to_string();
+ let Some(safe) = sanitize_entry_path(&raw) else {
+ skipped += 1;
+ continue;
+ };
+ let out = dest.join(&safe);
+ if let Some(parent) = out.parent() {
+ std::fs::create_dir_all(parent)?;
+ }
+ let mut buf = Vec::new();
+ // bounded read: `entry.size()` is the DECLARED size, so read via
+ // take() + counter and stop the whole extraction past the cap
+ let limit = MAX_EXTRACT_BYTES.saturating_sub(total);
+ let mut reader = (&mut entry).take(limit + 1);
+ reader.read_to_end(&mut buf).context("decompress entry")?;
+ total += buf.len() as u64;
+ if total > MAX_EXTRACT_BYTES {
+ bail!(
+ "zip {} expands past the {} cap",
+ zip.display(),
+ MAX_EXTRACT_BYTES
+ );
+ }
+ std::fs::write(&out, &buf)?;
+ }
+ Ok(skipped)
+}
+
+/// Reject absolute paths, drive letters, `..`, and anything not expressible
+/// as normal relative components (the zip-slip family). Returns the
+/// posix-style relative path. Windows-style names (`C:/x`, `a\\b`) are
+/// rejected on EVERY os — a pack may be built anywhere, and a name that is
+/// a harmless subdir on linux is an escape on windows.
+fn sanitize_entry_path(raw: &str) -> Option<String> {
+ let path = Path::new(raw);
+ let mut parts: Vec<&str> = Vec::new();
+ for c in path.components() {
+ match c {
+ Component::Normal(p) => {
+ let s = p.to_str()?;
+ if s.is_empty()
+ || s == "."
+ || s.contains('\0')
+ || s.contains(':')
+ || s.contains('\\')
+ {
+ return None;
+ }
+ parts.push(s);
+ }
+ // a leading `/` makes it absolute on unix; Prefix is a windows
+ // drive; CurDir/ParentDir have no business in an entry name
+ _ => return None,
+ }
+ }
+ if parts.is_empty() {
+ return None;
+ }
+ Some(parts.join("/"))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn write_zip(path: &Path, entries: &[(&str, &str)]) {
+ let f = std::fs::File::create(path).unwrap();
+ let mut z = zip::ZipWriter::new(f);
+ let opts: zip::write::SimpleFileOptions = zip::write::SimpleFileOptions::default()
+ .compression_method(zip::CompressionMethod::Deflated);
+ for (name, body) in entries {
+ z.start_file(*name, opts).unwrap();
+ use std::io::Write;
+ z.write_all(body.as_bytes()).unwrap();
+ }
+ z.finish().unwrap();
+ }
+
+ #[test]
+ fn zip_extracts_nested_and_replaces() {
+ let tmp = tempfile::tempdir().unwrap();
+ let zp = tmp.path().join("d.zip");
+ write_zip(
+ &zp,
+ &[
+ ("a.json", "{\"id\":\"x\"}"),
+ ("nested/b.json", "{\"id\":\"y\"}"),
+ ],
+ );
+ let dest = tmp.path().join("out");
+ assert_eq!(extract_zip(&zp, &dest).unwrap(), 0);
+ assert_eq!(
+ std::fs::read_to_string(dest.join("a.json")).unwrap(),
+ "{\"id\":\"x\"}"
+ );
+ assert_eq!(
+ std::fs::read_to_string(dest.join("nested/b.json")).unwrap(),
+ "{\"id\":\"y\"}"
+ );
+
+ // a re-extract with a renamed file must not leave the stale one
+ write_zip(&zp, &[("renamed.json", "{}")]);
+ extract_zip(&zp, &dest).unwrap();
+ assert!(!dest.join("a.json").exists());
+ assert!(dest.join("renamed.json").exists());
+ }
+
+ #[test]
+ fn zip_slip_and_absolute_are_skipped_counted() {
+ let tmp = tempfile::tempdir().unwrap();
+ let outside = tmp.path().join("outside.txt");
+ let zp = tmp.path().join("evil.zip");
+ write_zip(
+ &zp,
+ &[
+ ("ok.json", "{}"),
+ ("../outside.txt", "pwned"),
+ ("/abs.json", "{}"),
+ ("C:/windows/evil.txt", "{}"),
+ ],
+ );
+ let dest = tmp.path().join("out");
+ let skipped = extract_zip(&zp, &dest).unwrap();
+ assert!(dest.join("ok.json").exists());
+ assert!(!outside.exists(), "nothing escaped the extraction root");
+ assert_eq!(skipped, 3, "absolute + escape entries counted, not written");
+ }
+
+ #[test]
+ fn git_source_checks_out_and_watermarks() {
+ // a real local repo via `git init` — no daemon needed
+ let tmp = tempfile::tempdir().unwrap();
+ let origin = tmp.path().join("origin");
+ std::fs::create_dir_all(&origin).unwrap();
+ assert!(crate::xc::git(Some(&origin), &["init", "--quiet", "-b", "main"]).is_ok());
+ std::fs::write(origin.join("one.json"), r#"{"id":"R1","aliases":["C1"]}"#).unwrap();
+ assert!(crate::xc::git(Some(&origin), &["add", "."]).is_ok());
+ let git_user = |args: &[&str]| crate::xc::git(Some(&origin), args);
+ git_user(&[
+ "-c",
+ "user.email=t@t",
+ "-c",
+ "user.name=t",
+ "commit",
+ "--quiet",
+ "-m",
+ "one",
+ ])
+ .ok();
+
+ let cache = tmp.path().join("cache");
+ let spec = super::super::recipe::SourceSpec {
+ slug: "repo".into(),
+ kind: super::super::recipe::SourceKind::Git,
+ path: None,
+ url: Some(origin.to_string_lossy().to_string()),
+ rev: None,
+ glob: "**/*.json".into(),
+ format: super::super::recipe::Format::Flat,
+ licence: "CC0-1.0".into(),
+ };
+ let f1 = fetch(&spec, tmp.path(), &cache).unwrap();
+ assert!(f1.root.join("one.json").exists());
+ assert!(!f1.unchanged, "first fetch has no previous watermark");
+ let wm = f1.watermark.unwrap();
+ assert_eq!(wm.len(), 40, "HEAD sha");
+
+ save_watermark(&cache, &spec.slug, &wm).unwrap();
+ let f2 = fetch(&spec, tmp.path(), &cache).unwrap();
+ assert!(f2.unchanged, "same HEAD ⇒ skip the walk");
+ assert_eq!(f2.watermark.as_deref(), Some(wm.as_str()));
+
+ // a new commit upstream moves the watermark
+ std::fs::write(origin.join("two.json"), r#"{"id":"R2"}"#).unwrap();
+ crate::xc::git(Some(&origin), &["add", "."]).ok();
+ git_user(&[
+ "-c",
+ "user.email=t@t",
+ "-c",
+ "user.name=t",
+ "commit",
+ "--quiet",
+ "-m",
+ "two",
+ ])
+ .ok();
+ let f3 = fetch(&spec, tmp.path(), &cache).unwrap();
+ assert!(!f3.unchanged);
+ assert_ne!(f3.watermark.as_deref(), Some(wm.as_str()));
+ assert!(
+ f3.root.join("two.json").exists(),
+ "checkout moved to the new tip"
+ );
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/xc.rs b/engine/crates/xerj-autoindex/src/xc.rs
index 17be15f4b..b1ebe6697 100644
--- a/engine/crates/xerj-autoindex/src/xc.rs
+++ b/engine/crates/xerj-autoindex/src/xc.rs
@@ -236,7 +236,7 @@ deterministic pack under builds/<name>/ (see tools/xerj-code/);
index builds/verifies/switches the corpus (exit 3 skips junk — normal);
list shows what is loaded on this node.";
-fn git(dir: Option<&Path>, args: &[&str]) -> Result<(i32, String)> {
+pub(crate) fn git(dir: Option<&Path>, args: &[&str]) -> Result<(i32, String)> {
let mut c = Command::new("git");
if let Some(d) = dir {
c.current_dir(d).arg("-C").arg(d);
@@ -254,7 +254,7 @@ fn git(dir: Option<&Path>, args: &[&str]) -> Result<(i32, String)> {
/// Move a clone to `sha` (shallow). Never a shell string: the sha comes from
/// an untrusted manifest, and `Command` passes it as ONE argv element.
-fn checkout_at_sha(target: &Path, url: &str, sha: &str) -> Result<()> {
+pub(crate) fn checkout_at_sha(target: &Path, url: &str, sha: &str) -> Result<()> {
if !target.join(".git").exists() {
std::fs::create_dir_all(target)?;
git(Some(target), &["init", "--quiet"])?;