AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(autoindex): network sources for corpus build — http-zip, git, watermarks (M2)
xerj-org5bd29f38a18d
diff --git a/engine/crates/xerj-autoindex/src/harvest/httpget.rs b/engine/crates/xerj-autoindex/src/harvest/httpget.rsnew file mode 100644index 000000000..afe3bfff6--- /dev/null+++ b/engine/crates/xerj-autoindex/src/harvest/httpget.rs@@ -0,0 +1,153 @@+//! Blocking HTTP download for `http-zip` recipe sources.+//!+//! Deliberately small next to `esclient`'s retry loop: that one protects an+//! interactive indexing session against a live server's 429/503 storms; this+//! one fetches a handful of public dump URLs per run. The part worth copying+//! is the SHAPE — bounded attempts, exponential backoff, one-line errors that+//! name the URL and status, and an atomic tmp+rename write so a partial+//! download never exists under the cache's final name.++use std::path::Path;+use std::time::Duration;++use anyhow::{bail, Context, Result};++/// Refuse to buffer more than this on disk. Public advisory dumps are+/// megabytes-to-a-few-hundred-MB; a runaway URL should fail loudly, not fill+/// the disk. Decompression safety is enforced separately at zip-extract time+/// (a small zip can expand enormously — see `source::extract_zip`).+const MAX_DOWNLOAD_BYTES: u64 = 4 << 30;++const ATTEMPTS: u32 = 3;+const INITIAL_BACKOFF: Duration = Duration::from_secs(1);++/// Download `url` to `dest` (atomically). Returns the byte count.+pub fn download(url: &str, dest: &Path) -> Result<u64> {+ let client = reqwest::blocking::Client::builder()+ .timeout(Duration::from_secs(600))+ .connect_timeout(Duration::from_secs(30))+ .redirect(reqwest::redirect::Policy::limited(10))+ .build()+ .context("build http client")?;++ let mut last_err = String::new();+ for attempt in 0..ATTEMPTS {+ if attempt > 0 {+ std::thread::sleep(INITIAL_BACKOFF * (1 << (attempt - 1)));+ }+ match once(&client, url, dest) {+ Ok(n) => return Ok(n),+ // keep the FIRST error: later attempts against a dead endpoint+ // report less (bare connect failure vs "HTTP 404")+ Err(e) => {+ if last_err.is_empty() {+ last_err = e.to_string();+ }+ }+ }+ }+ bail!("download {url} failed after {ATTEMPTS} attempts: {last_err}")+}++fn once(client: &reqwest::blocking::Client, url: &str, dest: &Path) -> Result<u64> {+ use std::io::{Read, Write};+ let resp = client+ .get(url)+ .send()+ .with_context(|| format!("GET {url}"))?;+ let status = resp.status();+ if !status.is_success() {+ // not retry-worthy in the same way as a timeout, but cheap to retry+ // once upstream hiccuped — the loop above bounds it+ bail!("HTTP {status} for {url}");+ }+ if let Some(len) = resp.content_length() {+ if len > MAX_DOWNLOAD_BYTES {+ bail!("{url} declares {len} bytes, over the {MAX_DOWNLOAD_BYTES} cap");+ }+ }++ let tmp = dest.with_extension("part");+ if let Some(parent) = dest.parent() {+ std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?;+ }+ let mut file =+ std::fs::File::create(&tmp).with_context(|| format!("create {}", tmp.display()))?;+ let mut written: u64 = 0;+ // stream: `resp.bytes()` would buffer the entire body in memory before+ // any cap could fire+ let mut reader = resp.take(MAX_DOWNLOAD_BYTES + 1);+ let mut buf = vec![0u8; 64 * 1024];+ loop {+ let n = reader.read(&mut buf)?;+ if n == 0 {+ break;+ }+ written += n as u64;+ if written > MAX_DOWNLOAD_BYTES {+ let _ = std::fs::remove_file(&tmp);+ bail!("{url} exceeded the {MAX_DOWNLOAD_BYTES} download cap");+ }+ file.write_all(&buf[..n])?;+ }+ file.flush()?;+ drop(file);+ std::fs::rename(&tmp, dest).with_context(|| format!("move {} into place", tmp.display()))?;+ Ok(written)+}++#[cfg(test)]+mod tests {+ use super::*;++ /// One canned HTTP response over a raw TcpListener — the fake-node+ /// pattern from `failure_resume_http_tests.rs`. No reqwest server dep.+ /// Owned arguments: the serving thread must be 'static.+ fn serve_once(head: String, body: Vec<u8>) -> std::io::Result<std::net::SocketAddr> {+ let listener = std::net::TcpListener::bind("127.0.0.1:0")?;+ let addr = listener.local_addr()?;+ std::thread::spawn(move || {+ if let Ok((mut sock, _)) = listener.accept() {+ use std::io::{Read, Write};+ let mut buf = [0u8; 1024];+ let _ = sock.read(&mut buf); // discard the request+ let _ = sock.write_all(head.as_bytes());+ let _ = sock.write_all(&body);+ let _ = sock.flush();+ }+ });+ Ok(addr)+ }++ #[test]+ fn downloads_and_retries() {+ let tmp = tempfile::tempdir().unwrap();+ let dest = tmp.path().join("dump.zip");++ // success path+ let addr = serve_once(+ "HTTP/1.1 200 OK\r\nContent-Length: 6\r\nConnection: close\r\n\r\n".to_string(),+ b"hello!".to_vec(),+ )+ .unwrap();+ let n = download(&format!("http://{addr}/x.zip"), &dest).unwrap();+ assert_eq!(n, 6);+ assert_eq!(std::fs::read(&dest).unwrap(), b"hello!");+ assert!(+ !tmp.path().join("dump.part").exists(),+ "no partial left behind"+ );++ // failure path: 404 on every attempt → named error, no dest file+ let addr = serve_once(+ "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_string(),+ Vec::new(),+ )+ .unwrap();+ let err = download(&format!("http://{addr}/gone.zip"), &dest)+ .unwrap_err()+ .to_string();+ assert!(err.contains("404"), "{err}");+ assert!(err.contains("3 attempts"), "{err}");+ }+}diff --git a/engine/crates/xerj-autoindex/src/harvest/mod.rs b/engine/crates/xerj-autoindex/src/harvest/mod.rsindex b00936c52..696deb822 100644--- a/engine/crates/xerj-autoindex/src/harvest/mod.rs+++ b/engine/crates/xerj-autoindex/src/harvest/mod.rs@@ -23,10 +23,12 @@//! `sources[]`); the tool does not and must not enforce them — the pack//! author owns what goes in the pack.+pub mod httpget;pub mod identity;pub mod normalize;pub mod pack;pub mod recipe;+pub mod source;pub mod store;pub(crate) use store::MAX_FIELDS;@@ -123,24 +125,33 @@ fn run_build_inner(args: &[String], home: &Path) -> Result<i32> {.with_context(|| format!("--fresh: cannot clear {}", build_dir.display()))?;}let store_dir = build_dir.join("store");+ let cache_dir = build_dir.join("cache");// ── harvest into the content store ───────────────────────────────────let mut per_source = Vec::new();for src in &recipe.sources {- let SourceKind::Dir = src.kind;- let root = recipe_dir.join(src.path.as_deref().unwrap_or("."));- let files = walk(&root, &src.glob)?;+ let fetched = source::fetch(src, &recipe_dir, &cache_dir)?;let mut stat = SourceStat {slug: src.slug.clone(),- kind: "dir".to_string(),+ kind: kind_name(src.kind).to_string(),path: src.path.clone(),url: src.url.clone(),licence: src.licence.clone(),+ watermark: fetched.watermark.clone(),+ unchanged: fetched.unchanged,records: 0,new: 0,skipped: 0,pruned: 0,};+ if fetched.unchanged {+ // The whole upstream tree is byte-identical to the previous run:+ // skip the walk AND the prune — every envelope this source ever+ // stored is still live, and pruning needs the full key set.+ per_source.push(stat);+ continue;+ }+ let files = walk(&fetched.root, &src.glob)?;let mut keys = HashSet::new();for (rel, path) in files {for value in parse_file(&path)? {@@ -163,6 +174,11 @@ fn run_build_inner(args: &[String], home: &Path) -> Result<i32> {}}stat.pruned = store::prune(&store_dir, &src.slug, &keys);+ // persist the watermark only after the source's harvest fully+ // succeeded — a failed run must re-walk+ if let Some(wm) = &fetched.watermark {+ source::save_watermark(&cache_dir, &src.slug, wm)?;+ }per_source.push(stat);}@@ -194,11 +210,37 @@ fn run_build_inner(args: &[String], home: &Path) -> Result<i32> {};let result = pack::emit(&build_dir, &recipe, &recipe_toml, records, &stats)?;+ // run report: the machine-readable twin of the stdout summary, with the+ // watermarks a scheduled rebuild (M6) will diff against+ let report = serde_json::json!({+ "finished_at": chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true),+ "pack": recipe.name,+ "generation": result.generation,+ "records": result.records,+ "envelopes": stats.envelopes,+ "sources": stats.per_source.iter()+ .map(|s| serde_json::to_value(s).expect("SourceStat serializes"))+ .collect::<Vec<_>>(),+ });+ std::fs::write(+ build_dir.join("report.json"),+ serde_json::to_vec_pretty(&report)?,+ )+ .with_context(|| format!("write {}", build_dir.join("report.json").display()))?;+println!("built '{}' generation {}: {} envelopes → {} records, {} shard file(s)",recipe.name, result.generation, stats.envelopes, result.records, result.shards);for s in &stats.per_source {+ if s.unchanged {+ println!(+ " {}: unchanged (watermark {})",+ s.slug,+ s.watermark.clone().unwrap_or_default()+ );+ continue;+ }println!(" {}: {} record(s) — {} new, {} skipped, {} pruned ({})",s.slug, s.records, s.new, s.skipped, s.pruned, s.licence@@ -208,6 +250,14 @@ fn run_build_inner(args: &[String], home: &Path) -> Result<i32> {Ok(0)}+fn kind_name(kind: SourceKind) -> &'static str {+ match kind {+ SourceKind::Dir => "dir",+ SourceKind::HttpZip => "http-zip",+ SourceKind::Git => "git",+ }+}+// ── dir source ──────────────────────────────────────────────────────────────/// Recursively list files under `root` matching `glob`, as@@ -597,4 +647,94 @@ shards = 4);assert_eq!(read_records(&pack).len(), 1);}++ // ── end to end: a git source, watermark-skipped on the second run ─────++ #[test]+ fn build_from_git_source_and_skip_when_unchanged() {+ let home = tempfile::tempdir().unwrap();+ let origin = home.path().join("origin");+ std::fs::create_dir_all(&origin).unwrap();+ crate::xc::git(Some(&origin), &["init", "--quiet", "-b", "main"]).ok();+ std::fs::write(origin.join("a.json"), r#"{"id": "R1", "aliases": ["C1"]}"#).unwrap();+ crate::xc::git(Some(&origin), &["add", "."]).ok();+ crate::xc::git(+ Some(&origin),+ &[+ "-c",+ "user.email=t@t",+ "-c",+ "user.name=t",+ "commit",+ "--quiet",+ "-m",+ "one",+ ],+ )+ .ok();++ let recipe = format!(+ "[recipe]\nformat = 1\nname = \"gitdemo\"\n\n\+ [[sources]]\nslug = \"upstream\"\nkind = \"git\"\nurl = \"{}\"\nformat = \"flat\"\n\+ licence = \"CC0-1.0\"\n\n\+ [identity]\nedges = [{{ field = \"id\" }}]\n",+ origin.to_string_lossy()+ );+ std::fs::create_dir_all(home.path().join("recipes")).unwrap();+ std::fs::write(home.path().join("recipes/gitdemo.toml"), recipe).unwrap();++ let args: Vec<String> = vec!["gitdemo".to_string()];+ assert_eq!(run_build_inner(&args, home.path()).unwrap(), 0);+ let pack = home.path().join("builds/gitdemo/pack/gitdemo");+ assert_eq!(read_records(&pack).len(), 1);++ let manifest: Value =+ serde_json::from_str(&std::fs::read_to_string(pack.join("manifest.json")).unwrap())+ .unwrap();+ let s = &manifest["sources"][0];+ assert_eq!(s["kind"], json!("git"));+ assert_eq!(s["new"], json!(1));+ assert!(s["watermark"].as_str().is_some_and(|w| w.len() == 40));++ // second run: same HEAD → the source is not walked, the store is+ // untouched, the pack is rebuilt identically from the store+ assert_eq!(run_build_inner(&args, home.path()).unwrap(), 0);+ let manifest: Value =+ serde_json::from_str(&std::fs::read_to_string(pack.join("manifest.json")).unwrap())+ .unwrap();+ let s = &manifest["sources"][0];+ assert_eq!(s["unchanged"], json!(true));+ assert_eq!(s["records"], json!(0), "no walk happened");+ assert_eq!(+ read_records(&pack).len(),+ 1,+ "records survive via the store"+ );+ assert!(home.path().join("builds/gitdemo/report.json").is_file());++ // a new commit upstream re-activates the source+ std::fs::write(origin.join("b.json"), r#"{"id": "R2"}"#).unwrap();+ crate::xc::git(Some(&origin), &["add", "."]).ok();+ crate::xc::git(+ Some(&origin),+ &[+ "-c",+ "user.email=t@t",+ "-c",+ "user.name=t",+ "commit",+ "--quiet",+ "-m",+ "two",+ ],+ )+ .ok();+ assert_eq!(run_build_inner(&args, home.path()).unwrap(), 0);+ assert_eq!(read_records(&pack).len(), 2);+ let manifest: Value =+ serde_json::from_str(&std::fs::read_to_string(pack.join("manifest.json")).unwrap())+ .unwrap();+ assert_eq!(manifest["sources"][0]["unchanged"], json!(false));+ assert_eq!(manifest["sources"][0]["new"], json!(1));+ }}diff --git a/engine/crates/xerj-autoindex/src/harvest/pack.rs b/engine/crates/xerj-autoindex/src/harvest/pack.rsindex 580c8f3c7..357614967 100644--- a/engine/crates/xerj-autoindex/src/harvest/pack.rs+++ b/engine/crates/xerj-autoindex/src/harvest/pack.rs@@ -24,13 +24,18 @@ use xxhash_rust::xxh3::xxh3_64;use super::recipe::Recipe;-/// Per-source counts for the manifest: what this run actually did.+/// Per-source counts for the manifest and report: what this run actually+/// did. `unchanged` = watermark matched, so the source was not walked at+/// all (and must not be pruned either — everything it stored is live).+#[derive(serde::Serialize)]pub struct SourceStat {pub slug: String,pub kind: String,pub path: Option<String>,pub url: Option<String>,pub licence: String,+ pub watermark: Option<String>,+ pub unchanged: bool,pub records: usize,pub new: usize,pub skipped: usize,@@ -138,11 +143,11 @@ pub fn emit("records": records.len(),"shards": shards,},- "sources": stats.per_source.iter().map(|s| serde_json::json!({- "slug": s.slug, "kind": s.kind, "path": s.path, "url": s.url,- "licence": s.licence, "records": s.records,- "new": s.new, "skipped": s.skipped, "pruned": s.pruned,- })).collect::<Vec<_>>(),+ "sources": stats+ .per_source+ .iter()+ .map(|s| serde_json::to_value(s).expect("SourceStat serializes"))+ .collect::<Vec<_>>(),"files": Value::Object(files.clone().into_iter().collect()),"tool": { "name": "xerj", "version": env!("CARGO_PKG_VERSION") },});diff --git a/engine/crates/xerj-autoindex/src/harvest/recipe.rs b/engine/crates/xerj-autoindex/src/harvest/recipe.rsindex 4229dc5a5..0af8a9309 100644--- a/engine/crates/xerj-autoindex/src/harvest/recipe.rs+++ b/engine/crates/xerj-autoindex/src/harvest/recipe.rs@@ -86,7 +86,11 @@ pub struct SourceSpec {pub kind: SourceKind,/// `dir` sources: a path relative to the recipe file.pub path: Option<String>,+ /// `http-zip`/`git` sources: where the bytes come from. A git url may+ /// also be a local path (git itself accepts both — tests rely on it).pub url: Option<String>,+ /// `git` sources: a pinned commit sha, or absent to track remote HEAD.+ pub rev: Option<String>,pub glob: String,pub format: Format,/// Carried onto every record from this source as record DATA — the tool@@ -97,6 +101,8 @@ pub struct SourceSpec {#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub enum SourceKind {Dir,+ HttpZip,+ Git,}#[derive(Debug, Clone, Copy, PartialEq, Eq)]@@ -239,6 +245,8 @@ struct RawSource {path: Option<String>,#[serde(default)]url: Option<String>,+ #[serde(default)]+ rev: Option<String>,#[serde(default = "d_glob")]glob: String,format: String,@@ -372,12 +380,12 @@ fn compile(path: &Path, file: RawRecipe) -> Result<Recipe> {}let kind = match s.kind.as_str() {"dir" => SourceKind::Dir,- other @ ("http-zip" | "git") => bail!(at(&format!(- "source '{}' kind '{other}' is not supported yet (planned: incremental \- network sources)",+ "http-zip" => SourceKind::HttpZip,+ "git" => SourceKind::Git,+ other => bail!(at(&format!(+ "source '{}': unknown kind '{other}' (supported: dir, http-zip, git)",s.slug))),- other => bail!(at(&format!("source '{}': unknown kind '{other}'", s.slug))),};let format = match s.format.as_str() {"osv" => Format::Osv,@@ -387,17 +395,55 @@ fn compile(path: &Path, file: RawRecipe) -> Result<Recipe> {s.slug))),};- if kind == SourceKind::Dir {- if s.path.as_deref().unwrap_or("").is_empty() {- bail!(at(&format!(- "dir source '{}' needs a 'path' (relative to the recipe file)",- s.slug- )));+ let url = s.url.as_deref().unwrap_or_default().to_string();+ let rev = s.rev.as_deref().unwrap_or_default().to_string();+ match kind {+ SourceKind::Dir => {+ if s.path.as_deref().unwrap_or("").is_empty() {+ bail!(at(&format!(+ "dir source '{}' needs a 'path' (relative to the recipe file)",+ s.slug+ )));+ }+ if let Some(p) = &s.path {+ if Path::new(p).is_absolute() {+ bail!(at(&format!(+ "dir source '{}' path must be recipe-relative, got absolute '{p}'",+ s.slug+ )));+ }+ }+ if !url.is_empty() || !rev.is_empty() {+ bail!(at(&format!(+ "dir source '{}' takes 'path', not 'url'/'rev'",+ s.slug+ )));+ }}- if let Some(p) = &s.path {- if Path::new(p).is_absolute() {+ SourceKind::HttpZip => {+ if !url.starts_with("http://") && !url.starts_with("https://") {bail!(at(&format!(- "dir source '{}' path must be recipe-relative, got absolute '{p}'",+ "http-zip source '{}' needs an http(s) 'url', got '{url}'",+ s.slug+ )));+ }+ if s.path.is_some() || !rev.is_empty() {+ bail!(at(&format!(+ "http-zip source '{}' takes 'url' only — no 'path'/'rev'",+ s.slug+ )));+ }+ }+ SourceKind::Git => {+ if url.is_empty() {+ bail!(at(&format!(+ "git source '{}' needs a 'url' (a git URL or local path)",+ s.slug+ )));+ }+ if s.path.is_some() {+ bail!(at(&format!(+ "git source '{}' takes 'url' (+optional 'rev'), not 'path'",s.slug)));}@@ -409,6 +455,7 @@ fn compile(path: &Path, file: RawRecipe) -> Result<Recipe> {kind,path: s.path.clone(),url: s.url.clone(),+ rev: if rev.is_empty() { None } else { Some(rev) },glob: s.glob.clone(),format,licence: s.licence.clone(),@@ -634,12 +681,34 @@ edges = [{ field = "id" }]}#[test]- fn unknown_source_kind_names_the_planned_one() {+ fn unknown_source_kind_is_loud() {+ let tmp = tempfile::tempdir().unwrap();+ let bad = MINIMAL.replace("kind = \"dir\"", "kind = \"svn\"");+ let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();+ assert!(err.contains("unknown kind 'svn'"), "{err}");+ }++ #[test]+ fn network_kinds_validate_their_targets() {let tmp = tempfile::tempdir().unwrap();- let bad = MINIMAL.replace("kind = \"dir\"", "kind = \"git\"");+ // http-zip without an http(s) url+ let bad = MINIMAL.replace(+ "kind = \"dir\"\npath = \"data/a\"",+ "kind = \"http-zip\"\nurl = \"ftp://x/y.zip\"",+ );+ let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();+ assert!(err.contains("http(s) 'url'"), "{err}");+ // git with a stray path+ let bad = MINIMAL.replace("kind = \"dir\"", "kind = \"git\"\nurl = \"https://x/y\"");+ let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();+ assert!(err.contains("not 'path'"), "{err}");+ // dir with a url+ let bad = MINIMAL.replace(+ "path = \"data/a\"",+ "path = \"data/a\"\nurl = \"https://x/y\"",+ );let err = load(&write(tmp.path(), &bad)).unwrap_err().to_string();- assert!(err.contains("not supported yet"), "{err}");- assert!(err.contains("'git'"), "{err}");+ assert!(err.contains("not 'url'/'rev'"), "{err}");}#[test]diff --git a/engine/crates/xerj-autoindex/src/harvest/source.rs b/engine/crates/xerj-autoindex/src/harvest/source.rsnew file mode 100644index 000000000..dbda8910c--- /dev/null+++ b/engine/crates/xerj-autoindex/src/harvest/source.rs@@ -0,0 +1,361 @@+//! Source adapters: turn a recipe `[[sources]]` entry into a directory to+//! walk, plus the upstream watermark that lets an unchanged source be+//! skipped wholesale on the next run.+//!+//! Kinds:+//! - `dir` — walk a recipe-relative directory (the fixture/local kind)+//! - `http-zip`— download a zip, extract into the cache, walk the+//! extraction. No watermark: the zip is re-downloaded every run and the+//! content store dedups what did not change (correctness lives there,+//! watermarks are only an optimization).+//! - `git` — promisor shallow checkout at `rev` (pinned) or the remote+//! HEAD (tracking). Watermark = HEAD sha: unchanged tree ⇒ the walk is+//! skipped entirely this run.+//!+//! Zip extraction is defensive by default: entry names are normalized and+//! any path escaping the extraction root (zip-slip), absolute path, or+//! symlink entry is skipped and COUNTED, never trusted — dump URLs are+//! fetched from the open internet.++use std::io::Read;+use std::path::{Component, Path, PathBuf};++use anyhow::{bail, Context, Result};++use super::httpget;+use super::recipe::SourceSpec;++/// Total decompressed cap for one zip. Real advisory dumps are well under+/// this; the cap exists because a zip's declared sizes are untrusted and a+/// 3 MiB file can claim to expand to terabytes.+const MAX_EXTRACT_BYTES: u64 = 2 << 30;++pub struct Fetched {+ /// The directory the glob walk starts from.+ pub root: PathBuf,+ /// Current upstream version marker, if the kind has one.+ pub watermark: Option<String>,+ /// The watermark matches the previous run's: the caller should skip+ /// walking AND pruning this source (everything it stored is still live).+ pub unchanged: bool,+}++/// Where a source's watermark from the previous run is kept.+fn watermark_path(cache_dir: &Path, slug: &str) -> PathBuf {+ cache_dir.join(format!("{slug}.watermark"))+}++/// Materialize a source. `cache_dir` is `<build>/cache`; it only exists to+/// make network kinds incremental.+pub fn fetch(src: &SourceSpec, recipe_dir: &Path, cache_dir: &Path) -> Result<Fetched> {+ match src.kind {+ super::recipe::SourceKind::Dir => Ok(Fetched {+ root: recipe_dir.join(src.path.as_deref().unwrap_or(".")),+ watermark: None,+ unchanged: false,+ }),+ super::recipe::SourceKind::HttpZip => {+ let url = src.url.as_deref().unwrap_or_default();+ let zip_path = cache_dir.join(format!("{}.zip", src.slug));+ httpget::download(url, &zip_path)+ .with_context(|| format!("source '{}': fetch", src.slug))?;+ let root = cache_dir.join(&src.slug);+ extract_zip(&zip_path, &root)+ .with_context(|| format!("source '{}': extract", src.slug))?;+ Ok(Fetched {+ root,+ watermark: None,+ unchanged: false,+ })+ }+ super::recipe::SourceKind::Git => {+ let url = src.url.as_deref().unwrap_or_default();+ let root = cache_dir.join(&src.slug);+ let rev = src.rev.as_deref().unwrap_or("HEAD");+ let watermark = git_checkout(&root, url, rev, &src.slug)?;+ let prev = std::fs::read_to_string(watermark_path(cache_dir, &src.slug))+ .map(|s| s.trim().to_string())+ .unwrap_or_default();+ Ok(Fetched {+ root,+ watermark: Some(watermark.clone()),+ unchanged: !watermark.is_empty() && watermark == prev,+ })+ }+ }+}++/// Persist a source's watermark after its harvest fully succeeded (never+/// before — a failed run must re-walk).+pub fn save_watermark(cache_dir: &Path, slug: &str, watermark: &str) -> Result<()> {+ std::fs::create_dir_all(cache_dir)?;+ std::fs::write(watermark_path(cache_dir, slug), watermark)+ .with_context(|| format!("save watermark for '{slug}'"))+}++/// Shallow promisor checkout of `rev` (a sha or `HEAD`) into `target`,+/// reusing the corpus-clone machinery: init + blob:none filter so a pin at+/// an old sha costs one small fetch, never a full history. Returns HEAD.+fn git_checkout(target: &Path, url: &str, rev: &str, slug: &str) -> Result<String> {+ if !target.join(".git").exists() {+ std::fs::create_dir_all(target)+ .with_context(|| format!("source '{slug}': create {}", target.display()))?;+ crate::xc::git(Some(target), &["init", "--quiet"])?;+ crate::xc::git(Some(target), &["remote", "add", "origin", url]).ok();+ crate::xc::git(Some(target), &["config", "remote.origin.promisor", "true"]).ok();+ crate::xc::git(+ Some(target),+ &["config", "remote.origin.partialclonefilter", "blob:none"],+ )+ .ok();+ }+ // A pinned rev (full sha) can reuse checkout_at_sha; tracking HEAD goes+ // through FETCH_HEAD, which checkout_at_sha cannot name.+ let ok = |r: Result<(i32, String)>| r.map(|(rc, _)| rc == 0).unwrap_or(false);+ let looks_like_sha = rev.len() == 40 && rev.chars().all(|c| c.is_ascii_hexdigit());+ if looks_like_sha {+ crate::xc::checkout_at_sha(target, url, rev)+ .with_context(|| format!("source '{slug}': checkout {rev}"))?;+ } else {+ if !ok(crate::xc::git(+ Some(target),+ &["fetch", "--depth", "1", "origin", "HEAD"],+ )) {+ bail!("source '{slug}': git fetch HEAD from {url} failed");+ }+ if !ok(crate::xc::git(+ Some(target),+ &["checkout", "--quiet", "--force", "--detach", "FETCH_HEAD"],+ )) {+ bail!("source '{slug}': git checkout FETCH_HEAD failed");+ }+ crate::xc::git(Some(target), &["clean", "-qfd"]).ok();+ }+ let (_, head) = crate::xc::git(Some(target), &["rev-parse", "HEAD"])+ .with_context(|| format!("source '{slug}': rev-parse"))?;+ if head.is_empty() {+ bail!("source '{slug}': empty HEAD after checkout");+ }+ Ok(head)+}++/// Extract `zip` into `dest`, replacing any previous extraction of the same+/// source. Defensive: entry paths are sanitized, escapes are skipped and+/// counted (returned), total decompressed bytes are capped.+pub fn extract_zip(zip: &Path, dest: &Path) -> Result<usize> {+ let f = std::fs::File::open(zip).with_context(|| format!("open {}", zip.display()))?;+ let mut z = zip::ZipArchive::new(f).context("open zip")?;++ // clear the previous extraction first: a file renamed upstream must not+ // survive as a stale duplicate+ if dest.exists() {+ std::fs::remove_dir_all(dest).with_context(|| format!("clear {}", dest.display()))?;+ }+ std::fs::create_dir_all(dest)?;++ let mut skipped = 0usize;+ let mut total: u64 = 0;+ for i in 0..z.len() {+ let mut entry = z.by_index(i).context("zip entry")?;+ if entry.is_dir() {+ continue;+ }+ if entry.is_symlink() {+ skipped += 1;+ continue;+ }+ let raw = entry.name().to_string();+ let Some(safe) = sanitize_entry_path(&raw) else {+ skipped += 1;+ continue;+ };+ let out = dest.join(&safe);+ if let Some(parent) = out.parent() {+ std::fs::create_dir_all(parent)?;+ }+ let mut buf = Vec::new();+ // bounded read: `entry.size()` is the DECLARED size, so read via+ // take() + counter and stop the whole extraction past the cap+ let limit = MAX_EXTRACT_BYTES.saturating_sub(total);+ let mut reader = (&mut entry).take(limit + 1);+ reader.read_to_end(&mut buf).context("decompress entry")?;+ total += buf.len() as u64;+ if total > MAX_EXTRACT_BYTES {+ bail!(+ "zip {} expands past the {} cap",+ zip.display(),+ MAX_EXTRACT_BYTES+ );+ }+ std::fs::write(&out, &buf)?;+ }+ Ok(skipped)+}++/// Reject absolute paths, drive letters, `..`, and anything not expressible+/// as normal relative components (the zip-slip family). Returns the+/// posix-style relative path. Windows-style names (`C:/x`, `a\\b`) are+/// rejected on EVERY os — a pack may be built anywhere, and a name that is+/// a harmless subdir on linux is an escape on windows.+fn sanitize_entry_path(raw: &str) -> Option<String> {+ let path = Path::new(raw);+ let mut parts: Vec<&str> = Vec::new();+ for c in path.components() {+ match c {+ Component::Normal(p) => {+ let s = p.to_str()?;+ if s.is_empty()+ || s == "."+ || s.contains('\0')+ || s.contains(':')+ || s.contains('\\')+ {+ return None;+ }+ parts.push(s);+ }+ // a leading `/` makes it absolute on unix; Prefix is a windows+ // drive; CurDir/ParentDir have no business in an entry name+ _ => return None,+ }+ }+ if parts.is_empty() {+ return None;+ }+ Some(parts.join("/"))+}++#[cfg(test)]+mod tests {+ use super::*;++ fn write_zip(path: &Path, entries: &[(&str, &str)]) {+ let f = std::fs::File::create(path).unwrap();+ let mut z = zip::ZipWriter::new(f);+ let opts: zip::write::SimpleFileOptions = zip::write::SimpleFileOptions::default()+ .compression_method(zip::CompressionMethod::Deflated);+ for (name, body) in entries {+ z.start_file(*name, opts).unwrap();+ use std::io::Write;+ z.write_all(body.as_bytes()).unwrap();+ }+ z.finish().unwrap();+ }++ #[test]+ fn zip_extracts_nested_and_replaces() {+ let tmp = tempfile::tempdir().unwrap();+ let zp = tmp.path().join("d.zip");+ write_zip(+ &zp,+ &[+ ("a.json", "{\"id\":\"x\"}"),+ ("nested/b.json", "{\"id\":\"y\"}"),+ ],+ );+ let dest = tmp.path().join("out");+ assert_eq!(extract_zip(&zp, &dest).unwrap(), 0);+ assert_eq!(+ std::fs::read_to_string(dest.join("a.json")).unwrap(),+ "{\"id\":\"x\"}"+ );+ assert_eq!(+ std::fs::read_to_string(dest.join("nested/b.json")).unwrap(),+ "{\"id\":\"y\"}"+ );++ // a re-extract with a renamed file must not leave the stale one+ write_zip(&zp, &[("renamed.json", "{}")]);+ extract_zip(&zp, &dest).unwrap();+ assert!(!dest.join("a.json").exists());+ assert!(dest.join("renamed.json").exists());+ }++ #[test]+ fn zip_slip_and_absolute_are_skipped_counted() {+ let tmp = tempfile::tempdir().unwrap();+ let outside = tmp.path().join("outside.txt");+ let zp = tmp.path().join("evil.zip");+ write_zip(+ &zp,+ &[+ ("ok.json", "{}"),+ ("../outside.txt", "pwned"),+ ("/abs.json", "{}"),+ ("C:/windows/evil.txt", "{}"),+ ],+ );+ let dest = tmp.path().join("out");+ let skipped = extract_zip(&zp, &dest).unwrap();+ assert!(dest.join("ok.json").exists());+ assert!(!outside.exists(), "nothing escaped the extraction root");+ assert_eq!(skipped, 3, "absolute + escape entries counted, not written");+ }++ #[test]+ fn git_source_checks_out_and_watermarks() {+ // a real local repo via `git init` — no daemon needed+ let tmp = tempfile::tempdir().unwrap();+ let origin = tmp.path().join("origin");+ std::fs::create_dir_all(&origin).unwrap();+ assert!(crate::xc::git(Some(&origin), &["init", "--quiet", "-b", "main"]).is_ok());+ std::fs::write(origin.join("one.json"), r#"{"id":"R1","aliases":["C1"]}"#).unwrap();+ assert!(crate::xc::git(Some(&origin), &["add", "."]).is_ok());+ let git_user = |args: &[&str]| crate::xc::git(Some(&origin), args);+ git_user(&[+ "-c",+ "user.email=t@t",+ "-c",+ "user.name=t",+ "commit",+ "--quiet",+ "-m",+ "one",+ ])+ .ok();++ let cache = tmp.path().join("cache");+ let spec = super::super::recipe::SourceSpec {+ slug: "repo".into(),+ kind: super::super::recipe::SourceKind::Git,+ path: None,+ url: Some(origin.to_string_lossy().to_string()),+ rev: None,+ glob: "**/*.json".into(),+ format: super::super::recipe::Format::Flat,+ licence: "CC0-1.0".into(),+ };+ let f1 = fetch(&spec, tmp.path(), &cache).unwrap();+ assert!(f1.root.join("one.json").exists());+ assert!(!f1.unchanged, "first fetch has no previous watermark");+ let wm = f1.watermark.unwrap();+ assert_eq!(wm.len(), 40, "HEAD sha");++ save_watermark(&cache, &spec.slug, &wm).unwrap();+ let f2 = fetch(&spec, tmp.path(), &cache).unwrap();+ assert!(f2.unchanged, "same HEAD ⇒ skip the walk");+ assert_eq!(f2.watermark.as_deref(), Some(wm.as_str()));++ // a new commit upstream moves the watermark+ std::fs::write(origin.join("two.json"), r#"{"id":"R2"}"#).unwrap();+ crate::xc::git(Some(&origin), &["add", "."]).ok();+ git_user(&[+ "-c",+ "user.email=t@t",+ "-c",+ "user.name=t",+ "commit",+ "--quiet",+ "-m",+ "two",+ ])+ .ok();+ let f3 = fetch(&spec, tmp.path(), &cache).unwrap();+ assert!(!f3.unchanged);+ assert_ne!(f3.watermark.as_deref(), Some(wm.as_str()));+ assert!(+ f3.root.join("two.json").exists(),+ "checkout moved to the new tip"+ );+ }+}diff --git a/engine/crates/xerj-autoindex/src/xc.rs b/engine/crates/xerj-autoindex/src/xc.rsindex 17be15f4b..b1ebe6697 100644--- a/engine/crates/xerj-autoindex/src/xc.rs+++ b/engine/crates/xerj-autoindex/src/xc.rs@@ -236,7 +236,7 @@ deterministic pack under builds/<name>/ (see tools/xerj-code/);index builds/verifies/switches the corpus (exit 3 skips junk — normal);list shows what is loaded on this node.";-fn git(dir: Option<&Path>, args: &[&str]) -> Result<(i32, String)> {+pub(crate) fn git(dir: Option<&Path>, args: &[&str]) -> Result<(i32, String)> {let mut c = Command::new("git");if let Some(d) = dir {c.current_dir(d).arg("-C").arg(d);@@ -254,7 +254,7 @@ fn git(dir: Option<&Path>, args: &[&str]) -> Result<(i32, String)> {/// Move a clone to `sha` (shallow). Never a shell string: the sha comes from/// an untrusted manifest, and `Command` passes it as ONE argv element.-fn checkout_at_sha(target: &Path, url: &str, sha: &str) -> Result<()> {+pub(crate) fn checkout_at_sha(target: &Path, url: &str, sha: &str) -> Result<()> {if !target.join(".git").exists() {std::fs::create_dir_all(target)?;git(Some(target), &["init", "--quiet"])?;