AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
test(autoindex): pin where the resource policy and the progress stream meet
xerj-orgfd34ca2ca6a5
diff --git a/engine/crates/xerj-autoindex/src/content.rs b/engine/crates/xerj-autoindex/src/content.rsindex f934ca238..581e423b7 100644--- a/engine/crates/xerj-autoindex/src/content.rs+++ b/engine/crates/xerj-autoindex/src/content.rs@@ -47,8 +47,8 @@ pub fn resolve_reporting() -> Result<Inventory> {// Phase A belongs to the run's scan pool, not to rayon's global pool:// `--workers` has to bound the CPU-bound phase to mean anything (#240 §2).- // The progress callback therefore fires from scan-pool threads, which is- // also the pool whose width the ETA is computed against (#241).+ // The progress callback therefore fires from scan-pool threads — the ones+ // actually doing the hashing — rather than from rayon's global pool (#241).let digests: Vec<Result<String>> = crate::pool::install(|| {files.par_iter()diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rsindex addc4e737..b00e26f8e 100644--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs@@ -1213,3 +1213,85 @@ fn quiet_runs_emit_no_progress_stream_at_all() {String::from_utf8_lossy(&buffer.lock().unwrap()));}++/// Where #240 (the resource policy) and #241 (the progress stream) meet.+///+/// Two rules have to hold at once, and each is easy to break while honouring+/// the other:+///+/// * The resource decision is **announced on the progress surface**, not on a+/// bare `eprintln!` gated on `!quiet`. A raw write would survive+/// `--progress none` — which promises silence — and would inject a+/// non-JSON line into the middle of `--progress json`, which promises one+/// parseable stream. (`quiet_runs_emit_no_progress_stream_at_all` above is+/// the other side of this assertion: it fails if any of these lines escape+/// the surface.)+/// * The numbers announced are the ones the run **got**, not the ones it+/// asked for. `pool::configure` is first-call-wins, because rayon pools+/// cannot be resized, so a process that already indexed once — `xerj+/// brain` does exactly that — keeps its original phase-A width. Reporting+/// the request there would make the stream and the run summary a record of+/// an intention rather than of a run.+///+/// The second rule is made observable by asking for a width this process+/// provably cannot grant: the pool is built before the run, then one more+/// thread than it has is requested.+#[test]+fn the_resource_plan_is_announced_on_the_progress_surface_with_the_width_it_got() {+ let _guard = FAILPOINT_TEST_LOCK.lock().unwrap();+ let _sink_guard = crate::progress::SINK_TEST_LOCK.lock().unwrap();+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(+ corpus.path().join("rows.csv"),+ "id,value\n0,alpha\n1,beta\n",+ )+ .unwrap();++ // Build the process-global scan pool first, so the width below is one this+ // run cannot possibly be granted.+ let installed = crate::pool::scan_pool().current_num_threads();+ let endpoint = MockEndpoint::start(usize::MAX);+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ config.quiet = false;+ config.progress = crate::progress::ProgressMode::Plain;+ config.progress_interval = Some(std::time::Duration::from_secs(1));+ config.scan_workers = installed + 1;+ config.workers = 3;+ config.resource_notes = vec!["memory safe zone 256 MiB allows 3 index workers".to_string()];++ let buffer = Arc::new(Mutex::new(Vec::new()));+ let (_code, report) = {+ let _sink = crate::progress::install_test_sink(&buffer);+ run_index_report(config).unwrap()+ };+ let stream = String::from_utf8(buffer.lock().unwrap().clone()).unwrap();++ assert!(+ stream.contains(&format!("{installed} scan threads")),+ "the run must announce the phase-A width it actually got, not {}: {stream}",+ installed + 1+ );+ assert!(+ !stream.contains(&format!("{} scan threads", installed + 1)),+ "a width the pool refused must never be reported as fact: {stream}"+ );+ assert!(+ stream.contains("3 index workers"),+ "phase B must name the count the policy chose: {stream}"+ );+ assert!(+ stream.contains("memory safe zone 256 MiB allows 3 index workers"),+ "what the machine forced on the run belongs in the same stream: {stream}"+ );+ assert!(+ stream.contains(&format!("with {installed} threads")),+ "the phase-A banner must carry the real width too: {stream}"+ );++ // The summary is read after the fact by agents and by `xerj brain`; it has+ // to agree with the stream rather than repeat the request.+ let report = report.expect("a completed run produces a summary");+ assert_eq!(report["scan_workers"], serde_json::json!(installed));+ assert_eq!(report["workers"], serde_json::json!(3));+}