AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
Merge pull request #22 from probelabs/fix/autoindex-bulk-timeout
xerj-org122bd0ca86a5
diff --cc engine/crates/xerj-autoindex/src/cli.rsindex 13f902386,b3c915f88..09436f00c--- a/engine/crates/xerj-autoindex/src/cli.rs+++ b/engine/crates/xerj-autoindex/src/cli.rs@@@ -8,9 -8,8 +8,10 @@@ pub struct IndexCfgpub url: String,pub api_key: Option<String>,pub workers: usize,+ pub pdf_workers: usize,+ pub pdf_timeout_secs: u64,pub bulk_mb: usize,+ pub bulk_timeout_secs: u64,pub prefix: String,pub state_dir: Option<PathBuf>,pub fresh: bool,@@@ -61,9 -60,10 +62,12 @@@ pub fn print_help()--url <U> ES-compat endpoint (default http://localhost:9200)\n\--api-key <K> Authorization header (or env XERJ_API_KEY)\n\--workers <N> extract workers (default min(cores,8))\n\+ --pdf-workers <N> concurrent PDF parser processes (default min(cores,4); max 4)\n\+ --pdf-timeout-secs <N> per-PDF parser timeout (default 120; max 3600)\n\--bulk-mb <N> bulk cut size in MB (default 8)\n\+ --bulk-timeout-secs <N>\n\+ bulk HTTP request timeout in seconds (default 300;\n\+ valid range 1..=3600)\n\--prefix <P> index prefix (default ax)\n\--state-dir <PATH> resume journal location (default ~/.xerj/autoindex/<hash>/)\n\--fresh ignore existing journal, restart (ids stay idempotent)\n\@@@ -113,12 -102,9 +117,14 @@@ pub fn parse(args: Vec<String>) -> Resu.map(|n| n.get()).unwrap_or(8).min(8);+ let mut pdf_workers = std::thread::available_parallelism()+ .map(|n| n.get())+ .unwrap_or(4)+ .min(4);+ let mut pdf_timeout_secs = 120u64;let mut bulk_mb = 8usize;+ let mut bulk_timeout_secs = 300u64;+ let mut bulk_timeout_explicit = false;let mut prefix = "ax".to_string();let mut state_dir: Option<PathBuf> = None;let mut fresh = false;@@@ -220,9 -209,8 +243,10 @@@url,api_key,workers: workers.max(1),+ pdf_workers,+ pdf_timeout_secs,bulk_mb: bulk_mb.clamp(1, 24),+ bulk_timeout_secs,prefix,state_dir,fresh,diff --cc engine/crates/xerj-autoindex/src/lib.rsindex 9e1103ce4,39e487a16..d2cc3a1a0--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@@ -189,10 -189,14 +189,16 @@@ fn build_mapping(specs: &[infer::FieldS// ─── the main run ────────────────────────────────────────────────────────fn run_index(cfg: IndexCfg) -> Result<i32> {+ extract::pdf::configure_workers(cfg.pdf_workers);+ extract::pdf::configure_timeout(cfg.pdf_timeout_secs);let t0 = Instant::now();- let es = Es::new(&cfg.url, cfg.api_key.clone())?;+ if !cfg.quiet {+ eprintln!(+ "autoindex: bulk HTTP request timeout: {}s",+ cfg.bulk_timeout_secs+ );+ }+ let es = Es::with_bulk_timeout(&cfg.url, cfg.api_key.clone(), cfg.bulk_timeout_secs)?;es.ping()?;let files = walk::walk(&cfg.root, cfg.follow_symlinks)?;