Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

Merge pull request #22 from probelabs/fix/autoindex-bulk-timeout

xerj-org122bd0ca86a5
diff --cc engine/crates/xerj-autoindex/src/cli.rs
index 13f902386,b3c915f88..09436f00c
--- a/engine/crates/xerj-autoindex/src/cli.rs
+++ b/engine/crates/xerj-autoindex/src/cli.rs
@@@ -8,9 -8,8 +8,10 @@@ pub struct IndexCfg
pub url: String,
pub api_key: Option<String>,
pub workers: usize,
+ pub pdf_workers: usize,
+ pub pdf_timeout_secs: u64,
pub bulk_mb: usize,
+ pub bulk_timeout_secs: u64,
pub prefix: String,
pub state_dir: Option<PathBuf>,
pub fresh: bool,
@@@ -61,9 -60,10 +62,12 @@@ pub fn print_help()
--url <U> ES-compat endpoint (default http://localhost:9200)\n\
--api-key <K> Authorization header (or env XERJ_API_KEY)\n\
--workers <N> extract workers (default min(cores,8))\n\
+ --pdf-workers <N> concurrent PDF parser processes (default min(cores,4); max 4)\n\
+ --pdf-timeout-secs <N> per-PDF parser timeout (default 120; max 3600)\n\
--bulk-mb <N> bulk cut size in MB (default 8)\n\
+ --bulk-timeout-secs <N>\n\
+ bulk HTTP request timeout in seconds (default 300;\n\
+ valid range 1..=3600)\n\
--prefix <P> index prefix (default ax)\n\
--state-dir <PATH> resume journal location (default ~/.xerj/autoindex/<hash>/)\n\
--fresh ignore existing journal, restart (ids stay idempotent)\n\
@@@ -113,12 -102,9 +117,14 @@@ pub fn parse(args: Vec<String>) -> Resu
.map(|n| n.get())
.unwrap_or(8)
.min(8);
+ let mut pdf_workers = std::thread::available_parallelism()
+ .map(|n| n.get())
+ .unwrap_or(4)
+ .min(4);
+ let mut pdf_timeout_secs = 120u64;
let mut bulk_mb = 8usize;
+ let mut bulk_timeout_secs = 300u64;
+ let mut bulk_timeout_explicit = false;
let mut prefix = "ax".to_string();
let mut state_dir: Option<PathBuf> = None;
let mut fresh = false;
@@@ -220,9 -209,8 +243,10 @@@
url,
api_key,
workers: workers.max(1),
+ pdf_workers,
+ pdf_timeout_secs,
bulk_mb: bulk_mb.clamp(1, 24),
+ bulk_timeout_secs,
prefix,
state_dir,
fresh,
diff --cc engine/crates/xerj-autoindex/src/lib.rs
index 9e1103ce4,39e487a16..d2cc3a1a0
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@@ -189,10 -189,14 +189,16 @@@ fn build_mapping(specs: &[infer::FieldS
// ─── the main run ────────────────────────────────────────────────────────
fn run_index(cfg: IndexCfg) -> Result<i32> {
+ extract::pdf::configure_workers(cfg.pdf_workers);
+ extract::pdf::configure_timeout(cfg.pdf_timeout_secs);
let t0 = Instant::now();
- let es = Es::new(&cfg.url, cfg.api_key.clone())?;
+ if !cfg.quiet {
+ eprintln!(
+ "autoindex: bulk HTTP request timeout: {}s",
+ cfg.bulk_timeout_secs
+ );
+ }
+ let es = Es::with_bulk_timeout(&cfg.url, cfg.api_key.clone(), cfg.bulk_timeout_secs)?;
es.ping()?;
let files = walk::walk(&cfg.root, cfg.follow_symlinks)?;