AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): raise RLIMIT_NOFILE + tree-sitter AST code extraction (#82)
xerj-orge1b9e1a443c5
diff --git a/engine/Cargo.lock b/engine/Cargo.lockindex 0c59c2e74..02e9445f2 100644--- a/engine/Cargo.lock+++ b/engine/Cargo.lock@@ -4656,6 +4656,12 @@ version = "1.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"+[[package]]+name = "streaming-iterator"+version = "0.1.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520"+[[package]]name = "stringprep"version = "0.1.5"@@ -5317,6 +5323,146 @@ dependencies = ["tracing-serde",]+[[package]]+name = "tree-sitter"+version = "0.25.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "78f873475d258561b06f1c595d93308a7ed124d9977cb26b148c2084a4a3cc87"+dependencies = [+ "cc",+ "regex",+ "regex-syntax",+ "serde_json",+ "streaming-iterator",+ "tree-sitter-language",+]++[[package]]+name = "tree-sitter-bash"+version = "0.23.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "329a4d48623ac337d42b1df84e81a1c9dbb2946907c102ca72db158c1964a52e"+dependencies = [+ "cc",+ "tree-sitter-language",+]++[[package]]+name = "tree-sitter-c"+version = "0.23.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "afd2b1bf1585dc2ef6d69e87d01db8adb059006649dd5f96f31aa789ee6e9c71"+dependencies = [+ "cc",+ "tree-sitter-language",+]++[[package]]+name = "tree-sitter-c-sharp"+version = "0.23.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c1aac67f1ad71de1d6d39708d34811081c26dfa495658de6c14c34200849357c"+dependencies = [+ "cc",+ "tree-sitter-language",+]++[[package]]+name = "tree-sitter-cpp"+version = "0.23.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "df2196ea9d47b4ab4a31b9297eaa5a5d19a0b121dceb9f118f6790ad0ab94743"+dependencies = [+ "cc",+ "tree-sitter-language",+]++[[package]]+name = "tree-sitter-go"+version = "0.23.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b13d476345220dbe600147dd444165c5791bf85ef53e28acbedd46112ee18431"+dependencies = [+ "cc",+ "tree-sitter-language",+]++[[package]]+name = "tree-sitter-java"+version = "0.23.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0aa6cbcdc8c679b214e616fd3300da67da0e492e066df01bcf5a5921a71e90d6"+dependencies = [+ "cc",+ "tree-sitter-language",+]++[[package]]+name = "tree-sitter-javascript"+version = "0.23.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bf40bf599e0416c16c125c3cec10ee5ddc7d1bb8b0c60fa5c4de249ad34dc1b1"+dependencies = [+ "cc",+ "tree-sitter-language",+]++[[package]]+name = "tree-sitter-language"+version = "0.1.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782"++[[package]]+name = "tree-sitter-php"+version = "0.23.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f066e94e9272cfe4f1dcb07a1c50c66097eca648f2d7233d299c8ae9ed8c130c"+dependencies = [+ "cc",+ "tree-sitter-language",+]++[[package]]+name = "tree-sitter-python"+version = "0.23.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3d065aaa27f3aaceaf60c1f0e0ac09e1cb9eb8ed28e7bcdaa52129cffc7f4b04"+dependencies = [+ "cc",+ "tree-sitter-language",+]++[[package]]+name = "tree-sitter-ruby"+version = "0.23.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "be0484ea4ef6bb9c575b4fdabde7e31340a8d2dbc7d52b321ac83da703249f95"+dependencies = [+ "cc",+ "tree-sitter-language",+]++[[package]]+name = "tree-sitter-rust"+version = "0.23.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ca8ccb3e3a3495c8a943f6c3fd24c3804c471fd7f4f16087623c7fa4c0068e8a"+dependencies = [+ "cc",+ "tree-sitter-language",+]++[[package]]+name = "tree-sitter-typescript"+version = "0.23.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff"+dependencies = [+ "cc",+ "tree-sitter-language",+]+[[package]]name = "try-lock"version = "0.2.5"@@ -6273,7 +6419,21 @@ dependencies = ["serde","serde_json","serde_yaml",+ "streaming-iterator","tempfile",+ "tree-sitter",+ "tree-sitter-bash",+ "tree-sitter-c",+ "tree-sitter-c-sharp",+ "tree-sitter-cpp",+ "tree-sitter-go",+ "tree-sitter-java",+ "tree-sitter-javascript",+ "tree-sitter-php",+ "tree-sitter-python",+ "tree-sitter-ruby",+ "tree-sitter-rust",+ "tree-sitter-typescript","walkdir","xxhash-rust","zip 2.4.2",diff --git a/engine/crates/xerj-autoindex/Cargo.toml b/engine/crates/xerj-autoindex/Cargo.tomlindex d9d45ad49..55283c9e9 100644--- a/engine/crates/xerj-autoindex/Cargo.toml+++ b/engine/crates/xerj-autoindex/Cargo.toml@@ -24,6 +24,24 @@ fs2 = "0.4"# (std threads), invoked from the server binary via spawn_blocking.reqwest = { workspace = true, features = ["blocking"] }+# --- AST code extraction (tree-sitter). Grammars expose a tree-sitter-language+# LanguageFn, decoupled from the core version, so one core serves all. ---+tree-sitter = "0.25"+tree-sitter-python = "0.23"+tree-sitter-c = "0.23"+tree-sitter-rust = "0.23"+tree-sitter-javascript = "0.23"+tree-sitter-go = "0.23"+tree-sitter-typescript = "0.23"+tree-sitter-java = "0.23"+tree-sitter-cpp = "0.23"+tree-sitter-ruby = "0.23"+tree-sitter-php = "0.23"+tree-sitter-c-sharp = "0.23"+tree-sitter-bash = "0.23"+streaming-iterator = "0.1"++# Format-family extractors (new deps, isolated to this crate):flate2 = "1" # gzip streaming (rust backend)pdf_oxide = { version = "=0.3.75", default-features = false, features = ["legacy-crypto"] }diff --git a/engine/crates/xerj-autoindex/src/extract/code.rs b/engine/crates/xerj-autoindex/src/extract/code.rsnew file mode 100644index 000000000..44dc539e5--- /dev/null+++ b/engine/crates/xerj-autoindex/src/extract/code.rs@@ -0,0 +1,447 @@+//! Source code — AST-aware extraction via tree-sitter.+//!+//! Source files used to fall through to the plain-text extractor, so a repo was+//! searchable only as prose. This parses each file with the matching tree-sitter+//! grammar and captures its DEFINITIONS (functions, classes, methods, structs,+//! traits, interfaces, modules, …). Each file becomes one document carrying:+//! - `language` the detected language+//! - `symbols` a structured array of {name, kind, line}+//! - `defs` "kind name" per symbol, newline-joined so BM25 matches a query+//! like `class User` or `def save` to the file that owns it+//! - `body` the full source text (still full-text searchable)+//!+//! The capture-name in each query IS the symbol kind (`@function`, `@class`, …),+//! so adding a language is a grammar dep + one registry row. If a grammar fails+//! to parse a file, it is indexed as plain text rather than dropped.++use super::{ExtractStats, RawRecord, Sink};+use anyhow::Result;+use serde_json::{Map, Value};+use std::path::Path;+use std::sync::OnceLock;+use streaming_iterator::StreamingIterator;+use tree_sitter::{Language, Parser, Query, QueryCursor};++/// Skip machine-generated giants (minified bundles, generated parsers) — past a+/// couple MB a single "file" is not human code and only bloats the index.+const CODE_CAP: u64 = 2 << 20;++struct LangDef {+ name: &'static str,+ exts: &'static [&'static str],+ language: Language,+ query: Query,+}++/// Is this extension a language we AST-parse? Cheap check used by the sniffer.+pub fn is_code_ext(ext: &str) -> bool {+ let e = ext.to_ascii_lowercase();+ registry().iter().any(|d| d.exts.contains(&e.as_str()))+}++fn def(name: &'static str, exts: &'static [&'static str], lang: Language, q: &str) -> LangDef {+ let query = Query::new(&lang, q).unwrap_or_else(|e| panic!("bad {name} query: {e}"));+ LangDef {+ name,+ exts,+ language: lang,+ query,+ }+}++fn registry() -> &'static [LangDef] {+ static REG: OnceLock<Vec<LangDef>> = OnceLock::new();+ REG.get_or_init(|| {+ vec![+ def(+ "python",+ &["py", "pyi"],+ tree_sitter_python::LANGUAGE.into(),+ PYTHON_Q,+ ),+ def(+ "javascript",+ &["js", "jsx", "mjs", "cjs"],+ tree_sitter_javascript::LANGUAGE.into(),+ JS_Q,+ ),+ def(+ "typescript",+ &["ts"],+ tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),+ TS_Q,+ ),+ def(+ "tsx",+ &["tsx"],+ tree_sitter_typescript::LANGUAGE_TSX.into(),+ TS_Q,+ ),+ def("rust", &["rs"], tree_sitter_rust::LANGUAGE.into(), RUST_Q),+ def("go", &["go"], tree_sitter_go::LANGUAGE.into(), GO_Q),+ def("java", &["java"], tree_sitter_java::LANGUAGE.into(), JAVA_Q),+ def("c", &["c", "h"], tree_sitter_c::LANGUAGE.into(), C_Q),+ def(+ "cpp",+ &["cpp", "cc", "cxx", "hpp", "hh", "hxx"],+ tree_sitter_cpp::LANGUAGE.into(),+ CPP_Q,+ ),+ def("ruby", &["rb"], tree_sitter_ruby::LANGUAGE.into(), RUBY_Q),+ def("php", &["php"], tree_sitter_php::LANGUAGE_PHP.into(), PHP_Q),+ def(+ "csharp",+ &["cs"],+ tree_sitter_c_sharp::LANGUAGE.into(),+ CSHARP_Q,+ ),+ def(+ "bash",+ &["sh", "bash"],+ tree_sitter_bash::LANGUAGE.into(),+ BASH_Q,+ ),+ ]+ })+}++pub fn extract(path: &Path, sink: Sink) -> Result<ExtractStats> {+ let mut stats = ExtractStats::default();+ let ext = path+ .extension()+ .and_then(|e| e.to_str())+ .unwrap_or("")+ .to_ascii_lowercase();+ let Some(def) = registry().iter().find(|d| d.exts.contains(&ext.as_str())) else {+ stats.junk += 1;+ return Ok(stats);+ };+ let Some(bytes) = super::read_whole(path, false, CODE_CAP)? else {+ stats.junk += 1;+ return Ok(stats);+ };+ let (text, _) = crate::sniff::decode_text(&bytes);++ // Parse + capture definitions. A parse failure (or an over-deep tree) is not+ // fatal — index the file as plain text so its content is still searchable.+ let symbols = parse_symbols(def, &text).unwrap_or_default();+ emit_code_doc(path, def.name, &text, &symbols, &mut stats, sink);+ Ok(stats)+}++/// (name, kind, 1-based line)+type Symbol = (String, String, usize);++fn parse_symbols(def: &LangDef, text: &str) -> Option<Vec<Symbol>> {+ let mut parser = Parser::new();+ parser.set_language(&def.language).ok()?;+ let tree = parser.parse(text.as_bytes(), None)?;+ let names = def.query.capture_names();+ let mut out: Vec<Symbol> = Vec::new();+ let mut cursor = QueryCursor::new();+ let mut it = cursor.matches(&def.query, tree.root_node(), text.as_bytes());+ while let Some(m) = it.next() {+ for cap in m.captures {+ let kind = names[cap.index as usize]; // capture name == symbol kind+ let node = cap.node;+ let name = text.get(node.byte_range()).unwrap_or("").trim();+ if name.is_empty() || name.len() > 200 {+ continue;+ }+ out.push((+ name.to_string(),+ kind.to_string(),+ node.start_position().row + 1,+ ));+ if out.len() >= 5000 {+ return Some(out); // pathological generated file — enough+ }+ }+ }+ Some(out)+}++fn emit_code_doc(+ path: &Path,+ language: &str,+ text: &str,+ symbols: &[Symbol],+ stats: &mut ExtractStats,+ sink: Sink,+) -> bool {+ let title = path+ .file_name()+ .and_then(|n| n.to_str())+ .unwrap_or("source")+ .to_string();++ let mut fields = Map::new();+ fields.insert("title".into(), Value::String(title));+ fields.insert("language".into(), Value::String(language.to_string()));++ if !symbols.is_empty() {+ // Searchable "kind name" list — this is what makes `class User` /+ // `def save` retrieve the file, and it dedups repeated names.+ let mut seen = std::collections::HashSet::new();+ let defs: Vec<String> = symbols+ .iter()+ .filter(|(n, k, _)| seen.insert((k.clone(), n.clone())))+ .map(|(n, k, _)| format!("{k} {n}"))+ .collect();+ fields.insert("defs".into(), Value::String(defs.join("\n")));+ let arr: Vec<Value> = symbols+ .iter()+ .map(|(n, k, line)| {+ let mut m = Map::new();+ m.insert("name".into(), Value::String(n.clone()));+ m.insert("kind".into(), Value::String(k.clone()));+ m.insert("line".into(), Value::Number((*line as u64).into()));+ Value::Object(m)+ })+ .collect();+ fields.insert("symbols".into(), Value::Array(arr));+ fields.insert(+ "symbol_count".into(),+ Value::Number((symbols.len() as u64).into()),+ );+ }+ // The full source stays full-text searchable.+ fields.insert("body".into(), Value::String(text.to_string()));++ stats.records += 1;+ sink(RawRecord {+ fields,+ locator: "code".into(),+ group: None,+ })+}++// ── Per-language capture queries. Capture-name == emitted symbol kind. ─────────++const PYTHON_Q: &str = r#"+(function_definition name: (identifier) @function)+(class_definition name: (identifier) @class)+"#;++const JS_Q: &str = r#"+(function_declaration name: (identifier) @function)+(generator_function_declaration name: (identifier) @function)+(class_declaration name: (identifier) @class)+(method_definition name: (property_identifier) @method)+(variable_declarator name: (identifier) @function value: [(arrow_function) (function_expression)])+(export_statement declaration: (lexical_declaration (variable_declarator name: (identifier) @function value: (arrow_function))))+"#;++const TS_Q: &str = r#"+(function_declaration name: (identifier) @function)+(class_declaration name: (type_identifier) @class)+(method_definition name: (property_identifier) @method)+(interface_declaration name: (type_identifier) @interface)+(type_alias_declaration name: (type_identifier) @type)+(enum_declaration name: (identifier) @enum)+(variable_declarator name: (identifier) @function value: (arrow_function))+"#;++const RUST_Q: &str = r#"+(function_item name: (identifier) @function)+(struct_item name: (type_identifier) @struct)+(enum_item name: (type_identifier) @enum)+(trait_item name: (type_identifier) @trait)+(mod_item name: (identifier) @module)+(macro_definition name: (identifier) @macro)+(type_item name: (type_identifier) @type)+"#;++const GO_Q: &str = r#"+(function_declaration name: (identifier) @function)+(method_declaration name: (field_identifier) @method)+(type_declaration (type_spec name: (type_identifier) @type))+"#;++const JAVA_Q: &str = r#"+(class_declaration name: (identifier) @class)+(interface_declaration name: (identifier) @interface)+(enum_declaration name: (identifier) @enum)+(method_declaration name: (identifier) @method)+(constructor_declaration name: (identifier) @method)+"#;++const C_Q: &str = r#"+(function_definition declarator: (function_declarator declarator: (identifier) @function))+(struct_specifier name: (type_identifier) @struct)+(enum_specifier name: (type_identifier) @enum)+(type_definition declarator: (type_identifier) @type)+"#;++const CPP_Q: &str = r#"+(function_definition declarator: (function_declarator declarator: (identifier) @function))+(function_definition declarator: (function_declarator declarator: (field_identifier) @method))+(class_specifier name: (type_identifier) @class)+(struct_specifier name: (type_identifier) @struct)+(enum_specifier name: (type_identifier) @enum)+(namespace_definition name: (namespace_identifier) @module)+"#;++const RUBY_Q: &str = r#"+(method name: (identifier) @method)+(singleton_method name: (identifier) @method)+(class name: (constant) @class)+(module name: (constant) @module)+"#;++const PHP_Q: &str = r#"+(function_definition name: (name) @function)+(method_declaration name: (name) @method)+(class_declaration name: (name) @class)+(interface_declaration name: (name) @interface)+(trait_declaration name: (name) @trait)+"#;++const CSHARP_Q: &str = r#"+(class_declaration name: (identifier) @class)+(interface_declaration name: (identifier) @interface)+(struct_declaration name: (identifier) @struct)+(enum_declaration name: (identifier) @enum)+(method_declaration name: (identifier) @method)+(constructor_declaration name: (identifier) @method)+"#;++const BASH_Q: &str = r#"+(function_definition name: (word) @function)+"#;++#[cfg(test)]+mod tests {+ use super::*;++ fn syms(lang: &str, src: &str) -> Vec<Symbol> {+ let def = registry().iter().find(|d| d.name == lang).unwrap();+ parse_symbols(def, src).unwrap()+ }+ fn has(s: &[Symbol], name: &str, kind: &str) -> bool {+ s.iter().any(|(n, k, _)| n == name && k == kind)+ }++ #[test]+ fn all_queries_compile() {+ // registry() builds every Query; a malformed query panics here.+ assert!(registry().len() >= 13);+ }++ #[test]+ fn python() {+ let s = syms(+ "python",+ "def foo():\n pass\nclass Bar:\n def baz(self):\n pass\n",+ );+ assert!(has(&s, "foo", "function"));+ assert!(has(&s, "Bar", "class"));+ assert!(has(&s, "baz", "function"));+ }+ #[test]+ fn javascript() {+ let s = syms(+ "javascript",+ "function f(){}\nclass C{ m(){} }\nconst g = () => 1;\n",+ );+ assert!(has(&s, "f", "function"));+ assert!(has(&s, "C", "class"));+ assert!(has(&s, "m", "method"));+ assert!(has(&s, "g", "function"));+ }+ #[test]+ fn typescript() {+ let s = syms(+ "typescript",+ "interface I {}\ntype T = number;\nclass C { m(): void {} }\nfunction f(): void {}\n",+ );+ assert!(has(&s, "I", "interface"));+ assert!(has(&s, "T", "type"));+ assert!(has(&s, "C", "class"));+ assert!(has(&s, "m", "method"));+ assert!(has(&s, "f", "function"));+ }+ #[test]+ fn rust() {+ let s = syms(+ "rust",+ "fn f(){}\nstruct S;\nenum E{A}\ntrait T{}\nmod m{}\n",+ );+ assert!(has(&s, "f", "function"));+ assert!(has(&s, "S", "struct"));+ assert!(has(&s, "E", "enum"));+ assert!(has(&s, "T", "trait"));+ assert!(has(&s, "m", "module"));+ }+ #[test]+ fn go() {+ let s = syms(+ "go",+ "package p\nfunc F(){}\nfunc (r R) M(){}\ntype T struct{}\n",+ );+ assert!(has(&s, "F", "function"));+ assert!(has(&s, "M", "method"));+ assert!(has(&s, "T", "type"));+ }+ #[test]+ fn java() {+ let s = syms("java", "class C { void m(){} interface I {} }\n");+ assert!(has(&s, "C", "class"));+ assert!(has(&s, "m", "method"));+ assert!(has(&s, "I", "interface"));+ }+ #[test]+ fn c() {+ let s = syms(+ "c",+ "int f(int x){return x;}\nstruct S{int a;};\nenum E{A};\n",+ );+ assert!(has(&s, "f", "function"));+ assert!(has(&s, "S", "struct"));+ assert!(has(&s, "E", "enum"));+ }+ #[test]+ fn cpp() {+ let s = syms(+ "cpp",+ "class C { void m(){} };\nint f(){return 0;}\nnamespace n {}\n",+ );+ assert!(has(&s, "C", "class"));+ assert!(has(&s, "f", "function"));+ assert!(has(&s, "n", "module"));+ }+ #[test]+ fn ruby() {+ let s = syms("ruby", "class C\n def m\n end\nend\nmodule M\nend\n");+ assert!(has(&s, "C", "class"));+ assert!(has(&s, "m", "method"));+ assert!(has(&s, "M", "module"));+ }+ #[test]+ fn php() {+ let s = syms(+ "php",+ "<?php\nfunction f(){}\nclass C { function m(){} }\ninterface I {}\n",+ );+ assert!(has(&s, "f", "function"));+ assert!(has(&s, "C", "class"));+ assert!(has(&s, "m", "method"));+ assert!(has(&s, "I", "interface"));+ }+ #[test]+ fn csharp() {+ let s = syms(+ "csharp",+ "class C { void M(){} }\ninterface I {}\nenum E {A}\n",+ );+ assert!(has(&s, "C", "class"));+ assert!(has(&s, "M", "method"));+ assert!(has(&s, "I", "interface"));+ assert!(has(&s, "E", "enum"));+ }+ #[test]+ fn bash() {+ let s = syms("bash", "foo() { echo hi; }\nfunction bar { echo yo; }\n");+ assert!(has(&s, "foo", "function"));+ }+}diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rsindex 7c79edea4..3f207fb0d 100644--- a/engine/crates/xerj-autoindex/src/extract/mod.rs+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs@@ -2,6 +2,7 @@//! Every extractor is bounded-memory and never fatal: parse failures//! downgrade (family → txt → junk-with-metadata) and are counted.+pub mod code;pub mod csv_x;pub mod docx;pub mod html;@@ -110,6 +111,7 @@ pub fn extract(Family::Docx => docx::extract(path, sink),Family::Sqlite => sqlite_x::extract(path, limit_bytes.map(|_| 500), sink),Family::SqlDump => sqldump::extract(path, sn.gzip, limit_bytes, sink),+ Family::Code => code::extract(path, sink),Family::Binary => Ok(ExtractStats::default()),}}diff --git a/engine/crates/xerj-autoindex/src/sniff.rs b/engine/crates/xerj-autoindex/src/sniff.rsindex b43438732..a0d8c5485 100644--- a/engine/crates/xerj-autoindex/src/sniff.rs+++ b/engine/crates/xerj-autoindex/src/sniff.rs@@ -22,6 +22,8 @@ pub enum Family {Docx,Sqlite,SqlDump,+ /// Source code — AST-parsed by the matching tree-sitter grammar.+ Code,Binary,}@@ -41,6 +43,7 @@ impl Family {Family::Docx => "docx",Family::Sqlite => "sqlite",Family::SqlDump => "sqldump",+ Family::Code => "code",Family::Binary => "binary",}}@@ -165,6 +168,18 @@ fn sniff_bytes(prefix: &[u8], path: &Path, gzip: bool) -> Result<Sniffed> {return Ok(s);}+ // 2b. Source code: a known code extension whose content is text. We only+ // reach here after the binary guards above, so a text `.py`/`.rs`/`.go`/…+ // routes to the tree-sitter AST extractor (crate::extract::code). Extension+ // is the right signal — code vs prose is not reliably content-sniffable.+ if let Some(ext) = path.extension().and_then(|e| e.to_str()) {+ if crate::extract::code::is_code_ext(ext) {+ let mut s = mk(Family::Code);+ s.encoding = encoding;+ return Ok(s);+ }+ }+// 3. Text heuristics — complete lines only (last line may be truncated).let mut lines: Vec<&str> = text.lines().collect();if !text.ends_with('\n') && lines.len() > 1 {diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rsindex 831409bed..e624d559c 100644--- a/engine/crates/xerj-server/src/main.rs+++ b/engine/crates/xerj-server/src/main.rs@@ -44,11 +44,29 @@ static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;// 1 s dirty/muzzy decay purges freed pages continuously; the same load then// holds a flat working-set RSS. Ingest throughput cost measured: none// (within noise, ~36k docs/s single-index bulk either way).-#[cfg(all(not(target_env = "msvc"), not(feature = "debug-profiling")))]+#[cfg(all(+ not(target_env = "msvc"),+ not(feature = "debug-profiling"),+ target_os = "linux"+))]#[allow(non_upper_case_globals)]#[export_name = "_rjem_malloc_conf"]pub static malloc_conf: &[u8] = b"background_thread:true,dirty_decay_ms:1000,muzzy_decay_ms:1000\0";+// macOS (and other non-Linux unix) jemalloc has no pthread background_thread+// support, so `background_thread:true` only prints `<jemalloc>: option+// background_thread currently supports pthread only` on every launch (reported+// by users running `xerj autoindex` on macOS). Keep the aggressive decay policy+// — it still purges freed pages on allocator ticks — without the unsupported knob.+#[cfg(all(+ not(target_env = "msvc"),+ not(feature = "debug-profiling"),+ not(target_os = "linux")+))]+#[allow(non_upper_case_globals)]+#[export_name = "_rjem_malloc_conf"]+pub static malloc_conf: &[u8] = b"dirty_decay_ms:1000,muzzy_decay_ms:1000\0";+// Preserve the production decay policy. Merely compiling this feature enables// jemalloc's profiler; sampling remains inactive until the local controller// starts a bounded capture.@@ -1256,7 +1274,57 @@ async fn run_cli_index(cmd: IndexCmdArgs) -> Result<()> {////// Override with `TOKIO_WORKER_THREADS` (operators on very small or very large/// hosts, or wanting the stock `ncpus` behaviour, can set it explicitly).+/// Raise the process open-file limit toward the OS hard limit before anything+/// else runs. Every index holds segment mmaps, a WAL and a merge task, so+/// discovering a large source tree (Django infers ~73 datasets → ~73 indices)+/// opens thousands of descriptors. On a default macOS soft limit (often 256)+/// the server exhausts them mid-run and index creation fails with+/// `Too many open files` (EMFILE, os error 24) — reproduced against a real+/// `django` clone. Bump the soft limit to the hard limit, stepping down if the+/// kernel (`kern.maxfilesperproc` on macOS) rejects the target. Best-effort:+/// any failure here is non-fatal (the server just runs with the inherited cap).+#[cfg(unix)]+fn raise_nofile_limit() {+ unsafe {+ let mut lim = libc::rlimit {+ rlim_cur: 0,+ rlim_max: 0,+ };+ if libc::getrlimit(libc::RLIMIT_NOFILE, &mut lim) != 0 {+ return;+ }+ let cur = lim.rlim_cur;+ // An "infinity" hard limit (macOS launchd) must be capped to a concrete+ // value or setrlimit fails outright; 1,048,576 is generous headroom.+ let hard = if lim.rlim_max == libc::RLIM_INFINITY {+ 1_048_576+ } else {+ lim.rlim_max+ };+ if hard <= cur {+ return; // already at the ceiling+ }+ let mut want = hard;+ loop {+ let next = libc::rlimit {+ rlim_cur: want,+ rlim_max: lim.rlim_max,+ };+ if libc::setrlimit(libc::RLIMIT_NOFILE, &next) == 0 {+ return; // raised+ }+ if want <= cur.saturating_add(1) {+ return; // can't beat the inherited soft limit; leave it be+ }+ want = (want / 2).max(cur + 1);+ }+ }+}+fn main() -> Result<()> {+ #[cfg(unix)]+ raise_nofile_limit();+// Dispatch before creating Tokio's pool so each PDF worker stays small.if matches!(std::env::args().nth(1).as_deref(), Some("__extract-pdf")) {std::process::exit(xerj_autoindex::extract::pdf::run_worker_cli());