Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): raise RLIMIT_NOFILE + tree-sitter AST code extraction (#82)

xerj-orge1b9e1a443c5
diff --git a/engine/Cargo.lock b/engine/Cargo.lock
index 0c59c2e74..02e9445f2 100644
--- a/engine/Cargo.lock
+++ b/engine/Cargo.lock
@@ -4656,6 +4656,12 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+[[package]]
+name = "streaming-iterator"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520"
+
[[package]]
name = "stringprep"
version = "0.1.5"
@@ -5317,6 +5323,146 @@ dependencies = [
"tracing-serde",
]
+[[package]]
+name = "tree-sitter"
+version = "0.25.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78f873475d258561b06f1c595d93308a7ed124d9977cb26b148c2084a4a3cc87"
+dependencies = [
+ "cc",
+ "regex",
+ "regex-syntax",
+ "serde_json",
+ "streaming-iterator",
+ "tree-sitter-language",
+]
+
+[[package]]
+name = "tree-sitter-bash"
+version = "0.23.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "329a4d48623ac337d42b1df84e81a1c9dbb2946907c102ca72db158c1964a52e"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
+[[package]]
+name = "tree-sitter-c"
+version = "0.23.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "afd2b1bf1585dc2ef6d69e87d01db8adb059006649dd5f96f31aa789ee6e9c71"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
+[[package]]
+name = "tree-sitter-c-sharp"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1aac67f1ad71de1d6d39708d34811081c26dfa495658de6c14c34200849357c"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
+[[package]]
+name = "tree-sitter-cpp"
+version = "0.23.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df2196ea9d47b4ab4a31b9297eaa5a5d19a0b121dceb9f118f6790ad0ab94743"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
+[[package]]
+name = "tree-sitter-go"
+version = "0.23.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b13d476345220dbe600147dd444165c5791bf85ef53e28acbedd46112ee18431"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
+[[package]]
+name = "tree-sitter-java"
+version = "0.23.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0aa6cbcdc8c679b214e616fd3300da67da0e492e066df01bcf5a5921a71e90d6"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
+[[package]]
+name = "tree-sitter-javascript"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bf40bf599e0416c16c125c3cec10ee5ddc7d1bb8b0c60fa5c4de249ad34dc1b1"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
+[[package]]
+name = "tree-sitter-language"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782"
+
+[[package]]
+name = "tree-sitter-php"
+version = "0.23.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f066e94e9272cfe4f1dcb07a1c50c66097eca648f2d7233d299c8ae9ed8c130c"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
+[[package]]
+name = "tree-sitter-python"
+version = "0.23.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d065aaa27f3aaceaf60c1f0e0ac09e1cb9eb8ed28e7bcdaa52129cffc7f4b04"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
+[[package]]
+name = "tree-sitter-ruby"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be0484ea4ef6bb9c575b4fdabde7e31340a8d2dbc7d52b321ac83da703249f95"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
+[[package]]
+name = "tree-sitter-rust"
+version = "0.23.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca8ccb3e3a3495c8a943f6c3fd24c3804c471fd7f4f16087623c7fa4c0068e8a"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
+[[package]]
+name = "tree-sitter-typescript"
+version = "0.23.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
[[package]]
name = "try-lock"
version = "0.2.5"
@@ -6273,7 +6419,21 @@ dependencies = [
"serde",
"serde_json",
"serde_yaml",
+ "streaming-iterator",
"tempfile",
+ "tree-sitter",
+ "tree-sitter-bash",
+ "tree-sitter-c",
+ "tree-sitter-c-sharp",
+ "tree-sitter-cpp",
+ "tree-sitter-go",
+ "tree-sitter-java",
+ "tree-sitter-javascript",
+ "tree-sitter-php",
+ "tree-sitter-python",
+ "tree-sitter-ruby",
+ "tree-sitter-rust",
+ "tree-sitter-typescript",
"walkdir",
"xxhash-rust",
"zip 2.4.2",
diff --git a/engine/crates/xerj-autoindex/Cargo.toml b/engine/crates/xerj-autoindex/Cargo.toml
index d9d45ad49..55283c9e9 100644
--- a/engine/crates/xerj-autoindex/Cargo.toml
+++ b/engine/crates/xerj-autoindex/Cargo.toml
@@ -24,6 +24,24 @@ fs2 = "0.4"
# (std threads), invoked from the server binary via spawn_blocking.
reqwest = { workspace = true, features = ["blocking"] }
+# --- AST code extraction (tree-sitter). Grammars expose a tree-sitter-language
+# LanguageFn, decoupled from the core version, so one core serves all. ---
+tree-sitter = "0.25"
+tree-sitter-python = "0.23"
+tree-sitter-c = "0.23"
+tree-sitter-rust = "0.23"
+tree-sitter-javascript = "0.23"
+tree-sitter-go = "0.23"
+tree-sitter-typescript = "0.23"
+tree-sitter-java = "0.23"
+tree-sitter-cpp = "0.23"
+tree-sitter-ruby = "0.23"
+tree-sitter-php = "0.23"
+tree-sitter-c-sharp = "0.23"
+tree-sitter-bash = "0.23"
+streaming-iterator = "0.1"
+
+
# Format-family extractors (new deps, isolated to this crate):
flate2 = "1" # gzip streaming (rust backend)
pdf_oxide = { version = "=0.3.75", default-features = false, features = ["legacy-crypto"] }
diff --git a/engine/crates/xerj-autoindex/src/extract/code.rs b/engine/crates/xerj-autoindex/src/extract/code.rs
new file mode 100644
index 000000000..44dc539e5
--- /dev/null
+++ b/engine/crates/xerj-autoindex/src/extract/code.rs
@@ -0,0 +1,447 @@
+//! Source code — AST-aware extraction via tree-sitter.
+//!
+//! Source files used to fall through to the plain-text extractor, so a repo was
+//! searchable only as prose. This parses each file with the matching tree-sitter
+//! grammar and captures its DEFINITIONS (functions, classes, methods, structs,
+//! traits, interfaces, modules, …). Each file becomes one document carrying:
+//! - `language` the detected language
+//! - `symbols` a structured array of {name, kind, line}
+//! - `defs` "kind name" per symbol, newline-joined so BM25 matches a query
+//! like `class User` or `def save` to the file that owns it
+//! - `body` the full source text (still full-text searchable)
+//!
+//! The capture-name in each query IS the symbol kind (`@function`, `@class`, …),
+//! so adding a language is a grammar dep + one registry row. If a grammar fails
+//! to parse a file, it is indexed as plain text rather than dropped.
+
+use super::{ExtractStats, RawRecord, Sink};
+use anyhow::Result;
+use serde_json::{Map, Value};
+use std::path::Path;
+use std::sync::OnceLock;
+use streaming_iterator::StreamingIterator;
+use tree_sitter::{Language, Parser, Query, QueryCursor};
+
+/// Skip machine-generated giants (minified bundles, generated parsers) — past a
+/// couple MB a single "file" is not human code and only bloats the index.
+const CODE_CAP: u64 = 2 << 20;
+
+struct LangDef {
+ name: &'static str,
+ exts: &'static [&'static str],
+ language: Language,
+ query: Query,
+}
+
+/// Is this extension a language we AST-parse? Cheap check used by the sniffer.
+pub fn is_code_ext(ext: &str) -> bool {
+ let e = ext.to_ascii_lowercase();
+ registry().iter().any(|d| d.exts.contains(&e.as_str()))
+}
+
+fn def(name: &'static str, exts: &'static [&'static str], lang: Language, q: &str) -> LangDef {
+ let query = Query::new(&lang, q).unwrap_or_else(|e| panic!("bad {name} query: {e}"));
+ LangDef {
+ name,
+ exts,
+ language: lang,
+ query,
+ }
+}
+
+fn registry() -> &'static [LangDef] {
+ static REG: OnceLock<Vec<LangDef>> = OnceLock::new();
+ REG.get_or_init(|| {
+ vec![
+ def(
+ "python",
+ &["py", "pyi"],
+ tree_sitter_python::LANGUAGE.into(),
+ PYTHON_Q,
+ ),
+ def(
+ "javascript",
+ &["js", "jsx", "mjs", "cjs"],
+ tree_sitter_javascript::LANGUAGE.into(),
+ JS_Q,
+ ),
+ def(
+ "typescript",
+ &["ts"],
+ tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),
+ TS_Q,
+ ),
+ def(
+ "tsx",
+ &["tsx"],
+ tree_sitter_typescript::LANGUAGE_TSX.into(),
+ TS_Q,
+ ),
+ def("rust", &["rs"], tree_sitter_rust::LANGUAGE.into(), RUST_Q),
+ def("go", &["go"], tree_sitter_go::LANGUAGE.into(), GO_Q),
+ def("java", &["java"], tree_sitter_java::LANGUAGE.into(), JAVA_Q),
+ def("c", &["c", "h"], tree_sitter_c::LANGUAGE.into(), C_Q),
+ def(
+ "cpp",
+ &["cpp", "cc", "cxx", "hpp", "hh", "hxx"],
+ tree_sitter_cpp::LANGUAGE.into(),
+ CPP_Q,
+ ),
+ def("ruby", &["rb"], tree_sitter_ruby::LANGUAGE.into(), RUBY_Q),
+ def("php", &["php"], tree_sitter_php::LANGUAGE_PHP.into(), PHP_Q),
+ def(
+ "csharp",
+ &["cs"],
+ tree_sitter_c_sharp::LANGUAGE.into(),
+ CSHARP_Q,
+ ),
+ def(
+ "bash",
+ &["sh", "bash"],
+ tree_sitter_bash::LANGUAGE.into(),
+ BASH_Q,
+ ),
+ ]
+ })
+}
+
+pub fn extract(path: &Path, sink: Sink) -> Result<ExtractStats> {
+ let mut stats = ExtractStats::default();
+ let ext = path
+ .extension()
+ .and_then(|e| e.to_str())
+ .unwrap_or("")
+ .to_ascii_lowercase();
+ let Some(def) = registry().iter().find(|d| d.exts.contains(&ext.as_str())) else {
+ stats.junk += 1;
+ return Ok(stats);
+ };
+ let Some(bytes) = super::read_whole(path, false, CODE_CAP)? else {
+ stats.junk += 1;
+ return Ok(stats);
+ };
+ let (text, _) = crate::sniff::decode_text(&bytes);
+
+ // Parse + capture definitions. A parse failure (or an over-deep tree) is not
+ // fatal — index the file as plain text so its content is still searchable.
+ let symbols = parse_symbols(def, &text).unwrap_or_default();
+ emit_code_doc(path, def.name, &text, &symbols, &mut stats, sink);
+ Ok(stats)
+}
+
+/// (name, kind, 1-based line)
+type Symbol = (String, String, usize);
+
+fn parse_symbols(def: &LangDef, text: &str) -> Option<Vec<Symbol>> {
+ let mut parser = Parser::new();
+ parser.set_language(&def.language).ok()?;
+ let tree = parser.parse(text.as_bytes(), None)?;
+ let names = def.query.capture_names();
+ let mut out: Vec<Symbol> = Vec::new();
+ let mut cursor = QueryCursor::new();
+ let mut it = cursor.matches(&def.query, tree.root_node(), text.as_bytes());
+ while let Some(m) = it.next() {
+ for cap in m.captures {
+ let kind = names[cap.index as usize]; // capture name == symbol kind
+ let node = cap.node;
+ let name = text.get(node.byte_range()).unwrap_or("").trim();
+ if name.is_empty() || name.len() > 200 {
+ continue;
+ }
+ out.push((
+ name.to_string(),
+ kind.to_string(),
+ node.start_position().row + 1,
+ ));
+ if out.len() >= 5000 {
+ return Some(out); // pathological generated file — enough
+ }
+ }
+ }
+ Some(out)
+}
+
+fn emit_code_doc(
+ path: &Path,
+ language: &str,
+ text: &str,
+ symbols: &[Symbol],
+ stats: &mut ExtractStats,
+ sink: Sink,
+) -> bool {
+ let title = path
+ .file_name()
+ .and_then(|n| n.to_str())
+ .unwrap_or("source")
+ .to_string();
+
+ let mut fields = Map::new();
+ fields.insert("title".into(), Value::String(title));
+ fields.insert("language".into(), Value::String(language.to_string()));
+
+ if !symbols.is_empty() {
+ // Searchable "kind name" list — this is what makes `class User` /
+ // `def save` retrieve the file, and it dedups repeated names.
+ let mut seen = std::collections::HashSet::new();
+ let defs: Vec<String> = symbols
+ .iter()
+ .filter(|(n, k, _)| seen.insert((k.clone(), n.clone())))
+ .map(|(n, k, _)| format!("{k} {n}"))
+ .collect();
+ fields.insert("defs".into(), Value::String(defs.join("\n")));
+ let arr: Vec<Value> = symbols
+ .iter()
+ .map(|(n, k, line)| {
+ let mut m = Map::new();
+ m.insert("name".into(), Value::String(n.clone()));
+ m.insert("kind".into(), Value::String(k.clone()));
+ m.insert("line".into(), Value::Number((*line as u64).into()));
+ Value::Object(m)
+ })
+ .collect();
+ fields.insert("symbols".into(), Value::Array(arr));
+ fields.insert(
+ "symbol_count".into(),
+ Value::Number((symbols.len() as u64).into()),
+ );
+ }
+ // The full source stays full-text searchable.
+ fields.insert("body".into(), Value::String(text.to_string()));
+
+ stats.records += 1;
+ sink(RawRecord {
+ fields,
+ locator: "code".into(),
+ group: None,
+ })
+}
+
+// ── Per-language capture queries. Capture-name == emitted symbol kind. ─────────
+
+const PYTHON_Q: &str = r#"
+(function_definition name: (identifier) @function)
+(class_definition name: (identifier) @class)
+"#;
+
+const JS_Q: &str = r#"
+(function_declaration name: (identifier) @function)
+(generator_function_declaration name: (identifier) @function)
+(class_declaration name: (identifier) @class)
+(method_definition name: (property_identifier) @method)
+(variable_declarator name: (identifier) @function value: [(arrow_function) (function_expression)])
+(export_statement declaration: (lexical_declaration (variable_declarator name: (identifier) @function value: (arrow_function))))
+"#;
+
+const TS_Q: &str = r#"
+(function_declaration name: (identifier) @function)
+(class_declaration name: (type_identifier) @class)
+(method_definition name: (property_identifier) @method)
+(interface_declaration name: (type_identifier) @interface)
+(type_alias_declaration name: (type_identifier) @type)
+(enum_declaration name: (identifier) @enum)
+(variable_declarator name: (identifier) @function value: (arrow_function))
+"#;
+
+const RUST_Q: &str = r#"
+(function_item name: (identifier) @function)
+(struct_item name: (type_identifier) @struct)
+(enum_item name: (type_identifier) @enum)
+(trait_item name: (type_identifier) @trait)
+(mod_item name: (identifier) @module)
+(macro_definition name: (identifier) @macro)
+(type_item name: (type_identifier) @type)
+"#;
+
+const GO_Q: &str = r#"
+(function_declaration name: (identifier) @function)
+(method_declaration name: (field_identifier) @method)
+(type_declaration (type_spec name: (type_identifier) @type))
+"#;
+
+const JAVA_Q: &str = r#"
+(class_declaration name: (identifier) @class)
+(interface_declaration name: (identifier) @interface)
+(enum_declaration name: (identifier) @enum)
+(method_declaration name: (identifier) @method)
+(constructor_declaration name: (identifier) @method)
+"#;
+
+const C_Q: &str = r#"
+(function_definition declarator: (function_declarator declarator: (identifier) @function))
+(struct_specifier name: (type_identifier) @struct)
+(enum_specifier name: (type_identifier) @enum)
+(type_definition declarator: (type_identifier) @type)
+"#;
+
+const CPP_Q: &str = r#"
+(function_definition declarator: (function_declarator declarator: (identifier) @function))
+(function_definition declarator: (function_declarator declarator: (field_identifier) @method))
+(class_specifier name: (type_identifier) @class)
+(struct_specifier name: (type_identifier) @struct)
+(enum_specifier name: (type_identifier) @enum)
+(namespace_definition name: (namespace_identifier) @module)
+"#;
+
+const RUBY_Q: &str = r#"
+(method name: (identifier) @method)
+(singleton_method name: (identifier) @method)
+(class name: (constant) @class)
+(module name: (constant) @module)
+"#;
+
+const PHP_Q: &str = r#"
+(function_definition name: (name) @function)
+(method_declaration name: (name) @method)
+(class_declaration name: (name) @class)
+(interface_declaration name: (name) @interface)
+(trait_declaration name: (name) @trait)
+"#;
+
+const CSHARP_Q: &str = r#"
+(class_declaration name: (identifier) @class)
+(interface_declaration name: (identifier) @interface)
+(struct_declaration name: (identifier) @struct)
+(enum_declaration name: (identifier) @enum)
+(method_declaration name: (identifier) @method)
+(constructor_declaration name: (identifier) @method)
+"#;
+
+const BASH_Q: &str = r#"
+(function_definition name: (word) @function)
+"#;
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn syms(lang: &str, src: &str) -> Vec<Symbol> {
+ let def = registry().iter().find(|d| d.name == lang).unwrap();
+ parse_symbols(def, src).unwrap()
+ }
+ fn has(s: &[Symbol], name: &str, kind: &str) -> bool {
+ s.iter().any(|(n, k, _)| n == name && k == kind)
+ }
+
+ #[test]
+ fn all_queries_compile() {
+ // registry() builds every Query; a malformed query panics here.
+ assert!(registry().len() >= 13);
+ }
+
+ #[test]
+ fn python() {
+ let s = syms(
+ "python",
+ "def foo():\n pass\nclass Bar:\n def baz(self):\n pass\n",
+ );
+ assert!(has(&s, "foo", "function"));
+ assert!(has(&s, "Bar", "class"));
+ assert!(has(&s, "baz", "function"));
+ }
+ #[test]
+ fn javascript() {
+ let s = syms(
+ "javascript",
+ "function f(){}\nclass C{ m(){} }\nconst g = () => 1;\n",
+ );
+ assert!(has(&s, "f", "function"));
+ assert!(has(&s, "C", "class"));
+ assert!(has(&s, "m", "method"));
+ assert!(has(&s, "g", "function"));
+ }
+ #[test]
+ fn typescript() {
+ let s = syms(
+ "typescript",
+ "interface I {}\ntype T = number;\nclass C { m(): void {} }\nfunction f(): void {}\n",
+ );
+ assert!(has(&s, "I", "interface"));
+ assert!(has(&s, "T", "type"));
+ assert!(has(&s, "C", "class"));
+ assert!(has(&s, "m", "method"));
+ assert!(has(&s, "f", "function"));
+ }
+ #[test]
+ fn rust() {
+ let s = syms(
+ "rust",
+ "fn f(){}\nstruct S;\nenum E{A}\ntrait T{}\nmod m{}\n",
+ );
+ assert!(has(&s, "f", "function"));
+ assert!(has(&s, "S", "struct"));
+ assert!(has(&s, "E", "enum"));
+ assert!(has(&s, "T", "trait"));
+ assert!(has(&s, "m", "module"));
+ }
+ #[test]
+ fn go() {
+ let s = syms(
+ "go",
+ "package p\nfunc F(){}\nfunc (r R) M(){}\ntype T struct{}\n",
+ );
+ assert!(has(&s, "F", "function"));
+ assert!(has(&s, "M", "method"));
+ assert!(has(&s, "T", "type"));
+ }
+ #[test]
+ fn java() {
+ let s = syms("java", "class C { void m(){} interface I {} }\n");
+ assert!(has(&s, "C", "class"));
+ assert!(has(&s, "m", "method"));
+ assert!(has(&s, "I", "interface"));
+ }
+ #[test]
+ fn c() {
+ let s = syms(
+ "c",
+ "int f(int x){return x;}\nstruct S{int a;};\nenum E{A};\n",
+ );
+ assert!(has(&s, "f", "function"));
+ assert!(has(&s, "S", "struct"));
+ assert!(has(&s, "E", "enum"));
+ }
+ #[test]
+ fn cpp() {
+ let s = syms(
+ "cpp",
+ "class C { void m(){} };\nint f(){return 0;}\nnamespace n {}\n",
+ );
+ assert!(has(&s, "C", "class"));
+ assert!(has(&s, "f", "function"));
+ assert!(has(&s, "n", "module"));
+ }
+ #[test]
+ fn ruby() {
+ let s = syms("ruby", "class C\n def m\n end\nend\nmodule M\nend\n");
+ assert!(has(&s, "C", "class"));
+ assert!(has(&s, "m", "method"));
+ assert!(has(&s, "M", "module"));
+ }
+ #[test]
+ fn php() {
+ let s = syms(
+ "php",
+ "<?php\nfunction f(){}\nclass C { function m(){} }\ninterface I {}\n",
+ );
+ assert!(has(&s, "f", "function"));
+ assert!(has(&s, "C", "class"));
+ assert!(has(&s, "m", "method"));
+ assert!(has(&s, "I", "interface"));
+ }
+ #[test]
+ fn csharp() {
+ let s = syms(
+ "csharp",
+ "class C { void M(){} }\ninterface I {}\nenum E {A}\n",
+ );
+ assert!(has(&s, "C", "class"));
+ assert!(has(&s, "M", "method"));
+ assert!(has(&s, "I", "interface"));
+ assert!(has(&s, "E", "enum"));
+ }
+ #[test]
+ fn bash() {
+ let s = syms("bash", "foo() { echo hi; }\nfunction bar { echo yo; }\n");
+ assert!(has(&s, "foo", "function"));
+ }
+}
diff --git a/engine/crates/xerj-autoindex/src/extract/mod.rs b/engine/crates/xerj-autoindex/src/extract/mod.rs
index 7c79edea4..3f207fb0d 100644
--- a/engine/crates/xerj-autoindex/src/extract/mod.rs
+++ b/engine/crates/xerj-autoindex/src/extract/mod.rs
@@ -2,6 +2,7 @@
//! Every extractor is bounded-memory and never fatal: parse failures
//! downgrade (family → txt → junk-with-metadata) and are counted.
+pub mod code;
pub mod csv_x;
pub mod docx;
pub mod html;
@@ -110,6 +111,7 @@ pub fn extract(
Family::Docx => docx::extract(path, sink),
Family::Sqlite => sqlite_x::extract(path, limit_bytes.map(|_| 500), sink),
Family::SqlDump => sqldump::extract(path, sn.gzip, limit_bytes, sink),
+ Family::Code => code::extract(path, sink),
Family::Binary => Ok(ExtractStats::default()),
}
}
diff --git a/engine/crates/xerj-autoindex/src/sniff.rs b/engine/crates/xerj-autoindex/src/sniff.rs
index b43438732..a0d8c5485 100644
--- a/engine/crates/xerj-autoindex/src/sniff.rs
+++ b/engine/crates/xerj-autoindex/src/sniff.rs
@@ -22,6 +22,8 @@ pub enum Family {
Docx,
Sqlite,
SqlDump,
+ /// Source code — AST-parsed by the matching tree-sitter grammar.
+ Code,
Binary,
}
@@ -41,6 +43,7 @@ impl Family {
Family::Docx => "docx",
Family::Sqlite => "sqlite",
Family::SqlDump => "sqldump",
+ Family::Code => "code",
Family::Binary => "binary",
}
}
@@ -165,6 +168,18 @@ fn sniff_bytes(prefix: &[u8], path: &Path, gzip: bool) -> Result<Sniffed> {
return Ok(s);
}
+ // 2b. Source code: a known code extension whose content is text. We only
+ // reach here after the binary guards above, so a text `.py`/`.rs`/`.go`/…
+ // routes to the tree-sitter AST extractor (crate::extract::code). Extension
+ // is the right signal — code vs prose is not reliably content-sniffable.
+ if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
+ if crate::extract::code::is_code_ext(ext) {
+ let mut s = mk(Family::Code);
+ s.encoding = encoding;
+ return Ok(s);
+ }
+ }
+
// 3. Text heuristics — complete lines only (last line may be truncated).
let mut lines: Vec<&str> = text.lines().collect();
if !text.ends_with('\n') && lines.len() > 1 {
diff --git a/engine/crates/xerj-server/src/main.rs b/engine/crates/xerj-server/src/main.rs
index 831409bed..e624d559c 100644
--- a/engine/crates/xerj-server/src/main.rs
+++ b/engine/crates/xerj-server/src/main.rs
@@ -44,11 +44,29 @@ static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
// 1 s dirty/muzzy decay purges freed pages continuously; the same load then
// holds a flat working-set RSS. Ingest throughput cost measured: none
// (within noise, ~36k docs/s single-index bulk either way).
-#[cfg(all(not(target_env = "msvc"), not(feature = "debug-profiling")))]
+#[cfg(all(
+ not(target_env = "msvc"),
+ not(feature = "debug-profiling"),
+ target_os = "linux"
+))]
#[allow(non_upper_case_globals)]
#[export_name = "_rjem_malloc_conf"]
pub static malloc_conf: &[u8] = b"background_thread:true,dirty_decay_ms:1000,muzzy_decay_ms:1000\0";
+// macOS (and other non-Linux unix) jemalloc has no pthread background_thread
+// support, so `background_thread:true` only prints `<jemalloc>: option
+// background_thread currently supports pthread only` on every launch (reported
+// by users running `xerj autoindex` on macOS). Keep the aggressive decay policy
+// — it still purges freed pages on allocator ticks — without the unsupported knob.
+#[cfg(all(
+ not(target_env = "msvc"),
+ not(feature = "debug-profiling"),
+ not(target_os = "linux")
+))]
+#[allow(non_upper_case_globals)]
+#[export_name = "_rjem_malloc_conf"]
+pub static malloc_conf: &[u8] = b"dirty_decay_ms:1000,muzzy_decay_ms:1000\0";
+
// Preserve the production decay policy. Merely compiling this feature enables
// jemalloc's profiler; sampling remains inactive until the local controller
// starts a bounded capture.
@@ -1256,7 +1274,57 @@ async fn run_cli_index(cmd: IndexCmdArgs) -> Result<()> {
///
/// Override with `TOKIO_WORKER_THREADS` (operators on very small or very large
/// hosts, or wanting the stock `ncpus` behaviour, can set it explicitly).
+/// Raise the process open-file limit toward the OS hard limit before anything
+/// else runs. Every index holds segment mmaps, a WAL and a merge task, so
+/// discovering a large source tree (Django infers ~73 datasets → ~73 indices)
+/// opens thousands of descriptors. On a default macOS soft limit (often 256)
+/// the server exhausts them mid-run and index creation fails with
+/// `Too many open files` (EMFILE, os error 24) — reproduced against a real
+/// `django` clone. Bump the soft limit to the hard limit, stepping down if the
+/// kernel (`kern.maxfilesperproc` on macOS) rejects the target. Best-effort:
+/// any failure here is non-fatal (the server just runs with the inherited cap).
+#[cfg(unix)]
+fn raise_nofile_limit() {
+ unsafe {
+ let mut lim = libc::rlimit {
+ rlim_cur: 0,
+ rlim_max: 0,
+ };
+ if libc::getrlimit(libc::RLIMIT_NOFILE, &mut lim) != 0 {
+ return;
+ }
+ let cur = lim.rlim_cur;
+ // An "infinity" hard limit (macOS launchd) must be capped to a concrete
+ // value or setrlimit fails outright; 1,048,576 is generous headroom.
+ let hard = if lim.rlim_max == libc::RLIM_INFINITY {
+ 1_048_576
+ } else {
+ lim.rlim_max
+ };
+ if hard <= cur {
+ return; // already at the ceiling
+ }
+ let mut want = hard;
+ loop {
+ let next = libc::rlimit {
+ rlim_cur: want,
+ rlim_max: lim.rlim_max,
+ };
+ if libc::setrlimit(libc::RLIMIT_NOFILE, &next) == 0 {
+ return; // raised
+ }
+ if want <= cur.saturating_add(1) {
+ return; // can't beat the inherited soft limit; leave it be
+ }
+ want = (want / 2).max(cur + 1);
+ }
+ }
+}
+
fn main() -> Result<()> {
+ #[cfg(unix)]
+ raise_nofile_limit();
+
// Dispatch before creating Tokio's pool so each PDF worker stays small.
if matches!(std::env::args().nth(1).as_deref(), Some("__extract-pdf")) {
std::process::exit(xerj_autoindex::extract::pdf::run_worker_cli());