AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
feat(hub): extract_value + definitions for the defs index (step 3 WIP)
xerj-orge9bb64ee0bcb
diff --git a/engine/crates/xerj-autoindex/src/extract/code.rs b/engine/crates/xerj-autoindex/src/extract/code.rsindex cd40f07a0..5cc06dc33 100644--- a/engine/crates/xerj-autoindex/src/extract/code.rs+++ b/engine/crates/xerj-autoindex/src/extract/code.rs@@ -1090,6 +1090,93 @@ fn cpp_declarator_is_const_qualified(node: tree_sitter::Node, source: &[u8]) ->false}+/// The literal value of a constant / macro / field declaration, or `None` for+/// anything else (a function, a type, a declaration with no initialiser).+///+/// Applied to a definition's declaration text (the `code` slice of a [`Symbol`],+/// which is `DefRecord.text`). The per-language rules are fixed by+/// `docs/CORPUS_HUB_V2_IMPLEMENTATION.md` step 3. The captured value is trimmed+/// and loses a trailing `;` or `,`.+pub fn extract_value(lang: &str, decl: &str) -> Option<String> {+ fn clean(s: &str) -> Option<String> {+ let v = s.trim().trim_end_matches([';', ',']).trim();+ (!v.is_empty()).then(|| v.to_string())+ }+ use std::sync::OnceLock;+ static C_DEFINE: OnceLock<regex::Regex> = OnceLock::new();+ static C_CONST: OnceLock<regex::Regex> = OnceLock::new();+ static RUST: OnceLock<regex::Regex> = OnceLock::new();+ static GO: OnceLock<regex::Regex> = OnceLock::new();+ static UPPER: OnceLock<regex::Regex> = OnceLock::new();+ match lang {+ "c" | "cpp" | "objc" => {+ let def = C_DEFINE+ .get_or_init(|| regex::Regex::new(r"(?m)^\s*#\s*define\s+(\w+)\s+(.+?)\s*(?:/[/*].*)?$").unwrap());+ if let Some(c) = def.captures(decl) {+ if let Some(v) = clean(&c[2]) {+ return Some(v);+ }+ }+ let cst = C_CONST.get_or_init(|| {+ regex::Regex::new(r"\b(?:static\s+)?const\s+\w[\w\s\*]*\b(\w+)\s*=\s*(.+?)\s*;").unwrap()+ });+ cst.captures(decl).and_then(|c| clean(&c[2]))+ }+ "rust" => {+ let re = RUST.get_or_init(|| {+ regex::Regex::new(r"(?m)^\s*(?:pub(?:\([^)]*\))?\s+)?(?:const|static)\s+(\w+)\s*:\s*[^=]+=\s*(.+?)\s*;")+ .unwrap()+ });+ re.captures(decl).and_then(|c| clean(&c[2]))+ }+ "go" => {+ let re = GO.get_or_init(|| {+ regex::Regex::new(r"(?m)^\s*(?:const\s+)?(\w+)\s*(?:[\w.\[\]*]+\s+)?=\s*(.+?)\s*$").unwrap()+ });+ re.captures(decl).and_then(|c| clean(&c[2]))+ }+ "python" | "ruby" | "bash" | "sh" | "shell" => {+ let re =+ UPPER.get_or_init(|| regex::Regex::new(r"(?m)^\s*([A-Z_][A-Z0-9_]*)\s*=\s*(.+?)\s*$").unwrap());+ re.captures(decl).and_then(|c| clean(&c[2]))+ }+ _ => None,+ }+}++/// Every definition in one file, as [`DefRecord`]s for the hub pack's defs index.+///+/// `file_rel` is the path relative to the corpus root (forward slashes), `lang`+/// is a registry language name (`registry()`), `text` is the file's decoded+/// contents. Unknown languages yield no records. The declaration text is capped+/// at 400 chars; constants and macros carry their value ([`extract_value`]).+pub fn definitions(file_rel: &str, lang: &str, text: &str) -> Vec<xerj_common::hub::DefRecord> {+ let Some(def) = registry().iter().find(|d| d.name == lang) else {+ return Vec::new();+ };+ parse_symbols(def, text)+ .unwrap_or_default()+ .into_iter()+ .map(|s| {+ let decl: String = if s.code.chars().count() > 400 {+ s.code.chars().take(400).collect()+ } else {+ s.code+ };+ let value = extract_value(lang, &decl);+ xerj_common::hub::DefRecord {+ symbol: s.name,+ kind: s.kind,+ lang: lang.to_string(),+ file: file_rel.to_string(),+ line: s.line as u32,+ text: decl,+ value,+ }+ })+ .collect()+}+fn parse_symbols(def: &LangDef, text: &str) -> Option<Vec<Symbol>> {let mut parser = Parser::new();parser.set_language(&def.language).ok()?;