AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): capture separately exported factory constants
Jarvis3263e17eb37f
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 1a0045c9a..fb8388a86 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -35,6 +35,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- TS/JS factory constants exported by a separate local export clause or+ default identifier now get definition records at their declaration lines+ (refs #2579). Inline `export const` stays supported; private constants,+ function-local bindings, type-only exports and remote re-exports stay out.+ Existing unchanged files need re-extraction to gain these definitions.+- Corpus cloning validates existing checkouts and serializes writers (#2453).New checkouts are staged before publication. Ambiguous existing directoriesare preserved for recovery rather than deleted or reset automatically.diff --git a/engine/crates/xerj-autoindex/src/extract/code.rs b/engine/crates/xerj-autoindex/src/extract/code.rsindex 414d0dbd2..ce5f161cb 100644--- a/engine/crates/xerj-autoindex/src/extract/code.rs+++ b/engine/crates/xerj-autoindex/src/extract/code.rs@@ -1246,10 +1246,69 @@ pub fn definitions(file_rel: &str, lang: &str, text: &str) -> Vec<xerj_common::h.collect()}+/// Local names exported separately from their declaration (#2579). Do not+/// mistake `export { x } from 'other'` or a TS type-only export for exporting+/// this module's value. Names/aliases come from the grammar fields, not text+/// matching. Reference: tree-sitter-typescript 0.23.2 (MIT),+/// typescript/src/node-types.json:1924 (`name` vs `alias`) and :1958 (`source`,+/// `value`); JS 0.23.1 uses the same export fields.+fn js_separate_value_exports<'a>(+ root: Node<'_>,+ text: &'a str,+) -> std::collections::HashSet<&'a str> {+ fn has_token(node: Node<'_>, kind: &str) -> bool {+ let mut cursor = node.walk();+ let found = node.children(&mut cursor).any(|child| child.kind() == kind);+ found+ }++ let mut names = std::collections::HashSet::new();+ let mut cursor = root.walk();+ for export in root.named_children(&mut cursor) {+ if export.kind() != "export_statement"+ || export.child_by_field_name("source").is_some()+ || has_token(export, "type")+ {+ continue;+ }+ if has_token(export, "default") {+ if let Some(value) = export.child_by_field_name("value") {+ if value.kind() == "identifier" {+ if let Some(name) = text.get(value.byte_range()) {+ names.insert(name);+ }+ }+ }+ }+ let mut clause_cursor = export.walk();+ for clause in export.named_children(&mut clause_cursor) {+ if clause.kind() != "export_clause" {+ continue;+ }+ let mut spec_cursor = clause.walk();+ for spec in clause.named_children(&mut spec_cursor) {+ if has_token(spec, "type") {+ continue;+ }+ if let Some(name) = spec.child_by_field_name("name") {+ if name.kind() == "identifier" {+ if let Some(name) = text.get(name.byte_range()) {+ names.insert(name);+ }+ }+ }+ }+ }+ }+ names+}+fn parse_symbols(def: &LangDef, text: &str) -> Option<Vec<Symbol>> {let mut parser = Parser::new();parser.set_language(&def.language).ok()?;let tree = parser.parse(text.as_bytes(), None)?;+ let js_exports = matches!(def.name, "javascript" | "typescript" | "tsx")+ .then(|| js_separate_value_exports(tree.root_node(), text));let names = def.query.capture_names();// Row → source line, built once (not `lines().nth(row)` per symbol, which// would be O(symbols · file)).@@ -1267,6 +1326,21 @@ fn parse_symbols(def: &LangDef, text: &str) -> Option<Vec<Symbol>> {continue; // predicate-only capture (`@_kw`), not a symbol}let node = cap.node;+ // The additional JS/TS @const pattern admits only top-level+ // factory calls. Keep its candidates only when this module exports+ // the binding separately; private values stay out (#285).+ if let Some(exports) = &js_exports {+ if kind == "const"+ && node+ .parent()+ .and_then(|n| n.parent())+ .and_then(|n| n.parent())+ .is_some_and(|n| n.kind() == "program")+ && !exports.contains(text.get(node.byte_range()).unwrap_or(""))+ {+ continue;+ }+ }// #852: every C++ pattern targets file/namespace/class-scope API// symbols — none intends to capture a function-local definition. A// type/class/struct/enum or its enumerators DEFINED INSIDE a@@ -1648,12 +1722,14 @@ const PYTHON_Q: &str = r#"// `@function` and `@const`, where the TS equivalent only does so for arrows.// `defs` dedups on (kind, name), so each spelling still answers its own search.//-// The `.bind`-valued declarator (last pattern) is the JS twin of the one in+// The `.bind`-valued declarator is the JS twin of the one in// TS_Q — see the rationale there. `const add = game.root.add.bind(game.root)` is// how a framework exposes its global API surface, and that binding is the name// consumers call. NOT the #170 90%-locals trap (that is `const x = 0`); a// `.bind()`-valued const is always a deliberately named bound callable, and it// is captured `@function` because that is what it is.+// The final top-level call candidate is filtered by js_separate_value_exports:+// `const store = create(...); export { store };` is also a public definition.const JS_Q: &str = r#"(function_declaration name: (identifier) @function)(generator_function_declaration name: (identifier) @function)@@ -1663,9 +1739,10 @@ const JS_Q: &str = r#"(export_statement declaration: (lexical_declaration (variable_declarator name: (identifier) @function value: (arrow_function))))(program (export_statement declaration: (lexical_declaration "const" (variable_declarator name: (identifier) @const))))(variable_declarator name: (identifier) @function value: (call_expression function: (member_expression property: (property_identifier) @_bind)) (#eq? @_bind "bind"))+(program (lexical_declaration "const" (variable_declarator name: (identifier) @const value: (call_expression))))"#;-// The last pattern is #170's failure mode in the language where it is most+// The exported-const pattern addresses #170's failure mode where it is most// common. Modern TypeScript declares most of a module's public surface as// `export const x = someBuilder(...)` — schema/table definitions, routers,// config objects. That is not an `arrow_function`, so the arrow-valued pattern@@ -1702,7 +1779,7 @@ const JS_Q: &str = r#"// what a caller searching for either would expect; suppressing one would mean// choosing which of the two true statements to hide.//-// The last pattern (`.bind`-valued declarator) captures the re-binding idiom+// The `.bind`-valued declarator captures the re-binding idiom// `const tween = game.root.tween.bind(game.root)`. Whole libraries expose their// public API this way — a framework builds a context object of methods bound to// a root, and the ~40 lines `const add = ….add.bind(…)`, `const get = …`, `const@@ -1721,6 +1798,11 @@ const JS_Q: &str = r#"// ON PURPOSE — these bindings live INSIDE the context-builder function body,// where `program`-anchoring (used for the plain-const pattern above) would// exclude them.+//+// #2579: a factory const exported by a later export clause/default identifier+// is public too. The final pattern admits only top-level call initializers;+// parse_symbols checks their names against js_separate_value_exports so+// private constants, locals and mutable bindings keep their existing scope.const TS_Q: &str = r#"(function_declaration name: (identifier) @function)(class_declaration name: (type_identifier) @class)@@ -1731,6 +1813,7 @@ const TS_Q: &str = r#"(variable_declarator name: (identifier) @function value: (arrow_function))(program (export_statement declaration: (lexical_declaration "const" (variable_declarator name: (identifier) @const))))(variable_declarator name: (identifier) @function value: (call_expression function: (member_expression property: (property_identifier) @_bind)) (#eq? @_bind "bind"))+(program (lexical_declaration "const" (variable_declarator name: (identifier) @const value: (call_expression))))"#;// `const_item` / `static_item` are captured because a file whose whole content@@ -2386,6 +2469,152 @@ mod tests {let def = registry().iter().find(|d| d.name == lang).unwrap();parse_symbols(def, src).unwrap()}++ /// #2579's inline-export examples already match the existing const rule.+ /// Exercise sniffing and emitted records too: an AST capture alone does+ /// not prove that `def` has a per-symbol document to retrieve.+ #[test]+ fn issue_2579_exported_factory_constants() {+ for (lang, ext) in [("typescript", "ts"), ("tsx", "tsx"), ("javascript", "js")] {+ let generic = if lang == "javascript" {+ ""+ } else {+ "<UserState>"+ };+ let src = format!(+ "export const useUserStore = create{generic}()((set) => ({{ user: null }}));\n\+ export const userSlice = createSlice({{ name: 'user' }});\n\+ export const usePiniaStore = defineStore('user', {{}});\n\+ export const StyledButton = styled.button`color: red;`;\n"+ );+ let symbols = syms(lang, &src);+ for (i, name) in ["useUserStore", "userSlice", "usePiniaStore", "StyledButton"]+ .iter()+ .enumerate()+ {+ let found: Vec<_> = symbols+ .iter()+ .filter(|s| s.name == *name && s.kind == "const")+ .collect();+ assert_eq!(found.len(), 1, "{lang}: {name}: {symbols:?}");+ assert_eq!(found[0].line, i + 1);+ assert!(found[0].code.contains(name), "{lang}: {found:?}");+ }+ let dir = tempfile::tempdir().unwrap();+ let file = dir.path().join(format!("store.{ext}"));+ std::fs::write(&file, &src).unwrap();+ let sniffed = crate::sniff::sniff_with_name(&file, &file).unwrap();+ assert_eq!(sniffed.family, crate::sniff::Family::Code);+ let mut records = Vec::new();+ let stats = extract(&file, &sniffed, &mut |record| {+ records.push(record);+ true+ })+ .unwrap();+ assert_eq!((stats.records, stats.junk), (5, 0), "{lang}");+ assert_eq!(records.len(), 5, "{lang}");+ for (i, name) in ["useUserStore", "userSlice", "usePiniaStore", "StyledButton"]+ .iter()+ .enumerate()+ {+ assert!(+ records[0].fields["defs"]+ .as_str()+ .unwrap()+ .lines()+ .any(|line| line == format!("const {name}")),+ "{lang}: {name}"+ );+ let found: Vec<_> = records+ .iter()+ .filter(|r| r.fields.get("name").and_then(Value::as_str) == Some(*name))+ .collect();+ assert_eq!(found.len(), 1, "{lang}: {name}");+ assert_eq!(found[0].fields["kind"], "const");+ assert_eq!(found[0].fields["language"], lang);+ assert_eq!(found[0].fields["line"], i + 1);+ assert_eq!(found[0].locator, format!("code:{}:{name}", i + 1));+ }+ }+ }++ #[test]+ fn issue_2579_factory_export_scope_controls() {+ for lang in ["typescript", "tsx", "javascript"] {+ let source = "\+const privateStore = createStore({});+const namedStore = createStore({});+export { namedStore };+const defaultStore = createStore({});+export default defaultStore;+const renamedStore = createStore({});+export { renamedStore as publicStore };+const remoteStore = createStore({});+export { remoteStore } from './remote';+export let mutableStore = createStore({});+function outer() { const localStore = createStore({}); }+";+ let symbols = syms(lang, source);+ for name in ["namedStore", "defaultStore", "renamedStore"] {+ assert!(+ has(&symbols, name, "const"),+ "{lang}: missing {name}: {symbols:?}"+ );+ }+ for name in [+ "privateStore",+ "remoteStore",+ "publicStore",+ "mutableStore",+ "localStore",+ ] {+ assert!(!has(&symbols, name, "const"), "{lang}: {name}: {symbols:?}");+ }+ let mut stats = ExtractStats::default();+ let mut records = Vec::new();+ assert!(emit_code_doc(+ Path::new("store.ts"),+ lang,+ source,+ &symbols,+ &mut stats,+ &mut |record| {+ records.push(record);+ true+ }+ ));+ for (name, line) in [("namedStore", 2), ("defaultStore", 4), ("renamedStore", 6)] {+ let found: Vec<_> = records+ .iter()+ .filter(|r| r.fields.get("name").and_then(Value::as_str) == Some(name))+ .collect();+ assert_eq!(found.len(), 1, "{lang}: {name}");+ assert_eq!(found[0].locator, format!("code:{line}:{name}"));+ assert_eq!(found[0].fields["line"], line);+ }+ }+ }++ #[test]+ fn issue_2579_type_only_exports_do_not_publish_factory_values() {+ for lang in ["typescript", "tsx"] {+ let symbols = syms(+ lang,+ "\+const typeOnly = createStore({});+export type { typeOnly };+const specTypeOnly = createStore({});+export { type specTypeOnly };+const valueStore = createStore({});+export { type specTypeOnly as Alias, valueStore };+",+ );+ assert!(has(&symbols, "valueStore", "const"), "{lang}: {symbols:?}");+ for name in ["typeOnly", "specTypeOnly", "Alias"] {+ assert!(!has(&symbols, name, "const"), "{lang}: {symbols:?}");+ }+ }+ }fn has(s: &[Symbol], name: &str, kind: &str) -> bool {s.iter().any(|s| s.name == name && s.kind == kind)}diff --git a/tools/factory-definitions/smoke.py b/tools/factory-definitions/smoke.pynew file mode 100644index 000000000..0b4aad26c--- /dev/null+++ b/tools/factory-definitions/smoke.py@@ -0,0 +1,143 @@+#!/usr/bin/env python3+"""Real autoindex -> CLI/MCP def checks for separately exported factories.++python3 tools/factory-definitions/smoke.py --binary engine/target/ci-test/xerj+Only synthetic source and a temporary private node are used. No models needed.+"""+import argparse+import hashlib+import json+import os+from pathlib import Path+import signal+import shutil+import socket+import subprocess+import tempfile+import time+from urllib.request import build_opener, ProxyHandler+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument('--binary', type=Path, required=True)+ parser.add_argument('--runner', type=Path)+ parser.add_argument('--estimate-only', action='store_true')+ args = parser.parse_args()+ source_binary = args.binary.resolve(strict=True)+ runner = str(args.runner.resolve(strict=True)) if args.runner else None+ repo = Path(__file__).resolve().parents[2]+ env = {k: v for k, v in os.environ.items() if not k.startswith('XERJ_')}+ env.update(XERJ_DISABLE_FEEDBACK='true', XERJ_NUM_CPUS='4')+ opener = build_opener(ProxyHandler({}))+ with tempfile.TemporaryDirectory(prefix='xerj-factory-defs-') as temp:+ root = Path(temp)+ snapshot = root / ('xerj.exe' if os.name == 'nt' else 'xerj')+ shutil.copyfile(source_binary, snapshot)+ snapshot.chmod(0o700)+ binary = str(snapshot)+ print(f'Executable snapshot SHA-256: {hashlib.sha256(snapshot.read_bytes()).hexdigest()}', flush=True)+ env['XERJ_CONFIG'] = str(root / 'node.toml')+ # Tiny isolated fixture: do not inherit a shared laptop's 95% reserve.+ # Leave 1% of the volume reserved; never alter another node's settings.+ (root / 'node.toml').write_text('[server]\nauto_port = false\n\n[limits]\ndisk_flood_stage_percent = 99\n')+ with socket.socket() as sock:+ sock.bind(('127.0.0.1', 0))+ port = sock.getsockname()[1]+ url = f'http://127.0.0.1:{port}'+ source = root / 'source'+ source.mkdir()+ expected = []+ for ext, lang in [('ts', 'typescript'), ('tsx', 'tsx'), ('js', 'javascript')]:+ generic = '' if ext == 'js' else '<UserState>'+ names = [f'{ext}InlineStore', f'{ext}NamedStore', f'{ext}DefaultStore']+ (source / f'store.{ext}').write_text(+ f'export const {names[0]} = create{generic}()((set) => ({{ user: null }}));\n'+ f'const {names[1]} = createStore({{}});\n'+ f'export {{ {names[1]} as publicStore }};\n'+ f'const {names[2]} = styled.button`color: red;`;\n'+ f'export default {names[2]};\n'+ f'const {ext}PrivateStore = createStore({{}});\n')+ expected.extend((name, lang, f'store.{ext}', line)+ for name, line in zip(names, [1, 2, 4]))++ def run(command, *, stdin=None, timeout=120):+ result = subprocess.run(command, input=stdin, text=True, capture_output=True,+ cwd=root, env=env, timeout=timeout)+ if result.returncode:+ raise AssertionError(f'{command}: exit {result.returncode}\n{result.stdout}\n{result.stderr}')+ if 'autoindex' in command:+ print(result.stderr, end='', flush=True)+ return result.stdout++ print(run([binary, '--version']), end='', flush=True)++ def check_definitions():+ for name, lang, filename, line in expected:+ cli = json.loads(run([binary, 'def', name, '--prefix', 'fx2579',+ '--url', url, '--kind', 'const', '--lang', lang, '--json']))+ messages = [+ {'jsonrpc': '2.0', 'id': 1, 'method': 'initialize', 'params': {}},+ {'jsonrpc': '2.0', 'id': 2, 'method': 'tools/call', 'params': {+ 'name': 'xerj_def', 'arguments': {'symbol': name, 'prefix': 'fx2579',+ 'kind': 'const', 'lang': lang}}},+ ]+ raw = run([binary, 'mcp', '--url', url],+ stdin=''.join(json.dumps(m) + '\n' for m in messages))+ reply = next(json.loads(s)['result'] for s in raw.splitlines()+ if json.loads(s).get('id') == 2)+ assert not reply.get('isError'), reply+ mcp = json.loads(reply['content'][0]['text'])+ assert cli['hits'] == mcp['hits'], (name, cli, mcp)+ hits = cli['hits']['hits']+ assert hits, (name, cli)+ hit = hits[0]['_source']+ assert hit['name'] == name and hit['kind'] == 'const', hit+ assert hit['line'] == line and hit['ax_path'].endswith(filename), hit+ assert name in hit['code'], hit+ print(f'PASS {len(expected)} definitions: CLI/MCP parity and declaration citations', flush=True)++ for phase in ['first-run', 'restart']:+ with (root / f'{phase}.log').open('w') as log:+ node = subprocess.Popen([binary, '--insecure', '--data-dir', str(root / 'data'),+ '--port', str(port), '--embed-mode', 'lexical'],+ cwd=root, env=env, stdout=log, stderr=log)+ try:+ deadline = time.monotonic() + 60+ while True:+ assert node.poll() is None, (root / f'{phase}.log').read_text()+ try:+ with opener.open(url, timeout=1) as response:+ json.load(response)+ break+ except OSError:+ if time.monotonic() >= deadline:+ raise TimeoutError('private node did not become ready')+ time.sleep(0.1)+ if phase == 'first-run':+ if runner:+ print(run([runner, '--url', url, '--dir', str(repo / 'engine/tests/es-compat-yaml/yaml')], timeout=300), flush=True)+ command = [binary, 'autoindex', str(source), '--url', url, '--prefix', 'fx2579',+ '--state-dir', str(root / 'state'), '--no-graph', '--no-semantic', '--yes']+ print(run([*command, '--dry-run']), flush=True)+ if args.estimate_only:+ return+ print(run(command), flush=True)+ check_definitions()+ print(run(command), flush=True)+ check_definitions()+ finally:+ if node.poll() is None:+ node.send_signal(signal.SIGTERM)+ try:+ node.wait(timeout=30)+ except subprocess.TimeoutExpired:+ node.kill()+ node.wait()+ raise+ assert node.returncode == 0, (root / f'{phase}.log').read_text()+ print('PASS first index, unchanged rerun, graceful stop and durable restart', flush=True)+++if __name__ == '__main__':+ main()