Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): capture separately exported factory constants

Jarvis3263e17eb37f
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1a0045c9a..fb8388a86 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -35,6 +35,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- TS/JS factory constants exported by a separate local export clause or
+ default identifier now get definition records at their declaration lines
+ (refs #2579). Inline `export const` stays supported; private constants,
+ function-local bindings, type-only exports and remote re-exports stay out.
+ Existing unchanged files need re-extraction to gain these definitions.
+
- Corpus cloning validates existing checkouts and serializes writers (#2453).
New checkouts are staged before publication. Ambiguous existing directories
are preserved for recovery rather than deleted or reset automatically.
diff --git a/engine/crates/xerj-autoindex/src/extract/code.rs b/engine/crates/xerj-autoindex/src/extract/code.rs
index 414d0dbd2..ce5f161cb 100644
--- a/engine/crates/xerj-autoindex/src/extract/code.rs
+++ b/engine/crates/xerj-autoindex/src/extract/code.rs
@@ -1246,10 +1246,69 @@ pub fn definitions(file_rel: &str, lang: &str, text: &str) -> Vec<xerj_common::h
.collect()
}
+/// Local names exported separately from their declaration (#2579). Do not
+/// mistake `export { x } from 'other'` or a TS type-only export for exporting
+/// this module's value. Names/aliases come from the grammar fields, not text
+/// matching. Reference: tree-sitter-typescript 0.23.2 (MIT),
+/// typescript/src/node-types.json:1924 (`name` vs `alias`) and :1958 (`source`,
+/// `value`); JS 0.23.1 uses the same export fields.
+fn js_separate_value_exports<'a>(
+ root: Node<'_>,
+ text: &'a str,
+) -> std::collections::HashSet<&'a str> {
+ fn has_token(node: Node<'_>, kind: &str) -> bool {
+ let mut cursor = node.walk();
+ let found = node.children(&mut cursor).any(|child| child.kind() == kind);
+ found
+ }
+
+ let mut names = std::collections::HashSet::new();
+ let mut cursor = root.walk();
+ for export in root.named_children(&mut cursor) {
+ if export.kind() != "export_statement"
+ || export.child_by_field_name("source").is_some()
+ || has_token(export, "type")
+ {
+ continue;
+ }
+ if has_token(export, "default") {
+ if let Some(value) = export.child_by_field_name("value") {
+ if value.kind() == "identifier" {
+ if let Some(name) = text.get(value.byte_range()) {
+ names.insert(name);
+ }
+ }
+ }
+ }
+ let mut clause_cursor = export.walk();
+ for clause in export.named_children(&mut clause_cursor) {
+ if clause.kind() != "export_clause" {
+ continue;
+ }
+ let mut spec_cursor = clause.walk();
+ for spec in clause.named_children(&mut spec_cursor) {
+ if has_token(spec, "type") {
+ continue;
+ }
+ if let Some(name) = spec.child_by_field_name("name") {
+ if name.kind() == "identifier" {
+ if let Some(name) = text.get(name.byte_range()) {
+ names.insert(name);
+ }
+ }
+ }
+ }
+ }
+ }
+ names
+}
+
fn parse_symbols(def: &LangDef, text: &str) -> Option<Vec<Symbol>> {
let mut parser = Parser::new();
parser.set_language(&def.language).ok()?;
let tree = parser.parse(text.as_bytes(), None)?;
+ let js_exports = matches!(def.name, "javascript" | "typescript" | "tsx")
+ .then(|| js_separate_value_exports(tree.root_node(), text));
let names = def.query.capture_names();
// Row → source line, built once (not `lines().nth(row)` per symbol, which
// would be O(symbols · file)).
@@ -1267,6 +1326,21 @@ fn parse_symbols(def: &LangDef, text: &str) -> Option<Vec<Symbol>> {
continue; // predicate-only capture (`@_kw`), not a symbol
}
let node = cap.node;
+ // The additional JS/TS @const pattern admits only top-level
+ // factory calls. Keep its candidates only when this module exports
+ // the binding separately; private values stay out (#285).
+ if let Some(exports) = &js_exports {
+ if kind == "const"
+ && node
+ .parent()
+ .and_then(|n| n.parent())
+ .and_then(|n| n.parent())
+ .is_some_and(|n| n.kind() == "program")
+ && !exports.contains(text.get(node.byte_range()).unwrap_or(""))
+ {
+ continue;
+ }
+ }
// #852: every C++ pattern targets file/namespace/class-scope API
// symbols — none intends to capture a function-local definition. A
// type/class/struct/enum or its enumerators DEFINED INSIDE a
@@ -1648,12 +1722,14 @@ const PYTHON_Q: &str = r#"
// `@function` and `@const`, where the TS equivalent only does so for arrows.
// `defs` dedups on (kind, name), so each spelling still answers its own search.
//
-// The `.bind`-valued declarator (last pattern) is the JS twin of the one in
+// The `.bind`-valued declarator is the JS twin of the one in
// TS_Q — see the rationale there. `const add = game.root.add.bind(game.root)` is
// how a framework exposes its global API surface, and that binding is the name
// consumers call. NOT the #170 90%-locals trap (that is `const x = 0`); a
// `.bind()`-valued const is always a deliberately named bound callable, and it
// is captured `@function` because that is what it is.
+// The final top-level call candidate is filtered by js_separate_value_exports:
+// `const store = create(...); export { store };` is also a public definition.
const JS_Q: &str = r#"
(function_declaration name: (identifier) @function)
(generator_function_declaration name: (identifier) @function)
@@ -1663,9 +1739,10 @@ const JS_Q: &str = r#"
(export_statement declaration: (lexical_declaration (variable_declarator name: (identifier) @function value: (arrow_function))))
(program (export_statement declaration: (lexical_declaration "const" (variable_declarator name: (identifier) @const))))
(variable_declarator name: (identifier) @function value: (call_expression function: (member_expression property: (property_identifier) @_bind)) (#eq? @_bind "bind"))
+(program (lexical_declaration "const" (variable_declarator name: (identifier) @const value: (call_expression))))
"#;
-// The last pattern is #170's failure mode in the language where it is most
+// The exported-const pattern addresses #170's failure mode where it is most
// common. Modern TypeScript declares most of a module's public surface as
// `export const x = someBuilder(...)` — schema/table definitions, routers,
// config objects. That is not an `arrow_function`, so the arrow-valued pattern
@@ -1702,7 +1779,7 @@ const JS_Q: &str = r#"
// what a caller searching for either would expect; suppressing one would mean
// choosing which of the two true statements to hide.
//
-// The last pattern (`.bind`-valued declarator) captures the re-binding idiom
+// The `.bind`-valued declarator captures the re-binding idiom
// `const tween = game.root.tween.bind(game.root)`. Whole libraries expose their
// public API this way — a framework builds a context object of methods bound to
// a root, and the ~40 lines `const add = ….add.bind(…)`, `const get = …`, `const
@@ -1721,6 +1798,11 @@ const JS_Q: &str = r#"
// ON PURPOSE — these bindings live INSIDE the context-builder function body,
// where `program`-anchoring (used for the plain-const pattern above) would
// exclude them.
+//
+// #2579: a factory const exported by a later export clause/default identifier
+// is public too. The final pattern admits only top-level call initializers;
+// parse_symbols checks their names against js_separate_value_exports so
+// private constants, locals and mutable bindings keep their existing scope.
const TS_Q: &str = r#"
(function_declaration name: (identifier) @function)
(class_declaration name: (type_identifier) @class)
@@ -1731,6 +1813,7 @@ const TS_Q: &str = r#"
(variable_declarator name: (identifier) @function value: (arrow_function))
(program (export_statement declaration: (lexical_declaration "const" (variable_declarator name: (identifier) @const))))
(variable_declarator name: (identifier) @function value: (call_expression function: (member_expression property: (property_identifier) @_bind)) (#eq? @_bind "bind"))
+(program (lexical_declaration "const" (variable_declarator name: (identifier) @const value: (call_expression))))
"#;
// `const_item` / `static_item` are captured because a file whose whole content
@@ -2386,6 +2469,152 @@ mod tests {
let def = registry().iter().find(|d| d.name == lang).unwrap();
parse_symbols(def, src).unwrap()
}
+
+ /// #2579's inline-export examples already match the existing const rule.
+ /// Exercise sniffing and emitted records too: an AST capture alone does
+ /// not prove that `def` has a per-symbol document to retrieve.
+ #[test]
+ fn issue_2579_exported_factory_constants() {
+ for (lang, ext) in [("typescript", "ts"), ("tsx", "tsx"), ("javascript", "js")] {
+ let generic = if lang == "javascript" {
+ ""
+ } else {
+ "<UserState>"
+ };
+ let src = format!(
+ "export const useUserStore = create{generic}()((set) => ({{ user: null }}));\n\
+ export const userSlice = createSlice({{ name: 'user' }});\n\
+ export const usePiniaStore = defineStore('user', {{}});\n\
+ export const StyledButton = styled.button`color: red;`;\n"
+ );
+ let symbols = syms(lang, &src);
+ for (i, name) in ["useUserStore", "userSlice", "usePiniaStore", "StyledButton"]
+ .iter()
+ .enumerate()
+ {
+ let found: Vec<_> = symbols
+ .iter()
+ .filter(|s| s.name == *name && s.kind == "const")
+ .collect();
+ assert_eq!(found.len(), 1, "{lang}: {name}: {symbols:?}");
+ assert_eq!(found[0].line, i + 1);
+ assert!(found[0].code.contains(name), "{lang}: {found:?}");
+ }
+ let dir = tempfile::tempdir().unwrap();
+ let file = dir.path().join(format!("store.{ext}"));
+ std::fs::write(&file, &src).unwrap();
+ let sniffed = crate::sniff::sniff_with_name(&file, &file).unwrap();
+ assert_eq!(sniffed.family, crate::sniff::Family::Code);
+ let mut records = Vec::new();
+ let stats = extract(&file, &sniffed, &mut |record| {
+ records.push(record);
+ true
+ })
+ .unwrap();
+ assert_eq!((stats.records, stats.junk), (5, 0), "{lang}");
+ assert_eq!(records.len(), 5, "{lang}");
+ for (i, name) in ["useUserStore", "userSlice", "usePiniaStore", "StyledButton"]
+ .iter()
+ .enumerate()
+ {
+ assert!(
+ records[0].fields["defs"]
+ .as_str()
+ .unwrap()
+ .lines()
+ .any(|line| line == format!("const {name}")),
+ "{lang}: {name}"
+ );
+ let found: Vec<_> = records
+ .iter()
+ .filter(|r| r.fields.get("name").and_then(Value::as_str) == Some(*name))
+ .collect();
+ assert_eq!(found.len(), 1, "{lang}: {name}");
+ assert_eq!(found[0].fields["kind"], "const");
+ assert_eq!(found[0].fields["language"], lang);
+ assert_eq!(found[0].fields["line"], i + 1);
+ assert_eq!(found[0].locator, format!("code:{}:{name}", i + 1));
+ }
+ }
+ }
+
+ #[test]
+ fn issue_2579_factory_export_scope_controls() {
+ for lang in ["typescript", "tsx", "javascript"] {
+ let source = "\
+const privateStore = createStore({});
+const namedStore = createStore({});
+export { namedStore };
+const defaultStore = createStore({});
+export default defaultStore;
+const renamedStore = createStore({});
+export { renamedStore as publicStore };
+const remoteStore = createStore({});
+export { remoteStore } from './remote';
+export let mutableStore = createStore({});
+function outer() { const localStore = createStore({}); }
+";
+ let symbols = syms(lang, source);
+ for name in ["namedStore", "defaultStore", "renamedStore"] {
+ assert!(
+ has(&symbols, name, "const"),
+ "{lang}: missing {name}: {symbols:?}"
+ );
+ }
+ for name in [
+ "privateStore",
+ "remoteStore",
+ "publicStore",
+ "mutableStore",
+ "localStore",
+ ] {
+ assert!(!has(&symbols, name, "const"), "{lang}: {name}: {symbols:?}");
+ }
+ let mut stats = ExtractStats::default();
+ let mut records = Vec::new();
+ assert!(emit_code_doc(
+ Path::new("store.ts"),
+ lang,
+ source,
+ &symbols,
+ &mut stats,
+ &mut |record| {
+ records.push(record);
+ true
+ }
+ ));
+ for (name, line) in [("namedStore", 2), ("defaultStore", 4), ("renamedStore", 6)] {
+ let found: Vec<_> = records
+ .iter()
+ .filter(|r| r.fields.get("name").and_then(Value::as_str) == Some(name))
+ .collect();
+ assert_eq!(found.len(), 1, "{lang}: {name}");
+ assert_eq!(found[0].locator, format!("code:{line}:{name}"));
+ assert_eq!(found[0].fields["line"], line);
+ }
+ }
+ }
+
+ #[test]
+ fn issue_2579_type_only_exports_do_not_publish_factory_values() {
+ for lang in ["typescript", "tsx"] {
+ let symbols = syms(
+ lang,
+ "\
+const typeOnly = createStore({});
+export type { typeOnly };
+const specTypeOnly = createStore({});
+export { type specTypeOnly };
+const valueStore = createStore({});
+export { type specTypeOnly as Alias, valueStore };
+",
+ );
+ assert!(has(&symbols, "valueStore", "const"), "{lang}: {symbols:?}");
+ for name in ["typeOnly", "specTypeOnly", "Alias"] {
+ assert!(!has(&symbols, name, "const"), "{lang}: {symbols:?}");
+ }
+ }
+ }
fn has(s: &[Symbol], name: &str, kind: &str) -> bool {
s.iter().any(|s| s.name == name && s.kind == kind)
}
diff --git a/tools/factory-definitions/smoke.py b/tools/factory-definitions/smoke.py
new file mode 100644
index 000000000..0b4aad26c
--- /dev/null
+++ b/tools/factory-definitions/smoke.py
@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""Real autoindex -> CLI/MCP def checks for separately exported factories.
+
+python3 tools/factory-definitions/smoke.py --binary engine/target/ci-test/xerj
+Only synthetic source and a temporary private node are used. No models needed.
+"""
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import signal
+import shutil
+import socket
+import subprocess
+import tempfile
+import time
+from urllib.request import build_opener, ProxyHandler
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--binary', type=Path, required=True)
+ parser.add_argument('--runner', type=Path)
+ parser.add_argument('--estimate-only', action='store_true')
+ args = parser.parse_args()
+ source_binary = args.binary.resolve(strict=True)
+ runner = str(args.runner.resolve(strict=True)) if args.runner else None
+ repo = Path(__file__).resolve().parents[2]
+ env = {k: v for k, v in os.environ.items() if not k.startswith('XERJ_')}
+ env.update(XERJ_DISABLE_FEEDBACK='true', XERJ_NUM_CPUS='4')
+ opener = build_opener(ProxyHandler({}))
+ with tempfile.TemporaryDirectory(prefix='xerj-factory-defs-') as temp:
+ root = Path(temp)
+ snapshot = root / ('xerj.exe' if os.name == 'nt' else 'xerj')
+ shutil.copyfile(source_binary, snapshot)
+ snapshot.chmod(0o700)
+ binary = str(snapshot)
+ print(f'Executable snapshot SHA-256: {hashlib.sha256(snapshot.read_bytes()).hexdigest()}', flush=True)
+ env['XERJ_CONFIG'] = str(root / 'node.toml')
+ # Tiny isolated fixture: do not inherit a shared laptop's 95% reserve.
+ # Leave 1% of the volume reserved; never alter another node's settings.
+ (root / 'node.toml').write_text('[server]\nauto_port = false\n\n[limits]\ndisk_flood_stage_percent = 99\n')
+ with socket.socket() as sock:
+ sock.bind(('127.0.0.1', 0))
+ port = sock.getsockname()[1]
+ url = f'http://127.0.0.1:{port}'
+ source = root / 'source'
+ source.mkdir()
+ expected = []
+ for ext, lang in [('ts', 'typescript'), ('tsx', 'tsx'), ('js', 'javascript')]:
+ generic = '' if ext == 'js' else '<UserState>'
+ names = [f'{ext}InlineStore', f'{ext}NamedStore', f'{ext}DefaultStore']
+ (source / f'store.{ext}').write_text(
+ f'export const {names[0]} = create{generic}()((set) => ({{ user: null }}));\n'
+ f'const {names[1]} = createStore({{}});\n'
+ f'export {{ {names[1]} as publicStore }};\n'
+ f'const {names[2]} = styled.button`color: red;`;\n'
+ f'export default {names[2]};\n'
+ f'const {ext}PrivateStore = createStore({{}});\n')
+ expected.extend((name, lang, f'store.{ext}', line)
+ for name, line in zip(names, [1, 2, 4]))
+
+ def run(command, *, stdin=None, timeout=120):
+ result = subprocess.run(command, input=stdin, text=True, capture_output=True,
+ cwd=root, env=env, timeout=timeout)
+ if result.returncode:
+ raise AssertionError(f'{command}: exit {result.returncode}\n{result.stdout}\n{result.stderr}')
+ if 'autoindex' in command:
+ print(result.stderr, end='', flush=True)
+ return result.stdout
+
+ print(run([binary, '--version']), end='', flush=True)
+
+ def check_definitions():
+ for name, lang, filename, line in expected:
+ cli = json.loads(run([binary, 'def', name, '--prefix', 'fx2579',
+ '--url', url, '--kind', 'const', '--lang', lang, '--json']))
+ messages = [
+ {'jsonrpc': '2.0', 'id': 1, 'method': 'initialize', 'params': {}},
+ {'jsonrpc': '2.0', 'id': 2, 'method': 'tools/call', 'params': {
+ 'name': 'xerj_def', 'arguments': {'symbol': name, 'prefix': 'fx2579',
+ 'kind': 'const', 'lang': lang}}},
+ ]
+ raw = run([binary, 'mcp', '--url', url],
+ stdin=''.join(json.dumps(m) + '\n' for m in messages))
+ reply = next(json.loads(s)['result'] for s in raw.splitlines()
+ if json.loads(s).get('id') == 2)
+ assert not reply.get('isError'), reply
+ mcp = json.loads(reply['content'][0]['text'])
+ assert cli['hits'] == mcp['hits'], (name, cli, mcp)
+ hits = cli['hits']['hits']
+ assert hits, (name, cli)
+ hit = hits[0]['_source']
+ assert hit['name'] == name and hit['kind'] == 'const', hit
+ assert hit['line'] == line and hit['ax_path'].endswith(filename), hit
+ assert name in hit['code'], hit
+ print(f'PASS {len(expected)} definitions: CLI/MCP parity and declaration citations', flush=True)
+
+ for phase in ['first-run', 'restart']:
+ with (root / f'{phase}.log').open('w') as log:
+ node = subprocess.Popen([binary, '--insecure', '--data-dir', str(root / 'data'),
+ '--port', str(port), '--embed-mode', 'lexical'],
+ cwd=root, env=env, stdout=log, stderr=log)
+ try:
+ deadline = time.monotonic() + 60
+ while True:
+ assert node.poll() is None, (root / f'{phase}.log').read_text()
+ try:
+ with opener.open(url, timeout=1) as response:
+ json.load(response)
+ break
+ except OSError:
+ if time.monotonic() >= deadline:
+ raise TimeoutError('private node did not become ready')
+ time.sleep(0.1)
+ if phase == 'first-run':
+ if runner:
+ print(run([runner, '--url', url, '--dir', str(repo / 'engine/tests/es-compat-yaml/yaml')], timeout=300), flush=True)
+ command = [binary, 'autoindex', str(source), '--url', url, '--prefix', 'fx2579',
+ '--state-dir', str(root / 'state'), '--no-graph', '--no-semantic', '--yes']
+ print(run([*command, '--dry-run']), flush=True)
+ if args.estimate_only:
+ return
+ print(run(command), flush=True)
+ check_definitions()
+ print(run(command), flush=True)
+ check_definitions()
+ finally:
+ if node.poll() is None:
+ node.send_signal(signal.SIGTERM)
+ try:
+ node.wait(timeout=30)
+ except subprocess.TimeoutExpired:
+ node.kill()
+ node.wait()
+ raise
+ assert node.returncode == 0, (root / f'{phase}.log').read_text()
+ print('PASS first index, unchanged rerun, graceful stop and durable restart', flush=True)
+
+
+if __name__ == '__main__':
+ main()