AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): stop opening directories as files — `--no-graph` was dead on Windows (#482) (#907)
xerj-org377b60bbe56a
diff --git a/.github/scripts/autoindex-fd-smoke.sh b/.github/scripts/autoindex-fd-smoke.shindex 158b3365c..678f85e29 100755--- a/.github/scripts/autoindex-fd-smoke.sh+++ b/.github/scripts/autoindex-fd-smoke.sh@@ -1,5 +1,11 @@#!/usr/bin/env bash-# Per-platform regression guard for the autoindex "Too many open files" crash.+# Per-platform runtime gate for `xerj autoindex`. Two regressions live here:+#+# 1. the "Too many open files" (EMFILE) crash — described below;+# 2. #482, the `--no-graph` ERROR_ACCESS_DENIED abort on Windows — described+# at its own section further down.+#+# Both exist because nothing in CI used to RUN the binary on macOS/Windows.## The crash: `xerj autoindex` on a large repo infers hundreds of datasets =># hundreds of indices. Before the fix each index pinned one WAL file descriptor@@ -93,10 +99,66 @@ if [ "${CREATED:-0}" -lt 300 ]; thenFAIL=1fi+# ── the `--no-graph` generated path (#482) ─────────────────────────────────+# The default (graph) path above never touches `sync-snapshots/`. `--no-graph`+# does: it seals a durable source snapshot under the state directory and+# fsyncs the directories it just published. Sealing used `File::open(dir)+# .sync_all()`, a Unix-only idiom that returns ERROR_ACCESS_DENIED (os error+# 5) on EVERY Windows call, so `xerj autoindex <folder> --no-graph` aborted on+# Windows right after the journal was written, before one document was+# indexed — reported against rc.17 and invisible to this job because it only+# ever ran the graph path.+#+# Two runs, on purpose. The first exercises the seal (create_snapshot_inner);+# the second exercises snapshot GC over an existing `sync-snapshots/`+# directory, which was a second, unconditional open of the same kind — so+# once a Windows user had run `--no-graph` once, EVERY later run over that+# state directory died too.+NG_CORPUS="$ROOT/nograph"+mkdir -p "$NG_CORPUS"+printf '# beacon report\n\nA short markdown note mentioning a trojan.\n' > "$NG_CORPUS/t.md"+printf 'ng_id,ng_name,ng_val\n1,alpha,10\n2,beta,20\n' > "$NG_CORPUS/rows.csv"+NG_STATE="$ROOT/nograph-state"++for attempt in 1 2; do+ "$BIN" autoindex "$NG_CORPUS" --no-graph --max-minutes 0 \+ --state-dir "$NG_STATE" --prefix ng > "$DATA/ng-$attempt.log" 2>&1+ NGRC=$?+ echo "autoindex --no-graph (run $attempt) exit=$NGRC"+ tail -20 "$DATA/ng-$attempt.log" || true+ if [ "$NGRC" != "0" ] && [ "$NGRC" != "3" ]; then+ echo "::error::--no-graph autoindex run $attempt exited $NGRC (expected 0 or 3) on $(uname -s)"+ FAIL=1+ fi+ # The localised Windows message ("Acceso denegado", "Zugriff verweigert", …)+ # is why this matches the os error NUMBER and not the text.+ if grep -qiE 'os error 5([^0-9]|$)' "$DATA/ng-$attempt.log" "$DATA/server.log"; then+ echo "::error::--no-graph run $attempt hit os error 5 (ACCESS_DENIED) — #482 regression"+ FAIL=1+ fi+done++# A silent no-op must not pass. The seal is the operation that aborted, so+# assert its output exists: a sealed snapshot directory holding a manifest.+SEALED="$(find "$NG_STATE/sync-snapshots" -name manifest.json 2>/dev/null | head -1)"+if [ -z "$SEALED" ]; then+ echo "::error::--no-graph sealed no snapshot manifest under $NG_STATE/sync-snapshots"+ FAIL=1+else+ echo "sealed snapshot manifest: $SEALED"+fi+NG_CREATED="$(curl -s 'localhost:9200/_cat/indices?h=index' 2>/dev/null | grep -cE '(^|[[:space:]])ng-' || true)"+echo "ng-* indices created: $NG_CREATED"+if [ "${NG_CREATED:-0}" -lt 1 ]; then+ echo "::error::--no-graph created no ng-* index"+ FAIL=1+fi+kill "$SPID" 2>/dev/null || trueif [ "$FAIL" = 0 ]; then- echo "AUTOINDEX FD SMOKE PASSED on $(uname -s) ($CREATED datasets, no EMFILE)"+ echo "AUTOINDEX SMOKE PASSED on $(uname -s) ($CREATED datasets, no EMFILE; \+--no-graph sealed and reconciled, no ACCESS_DENIED)"else- echo "AUTOINDEX FD SMOKE FAILED on $(uname -s)"+ echo "AUTOINDEX SMOKE FAILED on $(uname -s)"exit 1fidiff --git a/CHANGELOG.md b/CHANGELOG.mdindex a0cd0f834..e03773e83 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -195,6 +195,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0separate work: it has to prove the #825 union/sum contract with tombstonespresent, and is not attempted here.+- **`xerj autoindex --no-graph` no longer aborts with `Access denied+ (os error 5)` on Windows**+ ([#482](https://github.com/xerj-org/xerj/issues/482)). The durable-snapshot+ path fsynced the directories it had just published with+ `File::open(dir)?.sync_all()?`. That is a Unix-only idiom: on Windows+ `File::open` on a *directory* returns `ERROR_ACCESS_DENIED` (os error 5) on+ every call, because `std` cannot pass `FILE_FLAG_BACKUP_SEMANTICS`. So the+ first such call — sealing the source snapshot, immediately after the journal+ was written and before a single document was indexed — killed the run with a+ context-free, locale-dependent io error (`Acceso denegado. (os error 5)` in+ the report). A second, unconditional open of the same kind in snapshot GC+ then failed *every later run* over that state directory, whether or not+ `--no-graph` was passed again. `xerj-autoindex` now routes directory+ durability through `xerj_common::fsio::fsync_dir`, which has carried the+ documented `#[cfg(windows)]` shim since the same mistake stopped the server+ creating any index at boot; the surrounding filesystem calls also carry+ context now, so a future failure names the operation instead of only its+ errno. This costs Windows no durability it had: the code it replaces flushed+ nothing there — it returned `Err` and ended the run. Windows keeps the+ engine-wide posture that directory-namespace changes carry no durability+ claim; file contents inside a snapshot are still fsynced. The+ windows-latest `autoindex-fd-smoke` CI job now runs the reported+ `--no-graph` flow twice over one state directory, which is what would have+ caught this: the job existed, but only ever ran the default graph path,+ which never touches `sync-snapshots/`.+- **A code symbol's retrievable unit is the whole declaration, not the singlephysical line the name sits on**([#500](https://github.com/xerj-org/xerj/issues/500)). Promoting eachdiff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rsindex f7f9bc0c4..4ee19d851 100644--- a/engine/crates/xerj-autoindex/src/sync_executor.rs+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs@@ -841,7 +841,10 @@ pub fn gc_snapshots(state_dir: &Path, journal: &Journal) -> Result<()> {let entries = match std::fs::read_dir(&root) {Ok(entries) => entries,Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),- Err(error) => return Err(error.into()),+ Err(error) => {+ return Err(anyhow::Error::from(error)+ .context(format!("list snapshot directory {}", root.display())));+ }};let mut entries = entries.collect::<std::io::Result<Vec<_>>>()?;entries.sort_by_key(|entry| entry.file_name());@@ -855,7 +858,6 @@ pub fn gc_snapshots(state_dir: &Path, journal: &Journal) -> Result<()> {entry.path().display());}- let directory = File::open(&root)?;for entry in entries.into_iter().take(SNAPSHOT_GC_BATCH_SIZE) {let metadata = entry.file_type()?;if !metadata.is_dir() {@@ -874,16 +876,23 @@ pub fn gc_snapshots(state_dir: &Path, journal: &Journal) -> Result<()> {"snapshot tombstone already exists: {}",tombstone.display());- std::fs::rename(entry.path(), &tombstone)?;- directory.sync_all()?;+ std::fs::rename(entry.path(), &tombstone).with_context(|| {+ format!(+ "rename snapshot {} to tombstone {}",+ entry.path().display(),+ tombstone.display()+ )+ })?;+ sync_dir(&root)?;#[cfg(test)]if GC_FAIL_AFTER_RENAME.swap(false, std::sync::atomic::Ordering::SeqCst) {anyhow::bail!("injected snapshot GC crash after durable tombstone rename");}tombstone};- std::fs::remove_dir_all(&tombstone)?;- directory.sync_all()?;+ std::fs::remove_dir_all(&tombstone)+ .with_context(|| format!("remove snapshot tombstone {}", tombstone.display()))?;+ sync_dir(&root)?;}Ok(())}@@ -1762,9 +1771,22 @@ fn write_synced_json(path: &Path, value: &impl Serialize) -> Result<()> {Ok(())}+/// fsync a directory so the entries created or renamed inside it survive+/// power loss.+///+/// #482: this used to be `File::open(path)?.sync_all()?` — a Unix-only idiom.+/// On Windows `File::open` on a *directory* fails with `ERROR_ACCESS_DENIED`+/// (os error 5) on **every** call, because `std` cannot pass+/// `FILE_FLAG_BACKUP_SEMANTICS`, so sealing a source snapshot aborted the+/// whole `--no-graph` run before a single document was indexed. Exactly the+/// same mistake had already been found and fixed once in+/// [`xerj_common::fsio::fsync_dir`]; route through it instead of re-deriving+/// it here. Its Windows body is a documented no-op that makes no durability+/// claim — which costs Windows nothing, because the code it replaces flushed+/// nothing either: it returned `Err` and killed the run.fn sync_dir(path: &Path) -> Result<()> {- File::open(path)?.sync_all()?;- Ok(())+ xerj_common::fsio::fsync_dir(path)+ .with_context(|| format!("fsync snapshot directory {}", path.display()))}#[cfg(test)]@@ -2920,3 +2942,89 @@ mod tests {assert!(journal.pending_sync.is_some());}}++/// Issue #482 regression guard.+///+/// The durable-snapshot path (`--no-graph`) fsyncs four directories per seal+/// plus two per GC step. `File::open` applied to a *directory* is a Unix-only+/// idiom: on Windows it returns `ERROR_ACCESS_DENIED` (os error 5) for every+/// call, because `std` cannot pass `FILE_FLAG_BACKUP_SEMANTICS`. Each of+/// those sites was therefore an unconditional abort of the whole run on that+/// platform. The engine had already learned this once —+/// `xerj_common::fsio::fsync_dir` carries the `#[cfg(windows)]` shim and the+/// write-up — and this module re-derived the broken form anyway.+///+/// The fix is `#[cfg(windows)]`-shaped, so no behavioural test compiled for+/// Unix can tell fixed code from unfixed. The behavioural coverage is the+/// windows-latest `autoindex-fd-smoke` job, which since this change runs the+/// reported `--no-graph` flow. This guard is what keeps the idiom from coming+/// back on the platform the unit suite is never compiled for.+#[cfg(test)]+mod windows_directory_open_guard {+ /// The body of the `fn` whose signature line starts with `header`,+ /// brace-matched out of this file's own source at compile time.+ fn function_body(source: &str, header: &str) -> String {+ let start = source+ .find(header)+ .unwrap_or_else(|| panic!("signature moved: {header}"))+ + header.len();+ let mut depth = 1usize;+ for (offset, ch) in source[start..].char_indices() {+ match ch {+ '{' => depth += 1,+ '}' => {+ depth -= 1;+ if depth == 0 {+ return source[start..start + offset].to_string();+ }+ }+ _ => {}+ }+ }+ panic!("unbalanced braces after {header}");+ }++ /// Every directory this module makes durable, named by the binding the+ /// code passes around. None of them may reach `File::open`.+ const DIRECTORY_BINDINGS: [&str; 6] = [+ "&root",+ "&staging",+ "&blobs",+ "&prepared_dir",+ "&final_dir",+ "&snapshot_dir",+ ];++ #[test]+ fn no_directory_is_opened_as_a_file() {+ const SRC: &str = include_str!("sync_executor.rs");+ for binding in DIRECTORY_BINDINGS {+ // Assembled, never written out literally, so this test's own+ // source can never satisfy the search it performs.+ let forbidden = format!("File::open({binding})");+ assert!(+ !SRC.contains(&forbidden),+ "#482: `{forbidden}` opens a directory as a file. That call returns \+ ERROR_ACCESS_DENIED (os error 5) on Windows every single time and \+ aborts the run. Use `sync_dir`, which routes through \+ `xerj_common::fsio::fsync_dir`."+ );+ }+ }++ #[test]+ fn sync_dir_delegates_to_the_platform_aware_shim() {+ const SRC: &str = include_str!("sync_executor.rs");+ let body = function_body(SRC, "fn sync_dir(path: &Path) -> Result<()> {");+ assert!(+ body.contains("xerj_common::fsio::fsync_dir(path)"),+ "#482: `sync_dir` must delegate to `xerj_common::fsio::fsync_dir`, whose \+ Windows body is a documented no-op; body was:\n{body}"+ );+ assert!(+ !body.contains("File::open"),+ "#482: `sync_dir` re-derived the Unix-only `File::open(dir).sync_all()` \+ idiom, which fails with os error 5 on every Windows call; body was:\n{body}"+ );+ }+}