Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): stop opening directories as files — `--no-graph` was dead on Windows (#482) (#907)

xerj-org377b60bbe56a
diff --git a/.github/scripts/autoindex-fd-smoke.sh b/.github/scripts/autoindex-fd-smoke.sh
index 158b3365c..678f85e29 100755
--- a/.github/scripts/autoindex-fd-smoke.sh
+++ b/.github/scripts/autoindex-fd-smoke.sh
@@ -1,5 +1,11 @@
#!/usr/bin/env bash
-# Per-platform regression guard for the autoindex "Too many open files" crash.
+# Per-platform runtime gate for `xerj autoindex`. Two regressions live here:
+#
+# 1. the "Too many open files" (EMFILE) crash — described below;
+# 2. #482, the `--no-graph` ERROR_ACCESS_DENIED abort on Windows — described
+# at its own section further down.
+#
+# Both exist because nothing in CI used to RUN the binary on macOS/Windows.
#
# The crash: `xerj autoindex` on a large repo infers hundreds of datasets =>
# hundreds of indices. Before the fix each index pinned one WAL file descriptor
@@ -93,10 +99,66 @@ if [ "${CREATED:-0}" -lt 300 ]; then
FAIL=1
fi
+# ── the `--no-graph` generated path (#482) ─────────────────────────────────
+# The default (graph) path above never touches `sync-snapshots/`. `--no-graph`
+# does: it seals a durable source snapshot under the state directory and
+# fsyncs the directories it just published. Sealing used `File::open(dir)
+# .sync_all()`, a Unix-only idiom that returns ERROR_ACCESS_DENIED (os error
+# 5) on EVERY Windows call, so `xerj autoindex <folder> --no-graph` aborted on
+# Windows right after the journal was written, before one document was
+# indexed — reported against rc.17 and invisible to this job because it only
+# ever ran the graph path.
+#
+# Two runs, on purpose. The first exercises the seal (create_snapshot_inner);
+# the second exercises snapshot GC over an existing `sync-snapshots/`
+# directory, which was a second, unconditional open of the same kind — so
+# once a Windows user had run `--no-graph` once, EVERY later run over that
+# state directory died too.
+NG_CORPUS="$ROOT/nograph"
+mkdir -p "$NG_CORPUS"
+printf '# beacon report\n\nA short markdown note mentioning a trojan.\n' > "$NG_CORPUS/t.md"
+printf 'ng_id,ng_name,ng_val\n1,alpha,10\n2,beta,20\n' > "$NG_CORPUS/rows.csv"
+NG_STATE="$ROOT/nograph-state"
+
+for attempt in 1 2; do
+ "$BIN" autoindex "$NG_CORPUS" --no-graph --max-minutes 0 \
+ --state-dir "$NG_STATE" --prefix ng > "$DATA/ng-$attempt.log" 2>&1
+ NGRC=$?
+ echo "autoindex --no-graph (run $attempt) exit=$NGRC"
+ tail -20 "$DATA/ng-$attempt.log" || true
+ if [ "$NGRC" != "0" ] && [ "$NGRC" != "3" ]; then
+ echo "::error::--no-graph autoindex run $attempt exited $NGRC (expected 0 or 3) on $(uname -s)"
+ FAIL=1
+ fi
+ # The localised Windows message ("Acceso denegado", "Zugriff verweigert", …)
+ # is why this matches the os error NUMBER and not the text.
+ if grep -qiE 'os error 5([^0-9]|$)' "$DATA/ng-$attempt.log" "$DATA/server.log"; then
+ echo "::error::--no-graph run $attempt hit os error 5 (ACCESS_DENIED) — #482 regression"
+ FAIL=1
+ fi
+done
+
+# A silent no-op must not pass. The seal is the operation that aborted, so
+# assert its output exists: a sealed snapshot directory holding a manifest.
+SEALED="$(find "$NG_STATE/sync-snapshots" -name manifest.json 2>/dev/null | head -1)"
+if [ -z "$SEALED" ]; then
+ echo "::error::--no-graph sealed no snapshot manifest under $NG_STATE/sync-snapshots"
+ FAIL=1
+else
+ echo "sealed snapshot manifest: $SEALED"
+fi
+NG_CREATED="$(curl -s 'localhost:9200/_cat/indices?h=index' 2>/dev/null | grep -cE '(^|[[:space:]])ng-' || true)"
+echo "ng-* indices created: $NG_CREATED"
+if [ "${NG_CREATED:-0}" -lt 1 ]; then
+ echo "::error::--no-graph created no ng-* index"
+ FAIL=1
+fi
+
kill "$SPID" 2>/dev/null || true
if [ "$FAIL" = 0 ]; then
- echo "AUTOINDEX FD SMOKE PASSED on $(uname -s) ($CREATED datasets, no EMFILE)"
+ echo "AUTOINDEX SMOKE PASSED on $(uname -s) ($CREATED datasets, no EMFILE; \
+--no-graph sealed and reconciled, no ACCESS_DENIED)"
else
- echo "AUTOINDEX FD SMOKE FAILED on $(uname -s)"
+ echo "AUTOINDEX SMOKE FAILED on $(uname -s)"
exit 1
fi
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a0cd0f834..e03773e83 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -195,6 +195,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
separate work: it has to prove the #825 union/sum contract with tombstones
present, and is not attempted here.
+- **`xerj autoindex --no-graph` no longer aborts with `Access denied
+ (os error 5)` on Windows**
+ ([#482](https://github.com/xerj-org/xerj/issues/482)). The durable-snapshot
+ path fsynced the directories it had just published with
+ `File::open(dir)?.sync_all()?`. That is a Unix-only idiom: on Windows
+ `File::open` on a *directory* returns `ERROR_ACCESS_DENIED` (os error 5) on
+ every call, because `std` cannot pass `FILE_FLAG_BACKUP_SEMANTICS`. So the
+ first such call — sealing the source snapshot, immediately after the journal
+ was written and before a single document was indexed — killed the run with a
+ context-free, locale-dependent io error (`Acceso denegado. (os error 5)` in
+ the report). A second, unconditional open of the same kind in snapshot GC
+ then failed *every later run* over that state directory, whether or not
+ `--no-graph` was passed again. `xerj-autoindex` now routes directory
+ durability through `xerj_common::fsio::fsync_dir`, which has carried the
+ documented `#[cfg(windows)]` shim since the same mistake stopped the server
+ creating any index at boot; the surrounding filesystem calls also carry
+ context now, so a future failure names the operation instead of only its
+ errno. This costs Windows no durability it had: the code it replaces flushed
+ nothing there — it returned `Err` and ended the run. Windows keeps the
+ engine-wide posture that directory-namespace changes carry no durability
+ claim; file contents inside a snapshot are still fsynced. The
+ windows-latest `autoindex-fd-smoke` CI job now runs the reported
+ `--no-graph` flow twice over one state directory, which is what would have
+ caught this: the job existed, but only ever ran the default graph path,
+ which never touches `sync-snapshots/`.
+
- **A code symbol's retrievable unit is the whole declaration, not the single
physical line the name sits on**
([#500](https://github.com/xerj-org/xerj/issues/500)). Promoting each
diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rs
index f7f9bc0c4..4ee19d851 100644
--- a/engine/crates/xerj-autoindex/src/sync_executor.rs
+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs
@@ -841,7 +841,10 @@ pub fn gc_snapshots(state_dir: &Path, journal: &Journal) -> Result<()> {
let entries = match std::fs::read_dir(&root) {
Ok(entries) => entries,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
- Err(error) => return Err(error.into()),
+ Err(error) => {
+ return Err(anyhow::Error::from(error)
+ .context(format!("list snapshot directory {}", root.display())));
+ }
};
let mut entries = entries.collect::<std::io::Result<Vec<_>>>()?;
entries.sort_by_key(|entry| entry.file_name());
@@ -855,7 +858,6 @@ pub fn gc_snapshots(state_dir: &Path, journal: &Journal) -> Result<()> {
entry.path().display()
);
}
- let directory = File::open(&root)?;
for entry in entries.into_iter().take(SNAPSHOT_GC_BATCH_SIZE) {
let metadata = entry.file_type()?;
if !metadata.is_dir() {
@@ -874,16 +876,23 @@ pub fn gc_snapshots(state_dir: &Path, journal: &Journal) -> Result<()> {
"snapshot tombstone already exists: {}",
tombstone.display()
);
- std::fs::rename(entry.path(), &tombstone)?;
- directory.sync_all()?;
+ std::fs::rename(entry.path(), &tombstone).with_context(|| {
+ format!(
+ "rename snapshot {} to tombstone {}",
+ entry.path().display(),
+ tombstone.display()
+ )
+ })?;
+ sync_dir(&root)?;
#[cfg(test)]
if GC_FAIL_AFTER_RENAME.swap(false, std::sync::atomic::Ordering::SeqCst) {
anyhow::bail!("injected snapshot GC crash after durable tombstone rename");
}
tombstone
};
- std::fs::remove_dir_all(&tombstone)?;
- directory.sync_all()?;
+ std::fs::remove_dir_all(&tombstone)
+ .with_context(|| format!("remove snapshot tombstone {}", tombstone.display()))?;
+ sync_dir(&root)?;
}
Ok(())
}
@@ -1762,9 +1771,22 @@ fn write_synced_json(path: &Path, value: &impl Serialize) -> Result<()> {
Ok(())
}
+/// fsync a directory so the entries created or renamed inside it survive
+/// power loss.
+///
+/// #482: this used to be `File::open(path)?.sync_all()?` — a Unix-only idiom.
+/// On Windows `File::open` on a *directory* fails with `ERROR_ACCESS_DENIED`
+/// (os error 5) on **every** call, because `std` cannot pass
+/// `FILE_FLAG_BACKUP_SEMANTICS`, so sealing a source snapshot aborted the
+/// whole `--no-graph` run before a single document was indexed. Exactly the
+/// same mistake had already been found and fixed once in
+/// [`xerj_common::fsio::fsync_dir`]; route through it instead of re-deriving
+/// it here. Its Windows body is a documented no-op that makes no durability
+/// claim — which costs Windows nothing, because the code it replaces flushed
+/// nothing either: it returned `Err` and killed the run.
fn sync_dir(path: &Path) -> Result<()> {
- File::open(path)?.sync_all()?;
- Ok(())
+ xerj_common::fsio::fsync_dir(path)
+ .with_context(|| format!("fsync snapshot directory {}", path.display()))
}
#[cfg(test)]
@@ -2920,3 +2942,89 @@ mod tests {
assert!(journal.pending_sync.is_some());
}
}
+
+/// Issue #482 regression guard.
+///
+/// The durable-snapshot path (`--no-graph`) fsyncs four directories per seal
+/// plus two per GC step. `File::open` applied to a *directory* is a Unix-only
+/// idiom: on Windows it returns `ERROR_ACCESS_DENIED` (os error 5) for every
+/// call, because `std` cannot pass `FILE_FLAG_BACKUP_SEMANTICS`. Each of
+/// those sites was therefore an unconditional abort of the whole run on that
+/// platform. The engine had already learned this once —
+/// `xerj_common::fsio::fsync_dir` carries the `#[cfg(windows)]` shim and the
+/// write-up — and this module re-derived the broken form anyway.
+///
+/// The fix is `#[cfg(windows)]`-shaped, so no behavioural test compiled for
+/// Unix can tell fixed code from unfixed. The behavioural coverage is the
+/// windows-latest `autoindex-fd-smoke` job, which since this change runs the
+/// reported `--no-graph` flow. This guard is what keeps the idiom from coming
+/// back on the platform the unit suite is never compiled for.
+#[cfg(test)]
+mod windows_directory_open_guard {
+ /// The body of the `fn` whose signature line starts with `header`,
+ /// brace-matched out of this file's own source at compile time.
+ fn function_body(source: &str, header: &str) -> String {
+ let start = source
+ .find(header)
+ .unwrap_or_else(|| panic!("signature moved: {header}"))
+ + header.len();
+ let mut depth = 1usize;
+ for (offset, ch) in source[start..].char_indices() {
+ match ch {
+ '{' => depth += 1,
+ '}' => {
+ depth -= 1;
+ if depth == 0 {
+ return source[start..start + offset].to_string();
+ }
+ }
+ _ => {}
+ }
+ }
+ panic!("unbalanced braces after {header}");
+ }
+
+ /// Every directory this module makes durable, named by the binding the
+ /// code passes around. None of them may reach `File::open`.
+ const DIRECTORY_BINDINGS: [&str; 6] = [
+ "&root",
+ "&staging",
+ "&blobs",
+ "&prepared_dir",
+ "&final_dir",
+ "&snapshot_dir",
+ ];
+
+ #[test]
+ fn no_directory_is_opened_as_a_file() {
+ const SRC: &str = include_str!("sync_executor.rs");
+ for binding in DIRECTORY_BINDINGS {
+ // Assembled, never written out literally, so this test's own
+ // source can never satisfy the search it performs.
+ let forbidden = format!("File::open({binding})");
+ assert!(
+ !SRC.contains(&forbidden),
+ "#482: `{forbidden}` opens a directory as a file. That call returns \
+ ERROR_ACCESS_DENIED (os error 5) on Windows every single time and \
+ aborts the run. Use `sync_dir`, which routes through \
+ `xerj_common::fsio::fsync_dir`."
+ );
+ }
+ }
+
+ #[test]
+ fn sync_dir_delegates_to_the_platform_aware_shim() {
+ const SRC: &str = include_str!("sync_executor.rs");
+ let body = function_body(SRC, "fn sync_dir(path: &Path) -> Result<()> {");
+ assert!(
+ body.contains("xerj_common::fsio::fsync_dir(path)"),
+ "#482: `sync_dir` must delegate to `xerj_common::fsio::fsync_dir`, whose \
+ Windows body is a documented no-op; body was:\n{body}"
+ );
+ assert!(
+ !body.contains("File::open"),
+ "#482: `sync_dir` re-derived the Unix-only `File::open(dir).sync_all()` \
+ idiom, which fails with os error 5 on every Windows call; body was:\n{body}"
+ );
+ }
+}