Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): refuse --fresh before it can destroy a generation

xerj-org3263388ec210
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
index 231d7d288..1c3af9254 100644
--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
@@ -399,6 +399,23 @@ fn paths(docs: &[Value]) -> Vec<String> {
paths
}
+fn snapshot_names(state_dir: &Path) -> Vec<String> {
+ let mut names: Vec<String> = fs::read_dir(state_dir.join("sync-snapshots"))
+ .map(|entries| {
+ entries
+ .filter_map(Result::ok)
+ .filter(|entry| {
+ entry.file_type().is_ok_and(|kind| kind.is_dir())
+ && !entry.file_name().to_string_lossy().starts_with('.')
+ })
+ .map(|entry| entry.file_name().to_string_lossy().into_owned())
+ .collect()
+ })
+ .unwrap_or_default();
+ names.sort();
+ names
+}
+
fn final_snapshot_count(state_dir: &Path) -> usize {
fs::read_dir(state_dir.join("sync-snapshots"))
.map(|entries| {
@@ -610,3 +627,126 @@ fn pending_semantic_identity_drift_rejects_before_any_further_bulk() {
);
assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);
}
+
+/// `--fresh` must be refused before it can destroy generation authority.
+///
+/// `Journal::open_after_preflight` deletes `journal.ndjson` whenever `fresh` is
+/// set, and `gc_snapshots` then sees an empty protected set and removes every
+/// sealed snapshot. Both run inside the generated branches, ahead of the legacy
+/// plan gate, so without the preflight refusal a single `--fresh` erases the
+/// committed manifest and every sealed source the next run needs.
+#[test]
+fn fresh_cannot_destroy_a_committed_generation() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();
+ fs::write(corpus.path().join("b.csv"), "id,value\n2,bravo\n").unwrap();
+ let endpoint = HttpEndpoint::start();
+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+ let committed_docs = endpoint.data_docs();
+ assert_eq!(paths(&committed_docs), ["a.csv", "b.csv"]);
+
+ let journal_path = state_dir.path().join("journal.ndjson");
+ let journal_before = fs::read(&journal_path).unwrap();
+ let snapshots_before = snapshot_names(state_dir.path());
+ assert_eq!(snapshots_before.len(), 1, "{snapshots_before:?}");
+ let requests_before = endpoint.state.lock().unwrap().requests.len();
+
+ // The folder also changed, which is exactly when a user reaches for
+ // --fresh.
+ fs::remove_file(corpus.path().join("b.csv")).unwrap();
+ config.fresh = true;
+ let error = run_index(config.clone()).unwrap_err();
+ let message = format!("{error:#}");
+ assert!(
+ message.contains("`--fresh` cannot discard an existing plan under the same destination"),
+ "{message}"
+ );
+
+ assert_eq!(
+ fs::read(&journal_path).unwrap(),
+ journal_before,
+ "a refused --fresh must not rewrite the journal"
+ );
+ assert_eq!(
+ snapshot_names(state_dir.path()),
+ snapshots_before,
+ "a refused --fresh must not garbage-collect sealed snapshots"
+ );
+ let attempted = endpoint.state.lock().unwrap().requests[requests_before..].to_vec();
+ assert_eq!(
+ attempted,
+ [("GET".to_owned(), "/".to_owned())],
+ "a refused --fresh may perform only the endpoint-readiness GET"
+ );
+
+ // The surviving authority still reconciles the real folder change.
+ config.fresh = false;
+ let (code, summary) = run_index_report(config).unwrap();
+ assert_eq!(code, 0);
+ assert_eq!(summary.unwrap()["generation"], 2);
+ assert_eq!(paths(&endpoint.data_docs()), ["a.csv"]);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 2);
+}
+
+/// The same refusal on a crashed generation, where the journal additionally
+/// holds the only record of the pending transaction and its sealed source.
+#[test]
+fn fresh_cannot_destroy_a_pending_generation() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap();
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK.lock().unwrap();
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ let source = corpus.path().join("a.csv");
+ fs::write(&source, "id,value\n1,committed\n").unwrap();
+ let endpoint = HttpEndpoint::start();
+ let mut config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+ assert_eq!(run_index(config.clone()).unwrap(), 0);
+
+ fs::write(&source, "id,value\n1,sealed-pending\n2,sealed-second\n").unwrap();
+ endpoint.state.lock().unwrap().fail_next_data_bulk = true;
+ assert!(run_index(config.clone()).is_err());
+ assert_eq!(journal_events(state_dir.path(), "sync_begin"), 2);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 1);
+
+ let journal_path = state_dir.path().join("journal.ndjson");
+ let journal_before = fs::read(&journal_path).unwrap();
+ let snapshots_before = snapshot_names(state_dir.path());
+ assert_eq!(snapshots_before.len(), 2, "{snapshots_before:?}");
+ let requests_before = endpoint.state.lock().unwrap().requests.len();
+
+ config.fresh = true;
+ let error = run_index(config.clone()).unwrap_err();
+ let message = format!("{error:#}");
+ assert!(
+ message.contains("`--fresh` cannot discard an existing plan under the same destination"),
+ "{message}"
+ );
+ assert_eq!(
+ fs::read(&journal_path).unwrap(),
+ journal_before,
+ "a refused --fresh must not rewrite a pending journal"
+ );
+ assert_eq!(
+ snapshot_names(state_dir.path()),
+ snapshots_before,
+ "a refused --fresh must not delete the pending sealed source"
+ );
+ let attempted = endpoint.state.lock().unwrap().requests[requests_before..].to_vec();
+ assert_eq!(
+ attempted,
+ [("GET".to_owned(), "/".to_owned())],
+ "a refused --fresh may perform only the endpoint-readiness GET"
+ );
+
+ // The pending transaction is still resumable from its sealed source.
+ config.fresh = false;
+ assert_eq!(run_index(config).unwrap(), 0);
+ assert_eq!(journal_events(state_dir.path(), "sync_commit"), 2);
+ let rendered = serde_json::to_string(&endpoint.data_docs()).unwrap();
+ assert!(rendered.contains("sealed-pending"), "{rendered}");
+ assert!(rendered.contains("sealed-second"), "{rendered}");
+}
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 47cf70f80..74c58dfe0 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -1404,7 +1404,7 @@ struct InventoryDeltaEntry {
path: String,
}
-#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
struct UnsupportedInventoryDelta {
added_content_groups: Vec<InventoryDeltaEntry>,
vanished_content_groups: Vec<InventoryDeltaEntry>,
@@ -1853,6 +1853,21 @@ pub fn run_index_report(cfg: IndexCfg) -> Result<(i32, Option<Value>)> {
.committed_manifest
.as_ref()
.is_some_and(|manifest| manifest.generation == 0 && manifest.groups.is_empty());
+ // `--fresh` must be refused before the journal is opened. `open_after_
+ // preflight` deletes journal.ndjson whenever `fresh` is set, and the
+ // `gc_snapshots` call that follows every open then sees an empty protected
+ // set and removes every sealed snapshot directory. The legacy plan gate
+ // further down cannot save a generated corpus: both generated branches are
+ // evaluated before it, so the committed manifest and any pending replay
+ // evidence would already be gone. Fail here, before anything is touched, so
+ // that `--fresh` really is "accepted only when the selected state directory
+ // has no durable plan" as the CLI help and docs state.
+ if cfg.fresh
+ && (preflight.pending_sync.is_some()
+ || (preflight.committed_manifest.is_some() && !genesis_recovery))
+ {
+ return Err(UnsupportedInventoryDelta::default().into_fresh_error());
+ }
// A durable sync_begin owns the desired generation. Never rediscover and
// replan from a mutable source tree while that transaction is pending.
// Operation handlers are deliberately not enabled by this foundation