Skip to content

armgit / xerj

Public Imported from GitHub
0

AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.

fix(autoindex): never journal a file past a record the server refused (#2807) (#2816)

Jake Elbirtf30ece8acfea
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0623ecd94..f3cdae6b0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -456,6 +456,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
XML, including `.xml`, `.xaml` and `.svg`, is unchanged. An index built
before this fix keeps its junked files until each one changes or the folder
is indexed again with `--fresh` (#2740).
+- When the server refuses a record or a graph edge, `xerj autoindex` now
+ names the file it came from and does not mark that file done. It used to
+ mark every file in the same bulk done, so the rerun the abort message asks
+ for resumed past the file, reported success and never sent the refused
+ record (#2807).
+- A file whose record the server refuses as over its 8 MiB per-document
+ limit is now set aside as junk, with its file name and the server's reason,
+ and the run completes with exit 3. Its other records are removed and it is
+ retried on the next run. It used to abort the whole run, every rerun, until
+ `--fresh`. On a `semantic_text` field the vectors the server adds count
+ toward that limit: a 2 MB generated `.d.ts` file grew to 16.6 MB this way.
+ `--no-semantic` may index such a file (#2807).
+- `xerj corpus index` and `xerj autoindex --no-graph` now name the file a
+ refused record came from, and for a size refusal they say how to index the
+ rest: under a new `--prefix` and `--state-dir`, with the file left out
+ (`--exclude`) or with `--no-semantic`. That route fixes its file list before
+ it sends anything, so it cannot set the file aside, and a rerun repeats the
+ refusal. It used to say only "prepared bulk contained 1 rejected items"
+ (#2807).
## [1.0.0-rc.100] - 2026-10-26
diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rs
index dc77d4402..2aa9d4fb7 100644
--- a/engine/crates/xerj-autoindex/src/esclient.rs
+++ b/engine/crates/xerj-autoindex/src/esclient.rs
@@ -964,6 +964,10 @@ fn is_whole_request_too_large(v: &Value, sent_actions: Option<usize>) -> bool {
== Some(413)
}
+/// The server's wording for a document over its per-document byte limit
+/// (`xerj_common::document_limit`), however the item's status reads.
+const DOCUMENT_LIMIT_REASON: &str = "byte limit for one indexed JSON document";
+
/// Read a bulk response body item by item.
fn parse_round(v: &Value) -> Round {
let items = v.get("items").and_then(|i| i.as_array());
@@ -1018,8 +1022,20 @@ fn parse_round(v: &Value) -> Round {
if round.outcome.first_server_error.is_none() {
round.outcome.first_server_error = Some(error_text());
}
- } else if round.first_settled_error.is_none() {
- round.first_settled_error = Some(op["error"].to_string().chars().take(300).collect());
+ } else {
+ if round.first_settled_error.is_none() {
+ round.first_settled_error =
+ Some(op["error"].to_string().chars().take(300).collect());
+ }
+ let reason = op["error"]
+ .get("reason")
+ .and_then(Value::as_str)
+ .unwrap_or("");
+ round.outcome.refused.push(RefusedItem {
+ id: op.get("_id").and_then(Value::as_str).map(str::to_owned),
+ error: error_text(),
+ too_large: item_status == 413 || reason.contains(DOCUMENT_LIMIT_REASON),
+ });
}
if round.outcome.first_error.is_none() {
round.outcome.first_error = Some(op["error"].to_string().chars().take(300).collect());
@@ -1028,9 +1044,22 @@ fn parse_round(v: &Value) -> Round {
round
}
+/// One item the server refused for the record itself (not a 429, 5xx or
+/// write block), with the `_id` the bulk action named.
+#[derive(Debug, Clone)]
+pub struct RefusedItem {
+ pub id: Option<String>,
+ pub error: String,
+ /// Refused because the stored document passed the server's per-document
+ /// byte limit, which a retry of the same record cannot clear (#2807).
+ pub too_large: bool,
+}
+
#[derive(Debug, Default)]
pub struct BulkOutcome {
pub item_errors: u64,
+ /// The record-level refusals counted in `item_errors`, in item order.
+ pub refused: Vec<RefusedItem>,
/// Per-item backend/admission failures — 5xx/429 statuses plus
/// cluster/index write blocks recognised by TYPE (see
/// [`is_index_block_error`]). These are not bad source records: the same
@@ -1054,6 +1083,7 @@ impl BulkOutcome {
/// Fold the outcome of a later window of the same body into this one.
fn absorb(&mut self, other: BulkOutcome) {
self.item_errors += other.item_errors;
+ self.refused.extend(other.refused);
self.server_errors += other.server_errors;
self.throttled_out += other.throttled_out;
if self.first_error.is_none() {
@@ -1940,6 +1970,9 @@ impl Es {
// 400): the record is not re-offered, its error is still reported.
let mut settled_item_errors = 0u64;
let mut settled_first_error: Option<String> = None;
+ // Their refused items too, so a refusal from an earlier round is still
+ // traced to its file after a 429 retry (#2807).
+ let mut settled_refused: Vec<RefusedItem> = Vec::new();
// Set once any round took an item. From then on the full patience
// applies, even if later rounds are all 429: a drain in progress
// (#944 held 57 s) must be allowed to finish.
@@ -1954,6 +1987,9 @@ impl Es {
}
let mut outcome = round.outcome;
outcome.item_errors += settled_item_errors;
+ let mut refused = std::mem::take(&mut settled_refused);
+ refused.append(&mut outcome.refused);
+ outcome.refused = refused;
if outcome.first_error.is_none() {
outcome.first_error = settled_first_error.clone();
}
@@ -2001,6 +2037,7 @@ impl Es {
settled_item_errors = outcome
.item_errors
.saturating_sub(round.throttled.len() as u64);
+ settled_refused = outcome.refused.clone();
if round.first_settled_error.is_some() {
settled_first_error = round.first_settled_error;
}
diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
index 53629e910..065dec612 100644
--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs
@@ -163,6 +163,9 @@ struct MockState {
/// and the type is not a block, so the run continues and must still
/// report the rejection.
reject_first_data_item: bool,
+ /// As `reject_first_data_item`, with the server's per-document size
+ /// refusal, which a retry cannot clear (#2807).
+ reject_first_data_item_as_too_large: bool,
/// Opt-in (#929): refuse the create-index PUT — which on this path carries
/// the dataset's whole mapping — of every index whose name contains one of
/// these, with the 400 a real engine returns for a field it cannot map.
@@ -916,15 +919,27 @@ fn bulk_response(body: &[u8], state: &Arc<Mutex<MockState>>) -> Value {
if locked.swallow_data_bulks {
return json!({"errors": false, "items": []});
}
- if locked.reject_first_data_item {
+ if locked.reject_first_data_item || locked.reject_first_data_item_as_too_large {
+ let too_large = locked.reject_first_data_item_as_too_large;
let mut items = Vec::new();
for (nth, pair) in lines.as_chunks::<2>().0.iter().enumerate() {
if nth == 0 {
+ let action: Value = serde_json::from_slice(pair[0]).unwrap();
items.push(json!({"index": {
+ "_id": action.pointer("/index/_id").cloned().unwrap_or(Value::Null),
"status": 400,
- "error": {
- "type": "document_parsing_exception",
- "reason": "refused: value could not be parsed"
+ "error": if too_large {
+ json!({
+ "type": "engine_exception",
+ "reason": "document is 16590136 bytes, over the 8388608 byte \
+ limit for one indexed JSON document; it was not \
+ indexed. Split it into smaller documents"
+ })
+ } else {
+ json!({
+ "type": "document_parsing_exception",
+ "reason": "refused: value could not be parsed"
+ })
}
}}));
continue;
@@ -3934,6 +3949,9 @@ fn write_block_rejections_are_backend_fatal_and_the_file_stays_pending() {
/// The rejection count is therefore reported in the one place a rejected run
/// ever reaches: the abort message, where it supplies the scale that
/// `bulk_errors`' first-five sample cannot.
+///
+/// The file that carried the refused record is named and stays un-journaled,
+/// so the rerun the message asks for really retries it (#2807).
#[test]
fn backend_rejected_records_abort_the_run_before_any_map_is_written() {
let _guard = FAILPOINT_TEST_LOCK
@@ -3966,23 +3984,24 @@ fn backend_rejected_records_abort_the_run_before_any_map_is_written() {
"the abort must state how many records were refused, not just sample the errors: {error}"
);
- // Measured, and deliberately pinned even though it is NOT what the abort
- // message implies: `record_bulk_outcome` treats a per-item rejection as
- // non-fatal to the worker (only `server_errors` set `send_err`), so the
- // file is journaled COMPLETE and the run aborts only at the end. A rerun
- // therefore resumes past this file and never retries the refused record,
- // while the message says failed sources "were not journaled complete".
- //
- // That divergence predates this branch — it is `main`'s behaviour for the
- // item-error class — and fixing it means changing when a worker treats a
- // rejection as fatal, which is a resume-semantics change with nothing to
- // do with map metadata. Recorded here so the next reader finds the fact
- // rather than the assumption; tracked in the PR body as a known gap.
+ // #2807: the refused record's source file is named and is NOT journaled
+ // complete, as the message says. It used to be journaled, so a rerun
+ // resumed past it, reported success and never sent the refused record.
+ assert!(
+ error.contains("records.csv"),
+ "the abort must name the file whose record was refused: {error}"
+ );
assert_eq!(
file_done_count(state_dir.path()),
- 1,
- "pre-existing: a per-item rejection still journals the source complete"
+ 0,
+ "a source with a refused record must not be journaled complete"
);
+
+ // With the refusal gone, the same command resumes and sends every record.
+ endpoint.state.lock().unwrap().reject_first_data_item = false;
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);
+ run_index_report(config).unwrap();
+ assert_eq!(file_done_count(state_dir.path()), 1);
}
/// #195 last-resort gate: a backend that ACCEPTS every bulk but persists
@@ -5211,3 +5230,59 @@ fn a_refused_dataset_on_the_graph_path_costs_that_dataset_not_the_run() {
.unwrap();
assert!(done.contains("datasets_refused=1"), "{done}");
}
+
+/// #2807: a record the server refuses as over its per-document size limit is
+/// refused again on every rerun, so its file is set aside as junk and the run
+/// completes (exit 3) instead of aborting. The file's accepted records are
+/// removed, it is not journaled, and it indexes once the refusal goes away.
+#[test]
+fn a_record_refused_as_too_large_sets_its_file_aside() {
+ let _guard = FAILPOINT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|p| p.into_inner());
+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|p| p.into_inner());
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(
+ corpus.path().join("records.csv"),
+ "id,value\n0,kept\n1,kept\n2,kept\n",
+ )
+ .unwrap();
+ let endpoint = MockEndpoint::start(usize::MAX);
+ endpoint
+ .state
+ .lock()
+ .unwrap()
+ .reject_first_data_item_as_too_large = true;
+
+ for round in 0..2 {
+ let (code, _) = run_index_report(cfg(corpus.path(), state_dir.path(), &endpoint.url))
+ .unwrap_or_else(|error| panic!("round {round} must not abort: {error:#}"));
+ assert_eq!(code, 3, "round {round}: completed with junk");
+ let state = endpoint.state.lock().unwrap();
+ let catalog = serde_json::to_string(&state.catalog_docs).unwrap();
+ assert!(
+ catalog.contains("set aside at index time") && catalog.contains("records.csv"),
+ "round {round}: the file is listed as junk with the reason: {catalog}"
+ );
+ assert!(
+ !serde_json::to_string(&state.docs)
+ .unwrap()
+ .contains("\"kept\""),
+ "round {round}: the file's accepted records are removed"
+ );
+ drop(state);
+ assert_eq!(file_done_count(state_dir.path()), 0, "round {round}");
+ }
+
+ endpoint
+ .state
+ .lock()
+ .unwrap()
+ .reject_first_data_item_as_too_large = false;
+ let (code, _) = run_index_report(cfg(corpus.path(), state_dir.path(), &endpoint.url)).unwrap();
+ assert_eq!(code, 0);
+ assert_eq!(file_done_count(state_dir.path()), 1);
+}
diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
index abbb577d6..dcd9e7211 100644
--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs
@@ -25,6 +25,9 @@ struct HttpState {
search_bodies: Vec<Value>,
data_bulk_requests: usize,
fail_next_data_bulk: bool,
+ /// #2807: refuse the first `index` action of the next data bulk as over
+ /// the per-document byte limit, echoing its `_id` as the server does.
+ refuse_next_data_item_as_too_large: bool,
/// Apply the first half of the next data bulk's items and *then* report
/// `errors: true`. This is the dangerous backend shape the legacy
/// replacement transaction was hardened against (visibility changed, but
@@ -588,6 +591,28 @@ fn bulk_http(body: &[u8], state: &Arc<Mutex<HttpState>>) -> (u16, Value) {
if numbered {
locked.fail_data_bulk_number = None;
}
+ if std::mem::take(&mut locked.refuse_next_data_item_as_too_large) {
+ let action: Value = serde_json::from_slice(lines[0]).unwrap();
+ let id = action["index"]["_id"]
+ .as_str()
+ .unwrap_or_default()
+ .to_owned();
+ return (
+ 200,
+ json!({
+ "errors": true,
+ "items": [{"index": {
+ "_id": id,
+ "status": 413,
+ "error": {
+ "type": "engine_exception",
+ "reason": "document is 16590136 bytes, over the 8388608 byte limit for one indexed JSON document; it was not indexed. Split it into smaller documents",
+ "status": 413
+ }
+ }}]
+ }),
+ );
+ }
if std::mem::take(&mut locked.fail_next_data_bulk) || numbered {
return (
200,
@@ -7676,3 +7701,38 @@ fn a_folder_of_templates_and_html_fragments_is_indexed_not_junked() {
);
}
}
+
+/// #2807 on the generated (`--no-graph`) path, which `xerj corpus index` uses.
+/// That path seals the generation before it sends, so a file the server
+/// refuses for size cannot be set aside there. The abort must at least name
+/// the file and a way out that works; it used to say only "prepared bulk
+/// contained 1 rejected items".
+#[test]
+fn a_record_refused_for_size_on_the_generated_path_names_its_file() {
+ let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());
+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK
+ .lock()
+ .unwrap_or_else(|p| p.into_inner());
+ let corpus = tempfile::tempdir().unwrap();
+ let state_dir = tempfile::tempdir().unwrap();
+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();
+ let endpoint = HttpEndpoint::start();
+ endpoint
+ .state
+ .lock()
+ .unwrap()
+ .refuse_next_data_item_as_too_large = true;
+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);
+
+ let error = format!("{:#}", run_index(config).unwrap_err());
+ assert!(
+ error.contains("from a.csv"),
+ "the file must be named: {error}"
+ );
+ assert!(error.contains("16590136 bytes"), "{error}");
+ assert!(
+ error.contains("--no-semantic") && error.contains("--exclude"),
+ "the abort must give a way out that works: {error}"
+ );
+ assert!(!error.contains("prepared bulk contained"), "{error}");
+}
diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rs
index 940f80071..878f499f3 100644
--- a/engine/crates/xerj-autoindex/src/lib.rs
+++ b/engine/crates/xerj-autoindex/src/lib.rs
@@ -3451,6 +3451,17 @@ fn record_bulk_outcome(
// ─── phase-B bulk coalescing (#1147) ─────────────────────────────────────
+/// One flushed bulk: the body, the tokens with records in it, and the
+/// token behind each `_id`.
+type PipeJob = (Vec<u8>, Vec<u64>, HashMap<String, u64>);
+
+/// The `_id` an NDJSON bulk action line names, if any.
+fn bulk_action_id(action: &[u8]) -> Option<String> {
+ let value: Value = serde_json::from_slice(action).ok()?;
+ let op = value.as_object()?.values().next()?;
+ op.get("_id")?.as_str().map(str::to_owned)
+}
+
/// Shared mutable state of [`BulkPipe`]. One lock covers the buffer and the
/// per-token bookkeeping so a flush is atomic against submissions.
#[derive(Default)]
@@ -3462,6 +3473,19 @@ struct PipeState {
/// Tokens with bytes in `buf`, in submission order. A token appears once
/// even across several chunk submissions (one file larger than the cut).
pending: Vec<u64>,
+ /// The token that submitted each `_id` in `buf`, so a refused item is
+ /// charged to its own file rather than to every file in the bulk (#2807).
+ owners: HashMap<String, u64>,
+ /// Tokens whose every refused record was over the server's per-document
+ /// byte limit, with the server's reason. A retry cannot clear that, so the
+ /// file is set aside instead of failing the run (#2807).
+ too_large: HashMap<u64, String>,
+ /// Tokens that also failed for any other reason. Never set aside: one
+ /// refusal a rerun could clear is enough to fail the run instead.
+ other_refusal: HashSet<u64>,
+ /// Bulks in flight per token, so a set-aside purge waits for the file's
+ /// last records to land before deleting them.
+ in_flight_tokens: HashMap<u64, usize>,
/// Terminal result per token, observed (and removed) by the token's
/// owner in [`BulkPipe::settle`].
done: std::collections::HashMap<u64, Result<(), String>>,
@@ -3539,7 +3563,6 @@ struct BulkPipe<'a, F> {
bulk_cut: usize,
doc_cap: usize,
rejected_records: &'a AtomicU64,
- bulk_errors: &'a Mutex<Vec<String>>,
state: Mutex<PipeState>,
ready: Condvar,
}
@@ -3553,7 +3576,6 @@ where
bulk_cut: usize,
doc_cap: usize,
rejected_records: &'a AtomicU64,
- bulk_errors: &'a Mutex<Vec<String>>,
workers: usize,
) -> Self {
Self {
@@ -3561,7 +3583,6 @@ where
bulk_cut: bulk_cut.max(1),
doc_cap: doc_cap.max(1),
rejected_records,
- bulk_errors,
state: Mutex::new(PipeState {
live_workers: workers,
..PipeState::default()
@@ -3601,6 +3622,11 @@ where
if !st.pending.contains(&token) {
st.pending.push(token);
}
+ for action in bytes.split(|b| *b == b'\n').step_by(2) {
+ if let Some(id) = bulk_action_id(action) {
+ st.owners.insert(id, token);
+ }
+ }
st.buf.extend_from_slice(&bytes);
st.docs += docs;
if st.buf.len() < self.bulk_cut && st.docs < self.doc_cap {
@@ -3623,7 +3649,43 @@ where
};
drop(st);
let outcome = (self.send)(job.0);
- self.deliver(job.1, outcome);
+ self.deliver(job.1, job.2, outcome);
+ }
+
+ /// The server's reason when this token failed only on records over the
+ /// per-document byte limit. Taken once, by the file's own worker.
+ fn take_too_large(&self, token: u64) -> Option<String> {
+ self.lock().too_large.remove(&token)
+ }
+
+ /// Block until no bulk holding this file's records is buffered or in
+ /// flight. A failed token settles at its first refusal, while its later
+ /// chunks may still be on the wire; a purge before they land would leave
+ /// a junk file half searchable.
+ fn quiesce(&self, token: u64) {
+ let mut st = self.lock();
+ loop {
+ let buffered = st.pending.contains(&token);
+ if !buffered && !st.in_flight_tokens.contains_key(&token) {
+ return;
+ }
+ if buffered {
+ if let Some(job) = self.take_job(&mut st) {
+ drop(st);
+ let outcome = (self.send)(job.0);
+ self.deliver(job.1, job.2, outcome);
+ st = self.lock();
+ continue;
+ }
+ }
+ st.parked_workers += 1;
+ let (guard, _) = self
+ .ready
+ .wait_timeout(st, std::time::Duration::from_secs(30))
+ .unwrap_or_else(|p| p.into_inner());
+ st = guard;
+ st.parked_workers -= 1;
+ }
}
/// Whether any bulk carrying this file's records has failed. The drain
@@ -3655,7 +3717,7 @@ where
if let Some(job) = self.take_job(&mut st) {
drop(st);
let outcome = (self.send)(job.0);
- self.deliver(job.1, outcome);
+ self.deliver(job.1, job.2, outcome);
st = self.lock();
continue;
}
@@ -3695,7 +3757,7 @@ where
if let Some(job) = self.take_job(&mut st) {
drop(st);
let outcome = (self.send)(job.0);
- self.deliver(job.1, outcome);
+ self.deliver(job.1, job.2, outcome);
st = self.lock();
continue;
}
@@ -3713,22 +3775,42 @@ where
/// Detach the buffer as a send job if a wire slot is free. Called with
/// the lock held; the caller drops the lock before sending.
- fn take_job(&self, st: &mut PipeState) -> Option<(Vec<u8>, Vec<u64>)> {
+ fn take_job(&self, st: &mut PipeState) -> Option<PipeJob> {
if st.buf.is_empty() || st.in_flight >= MAX_IN_FLIGHT {
return None;
}
st.in_flight += 1;
st.docs = 0;
- Some((std::mem::take(&mut st.buf), std::mem::take(&mut st.pending)))
+ for t in &st.pending {
+ *st.in_flight_tokens.entry(*t).or_default() += 1;
+ }
+ Some((
+ std::mem::take(&mut st.buf),
+ std::mem::take(&mut st.pending),
+ std::mem::take(&mut st.owners),
+ ))
}
/// Resolve a completed job: reclaim the wire slot, settle every token it
/// carried, wake the parked. Same outcome handling as
/// `record_bulk_outcome`, resolved per token instead of through the
/// caller's `send_err`.
- fn deliver(&self, tokens: Vec<u64>, outcome: anyhow::Result<crate::esclient::BulkOutcome>) {
+ fn deliver(
+ &self,
+ tokens: Vec<u64>,
+ owners: HashMap<String, u64>,
+ outcome: anyhow::Result<crate::esclient::BulkOutcome>,
+ ) {
let mut st = self.lock();
st.in_flight = st.in_flight.saturating_sub(1);
+ for t in &tokens {
+ if let Some(n) = st.in_flight_tokens.get_mut(t) {
+ *n -= 1;
+ if *n == 0 {
+ st.in_flight_tokens.remove(t);
+ }
+ }
+ }
match outcome {
Ok(outcome) => {
if outcome.server_errors > 0 {
@@ -3741,19 +3823,73 @@ where
);
for t in &tokens {
st.failed.insert(*t);
+ st.other_refusal.insert(*t);
+ st.too_large.remove(t);
st.done.insert(*t, Err(msg.clone()));
}
- } else {
- if outcome.item_errors > 0 {
- self.rejected_records
- .fetch_add(outcome.item_errors, Ordering::Relaxed);
- if let Some(error) = outcome.first_error {
- let mut errors = self.bulk_errors.lock().unwrap();
- if errors.len() < 5 {
- errors.push(error);
+ } else if outcome.item_errors > 0 {
+ // Charge each refused record to the file whose `_id` it
+ // carried: that file is not journaled, so the rerun the
+ // abort asks for re-sends it, and the files whose records
+ // were all accepted settle Ok. A refusal without a known
+ // `_id` fails every file in the bulk. Settling all of them
+ // Ok journaled a file past a record never stored (#2807).
+ let mut refused: HashMap<u64, Vec<String>> = HashMap::new();
+ let mut too_large: HashMap<u64, (bool, String)> = HashMap::new();
+ let mut unowned = outcome.refused.len() as u64 != outcome.item_errors;
+ for item in &outcome.refused {
+ match item.id.as_ref().and_then(|id| owners.get(id)) {
+ Some(token) => {
+ refused.entry(*token).or_default().push(item.error.clone());
+ let entry = too_large
+ .entry(*token)
+ .or_insert_with(|| (true, item.error.clone()));
+ entry.0 &= item.too_large;
}
+ None => unowned = true,
+ }
+ }
+ // A file set aside for size is junk, not a backend
+ // rejection, so it is not counted as one.
+ let mut set_aside = 0u64;
+ for (t, (all_too_large, reason)) in too_large {
+ if all_too_large && !unowned && !st.other_refusal.contains(&t) {
+ set_aside += refused.get(&t).map_or(0, |e| e.len() as u64);
+ st.too_large.insert(t, reason);
+ } else {
+ st.other_refusal.insert(t);
+ st.too_large.remove(&t);
}
}
+ self.rejected_records
+ .fetch_add(outcome.item_errors - set_aside, Ordering::Relaxed);
+ for t in &tokens {
+ let errors = refused.get(t);
+ if unowned || errors.is_some() {
+ let msg = match errors {
+ Some(errors) => format!(
+ "the backend refused {} of this file's record(s): {}",
+ errors.len(),
+ errors[0]
+ ),
+ None => format!(
+ "the backend refused {} record(s) in the bulk carrying \
+ this file: {}",
+ outcome.item_errors,
+ outcome.first_error.as_deref().unwrap_or("no reason given")
+ ),
+ };
+ if unowned {
+ st.other_refusal.insert(*t);
+ st.too_large.remove(t);
+ }
+ st.failed.insert(*t);
+ st.done.insert(*t, Err(msg));
+ } else if !st.failed.contains(t) {
+ st.done.insert(*t, Ok(()));
+ }
+ }
+ } else {
for t in &tokens {
if !st.failed.contains(t) {
st.done.insert(*t, Ok(()));
@@ -3765,6 +3901,8 @@ where
let msg = format!("{error:#}");
for t in &tokens {
st.failed.insert(*t);
+ st.other_refusal.insert(*t);
+ st.too_large.remove(t);
st.done.insert(*t, Err(msg.clone()));
}
}
@@ -11166,13 +11304,11 @@ fn run_index_report_inner(
let pipe = {
let es = &es;
let rejected = &rejected_records;
- let errors = &bulk_errors;
BulkPipe::new(
move |body| es.bulk(body),
bulk_cut,
5000,
rejected,
- errors,
worker_count,
)
};
@@ -11790,7 +11926,54 @@ fn run_index_report_inner(
// nor the journal commit may run.
if send_err.is_none() {
if let Err(e) = pipe.settle(file_token) {
- send_err = Some(e);
+ match pipe.take_too_large(file_token) {
+ // The server refused a record as too large, which the
+ // same record will hit on every rerun: set the file
+ // aside and keep going, as for any junk file (#2807).
+ // Its accepted records are removed first, so a junk
+ // file is never half searchable.
+ Some(reason) => {
+ pipe.quiesce(file_token);
+ let query = replacement_purge_query(key, &f.rel, None);
+ let indices: BTreeSet<String> = fa
+ .assignments
+ .iter()
+ .filter_map(|(_, slug)| ds_rt.get(slug).map(|rt| rt.index.clone()))
+ .collect();
+ for index in &indices {
+ if let Err(error) = es.delete_by_query(index, &query) {
+ send_err = Some(format!(
+ "{}: remove its records after the server refused one as \
+ too large: {error:#}",
+ f.rel
+ ));
+ break;
+ }
+ }
+ if send_err.is_none() {
+ extra_junk.lock().unwrap().push(JunkFile {
+ file_key: key.clone(),
+ rel: f.rel.clone(),
+ format: format_str(Some(&sn)),
+ status: "junk".into(),
+ reason: format!(
+ "set aside at index time: the server refused a record \
+ from this file: {reason}. On a semantic field the \
+ vectors the server adds count toward that limit; \
+ `--no-semantic` indexes the file without them"
+ ),
+ bytes: f.size,
+ });
+ pr.note(&format!(
+ "{}: set aside, the server refused a record from it as \
+ over its per-document size limit",
+ f.rel
+ ));
+ continue;
+ }
+ }
+ None => send_err = Some(format!("{}: {e}", f.rel)),
+ }
}
}
// Second-brain edges for this file (§6.7): only after the
@@ -11804,6 +11987,9 @@ fn run_index_report_inner(
let out = detect::assemble(&edge_drafts, &gr.edges_index, gr.created_at_ms);
gr.self_dropped
.fetch_add(out.self_dropped, Ordering::Relaxed);
+ // Counted per file, not from the shared total, which
+ // other workers move at the same time.
+ let edge_refused = AtomicU64::new(0);
let mut ebuf: Vec<u8> = Vec::new();
for edge in &out.edges {
ebuf.extend_from_slice(&edge.ndjson);
@@ -11811,7 +11997,7 @@ fn run_index_report_inner(
&& record_bulk_outcome(
&es,
std::mem::take(&mut ebuf),
- &rejected_records,
+ &edge_refused,
&bulk_errors,
&mut send_err,
)
@@ -11820,13 +12006,17 @@ fn run_index_report_inner(
}
}
if send_err.is_none() && !ebuf.is_empty() {
- record_bulk_outcome(
- &es,
- ebuf,
- &rejected_records,
- &bulk_errors,
- &mut send_err,
- );
+ record_bulk_outcome(&es, ebuf, &edge_refused, &bulk_errors, &mut send_err);
+ }
+ // A refused edge is a failed send too: leave the file
+ // un-journaled so the rerun republishes it (#2807).
+ let refused = edge_refused.load(Ordering::Relaxed);
+ rejected_records.fetch_add(refused, Ordering::Relaxed);
+ if send_err.is_none() && refused > 0 {
+ send_err = Some(format!(
+ "{}: the backend refused {refused} of this file's graph edge(s)",
+ f.rel
+ ));
}
if send_err.is_none() {
let mut written = gr.written.lock().unwrap();
@@ -16662,6 +16852,7 @@ mod bulk_pipe_tests {
fn outcome() -> BulkOutcome {
BulkOutcome {
item_errors: 0,
+ refused: Vec::new(),
server_errors: 0,
first_error: None,
first_server_error: None,
@@ -16687,15 +16878,7 @@ mod bulk_pipe_tests {
fn small_files_coalesce_into_one_bulk() {
let bodies = Mutex::new(Vec::new());
let rejected = AtomicU64::new(0);
- let errors = Mutex::new(Vec::new());
- let pipe = BulkPipe::new(
- recording_send(&bodies),
- 1 << 20,
- 5000,
- &rejected,
- &errors,
- 1,
- );
+ let pipe = BulkPipe::new(recording_send(&bodies), 1 << 20, 5000, &rejected, 1);
let t1 = pipe.token();
let t2 = pipe.token();
@@ -16731,8 +16914,7 @@ mod bulk_pipe_tests {
fn an_oversized_chunk_flushes_inline() {
let bodies = Mutex::new(Vec::new());
let rejected = AtomicU64::new(0);
- let errors = Mutex::new(Vec::new());
- let pipe = BulkPipe::new(recording_send(&bodies), 64, 5000, &rejected, &errors, 1);
+ let pipe = BulkPipe::new(recording_send(&bodies), 64, 5000, &rejected, 1);
let t = pipe.token();
let big: Vec<u8> = vec![b'x'; 200];
@@ -16754,8 +16936,7 @@ mod bulk_pipe_tests {
fn abandoned_workers_do_not_hold_back_the_residual_buffer() {
let bodies = Mutex::new(Vec::new());
let rejected = AtomicU64::new(0);
- let errors = Mutex::new(Vec::new());
- let pipe = BulkPipe::new(recording_send(&bodies), 64, 5000, &rejected, &errors, 4);
+ let pipe = BulkPipe::new(recording_send(&bodies), 64, 5000, &rejected, 4);
pipe.abandon_workers(3);
let t = pipe.token();
@@ -16778,13 +16959,11 @@ mod bulk_pipe_tests {
#[test]
fn a_failed_bulk_fails_every_file_it_carried() {
let rejected = AtomicU64::new(0);
- let errors = Mutex::new(Vec::new());
let pipe = BulkPipe::new(
|_body| Err(anyhow::anyhow!("connection reset by peer")),
1 << 20,
5000,
&rejected,
- &errors,
1,
);
let t1 = pipe.token();
@@ -16810,7 +16989,6 @@ mod bulk_pipe_tests {
#[test]
fn a_later_successful_bulk_does_not_erase_an_earlier_failure() {
let rejected = AtomicU64::new(0);
- let errors = Mutex::new(Vec::new());
let attempts = Mutex::new(0);
let pipe = BulkPipe::new(
{
@@ -16829,7 +17007,6 @@ mod bulk_pipe_tests {
1 << 20,
1,
&rejected,
- &errors,
1,
);
let big_file = pipe.token();
@@ -16858,7 +17035,6 @@ mod bulk_pipe_tests {
#[test]
fn server_errors_settle_err_without_poisoning_later_batches() {
let rejected = AtomicU64::new(0);
- let errors = Mutex::new(Vec::new());
let attempts = Mutex::new(0);
let pipe = BulkPipe::new(
{
@@ -16880,7 +17056,6 @@ mod bulk_pipe_tests {
1 << 20,
5000,
&rejected,
- &errors,
1,
);
let bad = pipe.token();
@@ -16894,16 +17069,87 @@ mod bulk_pipe_tests {
pipe.retire();
}
- /// Per-item rejections are counted and surfaced but stay non-fatal — the
- /// file's OTHER records are live and it journals complete.
+ /// A refused record fails only the file whose `_id` it carried, so that
+ /// file is not journaled; a neighbour in the same bulk settles Ok (#2807).
+ /// A refusal the pipe cannot attribute fails every file in the bulk.
+ /// #2807: a file is set aside only when every refusal it got was a size
+ /// refusal. A size refusal in one bulk and a mapping refusal in a later one
+ /// must fail the run, not set the file aside; the flag is not sticky.
#[test]
- fn item_errors_are_counted_and_non_fatal() {
+ fn a_file_with_any_other_refusal_is_not_set_aside() {
let rejected = AtomicU64::new(0);
- let errors = Mutex::new(Vec::new());
+ let calls = AtomicU64::new(0);
let pipe = BulkPipe::new(
|_body| {
+ let first = calls.fetch_add(1, Ordering::Relaxed) == 0;
Ok(BulkOutcome {
- item_errors: 2,
+ item_errors: 1,
+ refused: vec![crate::esclient::RefusedItem {
+ id: Some(if first { "a-1" } else { "a-2" }.into()),
+ error: if first { "too large" } else { "mapper refused" }.into(),
+ too_large: first,
+ }],
+ first_error: Some("refused".into()),
+ ..outcome()
+ })
+ },
+ 1,
+ 5000,
+ &rejected,
+ 1,
+ );
+ let a = pipe.token();
+ pipe.submit(a, b"{\"index\":{\"_id\":\"a-1\"}}\n{}\n".to_vec(), 1);
+ pipe.submit(a, b"{\"index\":{\"_id\":\"a-2\"}}\n{}\n".to_vec(), 1);
+ pipe.settle(a).unwrap_err();
+ pipe.quiesce(a);
+ assert_eq!(calls.load(Ordering::Relaxed), 2, "both bulks landed");
+ assert_eq!(pipe.take_too_large(a), None);
+ pipe.retire();
+ }
+
+ #[test]
+ fn a_refused_record_fails_only_its_own_file() {
+ let rejected = AtomicU64::new(0);
+ let refused = |id: Option<&str>| crate::esclient::RefusedItem {
+ id: id.map(str::to_owned),
+ error: "mapper refused field".into(),
+ too_large: false,
+ };
+ let pipe = BulkPipe::new(
+ |_body| {
+ Ok(BulkOutcome {
+ item_errors: 1,
+ refused: vec![refused(Some("a-1"))],
+ first_error: Some("mapper refused field".into()),
+ ..outcome()
+ })
+ },
+ 1 << 20,
+ 5000,
+ &rejected,
+ 2,
+ );
+ let a = pipe.token();
+ let b = pipe.token();
+ pipe.submit(a, b"{\"index\":{\"_id\":\"a-1\"}}\n{}\n".to_vec(), 1);
+ pipe.submit(b, b"{\"index\":{\"_id\":\"b-1\"}}\n{}\n".to_vec(), 1);
+ let error = pipe.settle(a).unwrap_err();
+ assert!(
+ error.contains("refused 1 of this file's record(s)"),
+ "{error}"
+ );
+ pipe.settle(b)
+ .expect("a file whose records were all accepted settles Ok");
+ assert_eq!(rejected.load(Ordering::Relaxed), 1);
+ pipe.retire();
+
+ let rejected = AtomicU64::new(0);
+ let pipe = BulkPipe::new(
+ |_body| {
+ Ok(BulkOutcome {
+ item_errors: 1,
+ refused: vec![refused(None)],
first_error: Some("mapper refused field".into()),
..outcome()
})
@@ -16911,14 +17157,14 @@ mod bulk_pipe_tests {
1 << 20,
5000,
&rejected,
- &errors,
1,
);
let t = pipe.token();
pipe.submit(t, b"a\n".to_vec(), 3);
- pipe.settle(t).expect("item errors do not fail the file");
- assert_eq!(rejected.load(Ordering::Relaxed), 2);
- assert_eq!(errors.lock().unwrap().len(), 1);
+ assert!(
+ pipe.settle(t).is_err(),
+ "an unattributed refusal fails the file"
+ );
pipe.retire();
}
@@ -16928,15 +17174,7 @@ mod bulk_pipe_tests {
fn a_file_with_no_records_settles_without_sending() {
let bodies = Mutex::new(Vec::new());
let rejected = AtomicU64::new(0);
- let errors = Mutex::new(Vec::new());
- let pipe = BulkPipe::new(
- recording_send(&bodies),
- 1 << 20,
- 5000,
- &rejected,
- &errors,
- 1,
- );
+ let pipe = BulkPipe::new(recording_send(&bodies), 1 << 20, 5000, &rejected, 1);
let t = pipe.token();
pipe.settle(t).expect("pre-resolved");
pipe.retire();
@@ -16952,7 +17190,6 @@ mod bulk_pipe_tests {
#[test]
fn wire_slots_overlap_instead_of_serializing() {
let rejected = AtomicU64::new(0);
- let errors = Mutex::new(Vec::new());
let concurrent = AtomicU64::new(0);
let peak = AtomicU64::new(0);
let pipe = BulkPipe::new(
@@ -16970,7 +17207,6 @@ mod bulk_pipe_tests {
64,
5000,
&rejected,
- &errors,
4,
);
let tokens: Vec<_> = (0..4).map(|_| pipe.token()).collect();
@@ -16999,7 +17235,6 @@ mod bulk_pipe_tests {
fn many_workers_many_files_coalesce_without_deadlock() {
let bodies = Mutex::new(Vec::new());
let rejected = AtomicU64::new(0);
- let errors = Mutex::new(Vec::new());
let pipe = BulkPipe::new(
{
let bodies = &bodies;
@@ -17012,7 +17247,6 @@ mod bulk_pipe_tests {
1 << 20,
5000,
&rejected,
- &errors,
8,
);
const WORKERS: usize = 8;
diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rs
index ee2c9dc2b..75ac2c763 100644
--- a/engine/crates/xerj-autoindex/src/sync_executor.rs
+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs
@@ -13,7 +13,7 @@ use crate::sync::{
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
-use std::collections::{BTreeMap, BTreeSet, HashMap};
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
use std::fs::{File, OpenOptions};
use std::io::{BufReader, BufWriter, Read, Write};
use std::path::Path;
@@ -2828,7 +2828,8 @@ fn checked_bulk(es: &crate::esclient::Es, body: Vec<u8>) -> Result<()> {
if body.is_empty() {
return Ok(());
}
- check_bulk_outcome(es, es.bulk(body)?)
+ let outcome = es.bulk(body.clone())?;
+ check_bulk_outcome(es, outcome, &body)
}
/// [`checked_bulk`] for a body nobody windowed — the catalog projection, which
@@ -2838,12 +2839,48 @@ fn checked_bulk_windowed(es: &crate::esclient::Es, body: Vec<u8>, bulk_bytes: us
if body.is_empty() {
return Ok(());
}
- check_bulk_outcome(es, es.bulk_windowed(body, bulk_bytes)?)
+ let outcome = es.bulk_windowed(body.clone(), bulk_bytes)?;
+ check_bulk_outcome(es, outcome, &body)
+}
+
+/// The `ax_path` of each refused record, read back from the body that was
+/// sent: a refusal reports only the record's `_id`, which is a hash.
+fn refused_paths(body: &[u8], outcome: &crate::esclient::BulkOutcome) -> Vec<String> {
+ let ids: HashSet<&str> = outcome
+ .refused
+ .iter()
+ .filter_map(|item| item.id.as_deref())
+ .collect();
+ let mut paths = Vec::new();
+ let mut lines = body.split(|byte| *byte == b'\n');
+ while let (Some(action), Some(doc)) = (lines.next(), lines.next()) {
+ let Ok(action) = serde_json::from_slice::<Value>(action) else {
+ continue;
+ };
+ let id = action
+ .as_object()
+ .and_then(|op| op.values().next())
+ .and_then(|op| op.get("_id"))
+ .and_then(Value::as_str);
+ if !id.is_some_and(|id| ids.contains(id)) {
+ continue;
+ }
+ if let Some(path) = serde_json::from_slice::<Value>(doc)
+ .ok()
+ .and_then(|doc| doc.get("ax_path")?.as_str().map(str::to_owned))
+ {
+ if !paths.contains(&path) {
+ paths.push(path);
+ }
+ }
+ }
+ paths
}
fn check_bulk_outcome(
es: &crate::esclient::Es,
outcome: crate::esclient::BulkOutcome,
+ body: &[u8],
) -> Result<()> {
// `Es::bulk` has already re-sent per-item 429s, and only those, for the
// whole of its patience (`THROTTLE_PATIENCE`, #944/#949, shortened to
@@ -2877,15 +2914,34 @@ fn check_bulk_outcome(
outcome.server_errors,
);
}
- anyhow::ensure!(
- outcome.item_errors == 0,
- "prepared bulk contained {} rejected items: {}",
+ if outcome.item_errors == 0 {
+ return Ok(());
+ }
+ // This route seals the generation before it sends, so a file can no
+ // longer leave the plan here, and a rerun replays the same refusal. Name
+ // the file and the ways out instead (#2807).
+ let paths = refused_paths(body, &outcome);
+ let source = if paths.is_empty() {
+ String::new()
+ } else {
+ format!(" from {}", paths.join(", "))
+ };
+ let too_large = outcome.refused.iter().any(|item| item.too_large);
+ let remedy = if too_large {
+ " The server adds vectors to a `semantic_text` field, and they count toward its \
+ per-document limit. A sealed generation cannot set a file aside, so rerunning \
+ repeats this refusal: index the folder under a new --prefix and --state-dir with \
+ the file left out (`--exclude`), or with `--no-semantic`"
+ } else {
+ ""
+ };
+ anyhow::bail!(
+ "the server refused {} record(s){source}: {}.{remedy}",
outcome.item_errors,
outcome
.first_error
.unwrap_or_else(|| "unknown error".into())
- );
- Ok(())
+ )
}
fn append_index_action(index: &str, id: &str, doc: &Value, body: &mut Vec<u8>) -> Result<()> {