AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
fix(autoindex): never journal a file past a record the server refused (#2807) (#2816)
Jake Elbirtf30ece8acfea
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 0623ecd94..f3cdae6b0 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -456,6 +456,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0XML, including `.xml`, `.xaml` and `.svg`, is unchanged. An index builtbefore this fix keeps its junked files until each one changes or the folderis indexed again with `--fresh` (#2740).+- When the server refuses a record or a graph edge, `xerj autoindex` now+ names the file it came from and does not mark that file done. It used to+ mark every file in the same bulk done, so the rerun the abort message asks+ for resumed past the file, reported success and never sent the refused+ record (#2807).+- A file whose record the server refuses as over its 8 MiB per-document+ limit is now set aside as junk, with its file name and the server's reason,+ and the run completes with exit 3. Its other records are removed and it is+ retried on the next run. It used to abort the whole run, every rerun, until+ `--fresh`. On a `semantic_text` field the vectors the server adds count+ toward that limit: a 2 MB generated `.d.ts` file grew to 16.6 MB this way.+ `--no-semantic` may index such a file (#2807).+- `xerj corpus index` and `xerj autoindex --no-graph` now name the file a+ refused record came from, and for a size refusal they say how to index the+ rest: under a new `--prefix` and `--state-dir`, with the file left out+ (`--exclude`) or with `--no-semantic`. That route fixes its file list before+ it sends anything, so it cannot set the file aside, and a rerun repeats the+ refusal. It used to say only "prepared bulk contained 1 rejected items"+ (#2807).## [1.0.0-rc.100] - 2026-10-26diff --git a/engine/crates/xerj-autoindex/src/esclient.rs b/engine/crates/xerj-autoindex/src/esclient.rsindex dc77d4402..2aa9d4fb7 100644--- a/engine/crates/xerj-autoindex/src/esclient.rs+++ b/engine/crates/xerj-autoindex/src/esclient.rs@@ -964,6 +964,10 @@ fn is_whole_request_too_large(v: &Value, sent_actions: Option<usize>) -> bool {== Some(413)}+/// The server's wording for a document over its per-document byte limit+/// (`xerj_common::document_limit`), however the item's status reads.+const DOCUMENT_LIMIT_REASON: &str = "byte limit for one indexed JSON document";+/// Read a bulk response body item by item.fn parse_round(v: &Value) -> Round {let items = v.get("items").and_then(|i| i.as_array());@@ -1018,8 +1022,20 @@ fn parse_round(v: &Value) -> Round {if round.outcome.first_server_error.is_none() {round.outcome.first_server_error = Some(error_text());}- } else if round.first_settled_error.is_none() {- round.first_settled_error = Some(op["error"].to_string().chars().take(300).collect());+ } else {+ if round.first_settled_error.is_none() {+ round.first_settled_error =+ Some(op["error"].to_string().chars().take(300).collect());+ }+ let reason = op["error"]+ .get("reason")+ .and_then(Value::as_str)+ .unwrap_or("");+ round.outcome.refused.push(RefusedItem {+ id: op.get("_id").and_then(Value::as_str).map(str::to_owned),+ error: error_text(),+ too_large: item_status == 413 || reason.contains(DOCUMENT_LIMIT_REASON),+ });}if round.outcome.first_error.is_none() {round.outcome.first_error = Some(op["error"].to_string().chars().take(300).collect());@@ -1028,9 +1044,22 @@ fn parse_round(v: &Value) -> Round {round}+/// One item the server refused for the record itself (not a 429, 5xx or+/// write block), with the `_id` the bulk action named.+#[derive(Debug, Clone)]+pub struct RefusedItem {+ pub id: Option<String>,+ pub error: String,+ /// Refused because the stored document passed the server's per-document+ /// byte limit, which a retry of the same record cannot clear (#2807).+ pub too_large: bool,+}+#[derive(Debug, Default)]pub struct BulkOutcome {pub item_errors: u64,+ /// The record-level refusals counted in `item_errors`, in item order.+ pub refused: Vec<RefusedItem>,/// Per-item backend/admission failures — 5xx/429 statuses plus/// cluster/index write blocks recognised by TYPE (see/// [`is_index_block_error`]). These are not bad source records: the same@@ -1054,6 +1083,7 @@ impl BulkOutcome {/// Fold the outcome of a later window of the same body into this one.fn absorb(&mut self, other: BulkOutcome) {self.item_errors += other.item_errors;+ self.refused.extend(other.refused);self.server_errors += other.server_errors;self.throttled_out += other.throttled_out;if self.first_error.is_none() {@@ -1940,6 +1970,9 @@ impl Es {// 400): the record is not re-offered, its error is still reported.let mut settled_item_errors = 0u64;let mut settled_first_error: Option<String> = None;+ // Their refused items too, so a refusal from an earlier round is still+ // traced to its file after a 429 retry (#2807).+ let mut settled_refused: Vec<RefusedItem> = Vec::new();// Set once any round took an item. From then on the full patience// applies, even if later rounds are all 429: a drain in progress// (#944 held 57 s) must be allowed to finish.@@ -1954,6 +1987,9 @@ impl Es {}let mut outcome = round.outcome;outcome.item_errors += settled_item_errors;+ let mut refused = std::mem::take(&mut settled_refused);+ refused.append(&mut outcome.refused);+ outcome.refused = refused;if outcome.first_error.is_none() {outcome.first_error = settled_first_error.clone();}@@ -2001,6 +2037,7 @@ impl Es {settled_item_errors = outcome.item_errors.saturating_sub(round.throttled.len() as u64);+ settled_refused = outcome.refused.clone();if round.first_settled_error.is_some() {settled_first_error = round.first_settled_error;}diff --git a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rsindex 53629e910..065dec612 100644--- a/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/failure_resume_http_tests.rs@@ -163,6 +163,9 @@ struct MockState {/// and the type is not a block, so the run continues and must still/// report the rejection.reject_first_data_item: bool,+ /// As `reject_first_data_item`, with the server's per-document size+ /// refusal, which a retry cannot clear (#2807).+ reject_first_data_item_as_too_large: bool,/// Opt-in (#929): refuse the create-index PUT — which on this path carries/// the dataset's whole mapping — of every index whose name contains one of/// these, with the 400 a real engine returns for a field it cannot map.@@ -916,15 +919,27 @@ fn bulk_response(body: &[u8], state: &Arc<Mutex<MockState>>) -> Value {if locked.swallow_data_bulks {return json!({"errors": false, "items": []});}- if locked.reject_first_data_item {+ if locked.reject_first_data_item || locked.reject_first_data_item_as_too_large {+ let too_large = locked.reject_first_data_item_as_too_large;let mut items = Vec::new();for (nth, pair) in lines.as_chunks::<2>().0.iter().enumerate() {if nth == 0 {+ let action: Value = serde_json::from_slice(pair[0]).unwrap();items.push(json!({"index": {+ "_id": action.pointer("/index/_id").cloned().unwrap_or(Value::Null),"status": 400,- "error": {- "type": "document_parsing_exception",- "reason": "refused: value could not be parsed"+ "error": if too_large {+ json!({+ "type": "engine_exception",+ "reason": "document is 16590136 bytes, over the 8388608 byte \+ limit for one indexed JSON document; it was not \+ indexed. Split it into smaller documents"+ })+ } else {+ json!({+ "type": "document_parsing_exception",+ "reason": "refused: value could not be parsed"+ })}}}));continue;@@ -3934,6 +3949,9 @@ fn write_block_rejections_are_backend_fatal_and_the_file_stays_pending() {/// The rejection count is therefore reported in the one place a rejected run/// ever reaches: the abort message, where it supplies the scale that/// `bulk_errors`' first-five sample cannot.+///+/// The file that carried the refused record is named and stays un-journaled,+/// so the rerun the message asks for really retries it (#2807).#[test]fn backend_rejected_records_abort_the_run_before_any_map_is_written() {let _guard = FAILPOINT_TEST_LOCK@@ -3966,23 +3984,24 @@ fn backend_rejected_records_abort_the_run_before_any_map_is_written() {"the abort must state how many records were refused, not just sample the errors: {error}");- // Measured, and deliberately pinned even though it is NOT what the abort- // message implies: `record_bulk_outcome` treats a per-item rejection as- // non-fatal to the worker (only `server_errors` set `send_err`), so the- // file is journaled COMPLETE and the run aborts only at the end. A rerun- // therefore resumes past this file and never retries the refused record,- // while the message says failed sources "were not journaled complete".- //- // That divergence predates this branch — it is `main`'s behaviour for the- // item-error class — and fixing it means changing when a worker treats a- // rejection as fatal, which is a resume-semantics change with nothing to- // do with map metadata. Recorded here so the next reader finds the fact- // rather than the assumption; tracked in the PR body as a known gap.+ // #2807: the refused record's source file is named and is NOT journaled+ // complete, as the message says. It used to be journaled, so a rerun+ // resumed past it, reported success and never sent the refused record.+ assert!(+ error.contains("records.csv"),+ "the abort must name the file whose record was refused: {error}"+ );assert_eq!(file_done_count(state_dir.path()),- 1,- "pre-existing: a per-item rejection still journals the source complete"+ 0,+ "a source with a refused record must not be journaled complete");++ // With the refusal gone, the same command resumes and sends every record.+ endpoint.state.lock().unwrap().reject_first_data_item = false;+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url);+ run_index_report(config).unwrap();+ assert_eq!(file_done_count(state_dir.path()), 1);}/// #195 last-resort gate: a backend that ACCEPTS every bulk but persists@@ -5211,3 +5230,59 @@ fn a_refused_dataset_on_the_graph_path_costs_that_dataset_not_the_run() {.unwrap();assert!(done.contains("datasets_refused=1"), "{done}");}++/// #2807: a record the server refuses as over its per-document size limit is+/// refused again on every rerun, so its file is set aside as junk and the run+/// completes (exit 3) instead of aborting. The file's accepted records are+/// removed, it is not journaled, and it indexes once the refusal goes away.+#[test]+fn a_record_refused_as_too_large_sets_its_file_aside() {+ let _guard = FAILPOINT_TEST_LOCK+ .lock()+ .unwrap_or_else(|p| p.into_inner());+ let _io_guard = state::FILE_DONE_IO_FAILPOINT_TEST_LOCK+ .lock()+ .unwrap_or_else(|p| p.into_inner());+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(+ corpus.path().join("records.csv"),+ "id,value\n0,kept\n1,kept\n2,kept\n",+ )+ .unwrap();+ let endpoint = MockEndpoint::start(usize::MAX);+ endpoint+ .state+ .lock()+ .unwrap()+ .reject_first_data_item_as_too_large = true;++ for round in 0..2 {+ let (code, _) = run_index_report(cfg(corpus.path(), state_dir.path(), &endpoint.url))+ .unwrap_or_else(|error| panic!("round {round} must not abort: {error:#}"));+ assert_eq!(code, 3, "round {round}: completed with junk");+ let state = endpoint.state.lock().unwrap();+ let catalog = serde_json::to_string(&state.catalog_docs).unwrap();+ assert!(+ catalog.contains("set aside at index time") && catalog.contains("records.csv"),+ "round {round}: the file is listed as junk with the reason: {catalog}"+ );+ assert!(+ !serde_json::to_string(&state.docs)+ .unwrap()+ .contains("\"kept\""),+ "round {round}: the file's accepted records are removed"+ );+ drop(state);+ assert_eq!(file_done_count(state_dir.path()), 0, "round {round}");+ }++ endpoint+ .state+ .lock()+ .unwrap()+ .reject_first_data_item_as_too_large = false;+ let (code, _) = run_index_report(cfg(corpus.path(), state_dir.path(), &endpoint.url)).unwrap();+ assert_eq!(code, 0);+ assert_eq!(file_done_count(state_dir.path()), 1);+}diff --git a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rsindex abbb577d6..dcd9e7211 100644--- a/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs+++ b/engine/crates/xerj-autoindex/src/incremental_reconcile_http_tests.rs@@ -25,6 +25,9 @@ struct HttpState {search_bodies: Vec<Value>,data_bulk_requests: usize,fail_next_data_bulk: bool,+ /// #2807: refuse the first `index` action of the next data bulk as over+ /// the per-document byte limit, echoing its `_id` as the server does.+ refuse_next_data_item_as_too_large: bool,/// Apply the first half of the next data bulk's items and *then* report/// `errors: true`. This is the dangerous backend shape the legacy/// replacement transaction was hardened against (visibility changed, but@@ -588,6 +591,28 @@ fn bulk_http(body: &[u8], state: &Arc<Mutex<HttpState>>) -> (u16, Value) {if numbered {locked.fail_data_bulk_number = None;}+ if std::mem::take(&mut locked.refuse_next_data_item_as_too_large) {+ let action: Value = serde_json::from_slice(lines[0]).unwrap();+ let id = action["index"]["_id"]+ .as_str()+ .unwrap_or_default()+ .to_owned();+ return (+ 200,+ json!({+ "errors": true,+ "items": [{"index": {+ "_id": id,+ "status": 413,+ "error": {+ "type": "engine_exception",+ "reason": "document is 16590136 bytes, over the 8388608 byte limit for one indexed JSON document; it was not indexed. Split it into smaller documents",+ "status": 413+ }+ }}]+ }),+ );+ }if std::mem::take(&mut locked.fail_next_data_bulk) || numbered {return (200,@@ -7676,3 +7701,38 @@ fn a_folder_of_templates_and_html_fragments_is_indexed_not_junked() {);}}++/// #2807 on the generated (`--no-graph`) path, which `xerj corpus index` uses.+/// That path seals the generation before it sends, so a file the server+/// refuses for size cannot be set aside there. The abort must at least name+/// the file and a way out that works; it used to say only "prepared bulk+/// contained 1 rejected items".+#[test]+fn a_record_refused_for_size_on_the_generated_path_names_its_file() {+ let _guard = HTTP_E2E_LOCK.lock().unwrap_or_else(|p| p.into_inner());+ let _replay_guard = sync_executor::REPLAY_FAILPOINT_TEST_LOCK+ .lock()+ .unwrap_or_else(|p| p.into_inner());+ let corpus = tempfile::tempdir().unwrap();+ let state_dir = tempfile::tempdir().unwrap();+ fs::write(corpus.path().join("a.csv"), "id,value\n1,alpha\n").unwrap();+ let endpoint = HttpEndpoint::start();+ endpoint+ .state+ .lock()+ .unwrap()+ .refuse_next_data_item_as_too_large = true;+ let config = cfg(corpus.path(), state_dir.path(), &endpoint.url, false);++ let error = format!("{:#}", run_index(config).unwrap_err());+ assert!(+ error.contains("from a.csv"),+ "the file must be named: {error}"+ );+ assert!(error.contains("16590136 bytes"), "{error}");+ assert!(+ error.contains("--no-semantic") && error.contains("--exclude"),+ "the abort must give a way out that works: {error}"+ );+ assert!(!error.contains("prepared bulk contained"), "{error}");+}diff --git a/engine/crates/xerj-autoindex/src/lib.rs b/engine/crates/xerj-autoindex/src/lib.rsindex 940f80071..878f499f3 100644--- a/engine/crates/xerj-autoindex/src/lib.rs+++ b/engine/crates/xerj-autoindex/src/lib.rs@@ -3451,6 +3451,17 @@ fn record_bulk_outcome(// ─── phase-B bulk coalescing (#1147) ─────────────────────────────────────+/// One flushed bulk: the body, the tokens with records in it, and the+/// token behind each `_id`.+type PipeJob = (Vec<u8>, Vec<u64>, HashMap<String, u64>);++/// The `_id` an NDJSON bulk action line names, if any.+fn bulk_action_id(action: &[u8]) -> Option<String> {+ let value: Value = serde_json::from_slice(action).ok()?;+ let op = value.as_object()?.values().next()?;+ op.get("_id")?.as_str().map(str::to_owned)+}+/// Shared mutable state of [`BulkPipe`]. One lock covers the buffer and the/// per-token bookkeeping so a flush is atomic against submissions.#[derive(Default)]@@ -3462,6 +3473,19 @@ struct PipeState {/// Tokens with bytes in `buf`, in submission order. A token appears once/// even across several chunk submissions (one file larger than the cut).pending: Vec<u64>,+ /// The token that submitted each `_id` in `buf`, so a refused item is+ /// charged to its own file rather than to every file in the bulk (#2807).+ owners: HashMap<String, u64>,+ /// Tokens whose every refused record was over the server's per-document+ /// byte limit, with the server's reason. A retry cannot clear that, so the+ /// file is set aside instead of failing the run (#2807).+ too_large: HashMap<u64, String>,+ /// Tokens that also failed for any other reason. Never set aside: one+ /// refusal a rerun could clear is enough to fail the run instead.+ other_refusal: HashSet<u64>,+ /// Bulks in flight per token, so a set-aside purge waits for the file's+ /// last records to land before deleting them.+ in_flight_tokens: HashMap<u64, usize>,/// Terminal result per token, observed (and removed) by the token's/// owner in [`BulkPipe::settle`].done: std::collections::HashMap<u64, Result<(), String>>,@@ -3539,7 +3563,6 @@ struct BulkPipe<'a, F> {bulk_cut: usize,doc_cap: usize,rejected_records: &'a AtomicU64,- bulk_errors: &'a Mutex<Vec<String>>,state: Mutex<PipeState>,ready: Condvar,}@@ -3553,7 +3576,6 @@ wherebulk_cut: usize,doc_cap: usize,rejected_records: &'a AtomicU64,- bulk_errors: &'a Mutex<Vec<String>>,workers: usize,) -> Self {Self {@@ -3561,7 +3583,6 @@ wherebulk_cut: bulk_cut.max(1),doc_cap: doc_cap.max(1),rejected_records,- bulk_errors,state: Mutex::new(PipeState {live_workers: workers,..PipeState::default()@@ -3601,6 +3622,11 @@ whereif !st.pending.contains(&token) {st.pending.push(token);}+ for action in bytes.split(|b| *b == b'\n').step_by(2) {+ if let Some(id) = bulk_action_id(action) {+ st.owners.insert(id, token);+ }+ }st.buf.extend_from_slice(&bytes);st.docs += docs;if st.buf.len() < self.bulk_cut && st.docs < self.doc_cap {@@ -3623,7 +3649,43 @@ where};drop(st);let outcome = (self.send)(job.0);- self.deliver(job.1, outcome);+ self.deliver(job.1, job.2, outcome);+ }++ /// The server's reason when this token failed only on records over the+ /// per-document byte limit. Taken once, by the file's own worker.+ fn take_too_large(&self, token: u64) -> Option<String> {+ self.lock().too_large.remove(&token)+ }++ /// Block until no bulk holding this file's records is buffered or in+ /// flight. A failed token settles at its first refusal, while its later+ /// chunks may still be on the wire; a purge before they land would leave+ /// a junk file half searchable.+ fn quiesce(&self, token: u64) {+ let mut st = self.lock();+ loop {+ let buffered = st.pending.contains(&token);+ if !buffered && !st.in_flight_tokens.contains_key(&token) {+ return;+ }+ if buffered {+ if let Some(job) = self.take_job(&mut st) {+ drop(st);+ let outcome = (self.send)(job.0);+ self.deliver(job.1, job.2, outcome);+ st = self.lock();+ continue;+ }+ }+ st.parked_workers += 1;+ let (guard, _) = self+ .ready+ .wait_timeout(st, std::time::Duration::from_secs(30))+ .unwrap_or_else(|p| p.into_inner());+ st = guard;+ st.parked_workers -= 1;+ }}/// Whether any bulk carrying this file's records has failed. The drain@@ -3655,7 +3717,7 @@ whereif let Some(job) = self.take_job(&mut st) {drop(st);let outcome = (self.send)(job.0);- self.deliver(job.1, outcome);+ self.deliver(job.1, job.2, outcome);st = self.lock();continue;}@@ -3695,7 +3757,7 @@ whereif let Some(job) = self.take_job(&mut st) {drop(st);let outcome = (self.send)(job.0);- self.deliver(job.1, outcome);+ self.deliver(job.1, job.2, outcome);st = self.lock();continue;}@@ -3713,22 +3775,42 @@ where/// Detach the buffer as a send job if a wire slot is free. Called with/// the lock held; the caller drops the lock before sending.- fn take_job(&self, st: &mut PipeState) -> Option<(Vec<u8>, Vec<u64>)> {+ fn take_job(&self, st: &mut PipeState) -> Option<PipeJob> {if st.buf.is_empty() || st.in_flight >= MAX_IN_FLIGHT {return None;}st.in_flight += 1;st.docs = 0;- Some((std::mem::take(&mut st.buf), std::mem::take(&mut st.pending)))+ for t in &st.pending {+ *st.in_flight_tokens.entry(*t).or_default() += 1;+ }+ Some((+ std::mem::take(&mut st.buf),+ std::mem::take(&mut st.pending),+ std::mem::take(&mut st.owners),+ ))}/// Resolve a completed job: reclaim the wire slot, settle every token it/// carried, wake the parked. Same outcome handling as/// `record_bulk_outcome`, resolved per token instead of through the/// caller's `send_err`.- fn deliver(&self, tokens: Vec<u64>, outcome: anyhow::Result<crate::esclient::BulkOutcome>) {+ fn deliver(+ &self,+ tokens: Vec<u64>,+ owners: HashMap<String, u64>,+ outcome: anyhow::Result<crate::esclient::BulkOutcome>,+ ) {let mut st = self.lock();st.in_flight = st.in_flight.saturating_sub(1);+ for t in &tokens {+ if let Some(n) = st.in_flight_tokens.get_mut(t) {+ *n -= 1;+ if *n == 0 {+ st.in_flight_tokens.remove(t);+ }+ }+ }match outcome {Ok(outcome) => {if outcome.server_errors > 0 {@@ -3741,19 +3823,73 @@ where);for t in &tokens {st.failed.insert(*t);+ st.other_refusal.insert(*t);+ st.too_large.remove(t);st.done.insert(*t, Err(msg.clone()));}- } else {- if outcome.item_errors > 0 {- self.rejected_records- .fetch_add(outcome.item_errors, Ordering::Relaxed);- if let Some(error) = outcome.first_error {- let mut errors = self.bulk_errors.lock().unwrap();- if errors.len() < 5 {- errors.push(error);+ } else if outcome.item_errors > 0 {+ // Charge each refused record to the file whose `_id` it+ // carried: that file is not journaled, so the rerun the+ // abort asks for re-sends it, and the files whose records+ // were all accepted settle Ok. A refusal without a known+ // `_id` fails every file in the bulk. Settling all of them+ // Ok journaled a file past a record never stored (#2807).+ let mut refused: HashMap<u64, Vec<String>> = HashMap::new();+ let mut too_large: HashMap<u64, (bool, String)> = HashMap::new();+ let mut unowned = outcome.refused.len() as u64 != outcome.item_errors;+ for item in &outcome.refused {+ match item.id.as_ref().and_then(|id| owners.get(id)) {+ Some(token) => {+ refused.entry(*token).or_default().push(item.error.clone());+ let entry = too_large+ .entry(*token)+ .or_insert_with(|| (true, item.error.clone()));+ entry.0 &= item.too_large;}+ None => unowned = true,+ }+ }+ // A file set aside for size is junk, not a backend+ // rejection, so it is not counted as one.+ let mut set_aside = 0u64;+ for (t, (all_too_large, reason)) in too_large {+ if all_too_large && !unowned && !st.other_refusal.contains(&t) {+ set_aside += refused.get(&t).map_or(0, |e| e.len() as u64);+ st.too_large.insert(t, reason);+ } else {+ st.other_refusal.insert(t);+ st.too_large.remove(&t);}}+ self.rejected_records+ .fetch_add(outcome.item_errors - set_aside, Ordering::Relaxed);+ for t in &tokens {+ let errors = refused.get(t);+ if unowned || errors.is_some() {+ let msg = match errors {+ Some(errors) => format!(+ "the backend refused {} of this file's record(s): {}",+ errors.len(),+ errors[0]+ ),+ None => format!(+ "the backend refused {} record(s) in the bulk carrying \+ this file: {}",+ outcome.item_errors,+ outcome.first_error.as_deref().unwrap_or("no reason given")+ ),+ };+ if unowned {+ st.other_refusal.insert(*t);+ st.too_large.remove(t);+ }+ st.failed.insert(*t);+ st.done.insert(*t, Err(msg));+ } else if !st.failed.contains(t) {+ st.done.insert(*t, Ok(()));+ }+ }+ } else {for t in &tokens {if !st.failed.contains(t) {st.done.insert(*t, Ok(()));@@ -3765,6 +3901,8 @@ wherelet msg = format!("{error:#}");for t in &tokens {st.failed.insert(*t);+ st.other_refusal.insert(*t);+ st.too_large.remove(t);st.done.insert(*t, Err(msg.clone()));}}@@ -11166,13 +11304,11 @@ fn run_index_report_inner(let pipe = {let es = &es;let rejected = &rejected_records;- let errors = &bulk_errors;BulkPipe::new(move |body| es.bulk(body),bulk_cut,5000,rejected,- errors,worker_count,)};@@ -11790,7 +11926,54 @@ fn run_index_report_inner(// nor the journal commit may run.if send_err.is_none() {if let Err(e) = pipe.settle(file_token) {- send_err = Some(e);+ match pipe.take_too_large(file_token) {+ // The server refused a record as too large, which the+ // same record will hit on every rerun: set the file+ // aside and keep going, as for any junk file (#2807).+ // Its accepted records are removed first, so a junk+ // file is never half searchable.+ Some(reason) => {+ pipe.quiesce(file_token);+ let query = replacement_purge_query(key, &f.rel, None);+ let indices: BTreeSet<String> = fa+ .assignments+ .iter()+ .filter_map(|(_, slug)| ds_rt.get(slug).map(|rt| rt.index.clone()))+ .collect();+ for index in &indices {+ if let Err(error) = es.delete_by_query(index, &query) {+ send_err = Some(format!(+ "{}: remove its records after the server refused one as \+ too large: {error:#}",+ f.rel+ ));+ break;+ }+ }+ if send_err.is_none() {+ extra_junk.lock().unwrap().push(JunkFile {+ file_key: key.clone(),+ rel: f.rel.clone(),+ format: format_str(Some(&sn)),+ status: "junk".into(),+ reason: format!(+ "set aside at index time: the server refused a record \+ from this file: {reason}. On a semantic field the \+ vectors the server adds count toward that limit; \+ `--no-semantic` indexes the file without them"+ ),+ bytes: f.size,+ });+ pr.note(&format!(+ "{}: set aside, the server refused a record from it as \+ over its per-document size limit",+ f.rel+ ));+ continue;+ }+ }+ None => send_err = Some(format!("{}: {e}", f.rel)),+ }}}// Second-brain edges for this file (§6.7): only after the@@ -11804,6 +11987,9 @@ fn run_index_report_inner(let out = detect::assemble(&edge_drafts, &gr.edges_index, gr.created_at_ms);gr.self_dropped.fetch_add(out.self_dropped, Ordering::Relaxed);+ // Counted per file, not from the shared total, which+ // other workers move at the same time.+ let edge_refused = AtomicU64::new(0);let mut ebuf: Vec<u8> = Vec::new();for edge in &out.edges {ebuf.extend_from_slice(&edge.ndjson);@@ -11811,7 +11997,7 @@ fn run_index_report_inner(&& record_bulk_outcome(&es,std::mem::take(&mut ebuf),- &rejected_records,+ &edge_refused,&bulk_errors,&mut send_err,)@@ -11820,13 +12006,17 @@ fn run_index_report_inner(}}if send_err.is_none() && !ebuf.is_empty() {- record_bulk_outcome(- &es,- ebuf,- &rejected_records,- &bulk_errors,- &mut send_err,- );+ record_bulk_outcome(&es, ebuf, &edge_refused, &bulk_errors, &mut send_err);+ }+ // A refused edge is a failed send too: leave the file+ // un-journaled so the rerun republishes it (#2807).+ let refused = edge_refused.load(Ordering::Relaxed);+ rejected_records.fetch_add(refused, Ordering::Relaxed);+ if send_err.is_none() && refused > 0 {+ send_err = Some(format!(+ "{}: the backend refused {refused} of this file's graph edge(s)",+ f.rel+ ));}if send_err.is_none() {let mut written = gr.written.lock().unwrap();@@ -16662,6 +16852,7 @@ mod bulk_pipe_tests {fn outcome() -> BulkOutcome {BulkOutcome {item_errors: 0,+ refused: Vec::new(),server_errors: 0,first_error: None,first_server_error: None,@@ -16687,15 +16878,7 @@ mod bulk_pipe_tests {fn small_files_coalesce_into_one_bulk() {let bodies = Mutex::new(Vec::new());let rejected = AtomicU64::new(0);- let errors = Mutex::new(Vec::new());- let pipe = BulkPipe::new(- recording_send(&bodies),- 1 << 20,- 5000,- &rejected,- &errors,- 1,- );+ let pipe = BulkPipe::new(recording_send(&bodies), 1 << 20, 5000, &rejected, 1);let t1 = pipe.token();let t2 = pipe.token();@@ -16731,8 +16914,7 @@ mod bulk_pipe_tests {fn an_oversized_chunk_flushes_inline() {let bodies = Mutex::new(Vec::new());let rejected = AtomicU64::new(0);- let errors = Mutex::new(Vec::new());- let pipe = BulkPipe::new(recording_send(&bodies), 64, 5000, &rejected, &errors, 1);+ let pipe = BulkPipe::new(recording_send(&bodies), 64, 5000, &rejected, 1);let t = pipe.token();let big: Vec<u8> = vec![b'x'; 200];@@ -16754,8 +16936,7 @@ mod bulk_pipe_tests {fn abandoned_workers_do_not_hold_back_the_residual_buffer() {let bodies = Mutex::new(Vec::new());let rejected = AtomicU64::new(0);- let errors = Mutex::new(Vec::new());- let pipe = BulkPipe::new(recording_send(&bodies), 64, 5000, &rejected, &errors, 4);+ let pipe = BulkPipe::new(recording_send(&bodies), 64, 5000, &rejected, 4);pipe.abandon_workers(3);let t = pipe.token();@@ -16778,13 +16959,11 @@ mod bulk_pipe_tests {#[test]fn a_failed_bulk_fails_every_file_it_carried() {let rejected = AtomicU64::new(0);- let errors = Mutex::new(Vec::new());let pipe = BulkPipe::new(|_body| Err(anyhow::anyhow!("connection reset by peer")),1 << 20,5000,&rejected,- &errors,1,);let t1 = pipe.token();@@ -16810,7 +16989,6 @@ mod bulk_pipe_tests {#[test]fn a_later_successful_bulk_does_not_erase_an_earlier_failure() {let rejected = AtomicU64::new(0);- let errors = Mutex::new(Vec::new());let attempts = Mutex::new(0);let pipe = BulkPipe::new({@@ -16829,7 +17007,6 @@ mod bulk_pipe_tests {1 << 20,1,&rejected,- &errors,1,);let big_file = pipe.token();@@ -16858,7 +17035,6 @@ mod bulk_pipe_tests {#[test]fn server_errors_settle_err_without_poisoning_later_batches() {let rejected = AtomicU64::new(0);- let errors = Mutex::new(Vec::new());let attempts = Mutex::new(0);let pipe = BulkPipe::new({@@ -16880,7 +17056,6 @@ mod bulk_pipe_tests {1 << 20,5000,&rejected,- &errors,1,);let bad = pipe.token();@@ -16894,16 +17069,87 @@ mod bulk_pipe_tests {pipe.retire();}- /// Per-item rejections are counted and surfaced but stay non-fatal — the- /// file's OTHER records are live and it journals complete.+ /// A refused record fails only the file whose `_id` it carried, so that+ /// file is not journaled; a neighbour in the same bulk settles Ok (#2807).+ /// A refusal the pipe cannot attribute fails every file in the bulk.+ /// #2807: a file is set aside only when every refusal it got was a size+ /// refusal. A size refusal in one bulk and a mapping refusal in a later one+ /// must fail the run, not set the file aside; the flag is not sticky.#[test]- fn item_errors_are_counted_and_non_fatal() {+ fn a_file_with_any_other_refusal_is_not_set_aside() {let rejected = AtomicU64::new(0);- let errors = Mutex::new(Vec::new());+ let calls = AtomicU64::new(0);let pipe = BulkPipe::new(|_body| {+ let first = calls.fetch_add(1, Ordering::Relaxed) == 0;Ok(BulkOutcome {- item_errors: 2,+ item_errors: 1,+ refused: vec![crate::esclient::RefusedItem {+ id: Some(if first { "a-1" } else { "a-2" }.into()),+ error: if first { "too large" } else { "mapper refused" }.into(),+ too_large: first,+ }],+ first_error: Some("refused".into()),+ ..outcome()+ })+ },+ 1,+ 5000,+ &rejected,+ 1,+ );+ let a = pipe.token();+ pipe.submit(a, b"{\"index\":{\"_id\":\"a-1\"}}\n{}\n".to_vec(), 1);+ pipe.submit(a, b"{\"index\":{\"_id\":\"a-2\"}}\n{}\n".to_vec(), 1);+ pipe.settle(a).unwrap_err();+ pipe.quiesce(a);+ assert_eq!(calls.load(Ordering::Relaxed), 2, "both bulks landed");+ assert_eq!(pipe.take_too_large(a), None);+ pipe.retire();+ }++ #[test]+ fn a_refused_record_fails_only_its_own_file() {+ let rejected = AtomicU64::new(0);+ let refused = |id: Option<&str>| crate::esclient::RefusedItem {+ id: id.map(str::to_owned),+ error: "mapper refused field".into(),+ too_large: false,+ };+ let pipe = BulkPipe::new(+ |_body| {+ Ok(BulkOutcome {+ item_errors: 1,+ refused: vec![refused(Some("a-1"))],+ first_error: Some("mapper refused field".into()),+ ..outcome()+ })+ },+ 1 << 20,+ 5000,+ &rejected,+ 2,+ );+ let a = pipe.token();+ let b = pipe.token();+ pipe.submit(a, b"{\"index\":{\"_id\":\"a-1\"}}\n{}\n".to_vec(), 1);+ pipe.submit(b, b"{\"index\":{\"_id\":\"b-1\"}}\n{}\n".to_vec(), 1);+ let error = pipe.settle(a).unwrap_err();+ assert!(+ error.contains("refused 1 of this file's record(s)"),+ "{error}"+ );+ pipe.settle(b)+ .expect("a file whose records were all accepted settles Ok");+ assert_eq!(rejected.load(Ordering::Relaxed), 1);+ pipe.retire();++ let rejected = AtomicU64::new(0);+ let pipe = BulkPipe::new(+ |_body| {+ Ok(BulkOutcome {+ item_errors: 1,+ refused: vec![refused(None)],first_error: Some("mapper refused field".into()),..outcome()})@@ -16911,14 +17157,14 @@ mod bulk_pipe_tests {1 << 20,5000,&rejected,- &errors,1,);let t = pipe.token();pipe.submit(t, b"a\n".to_vec(), 3);- pipe.settle(t).expect("item errors do not fail the file");- assert_eq!(rejected.load(Ordering::Relaxed), 2);- assert_eq!(errors.lock().unwrap().len(), 1);+ assert!(+ pipe.settle(t).is_err(),+ "an unattributed refusal fails the file"+ );pipe.retire();}@@ -16928,15 +17174,7 @@ mod bulk_pipe_tests {fn a_file_with_no_records_settles_without_sending() {let bodies = Mutex::new(Vec::new());let rejected = AtomicU64::new(0);- let errors = Mutex::new(Vec::new());- let pipe = BulkPipe::new(- recording_send(&bodies),- 1 << 20,- 5000,- &rejected,- &errors,- 1,- );+ let pipe = BulkPipe::new(recording_send(&bodies), 1 << 20, 5000, &rejected, 1);let t = pipe.token();pipe.settle(t).expect("pre-resolved");pipe.retire();@@ -16952,7 +17190,6 @@ mod bulk_pipe_tests {#[test]fn wire_slots_overlap_instead_of_serializing() {let rejected = AtomicU64::new(0);- let errors = Mutex::new(Vec::new());let concurrent = AtomicU64::new(0);let peak = AtomicU64::new(0);let pipe = BulkPipe::new(@@ -16970,7 +17207,6 @@ mod bulk_pipe_tests {64,5000,&rejected,- &errors,4,);let tokens: Vec<_> = (0..4).map(|_| pipe.token()).collect();@@ -16999,7 +17235,6 @@ mod bulk_pipe_tests {fn many_workers_many_files_coalesce_without_deadlock() {let bodies = Mutex::new(Vec::new());let rejected = AtomicU64::new(0);- let errors = Mutex::new(Vec::new());let pipe = BulkPipe::new({let bodies = &bodies;@@ -17012,7 +17247,6 @@ mod bulk_pipe_tests {1 << 20,5000,&rejected,- &errors,8,);const WORKERS: usize = 8;diff --git a/engine/crates/xerj-autoindex/src/sync_executor.rs b/engine/crates/xerj-autoindex/src/sync_executor.rsindex ee2c9dc2b..75ac2c763 100644--- a/engine/crates/xerj-autoindex/src/sync_executor.rs+++ b/engine/crates/xerj-autoindex/src/sync_executor.rs@@ -13,7 +13,7 @@ use crate::sync::{use anyhow::{Context, Result};use serde::{Deserialize, Serialize};use serde_json::{Map, Value};-use std::collections::{BTreeMap, BTreeSet, HashMap};+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};use std::fs::{File, OpenOptions};use std::io::{BufReader, BufWriter, Read, Write};use std::path::Path;@@ -2828,7 +2828,8 @@ fn checked_bulk(es: &crate::esclient::Es, body: Vec<u8>) -> Result<()> {if body.is_empty() {return Ok(());}- check_bulk_outcome(es, es.bulk(body)?)+ let outcome = es.bulk(body.clone())?;+ check_bulk_outcome(es, outcome, &body)}/// [`checked_bulk`] for a body nobody windowed — the catalog projection, which@@ -2838,12 +2839,48 @@ fn checked_bulk_windowed(es: &crate::esclient::Es, body: Vec<u8>, bulk_bytes: usif body.is_empty() {return Ok(());}- check_bulk_outcome(es, es.bulk_windowed(body, bulk_bytes)?)+ let outcome = es.bulk_windowed(body.clone(), bulk_bytes)?;+ check_bulk_outcome(es, outcome, &body)+}++/// The `ax_path` of each refused record, read back from the body that was+/// sent: a refusal reports only the record's `_id`, which is a hash.+fn refused_paths(body: &[u8], outcome: &crate::esclient::BulkOutcome) -> Vec<String> {+ let ids: HashSet<&str> = outcome+ .refused+ .iter()+ .filter_map(|item| item.id.as_deref())+ .collect();+ let mut paths = Vec::new();+ let mut lines = body.split(|byte| *byte == b'\n');+ while let (Some(action), Some(doc)) = (lines.next(), lines.next()) {+ let Ok(action) = serde_json::from_slice::<Value>(action) else {+ continue;+ };+ let id = action+ .as_object()+ .and_then(|op| op.values().next())+ .and_then(|op| op.get("_id"))+ .and_then(Value::as_str);+ if !id.is_some_and(|id| ids.contains(id)) {+ continue;+ }+ if let Some(path) = serde_json::from_slice::<Value>(doc)+ .ok()+ .and_then(|doc| doc.get("ax_path")?.as_str().map(str::to_owned))+ {+ if !paths.contains(&path) {+ paths.push(path);+ }+ }+ }+ paths}fn check_bulk_outcome(es: &crate::esclient::Es,outcome: crate::esclient::BulkOutcome,+ body: &[u8],) -> Result<()> {// `Es::bulk` has already re-sent per-item 429s, and only those, for the// whole of its patience (`THROTTLE_PATIENCE`, #944/#949, shortened to@@ -2877,15 +2914,34 @@ fn check_bulk_outcome(outcome.server_errors,);}- anyhow::ensure!(- outcome.item_errors == 0,- "prepared bulk contained {} rejected items: {}",+ if outcome.item_errors == 0 {+ return Ok(());+ }+ // This route seals the generation before it sends, so a file can no+ // longer leave the plan here, and a rerun replays the same refusal. Name+ // the file and the ways out instead (#2807).+ let paths = refused_paths(body, &outcome);+ let source = if paths.is_empty() {+ String::new()+ } else {+ format!(" from {}", paths.join(", "))+ };+ let too_large = outcome.refused.iter().any(|item| item.too_large);+ let remedy = if too_large {+ " The server adds vectors to a `semantic_text` field, and they count toward its \+ per-document limit. A sealed generation cannot set a file aside, so rerunning \+ repeats this refusal: index the folder under a new --prefix and --state-dir with \+ the file left out (`--exclude`), or with `--no-semantic`"+ } else {+ ""+ };+ anyhow::bail!(+ "the server refused {} record(s){source}: {}.{remedy}",outcome.item_errors,outcome.first_error.unwrap_or_else(|| "unknown error".into())- );- Ok(())+ )}fn append_index_action(index: &str, id: &str, doc: &Value, body: &mut Vec<u8>) -> Result<()> {