Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(batch-import-worker): re-send a chunk a crashed worker left mid-commit (#125954)

Eli Reisman3f09b8d0203f
diff --git a/rust/batch-import-worker/src/job/mod.rs b/rust/batch-import-worker/src/job/mod.rs
index 2bf17c318d36..11de42dc00af 100644
--- a/rust/batch-import-worker/src/job/mod.rs
+++ b/rust/batch-import-worker/src/job/mod.rs
@@ -793,7 +793,8 @@ impl Job {
self.shutdown_guard()?;
txn.emit(&parsed.data).await?;
// Save the advanced offset and a pending-commit marker before the sink commit. After a crash
- // between the two, the next claim finds the marker and pauses the job for review.
+ // between the two, the next claim finds the marker and sends the chunk again, or pauses the
+ // job for review after `MAX_INTERRUPTED_SENDS` interruptions or an offset mismatch.
self.shutdown_guard()?;
info!(job_id = %self.job_id, "Beginning PG part commit");
if self
@@ -915,10 +916,18 @@ impl Job {
};
model_state.events_emitted = model_state.events_emitted.saturating_add(events as u64);
+ let from_offset = new_offset.saturating_sub(consumed as u64);
+ let interrupted = model
+ .recovered_commit
+ .take()
+ .filter(|recovered| recovered.key == key && recovered.from_offset == from_offset)
+ .map_or(0, |recovered| recovered.interrupted);
model.pending_commit = Some(model::PendingCommit {
key: key.to_string(),
- from_offset: new_offset.saturating_sub(consumed as u64),
+ from_offset,
to_offset: new_offset,
+ interrupted,
+ events: events as u64,
});
let written = model.save_progress(&self.context.db, true, true).await?;
@@ -1319,6 +1328,7 @@ mod tests {
secrets: std::collections::HashMap::new(),
},
was_leased: false,
+ recovered_commit: None,
backoff_attempt: 5,
backoff_until: None,
pending_commit: None,
@@ -1665,6 +1675,29 @@ mod tests {
}))
}
+ fn one_event_per_chunk() -> Arc<ParserFn> {
+ Arc::new(Box::new(|bytes: Vec<u8>| {
+ let inner = common_types::CapturedEvent {
+ uuid: Uuid::now_v7(),
+ distinct_id: "user".to_string(),
+ session_id: None,
+ ip: "127.0.0.1".to_string(),
+ data: "{}".to_string(),
+ now: chrono::Utc::now().to_rfc3339(),
+ sent_at: None,
+ token: "token".to_string(),
+ event: "event".to_string(),
+ timestamp: chrono::Utc::now(),
+ is_cookieless_mode: false,
+ historical_migration: true,
+ };
+ Ok(Parsed {
+ consumed: bytes.len(),
+ data: vec![InternallyCapturedEvent { team_id: 1, inner }],
+ })
+ }))
+ }
+
fn make_handle() -> lifecycle::Handle {
let mut manager = lifecycle::Manager::builder("test")
.with_trap_signals(false)
@@ -1685,6 +1718,21 @@ mod tests {
/// Run a three-part job on a fresh row, calling `before_process` between commits and the
/// `on_send` hook during deliveries.
async fn run_job<H, B, BF>(on_send: H, before_process: B, fail_sends: bool) -> Run
+ where
+ H: FnOnce(Uuid, PgPool) -> Option<SendHook>,
+ B: Fn(usize, Uuid, Arc<AppContext>) -> BF,
+ BF: Future<Output = ()>,
+ {
+ run_job_after_claim(None, consume_all(), on_send, before_process, fail_sends).await
+ }
+
+ async fn run_job_after_claim<H, B, BF>(
+ recovered: Option<serde_json::Value>,
+ transform: Arc<ParserFn>,
+ on_send: H,
+ before_process: B,
+ fail_sends: bool,
+ ) -> Run
where
H: FnOnce(Uuid, PgPool) -> Option<SendHook>,
B: Fn(usize, Uuid, Arc<AppContext>) -> BF,
@@ -1704,14 +1752,24 @@ mod tests {
trial: None,
events_emitted: 0,
};
- let row = test_db::JobRowFixture {
+ let mut row = test_db::JobRowFixture {
status: JobStatus::Running,
lease_id: "lease-from-claim",
state: state.clone(),
sink: serde_json::json!({"type": "capture", "send_rate": SEND_RATE}),
pending_commit: None,
};
+ if recovered.is_some() {
+ row.state.parts[0].current_offset = PART_BYTES as u64;
+ row.state.events_emitted = 1;
+ row.pending_commit = recovered;
+ }
let id = test_db::insert_job(&context.db, &row).await;
+ let mut model = test_db::claimed_model(id, &row);
+ assert!(!model
+ .recover_interrupted_commit(context.clone())
+ .await
+ .unwrap());
let sends = Arc::new(std::sync::Mutex::new(Vec::new()));
let aborts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let mut next = Some(Job {
@@ -1720,9 +1778,9 @@ mod tests {
job_id: id,
chunk_size: PART_BYTES,
state: Mutex::new(state),
- model: Mutex::new(test_db::claimed_model(id, &row)),
+ model: Mutex::new(model),
source: Box::new(MemorySource { parts }),
- transform: consume_all(),
+ transform,
sink: Mutex::new(Box::new(RecordingEmitter {
send_rate: SEND_RATE,
sends: sends.clone(),
@@ -1787,6 +1845,53 @@ mod tests {
async fn no_op(_step: usize, _id: Uuid, _context: Arc<AppContext>) {}
+ #[tokio::test]
+ async fn test_resent_chunk_keeps_its_interruption_count() {
+ let markers = Arc::new(std::sync::Mutex::new(Vec::new()));
+ let recorded = markers.clone();
+ let run = run_job_after_claim(
+ Some(serde_json::json!(
+ {"key": "part-1", "from_offset": 0, "to_offset": PART_BYTES, "interrupted": 1, "events": 1}
+ )),
+ one_event_per_chunk(),
+ move |id, pool| {
+ let hook: SendHook = Arc::new(move |_index| {
+ let (pool, recorded) = (pool.clone(), recorded.clone());
+ Box::pin(async move {
+ let marker = test_db::read_job(&pool, id).await.pending_commit.unwrap();
+ recorded
+ .lock()
+ .unwrap()
+ .push((marker["interrupted"].clone(), marker["events"].clone()));
+ })
+ });
+ Some(hook)
+ },
+ no_op,
+ false,
+ )
+ .await;
+
+ run.result.unwrap();
+ assert_eq!(
+ *markers.lock().unwrap(),
+ vec![
+ (serde_json::json!(2), serde_json::json!(1)),
+ (serde_json::json!(0), serde_json::json!(1)),
+ (serde_json::json!(0), serde_json::json!(1))
+ ],
+ "only the re-sent chunk carries the count, so a crash loop on it reaches the cap"
+ );
+ let stored = test_db::read_job(&run.context.db, run.id).await;
+ assert_eq!(stored.status, "completed");
+ assert_eq!(
+ stored.state.events_emitted, 3,
+ "the re-sent chunk's events count once"
+ );
+ assert_eq!(stored.pending_commit, None);
+ test_db::delete_job(&run.context.db, run.id).await;
+ }
+
#[tokio::test]
async fn test_send_rate_change_applies_from_the_next_chunk() {
let run = run_job(
@@ -2119,6 +2224,7 @@ mod tests {
backoff_until: None,
pending_commit: None,
was_leased: false,
+ recovered_commit: None,
}
}
diff --git a/rust/batch-import-worker/src/job/model.rs b/rust/batch-import-worker/src/job/model.rs
index 5e16b41c8d4d..36ffbd196d18 100644
--- a/rust/batch-import-worker/src/job/model.rs
+++ b/rust/batch-import-worker/src/job/model.rs
@@ -40,6 +40,10 @@ pub struct JobModel {
// Not actually in the model, but we calculate it on fetch to let us reason about whether
// we're resuming an interrupted job
pub was_leased: bool,
+
+ /// The chunk a claim rewound for this worker to send again, kept out of the row so that a stop
+ /// before the re-send starts (backoff, shutdown) does not count as another interrupted send.
+ pub recovered_commit: Option<PendingCommit>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -51,12 +55,22 @@ pub enum JobStatus {
Completed,
}
+/// After this many interrupted sends of one chunk, a claim pauses the job for review instead of
+/// re-sending, so a chunk that crashes every worker cannot loop forever.
+pub const MAX_INTERRUPTED_SENDS: u32 = 3;
+
/// The chunk a worker is sending; a value left on the row means a worker stopped mid-commit.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct PendingCommit {
pub key: String,
pub from_offset: u64,
pub to_offset: u64,
+ /// Earlier sends of this chunk that a worker stopped in the middle of.
+ #[serde(default)]
+ pub interrupted: u32,
+ /// Events the chunk added to `events_emitted`, so a rewind can take them back out.
+ #[serde(default)]
+ pub events: u64,
}
/// The result of a worker write to its job row.
@@ -132,6 +146,23 @@ impl JobState {
Some(part.current_offset)
}
+ /// Move the part that `pending` names back to the start of its chunk, and take the chunk's
+ /// events out of `events_emitted` when the part moves. Returns false and changes nothing when
+ /// the part's offset is at neither end of the chunk.
+ pub fn rewind_to_commit_start(&mut self, pending: &PendingCommit) -> bool {
+ let Some(part) = self.parts.iter_mut().find(|p| p.key == pending.key) else {
+ return false;
+ };
+ if part.current_offset != pending.to_offset && part.current_offset != pending.from_offset {
+ return false;
+ }
+ if part.current_offset != pending.from_offset {
+ part.current_offset = pending.from_offset;
+ self.events_emitted = self.events_emitted.saturating_sub(pending.events);
+ }
+ true
+ }
+
/// Return why the job must not complete, or `None` when it can. The read loop
/// decides completion from its in-memory parts, so this checks the stored parts
/// as well: a job must never report success for a part it did not finish, or
@@ -365,29 +396,75 @@ impl JobModel {
Ok(())
}
- /// Pause a job left mid-commit for review, because its `pending_commit` chunk may not have
- /// reached the sink, and release the lease so a resume is not blocked. Returns true when paused.
- pub async fn pause_if_interrupted_commit(
+ /// Rewind a part a worker left mid-commit so this worker sends the chunk again, or pause for
+ /// review at the cap or on an offset mismatch. Returns true when this worker must skip the job.
+ pub async fn recover_interrupted_commit(
&mut self,
context: Arc<AppContext>,
) -> Result<bool, Error> {
let Some(pending) = self.pending_commit.clone() else {
return Ok(false);
};
+ let interrupted = pending.interrupted.saturating_add(1);
+ let rewound = interrupted < MAX_INTERRUPTED_SENDS
+ && self
+ .state
+ .as_mut()
+ .is_some_and(|state| state.rewind_to_commit_start(&pending));
+ if !rewound {
+ return self
+ .pause_interrupted_commit(context, &pending, interrupted)
+ .await;
+ }
+
+ self.pending_commit = None;
+ if !self.save_recovered_commit(&context.db).await? {
+ self.release_lease(&context.db).await?;
+ return Ok(true);
+ }
+ self.recovered_commit = Some(PendingCommit {
+ interrupted,
+ ..pending.clone()
+ });
warn!(
job_id = %self.id,
key = %pending.key,
from_offset = pending.from_offset,
to_offset = pending.to_offset,
+ interrupted,
+ "job was interrupted mid-commit, re-sending the chunk"
+ );
+ metrics::interrupted_commit_resent();
+ Ok(false)
+ }
+
+ async fn pause_interrupted_commit(
+ &mut self,
+ context: Arc<AppContext>,
+ pending: &PendingCommit,
+ interrupted: u32,
+ ) -> Result<bool, Error> {
+ warn!(
+ job_id = %self.id,
+ key = %pending.key,
+ from_offset = pending.from_offset,
+ to_offset = pending.to_offset,
+ interrupted,
"job was interrupted mid-commit, pausing for review"
);
metrics::interrupted_commit_paused();
+ let chunk = format!(
+ "part {} (offsets {} to {})",
+ pending.key, pending.from_offset, pending.to_offset
+ );
+ let reason = if interrupted >= MAX_INTERRUPTED_SENDS {
+ format!("Workers stopped {interrupted} times while committing {chunk}.")
+ } else {
+ format!("A worker stopped while committing {chunk}, and the saved offset does not match that chunk.")
+ };
self.pause(
context.clone(),
- format!(
- "A worker stopped while committing part {} (offsets {} to {}). The chunk may not have reached the sink.",
- pending.key, pending.from_offset, pending.to_offset
- ),
+ format!("{reason} The chunk may not have reached the sink."),
Some("Job paused after an interrupted commit. Resume it to continue.".to_string()),
)
.await?;
@@ -395,6 +472,31 @@ impl JobModel {
Ok(true)
}
+ /// Save a rewound part and clear its marker without changing the claim lease, which stays
+ /// long because job init is slow. Returns false when the row is no longer running.
+ async fn save_recovered_commit(&mut self, pool: &PgPool) -> Result<bool, Error> {
+ let Some(lease) = self.lease_id.clone() else {
+ anyhow::bail!("Cannot write a job with no lease")
+ };
+ let result = sqlx::query(
+ r#"
+ UPDATE posthog_batchimport
+ SET state = $2, pending_commit = NULL, updated_at = now()
+ WHERE id = $1 AND lease_id = $3 AND status = 'running'
+ "#,
+ )
+ .bind(self.id)
+ .bind(serde_json::to_value(&self.state)?)
+ .bind(&lease)
+ .execute(pool)
+ .await?;
+ if result.rows_affected() == 1 {
+ return Ok(true);
+ }
+ self.status = self.explain_missed_write(pool, &lease).await?;
+ Ok(false)
+ }
+
/// Give up the lease so the row shows that no worker holds it.
pub async fn release_lease(&mut self, pool: &PgPool) -> Result<(), Error> {
let Some(lease) = self.lease_id.take() else {
@@ -666,6 +768,7 @@ impl TryFrom<(JobRow, &[String], String)> for JobModel {
backoff_until: row.backoff_until,
pending_commit,
was_leased: row.previous_lease_id.is_some(),
+ recovered_commit: None,
})
}
}
@@ -716,6 +819,7 @@ mod tests {
backoff_until: None,
pending_commit: None,
was_leased: false,
+ recovered_commit: None,
}
}
@@ -1046,38 +1150,153 @@ mod tests {
}
}
+ struct RecoveryCase {
+ name: &'static str,
+ part_offset: u64,
+ pending: Option<serde_json::Value>,
+ operator: Option<JobStatus>,
+ skipped: bool,
+ status: &'static str,
+ offset: u64,
+ stored_pending: Option<serde_json::Value>,
+ recovered_interrupted: Option<u32>,
+ events_emitted: u64,
+ message: Option<&'static str>,
+ }
+
#[tokio::test]
- async fn test_claim_with_pending_commit_pauses_for_review() {
+ async fn test_claim_recovers_an_interrupted_commit() {
let context = test_db::context().await;
- for pending in [
- None,
- Some(serde_json::json!({"key": "part-1", "from_offset": 100, "to_offset": 250})),
- ] {
- let row = fixture(JobStatus::Running, pending.clone());
+ let marker = |key: &str, interrupted: u32| {
+ Some(serde_json::json!(
+ {"key": key, "from_offset": 100, "to_offset": 250, "interrupted": interrupted, "events": 40}
+ ))
+ };
+ let resent = |name, part_offset, interrupted| RecoveryCase {
+ name,
+ part_offset,
+ pending: marker("part-1", interrupted),
+ operator: None,
+ skipped: false,
+ status: "running",
+ offset: 100,
+ stored_pending: None,
+ recovered_interrupted: Some(interrupted + 1),
+ events_emitted: if part_offset == 250 { 60 } else { 100 },
+ message: None,
+ };
+ let paused =
+ |name, part_offset, pending: Option<serde_json::Value>, message| RecoveryCase {
+ name,
+ part_offset,
+ pending: pending.clone(),
+ operator: None,
+ skipped: true,
+ status: "paused",
+ offset: part_offset,
+ stored_pending: pending,
+ recovered_interrupted: None,
+ events_emitted: 100,
+ message: Some(message),
+ };
+ let cases = [
+ RecoveryCase {
+ pending: None,
+ offset: 250,
+ recovered_interrupted: None,
+ events_emitted: 100,
+ ..resent("no marker", 250, 0)
+ },
+ resent("first crash re-sends", 250, 0),
+ resent("part already at the chunk start", 100, 1),
+ paused("cap pauses", 250, marker("part-1", 2), "3 times"),
+ paused(
+ "offset mismatch pauses",
+ 300,
+ marker("part-1", 0),
+ "does not match",
+ ),
+ paused(
+ "unknown part pauses",
+ 250,
+ marker("part-9", 0),
+ "does not match",
+ ),
+ RecoveryCase {
+ operator: Some(JobStatus::Paused),
+ ..paused(
+ "operator stopped it after the claim",
+ 250,
+ marker("part-1", 0),
+ test_db::OPERATOR_MESSAGE,
+ )
+ },
+ ];
+ for case in cases {
+ let name = case.name;
+ let row = test_db::JobRowFixture {
+ state: JobState {
+ events_emitted: 100,
+ ..make_job_state(&[("part-1", case.part_offset)])
+ },
+ ..fixture(JobStatus::Running, case.pending)
+ };
let id = test_db::insert_job(&context.db, &row).await;
let mut model = test_db::claimed_model(id, &row);
+ if let Some(operator_status) = case.operator {
+ test_db::operator_sets(&context.db, id, operator_status).await;
+ }
+ let claimed = test_db::read_job(&context.db, id).await;
- let paused = model
- .pause_if_interrupted_commit(context.clone())
+ let skipped = model
+ .recover_interrupted_commit(context.clone())
.await
.unwrap();
let stored = test_db::read_job(&context.db, id).await;
- assert_eq!(paused, pending.is_some());
- if pending.is_some() {
- assert_eq!(stored.status, "paused");
- let message = stored.status_message.unwrap();
- assert!(message.contains("part-1"), "{message}");
- assert!(message.contains("100 to 250"), "{message}");
- assert!(stored.display_status_message.is_some());
- assert_eq!(stored.pending_commit, pending);
+ assert_eq!(skipped, case.skipped, "case '{name}'");
+ assert_eq!(stored.status, case.status, "case '{name}'");
+ assert_eq!(
+ stored.state.parts[0].current_offset, case.offset,
+ "case '{name}'"
+ );
+ assert_eq!(
+ stored.state.events_emitted, case.events_emitted,
+ "case '{name}': only a part that moves back gives up its chunk's events"
+ );
+ assert_eq!(
+ stored.pending_commit, case.stored_pending,
+ "case '{name}': a re-send clears the marker, so a stop before it starts adds no interruption"
+ );
+ assert_eq!(
+ model.recovered_commit.as_ref().map(|r| r.interrupted),
+ case.recovered_interrupted,
+ "case '{name}'"
+ );
+ if case.skipped {
assert_eq!(
stored.lease_id, None,
- "a resume must not wait for this lease"
+ "case '{name}': a skipped job frees the row"
);
} else {
- assert_eq!(stored.status, "running");
+ assert_eq!(
+ (stored.lease_id, stored.leased_until),
+ (claimed.lease_id, claimed.leased_until),
+ "case '{name}': the worker keeps its claim lease while job init runs"
+ );
}
+ if let Some(message) = case.message {
+ let stored_message = stored.status_message.unwrap();
+ assert!(
+ stored_message.contains(message),
+ "case '{name}': {stored_message}"
+ );
+ }
+ assert_eq!(
+ stored.display_status_message.is_some(),
+ case.skipped && case.operator.is_none(),
+ "case '{name}': only a review pause tells the customer to resume"
+ );
test_db::delete_job(&context.db, id).await;
}
}
@@ -1127,7 +1346,9 @@ mod tests {
Some(PendingCommit {
key: "k".to_string(),
from_offset: 1,
- to_offset: 2
+ to_offset: 2,
+ interrupted: 0,
+ events: 0,
})
);
}
diff --git a/rust/batch-import-worker/src/job/test_db.rs b/rust/batch-import-worker/src/job/test_db.rs
index da3ae543583b..d19028f3fa23 100644
--- a/rust/batch-import-worker/src/job/test_db.rs
+++ b/rust/batch-import-worker/src/job/test_db.rs
@@ -81,6 +81,7 @@ pub struct StoredJob {
pub status_message: Option<String>,
pub display_status_message: Option<String>,
pub lease_id: Option<String>,
+ pub leased_until: Option<DateTime<Utc>>,
pub state: JobState,
pub pending_commit: Option<Value>,
pub backoff_until: Option<DateTime<Utc>>,
@@ -89,8 +90,8 @@ pub struct StoredJob {
pub async fn read_job(pool: &PgPool, id: Uuid) -> StoredJob {
let row = sqlx::query(
r#"
- SELECT status, status_message, display_status_message, lease_id, state, pending_commit,
- backoff_until
+ SELECT status, status_message, display_status_message, lease_id, leased_until, state,
+ pending_commit, backoff_until
FROM posthog_batchimport WHERE id = $1
"#,
)
@@ -103,6 +104,7 @@ pub async fn read_job(pool: &PgPool, id: Uuid) -> StoredJob {
status_message: row.get("status_message"),
display_status_message: row.get("display_status_message"),
lease_id: row.get("lease_id"),
+ leased_until: row.get("leased_until"),
state: serde_json::from_value(row.get("state")).expect("state json"),
pending_commit: row.get("pending_commit"),
backoff_until: row.get("backoff_until"),
@@ -179,5 +181,6 @@ pub fn claimed_model(id: Uuid, row: &JobRowFixture) -> JobModel {
.clone()
.map(|v| serde_json::from_value(v).expect("pending commit")),
was_leased: false,
+ recovered_commit: None,
}
}
diff --git a/rust/batch-import-worker/src/main.rs b/rust/batch-import-worker/src/main.rs
index 17a346a57462..3fd39b396668 100644
--- a/rust/batch-import-worker/src/main.rs
+++ b/rust/batch-import-worker/src/main.rs
@@ -209,11 +209,11 @@ pub async fn main() -> Result<(), Error> {
info!("Claimed job: {:?}", model.id);
- match model.pause_if_interrupted_commit(context.clone()).await {
+ match model.recover_interrupted_commit(context.clone()).await {
Ok(true) => continue,
Ok(false) => {}
Err(e) => {
- error!("Failed to pause interrupted job {}: {:?}", model.id, e);
+ error!("Failed to recover interrupted job {}: {:?}", model.id, e);
continue;
}
}
diff --git a/rust/batch-import-worker/src/metrics.rs b/rust/batch-import-worker/src/metrics.rs
index b8e61dcb4ed4..6edf6e0706b5 100644
--- a/rust/batch-import-worker/src/metrics.rs
+++ b/rust/batch-import-worker/src/metrics.rs
@@ -18,6 +18,7 @@ pub const COMPLETED_JOB_EVENTS: &str = "batch_import_completed_job_events";
pub const COMPLETED_JOB_PARTS: &str = "batch_import_completed_job_parts";
pub const OPERATOR_STOP_TOTAL: &str = "batch_import_operator_stop_total";
pub const INTERRUPTED_COMMIT_PAUSED_TOTAL: &str = "batch_import_interrupted_commit_paused_total";
+pub const INTERRUPTED_COMMIT_RESENT_TOTAL: &str = "batch_import_interrupted_commit_resent_total";
use metrics::{counter, gauge, histogram};
@@ -41,6 +42,11 @@ pub fn interrupted_commit_paused() {
counter!(INTERRUPTED_COMMIT_PAUSED_TOTAL).increment(1);
}
+/// Count jobs rewound at claim to re-send a chunk an earlier worker stopped sending.
+pub fn interrupted_commit_resent() {
+ counter!(INTERRUPTED_COMMIT_RESENT_TOTAL).increment(1);
+}
+
pub fn staging_sweep_removed(count: u64) {
counter!(STAGING_SWEEP_REMOVED).increment(count);
}
diff --git a/rust/batch-import-worker/tests/common/harness.rs b/rust/batch-import-worker/tests/common/harness.rs
index 2cbbcdf84f18..2435b75579f1 100644
--- a/rust/batch-import-worker/tests/common/harness.rs
+++ b/rust/batch-import-worker/tests/common/harness.rs
@@ -113,6 +113,7 @@ impl Harness {
backoff_until: None,
pending_commit: None,
was_leased: false,
+ recovered_commit: None,
};
Self {
diff --git a/rust/batch-import-worker/tests/trial_e2e_test.rs b/rust/batch-import-worker/tests/trial_e2e_test.rs
index 95bfb2e02488..6d05a70c91d2 100644
--- a/rust/batch-import-worker/tests/trial_e2e_test.rs
+++ b/rust/batch-import-worker/tests/trial_e2e_test.rs
@@ -185,6 +185,7 @@ impl TrialRun {
backoff_until: None,
pending_commit: None,
was_leased: false,
+ recovered_commit: None,
};
let prefix = format!("trial-e2e-test/team_{TEAM_ID}/job_{job_id}");