:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(batch-import-worker): re-send a chunk a crashed worker left mid-commit (#125954)
Eli Reisman3f09b8d0203f
diff --git a/rust/batch-import-worker/src/job/mod.rs b/rust/batch-import-worker/src/job/mod.rsindex 2bf17c318d36..11de42dc00af 100644--- a/rust/batch-import-worker/src/job/mod.rs+++ b/rust/batch-import-worker/src/job/mod.rs@@ -793,7 +793,8 @@ impl Job {self.shutdown_guard()?;txn.emit(&parsed.data).await?;// Save the advanced offset and a pending-commit marker before the sink commit. After a crash- // between the two, the next claim finds the marker and pauses the job for review.+ // between the two, the next claim finds the marker and sends the chunk again, or pauses the+ // job for review after `MAX_INTERRUPTED_SENDS` interruptions or an offset mismatch.self.shutdown_guard()?;info!(job_id = %self.job_id, "Beginning PG part commit");if self@@ -915,10 +916,18 @@ impl Job {};model_state.events_emitted = model_state.events_emitted.saturating_add(events as u64);+ let from_offset = new_offset.saturating_sub(consumed as u64);+ let interrupted = model+ .recovered_commit+ .take()+ .filter(|recovered| recovered.key == key && recovered.from_offset == from_offset)+ .map_or(0, |recovered| recovered.interrupted);model.pending_commit = Some(model::PendingCommit {key: key.to_string(),- from_offset: new_offset.saturating_sub(consumed as u64),+ from_offset,to_offset: new_offset,+ interrupted,+ events: events as u64,});let written = model.save_progress(&self.context.db, true, true).await?;@@ -1319,6 +1328,7 @@ mod tests {secrets: std::collections::HashMap::new(),},was_leased: false,+ recovered_commit: None,backoff_attempt: 5,backoff_until: None,pending_commit: None,@@ -1665,6 +1675,29 @@ mod tests {}))}+ fn one_event_per_chunk() -> Arc<ParserFn> {+ Arc::new(Box::new(|bytes: Vec<u8>| {+ let inner = common_types::CapturedEvent {+ uuid: Uuid::now_v7(),+ distinct_id: "user".to_string(),+ session_id: None,+ ip: "127.0.0.1".to_string(),+ data: "{}".to_string(),+ now: chrono::Utc::now().to_rfc3339(),+ sent_at: None,+ token: "token".to_string(),+ event: "event".to_string(),+ timestamp: chrono::Utc::now(),+ is_cookieless_mode: false,+ historical_migration: true,+ };+ Ok(Parsed {+ consumed: bytes.len(),+ data: vec![InternallyCapturedEvent { team_id: 1, inner }],+ })+ }))+ }+fn make_handle() -> lifecycle::Handle {let mut manager = lifecycle::Manager::builder("test").with_trap_signals(false)@@ -1685,6 +1718,21 @@ mod tests {/// Run a three-part job on a fresh row, calling `before_process` between commits and the/// `on_send` hook during deliveries.async fn run_job<H, B, BF>(on_send: H, before_process: B, fail_sends: bool) -> Run+ where+ H: FnOnce(Uuid, PgPool) -> Option<SendHook>,+ B: Fn(usize, Uuid, Arc<AppContext>) -> BF,+ BF: Future<Output = ()>,+ {+ run_job_after_claim(None, consume_all(), on_send, before_process, fail_sends).await+ }++ async fn run_job_after_claim<H, B, BF>(+ recovered: Option<serde_json::Value>,+ transform: Arc<ParserFn>,+ on_send: H,+ before_process: B,+ fail_sends: bool,+ ) -> RunwhereH: FnOnce(Uuid, PgPool) -> Option<SendHook>,B: Fn(usize, Uuid, Arc<AppContext>) -> BF,@@ -1704,14 +1752,24 @@ mod tests {trial: None,events_emitted: 0,};- let row = test_db::JobRowFixture {+ let mut row = test_db::JobRowFixture {status: JobStatus::Running,lease_id: "lease-from-claim",state: state.clone(),sink: serde_json::json!({"type": "capture", "send_rate": SEND_RATE}),pending_commit: None,};+ if recovered.is_some() {+ row.state.parts[0].current_offset = PART_BYTES as u64;+ row.state.events_emitted = 1;+ row.pending_commit = recovered;+ }let id = test_db::insert_job(&context.db, &row).await;+ let mut model = test_db::claimed_model(id, &row);+ assert!(!model+ .recover_interrupted_commit(context.clone())+ .await+ .unwrap());let sends = Arc::new(std::sync::Mutex::new(Vec::new()));let aborts = Arc::new(std::sync::atomic::AtomicUsize::new(0));let mut next = Some(Job {@@ -1720,9 +1778,9 @@ mod tests {job_id: id,chunk_size: PART_BYTES,state: Mutex::new(state),- model: Mutex::new(test_db::claimed_model(id, &row)),+ model: Mutex::new(model),source: Box::new(MemorySource { parts }),- transform: consume_all(),+ transform,sink: Mutex::new(Box::new(RecordingEmitter {send_rate: SEND_RATE,sends: sends.clone(),@@ -1787,6 +1845,53 @@ mod tests {async fn no_op(_step: usize, _id: Uuid, _context: Arc<AppContext>) {}+ #[tokio::test]+ async fn test_resent_chunk_keeps_its_interruption_count() {+ let markers = Arc::new(std::sync::Mutex::new(Vec::new()));+ let recorded = markers.clone();+ let run = run_job_after_claim(+ Some(serde_json::json!(+ {"key": "part-1", "from_offset": 0, "to_offset": PART_BYTES, "interrupted": 1, "events": 1}+ )),+ one_event_per_chunk(),+ move |id, pool| {+ let hook: SendHook = Arc::new(move |_index| {+ let (pool, recorded) = (pool.clone(), recorded.clone());+ Box::pin(async move {+ let marker = test_db::read_job(&pool, id).await.pending_commit.unwrap();+ recorded+ .lock()+ .unwrap()+ .push((marker["interrupted"].clone(), marker["events"].clone()));+ })+ });+ Some(hook)+ },+ no_op,+ false,+ )+ .await;++ run.result.unwrap();+ assert_eq!(+ *markers.lock().unwrap(),+ vec![+ (serde_json::json!(2), serde_json::json!(1)),+ (serde_json::json!(0), serde_json::json!(1)),+ (serde_json::json!(0), serde_json::json!(1))+ ],+ "only the re-sent chunk carries the count, so a crash loop on it reaches the cap"+ );+ let stored = test_db::read_job(&run.context.db, run.id).await;+ assert_eq!(stored.status, "completed");+ assert_eq!(+ stored.state.events_emitted, 3,+ "the re-sent chunk's events count once"+ );+ assert_eq!(stored.pending_commit, None);+ test_db::delete_job(&run.context.db, run.id).await;+ }+#[tokio::test]async fn test_send_rate_change_applies_from_the_next_chunk() {let run = run_job(@@ -2119,6 +2224,7 @@ mod tests {backoff_until: None,pending_commit: None,was_leased: false,+ recovered_commit: None,}}diff --git a/rust/batch-import-worker/src/job/model.rs b/rust/batch-import-worker/src/job/model.rsindex 5e16b41c8d4d..36ffbd196d18 100644--- a/rust/batch-import-worker/src/job/model.rs+++ b/rust/batch-import-worker/src/job/model.rs@@ -40,6 +40,10 @@ pub struct JobModel {// Not actually in the model, but we calculate it on fetch to let us reason about whether// we're resuming an interrupted jobpub was_leased: bool,++ /// The chunk a claim rewound for this worker to send again, kept out of the row so that a stop+ /// before the re-send starts (backoff, shutdown) does not count as another interrupted send.+ pub recovered_commit: Option<PendingCommit>,}#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]@@ -51,12 +55,22 @@ pub enum JobStatus {Completed,}+/// After this many interrupted sends of one chunk, a claim pauses the job for review instead of+/// re-sending, so a chunk that crashes every worker cannot loop forever.+pub const MAX_INTERRUPTED_SENDS: u32 = 3;+/// The chunk a worker is sending; a value left on the row means a worker stopped mid-commit.#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]pub struct PendingCommit {pub key: String,pub from_offset: u64,pub to_offset: u64,+ /// Earlier sends of this chunk that a worker stopped in the middle of.+ #[serde(default)]+ pub interrupted: u32,+ /// Events the chunk added to `events_emitted`, so a rewind can take them back out.+ #[serde(default)]+ pub events: u64,}/// The result of a worker write to its job row.@@ -132,6 +146,23 @@ impl JobState {Some(part.current_offset)}+ /// Move the part that `pending` names back to the start of its chunk, and take the chunk's+ /// events out of `events_emitted` when the part moves. Returns false and changes nothing when+ /// the part's offset is at neither end of the chunk.+ pub fn rewind_to_commit_start(&mut self, pending: &PendingCommit) -> bool {+ let Some(part) = self.parts.iter_mut().find(|p| p.key == pending.key) else {+ return false;+ };+ if part.current_offset != pending.to_offset && part.current_offset != pending.from_offset {+ return false;+ }+ if part.current_offset != pending.from_offset {+ part.current_offset = pending.from_offset;+ self.events_emitted = self.events_emitted.saturating_sub(pending.events);+ }+ true+ }+/// Return why the job must not complete, or `None` when it can. The read loop/// decides completion from its in-memory parts, so this checks the stored parts/// as well: a job must never report success for a part it did not finish, or@@ -365,29 +396,75 @@ impl JobModel {Ok(())}- /// Pause a job left mid-commit for review, because its `pending_commit` chunk may not have- /// reached the sink, and release the lease so a resume is not blocked. Returns true when paused.- pub async fn pause_if_interrupted_commit(+ /// Rewind a part a worker left mid-commit so this worker sends the chunk again, or pause for+ /// review at the cap or on an offset mismatch. Returns true when this worker must skip the job.+ pub async fn recover_interrupted_commit(&mut self,context: Arc<AppContext>,) -> Result<bool, Error> {let Some(pending) = self.pending_commit.clone() else {return Ok(false);};+ let interrupted = pending.interrupted.saturating_add(1);+ let rewound = interrupted < MAX_INTERRUPTED_SENDS+ && self+ .state+ .as_mut()+ .is_some_and(|state| state.rewind_to_commit_start(&pending));+ if !rewound {+ return self+ .pause_interrupted_commit(context, &pending, interrupted)+ .await;+ }++ self.pending_commit = None;+ if !self.save_recovered_commit(&context.db).await? {+ self.release_lease(&context.db).await?;+ return Ok(true);+ }+ self.recovered_commit = Some(PendingCommit {+ interrupted,+ ..pending.clone()+ });warn!(job_id = %self.id,key = %pending.key,from_offset = pending.from_offset,to_offset = pending.to_offset,+ interrupted,+ "job was interrupted mid-commit, re-sending the chunk"+ );+ metrics::interrupted_commit_resent();+ Ok(false)+ }++ async fn pause_interrupted_commit(+ &mut self,+ context: Arc<AppContext>,+ pending: &PendingCommit,+ interrupted: u32,+ ) -> Result<bool, Error> {+ warn!(+ job_id = %self.id,+ key = %pending.key,+ from_offset = pending.from_offset,+ to_offset = pending.to_offset,+ interrupted,"job was interrupted mid-commit, pausing for review");metrics::interrupted_commit_paused();+ let chunk = format!(+ "part {} (offsets {} to {})",+ pending.key, pending.from_offset, pending.to_offset+ );+ let reason = if interrupted >= MAX_INTERRUPTED_SENDS {+ format!("Workers stopped {interrupted} times while committing {chunk}.")+ } else {+ format!("A worker stopped while committing {chunk}, and the saved offset does not match that chunk.")+ };self.pause(context.clone(),- format!(- "A worker stopped while committing part {} (offsets {} to {}). The chunk may not have reached the sink.",- pending.key, pending.from_offset, pending.to_offset- ),+ format!("{reason} The chunk may not have reached the sink."),Some("Job paused after an interrupted commit. Resume it to continue.".to_string()),).await?;@@ -395,6 +472,31 @@ impl JobModel {Ok(true)}+ /// Save a rewound part and clear its marker without changing the claim lease, which stays+ /// long because job init is slow. Returns false when the row is no longer running.+ async fn save_recovered_commit(&mut self, pool: &PgPool) -> Result<bool, Error> {+ let Some(lease) = self.lease_id.clone() else {+ anyhow::bail!("Cannot write a job with no lease")+ };+ let result = sqlx::query(+ r#"+ UPDATE posthog_batchimport+ SET state = $2, pending_commit = NULL, updated_at = now()+ WHERE id = $1 AND lease_id = $3 AND status = 'running'+ "#,+ )+ .bind(self.id)+ .bind(serde_json::to_value(&self.state)?)+ .bind(&lease)+ .execute(pool)+ .await?;+ if result.rows_affected() == 1 {+ return Ok(true);+ }+ self.status = self.explain_missed_write(pool, &lease).await?;+ Ok(false)+ }+/// Give up the lease so the row shows that no worker holds it.pub async fn release_lease(&mut self, pool: &PgPool) -> Result<(), Error> {let Some(lease) = self.lease_id.take() else {@@ -666,6 +768,7 @@ impl TryFrom<(JobRow, &[String], String)> for JobModel {backoff_until: row.backoff_until,pending_commit,was_leased: row.previous_lease_id.is_some(),+ recovered_commit: None,})}}@@ -716,6 +819,7 @@ mod tests {backoff_until: None,pending_commit: None,was_leased: false,+ recovered_commit: None,}}@@ -1046,38 +1150,153 @@ mod tests {}}+ struct RecoveryCase {+ name: &'static str,+ part_offset: u64,+ pending: Option<serde_json::Value>,+ operator: Option<JobStatus>,+ skipped: bool,+ status: &'static str,+ offset: u64,+ stored_pending: Option<serde_json::Value>,+ recovered_interrupted: Option<u32>,+ events_emitted: u64,+ message: Option<&'static str>,+ }+#[tokio::test]- async fn test_claim_with_pending_commit_pauses_for_review() {+ async fn test_claim_recovers_an_interrupted_commit() {let context = test_db::context().await;- for pending in [- None,- Some(serde_json::json!({"key": "part-1", "from_offset": 100, "to_offset": 250})),- ] {- let row = fixture(JobStatus::Running, pending.clone());+ let marker = |key: &str, interrupted: u32| {+ Some(serde_json::json!(+ {"key": key, "from_offset": 100, "to_offset": 250, "interrupted": interrupted, "events": 40}+ ))+ };+ let resent = |name, part_offset, interrupted| RecoveryCase {+ name,+ part_offset,+ pending: marker("part-1", interrupted),+ operator: None,+ skipped: false,+ status: "running",+ offset: 100,+ stored_pending: None,+ recovered_interrupted: Some(interrupted + 1),+ events_emitted: if part_offset == 250 { 60 } else { 100 },+ message: None,+ };+ let paused =+ |name, part_offset, pending: Option<serde_json::Value>, message| RecoveryCase {+ name,+ part_offset,+ pending: pending.clone(),+ operator: None,+ skipped: true,+ status: "paused",+ offset: part_offset,+ stored_pending: pending,+ recovered_interrupted: None,+ events_emitted: 100,+ message: Some(message),+ };+ let cases = [+ RecoveryCase {+ pending: None,+ offset: 250,+ recovered_interrupted: None,+ events_emitted: 100,+ ..resent("no marker", 250, 0)+ },+ resent("first crash re-sends", 250, 0),+ resent("part already at the chunk start", 100, 1),+ paused("cap pauses", 250, marker("part-1", 2), "3 times"),+ paused(+ "offset mismatch pauses",+ 300,+ marker("part-1", 0),+ "does not match",+ ),+ paused(+ "unknown part pauses",+ 250,+ marker("part-9", 0),+ "does not match",+ ),+ RecoveryCase {+ operator: Some(JobStatus::Paused),+ ..paused(+ "operator stopped it after the claim",+ 250,+ marker("part-1", 0),+ test_db::OPERATOR_MESSAGE,+ )+ },+ ];+ for case in cases {+ let name = case.name;+ let row = test_db::JobRowFixture {+ state: JobState {+ events_emitted: 100,+ ..make_job_state(&[("part-1", case.part_offset)])+ },+ ..fixture(JobStatus::Running, case.pending)+ };let id = test_db::insert_job(&context.db, &row).await;let mut model = test_db::claimed_model(id, &row);+ if let Some(operator_status) = case.operator {+ test_db::operator_sets(&context.db, id, operator_status).await;+ }+ let claimed = test_db::read_job(&context.db, id).await;- let paused = model- .pause_if_interrupted_commit(context.clone())+ let skipped = model+ .recover_interrupted_commit(context.clone()).await.unwrap();let stored = test_db::read_job(&context.db, id).await;- assert_eq!(paused, pending.is_some());- if pending.is_some() {- assert_eq!(stored.status, "paused");- let message = stored.status_message.unwrap();- assert!(message.contains("part-1"), "{message}");- assert!(message.contains("100 to 250"), "{message}");- assert!(stored.display_status_message.is_some());- assert_eq!(stored.pending_commit, pending);+ assert_eq!(skipped, case.skipped, "case '{name}'");+ assert_eq!(stored.status, case.status, "case '{name}'");+ assert_eq!(+ stored.state.parts[0].current_offset, case.offset,+ "case '{name}'"+ );+ assert_eq!(+ stored.state.events_emitted, case.events_emitted,+ "case '{name}': only a part that moves back gives up its chunk's events"+ );+ assert_eq!(+ stored.pending_commit, case.stored_pending,+ "case '{name}': a re-send clears the marker, so a stop before it starts adds no interruption"+ );+ assert_eq!(+ model.recovered_commit.as_ref().map(|r| r.interrupted),+ case.recovered_interrupted,+ "case '{name}'"+ );+ if case.skipped {assert_eq!(stored.lease_id, None,- "a resume must not wait for this lease"+ "case '{name}': a skipped job frees the row");} else {- assert_eq!(stored.status, "running");+ assert_eq!(+ (stored.lease_id, stored.leased_until),+ (claimed.lease_id, claimed.leased_until),+ "case '{name}': the worker keeps its claim lease while job init runs"+ );}+ if let Some(message) = case.message {+ let stored_message = stored.status_message.unwrap();+ assert!(+ stored_message.contains(message),+ "case '{name}': {stored_message}"+ );+ }+ assert_eq!(+ stored.display_status_message.is_some(),+ case.skipped && case.operator.is_none(),+ "case '{name}': only a review pause tells the customer to resume"+ );test_db::delete_job(&context.db, id).await;}}@@ -1127,7 +1346,9 @@ mod tests {Some(PendingCommit {key: "k".to_string(),from_offset: 1,- to_offset: 2+ to_offset: 2,+ interrupted: 0,+ events: 0,}));}diff --git a/rust/batch-import-worker/src/job/test_db.rs b/rust/batch-import-worker/src/job/test_db.rsindex da3ae543583b..d19028f3fa23 100644--- a/rust/batch-import-worker/src/job/test_db.rs+++ b/rust/batch-import-worker/src/job/test_db.rs@@ -81,6 +81,7 @@ pub struct StoredJob {pub status_message: Option<String>,pub display_status_message: Option<String>,pub lease_id: Option<String>,+ pub leased_until: Option<DateTime<Utc>>,pub state: JobState,pub pending_commit: Option<Value>,pub backoff_until: Option<DateTime<Utc>>,@@ -89,8 +90,8 @@ pub struct StoredJob {pub async fn read_job(pool: &PgPool, id: Uuid) -> StoredJob {let row = sqlx::query(r#"- SELECT status, status_message, display_status_message, lease_id, state, pending_commit,- backoff_until+ SELECT status, status_message, display_status_message, lease_id, leased_until, state,+ pending_commit, backoff_untilFROM posthog_batchimport WHERE id = $1"#,)@@ -103,6 +104,7 @@ pub async fn read_job(pool: &PgPool, id: Uuid) -> StoredJob {status_message: row.get("status_message"),display_status_message: row.get("display_status_message"),lease_id: row.get("lease_id"),+ leased_until: row.get("leased_until"),state: serde_json::from_value(row.get("state")).expect("state json"),pending_commit: row.get("pending_commit"),backoff_until: row.get("backoff_until"),@@ -179,5 +181,6 @@ pub fn claimed_model(id: Uuid, row: &JobRowFixture) -> JobModel {.clone().map(|v| serde_json::from_value(v).expect("pending commit")),was_leased: false,+ recovered_commit: None,}}diff --git a/rust/batch-import-worker/src/main.rs b/rust/batch-import-worker/src/main.rsindex 17a346a57462..3fd39b396668 100644--- a/rust/batch-import-worker/src/main.rs+++ b/rust/batch-import-worker/src/main.rs@@ -209,11 +209,11 @@ pub async fn main() -> Result<(), Error> {info!("Claimed job: {:?}", model.id);- match model.pause_if_interrupted_commit(context.clone()).await {+ match model.recover_interrupted_commit(context.clone()).await {Ok(true) => continue,Ok(false) => {}Err(e) => {- error!("Failed to pause interrupted job {}: {:?}", model.id, e);+ error!("Failed to recover interrupted job {}: {:?}", model.id, e);continue;}}diff --git a/rust/batch-import-worker/src/metrics.rs b/rust/batch-import-worker/src/metrics.rsindex b8e61dcb4ed4..6edf6e0706b5 100644--- a/rust/batch-import-worker/src/metrics.rs+++ b/rust/batch-import-worker/src/metrics.rs@@ -18,6 +18,7 @@ pub const COMPLETED_JOB_EVENTS: &str = "batch_import_completed_job_events";pub const COMPLETED_JOB_PARTS: &str = "batch_import_completed_job_parts";pub const OPERATOR_STOP_TOTAL: &str = "batch_import_operator_stop_total";pub const INTERRUPTED_COMMIT_PAUSED_TOTAL: &str = "batch_import_interrupted_commit_paused_total";+pub const INTERRUPTED_COMMIT_RESENT_TOTAL: &str = "batch_import_interrupted_commit_resent_total";use metrics::{counter, gauge, histogram};@@ -41,6 +42,11 @@ pub fn interrupted_commit_paused() {counter!(INTERRUPTED_COMMIT_PAUSED_TOTAL).increment(1);}+/// Count jobs rewound at claim to re-send a chunk an earlier worker stopped sending.+pub fn interrupted_commit_resent() {+ counter!(INTERRUPTED_COMMIT_RESENT_TOTAL).increment(1);+}+pub fn staging_sweep_removed(count: u64) {counter!(STAGING_SWEEP_REMOVED).increment(count);}diff --git a/rust/batch-import-worker/tests/common/harness.rs b/rust/batch-import-worker/tests/common/harness.rsindex 2cbbcdf84f18..2435b75579f1 100644--- a/rust/batch-import-worker/tests/common/harness.rs+++ b/rust/batch-import-worker/tests/common/harness.rs@@ -113,6 +113,7 @@ impl Harness {backoff_until: None,pending_commit: None,was_leased: false,+ recovered_commit: None,};Self {diff --git a/rust/batch-import-worker/tests/trial_e2e_test.rs b/rust/batch-import-worker/tests/trial_e2e_test.rsindex 95bfb2e02488..6d05a70c91d2 100644--- a/rust/batch-import-worker/tests/trial_e2e_test.rs+++ b/rust/batch-import-worker/tests/trial_e2e_test.rs@@ -185,6 +185,7 @@ impl TrialRun {backoff_until: None,pending_commit: None,was_leased: false,+ recovered_commit: None,};let prefix = format!("trial-e2e-test/team_{TEAM_ID}/job_{job_id}");