:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(batch-imports): stop marking empty or unfinished imports complete (#118289)
posthog[bot]734468db6f95
diff --git a/products/managed_migrations/frontend/ManagedMigration.tsx b/products/managed_migrations/frontend/ManagedMigration.tsxindex 4ee76a9e526c..3e1c17a3d1c2 100644--- a/products/managed_migrations/frontend/ManagedMigration.tsx+++ b/products/managed_migrations/frontend/ManagedMigration.tsx@@ -17,6 +17,7 @@ import { LemonProgress } from 'lib/lemon-ui/LemonProgress'import { LemonSegmentedButton } from 'lib/lemon-ui/LemonSegmentedButton'import { LemonSelect } from 'lib/lemon-ui/LemonSelect'import { ProfilePicture } from 'lib/lemon-ui/ProfilePicture'+import { humanFriendlyNumber } from 'lib/utils/numbers'import { sceneConfigurations } from 'scenes/scenes'import { Scene, SceneExport } from 'scenes/sceneTypes'import { urls } from 'scenes/urls'@@ -454,6 +455,17 @@ export function ManagedMigration(): JSX.Element {)}+function completedLabel(migration: ManagedMigrationData): string {+ const eventsEmitted = migration.state?.events_emitted+ if (eventsEmitted === undefined) {+ return 'Complete'+ }+ if (eventsEmitted === 0) {+ return 'Complete, no events imported'+ }+ return `Complete, ${humanFriendlyNumber(eventsEmitted)} events imported`+}+export function ManagedMigrations(): JSX.Element {const { managedMigrationId, migrations, migrationsLoading, promotingMigrationId } = useValues(managedMigrationLogic)const { pauseMigration, resumeMigration, promoteTrial, viewTrialResults } = useActions(managedMigrationLogic)@@ -465,7 +477,7 @@ export function ManagedMigrations(): JSX.Element {const parts = migration.state.partsconst totalParts = parts.lengthconst completedParts = parts.filter(- (part) => part.total_size !== null && part.total_size === part.current_offset+ (part) => part.total_size !== null && part.current_offset >= part.total_size).lengthreturn {progress: totalParts > 0 ? (completedParts / totalParts) * 100 : 0,@@ -609,7 +621,7 @@ export function ManagedMigrations(): JSX.Element {/><span className="text-xs text-muted">{migration.display_status === 'completed'- ? 'Complete'+ ? completedLabel(migration): migration.display_status === 'paused'? 'Paused': migration.display_status === 'waiting_to_start'diff --git a/products/managed_migrations/frontend/types.ts b/products/managed_migrations/frontend/types.tsindex 621811f6e4f3..1cc7645eee74 100644--- a/products/managed_migrations/frontend/types.ts+++ b/products/managed_migrations/frontend/types.ts@@ -58,6 +58,7 @@ export interface BaseManagedMigration {current_offset: number}>trial?: TrialProgress+ events_emitted?: number}}diff --git a/rust/batch-import-worker/src/job/mod.rs b/rust/batch-import-worker/src/job/mod.rsindex 72ff92b04464..509b4810febe 100644--- a/rust/batch-import-worker/src/job/mod.rs+++ b/rust/batch-import-worker/src/job/mod.rs@@ -508,6 +508,16 @@ pub(crate) async fn load_or_init_state(.keys().await.with_context(|| "Failed to get source keys".to_string())?;+ // An empty listing would leave nothing to process, and the job would complete+ // at once with nothing imported. Pause instead, so the user can fix the source.+ if keys.is_empty() {+ return Err(+ Error::msg("source listed no keys to import").context(UserError::new(+ "No files were found to import. Check the source settings (for example the \+ bucket, the prefix and the file extension), then resume the import.",+ )),+ );+ }for key in keys {let size = source.size(&key)@@ -759,7 +769,8 @@ impl Job {// confirms whether the sink commit landed via the last event written or the logs.self.shutdown_guard()?;info!(job_id = %self.job_id, "Beginning PG part commit");- self.begin_part_commit(&key, parsed.consumed).await?;+ self.begin_part_commit(&key, parsed.consumed, parsed.data.len())+ .await?;info!(job_id = %self.job_id, "Beginning emitter part commit");// A *returned* error (as opposed to a crash) means the sink definitively rejected the@@ -768,7 +779,8 @@ impl Job {let to_sleep = match txn.commit_write().await {Ok(to_sleep) => to_sleep,Err(e) => {- self.rollback_part_commit(&key, parsed.consumed, &e).await?;+ self.rollback_part_commit(&key, parsed.consumed, parsed.data.len(), &e)+ .await?;return Err(e);}};@@ -800,16 +812,48 @@ impl Job {async fn successfully_complete(self) -> Result<(), Error> {let mut model = self.model.lock().await;+ let stored = model.state.clone().unwrap_or_default();++ if let Some(blocker) = stored.completion_blocker() {+ error!(job_id = %self.job_id, "Refusing to complete job: {blocker}");+ metric_emit::job_completion("blocked");+ return model+ .pause(+ self.context.clone(),+ format!("Refused to complete job: {blocker}"),+ Some(+ "The import stopped before all files were processed. Resume the import \+ to continue."+ .to_string(),+ ),+ )+ .await;+ }+let result = model.complete(&self.context.db).await;if result.is_ok() {- info!(job_id = %self.job_id, "Batch import job complete");+ let parts = stored.parts.len();+ let events = stored.events_emitted;+ metric_emit::completed_job_size(parts, events);+ if events == 0 {+ metric_emit::job_completion("completed_empty");+ warn!(job_id = %self.job_id, parts, "Batch import job complete, but it emitted no events");+ } else {+ metric_emit::job_completion("completed");+ info!(job_id = %self.job_id, parts, events, "Batch import job complete");+ }}result}// Writes the new partstate to the DB, and sets the job status to paused, such that if there's an issue with the sink commit, the job// will be paused, and manual intervention will be required to resume it- async fn begin_part_commit(&self, key: &str, consumed: usize) -> Result<(), Error> {+ async fn begin_part_commit(+ &self,+ key: &str,+ consumed: usize,+ events: usize,+ ) -> Result<(), Error> {let mut model = self.model.lock().await;let Some(model_state) = &mut model.state else {return Err(Error::msg("No model state found"));@@ -818,6 +862,7 @@ impl Job {let Some(new_offset) = model_state.advance_part_offset(key, consumed as u64) else {return Err(Error::msg(format!("No part found with key {key}")));};+ model_state.events_emitted = model_state.events_emitted.saturating_add(events as u64);let status_message = format!("Starting commit of part {key} to offset {new_offset}, consumed {consumed} additional bytes"@@ -839,6 +884,7 @@ impl Job {&self,key: &str,consumed: usize,+ events: usize,err: &Error,) -> Result<(), Error> {let mut model = self.model.lock().await;@@ -849,6 +895,7 @@ impl Job {let Some(new_offset) = model_state.revert_part_offset(key, consumed as u64) else {return Err(Error::msg(format!("No part found with key {key}")));};+ model_state.events_emitted = model_state.events_emitted.saturating_sub(events as u64);let status_message =format!("Commit of part {key} failed, rolled back to offset {new_offset}: {err:#}");@@ -1190,6 +1237,7 @@ mod tests {state: Some(JobState {parts: vec![],trial: None,+ events_emitted: 0,}),import_config: super::config::JobConfig {// Construct a trivially valid config that won't be used by this test@@ -1667,6 +1715,7 @@ mod tests {}).collect(),trial: None,+ events_emitted: 0,}}@@ -1878,6 +1927,7 @@ mod tests {},],trial: None,+ events_emitted: 0,};let state = Mutex::new(poisoned.clone());let model = Mutex::new(dummy_model(poisoned));@@ -2523,6 +2573,19 @@ mod tests {}}+ #[tokio::test]+ async fn test_empty_source_listing_fails_init_with_user_message() {+ let source = MockDataSourceWithoutDateRange { keys: vec![] };+ let mut model = staging_cleanup_invariant_tests::dummy_model(JobState::default());++ let err = load_or_init_state(&source, &mut model)+ .await+ .expect_err("an empty listing must not start a job that completes at once");++ assert!(get_user_message(&err).contains("No files were found to import"));+ assert!(model.state.unwrap().parts.is_empty());+ }+/// Streaming sources discover a part's decompressed size only on the read/// that observes end-of-stream. When that stream turns out to be empty (or/// already fully consumed at the resume offset), the first read returns an@@ -2657,6 +2720,7 @@ mod tests {total_size: None,}],trial: None,+ events_emitted: 0,};let state = Mutex::new(resumed.clone());let model = Mutex::new(dummy_model(resumed));@@ -2737,6 +2801,7 @@ mod tests {total_size: part_total,}],trial: None,+ events_emitted: 0,};let state = Mutex::new(part_state.clone());let model = Mutex::new(dummy_model(part_state));@@ -2938,6 +3003,7 @@ mod tests {},],trial: None,+ events_emitted: 0,};let state = Mutex::new(poisoned.clone());let model = Mutex::new(dummy_model(poisoned));diff --git a/rust/batch-import-worker/src/job/model.rs b/rust/batch-import-worker/src/job/model.rsindex feea3478dae0..cfda97751818 100644--- a/rust/batch-import-worker/src/job/model.rs+++ b/rust/batch-import-worker/src/job/model.rs@@ -60,6 +60,10 @@ pub struct JobState {// and running summary instead of starting over.#[serde(default, skip_serializing_if = "Option::is_none")]pub trial: Option<crate::trial::TrialProgress>,+ // Events committed to the sink across all parts. Old states have no value,+ // so it defaults to 0.+ #[serde(default)]+ pub events_emitted: u64,}#[derive(Debug, Clone, Serialize, Deserialize)]@@ -99,6 +103,26 @@ impl JobState {part.current_offset = part.current_offset.saturating_sub(consumed);Some(part.current_offset)}++ /// Return why the job must not complete, or `None` when it can. The read loop+ /// decides completion from its in-memory parts, so this checks the stored parts+ /// as well: a job must never report success for a part it did not finish, or+ /// for a source that listed no parts at all.+ pub fn completion_blocker(&self) -> Option<String> {+ if self.parts.is_empty() {+ return Some("job has no parts to import".to_string());+ }+ let unfinished: Vec<&PartState> = self.parts.iter().filter(|p| !p.is_done()).collect();+ let first = unfinished.first()?;+ Some(format!(+ "{} of {} parts are not finished in the stored state, first is {} at offset {} of {:?}",+ unfinished.len(),+ self.parts.len(),+ first.key,+ first.current_offset,+ first.total_size+ ))+ }}impl JobModel {@@ -421,6 +445,7 @@ impl TryFrom<(JobRow, &[String], String)> for JobModel {None => JobState {parts: vec![],trial: None,+ events_emitted: 0,},};@@ -522,6 +547,7 @@ mod tests {}).collect(),trial: None,+ events_emitted: 0,}}@@ -587,6 +613,42 @@ mod tests {}}+ #[test]+ fn test_completion_blocker() {+ let part = |offset: u64, total: Option<u64>| PartState {+ key: "k".to_string(),+ current_offset: offset,+ total_size: total,+ };+ let cases: [(&str, Vec<PartState>, bool); 5] = [+ ("no parts", vec![], true),+ (+ "all done",+ vec![part(10, Some(10)), part(0, Some(0))],+ false,+ ),+ ("overshot counts as done", vec![part(11, Some(10))], false),+ (+ "one part behind",+ vec![part(10, Some(10)), part(4, Some(10))],+ true,+ ),+ ("unknown size", vec![part(10, None)], true),+ ];+ for (name, parts, blocked) in cases {+ let state = JobState {+ parts,+ trial: None,+ events_emitted: 0,+ };+ assert_eq!(+ state.completion_blocker().is_some(),+ blocked,+ "case '{name}'"+ );+ }+ }+#[test]fn test_advance_part_offset_increments_and_returns_new_offset() {let mut state = make_job_state(&[("a", 100), ("b", 0)]);diff --git a/rust/batch-import-worker/src/metrics.rs b/rust/batch-import-worker/src/metrics.rsindex 009ddefe7a2f..68e896e1180b 100644--- a/rust/batch-import-worker/src/metrics.rs+++ b/rust/batch-import-worker/src/metrics.rs@@ -13,6 +13,9 @@ pub const TEMP_BUCKET_READ_DURATION_SECONDS: &str =pub const STAGED_PLAINTEXT_CEILING_TRIPPED: &str ="batch_import_staged_plaintext_ceiling_tripped_total";pub const PART_CLEANUP_TOTAL: &str = "batch_import_part_cleanup_total";+pub const JOB_COMPLETION_TOTAL: &str = "batch_import_job_completion_total";+pub const COMPLETED_JOB_EVENTS: &str = "batch_import_completed_job_events";+pub const COMPLETED_JOB_PARTS: &str = "batch_import_completed_job_parts";use metrics::{counter, gauge, histogram};@@ -65,3 +68,15 @@ pub fn staged_plaintext_ceiling_tripped() {pub fn part_cleanup(outcome: &'static str) {counter!(PART_CLEANUP_TOTAL, "outcome" => outcome).increment(1);}++/// Count a job that reached the end of its parts, by outcome: "completed",+/// "completed_empty" (no events emitted) or "blocked" (paused instead of completed).+pub fn job_completion(outcome: &'static str) {+ counter!(JOB_COMPLETION_TOTAL, "outcome" => outcome).increment(1);+}++/// Record the size of a completed job, so silent partial imports show up as outliers.+pub fn completed_job_size(parts: usize, events: u64) {+ histogram!(COMPLETED_JOB_PARTS).record(parts as f64);+ histogram!(COMPLETED_JOB_EVENTS).record(events as f64);+}diff --git a/rust/batch-import-worker/tests/common/harness.rs b/rust/batch-import-worker/tests/common/harness.rsindex ef598bed1cbb..4f7ba6eddae2 100644--- a/rust/batch-import-worker/tests/common/harness.rs+++ b/rust/batch-import-worker/tests/common/harness.rs@@ -90,7 +90,11 @@ impl Harness {total_size: None,}).collect();- let job_state = JobState { parts, trial: None };+ let job_state = JobState {+ parts,+ trial: None,+ events_emitted: 0,+ };let model = JobModel {id: job_id,