Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(batch-imports): stop marking empty or unfinished imports complete (#118289)

posthog[bot]734468db6f95
diff --git a/products/managed_migrations/frontend/ManagedMigration.tsx b/products/managed_migrations/frontend/ManagedMigration.tsx
index 4ee76a9e526c..3e1c17a3d1c2 100644
--- a/products/managed_migrations/frontend/ManagedMigration.tsx
+++ b/products/managed_migrations/frontend/ManagedMigration.tsx
@@ -17,6 +17,7 @@ import { LemonProgress } from 'lib/lemon-ui/LemonProgress'
import { LemonSegmentedButton } from 'lib/lemon-ui/LemonSegmentedButton'
import { LemonSelect } from 'lib/lemon-ui/LemonSelect'
import { ProfilePicture } from 'lib/lemon-ui/ProfilePicture'
+import { humanFriendlyNumber } from 'lib/utils/numbers'
import { sceneConfigurations } from 'scenes/scenes'
import { Scene, SceneExport } from 'scenes/sceneTypes'
import { urls } from 'scenes/urls'
@@ -454,6 +455,17 @@ export function ManagedMigration(): JSX.Element {
)
}
+function completedLabel(migration: ManagedMigrationData): string {
+ const eventsEmitted = migration.state?.events_emitted
+ if (eventsEmitted === undefined) {
+ return 'Complete'
+ }
+ if (eventsEmitted === 0) {
+ return 'Complete, no events imported'
+ }
+ return `Complete, ${humanFriendlyNumber(eventsEmitted)} events imported`
+}
+
export function ManagedMigrations(): JSX.Element {
const { managedMigrationId, migrations, migrationsLoading, promotingMigrationId } = useValues(managedMigrationLogic)
const { pauseMigration, resumeMigration, promoteTrial, viewTrialResults } = useActions(managedMigrationLogic)
@@ -465,7 +477,7 @@ export function ManagedMigrations(): JSX.Element {
const parts = migration.state.parts
const totalParts = parts.length
const completedParts = parts.filter(
- (part) => part.total_size !== null && part.total_size === part.current_offset
+ (part) => part.total_size !== null && part.current_offset >= part.total_size
).length
return {
progress: totalParts > 0 ? (completedParts / totalParts) * 100 : 0,
@@ -609,7 +621,7 @@ export function ManagedMigrations(): JSX.Element {
/>
<span className="text-xs text-muted">
{migration.display_status === 'completed'
- ? 'Complete'
+ ? completedLabel(migration)
: migration.display_status === 'paused'
? 'Paused'
: migration.display_status === 'waiting_to_start'
diff --git a/products/managed_migrations/frontend/types.ts b/products/managed_migrations/frontend/types.ts
index 621811f6e4f3..1cc7645eee74 100644
--- a/products/managed_migrations/frontend/types.ts
+++ b/products/managed_migrations/frontend/types.ts
@@ -58,6 +58,7 @@ export interface BaseManagedMigration {
current_offset: number
}>
trial?: TrialProgress
+ events_emitted?: number
}
}
diff --git a/rust/batch-import-worker/src/job/mod.rs b/rust/batch-import-worker/src/job/mod.rs
index 72ff92b04464..509b4810febe 100644
--- a/rust/batch-import-worker/src/job/mod.rs
+++ b/rust/batch-import-worker/src/job/mod.rs
@@ -508,6 +508,16 @@ pub(crate) async fn load_or_init_state(
.keys()
.await
.with_context(|| "Failed to get source keys".to_string())?;
+ // An empty listing would leave nothing to process, and the job would complete
+ // at once with nothing imported. Pause instead, so the user can fix the source.
+ if keys.is_empty() {
+ return Err(
+ Error::msg("source listed no keys to import").context(UserError::new(
+ "No files were found to import. Check the source settings (for example the \
+ bucket, the prefix and the file extension), then resume the import.",
+ )),
+ );
+ }
for key in keys {
let size = source
.size(&key)
@@ -759,7 +769,8 @@ impl Job {
// confirms whether the sink commit landed via the last event written or the logs.
self.shutdown_guard()?;
info!(job_id = %self.job_id, "Beginning PG part commit");
- self.begin_part_commit(&key, parsed.consumed).await?;
+ self.begin_part_commit(&key, parsed.consumed, parsed.data.len())
+ .await?;
info!(job_id = %self.job_id, "Beginning emitter part commit");
// A *returned* error (as opposed to a crash) means the sink definitively rejected the
@@ -768,7 +779,8 @@ impl Job {
let to_sleep = match txn.commit_write().await {
Ok(to_sleep) => to_sleep,
Err(e) => {
- self.rollback_part_commit(&key, parsed.consumed, &e).await?;
+ self.rollback_part_commit(&key, parsed.consumed, parsed.data.len(), &e)
+ .await?;
return Err(e);
}
};
@@ -800,16 +812,48 @@ impl Job {
async fn successfully_complete(self) -> Result<(), Error> {
let mut model = self.model.lock().await;
+ let stored = model.state.clone().unwrap_or_default();
+
+ if let Some(blocker) = stored.completion_blocker() {
+ error!(job_id = %self.job_id, "Refusing to complete job: {blocker}");
+ metric_emit::job_completion("blocked");
+ return model
+ .pause(
+ self.context.clone(),
+ format!("Refused to complete job: {blocker}"),
+ Some(
+ "The import stopped before all files were processed. Resume the import \
+ to continue."
+ .to_string(),
+ ),
+ )
+ .await;
+ }
+
let result = model.complete(&self.context.db).await;
if result.is_ok() {
- info!(job_id = %self.job_id, "Batch import job complete");
+ let parts = stored.parts.len();
+ let events = stored.events_emitted;
+ metric_emit::completed_job_size(parts, events);
+ if events == 0 {
+ metric_emit::job_completion("completed_empty");
+ warn!(job_id = %self.job_id, parts, "Batch import job complete, but it emitted no events");
+ } else {
+ metric_emit::job_completion("completed");
+ info!(job_id = %self.job_id, parts, events, "Batch import job complete");
+ }
}
result
}
// Writes the new partstate to the DB, and sets the job status to paused, such that if there's an issue with the sink commit, the job
// will be paused, and manual intervention will be required to resume it
- async fn begin_part_commit(&self, key: &str, consumed: usize) -> Result<(), Error> {
+ async fn begin_part_commit(
+ &self,
+ key: &str,
+ consumed: usize,
+ events: usize,
+ ) -> Result<(), Error> {
let mut model = self.model.lock().await;
let Some(model_state) = &mut model.state else {
return Err(Error::msg("No model state found"));
@@ -818,6 +862,7 @@ impl Job {
let Some(new_offset) = model_state.advance_part_offset(key, consumed as u64) else {
return Err(Error::msg(format!("No part found with key {key}")));
};
+ model_state.events_emitted = model_state.events_emitted.saturating_add(events as u64);
let status_message = format!(
"Starting commit of part {key} to offset {new_offset}, consumed {consumed} additional bytes"
@@ -839,6 +884,7 @@ impl Job {
&self,
key: &str,
consumed: usize,
+ events: usize,
err: &Error,
) -> Result<(), Error> {
let mut model = self.model.lock().await;
@@ -849,6 +895,7 @@ impl Job {
let Some(new_offset) = model_state.revert_part_offset(key, consumed as u64) else {
return Err(Error::msg(format!("No part found with key {key}")));
};
+ model_state.events_emitted = model_state.events_emitted.saturating_sub(events as u64);
let status_message =
format!("Commit of part {key} failed, rolled back to offset {new_offset}: {err:#}");
@@ -1190,6 +1237,7 @@ mod tests {
state: Some(JobState {
parts: vec![],
trial: None,
+ events_emitted: 0,
}),
import_config: super::config::JobConfig {
// Construct a trivially valid config that won't be used by this test
@@ -1667,6 +1715,7 @@ mod tests {
})
.collect(),
trial: None,
+ events_emitted: 0,
}
}
@@ -1878,6 +1927,7 @@ mod tests {
},
],
trial: None,
+ events_emitted: 0,
};
let state = Mutex::new(poisoned.clone());
let model = Mutex::new(dummy_model(poisoned));
@@ -2523,6 +2573,19 @@ mod tests {
}
}
+ #[tokio::test]
+ async fn test_empty_source_listing_fails_init_with_user_message() {
+ let source = MockDataSourceWithoutDateRange { keys: vec![] };
+ let mut model = staging_cleanup_invariant_tests::dummy_model(JobState::default());
+
+ let err = load_or_init_state(&source, &mut model)
+ .await
+ .expect_err("an empty listing must not start a job that completes at once");
+
+ assert!(get_user_message(&err).contains("No files were found to import"));
+ assert!(model.state.unwrap().parts.is_empty());
+ }
+
/// Streaming sources discover a part's decompressed size only on the read
/// that observes end-of-stream. When that stream turns out to be empty (or
/// already fully consumed at the resume offset), the first read returns an
@@ -2657,6 +2720,7 @@ mod tests {
total_size: None,
}],
trial: None,
+ events_emitted: 0,
};
let state = Mutex::new(resumed.clone());
let model = Mutex::new(dummy_model(resumed));
@@ -2737,6 +2801,7 @@ mod tests {
total_size: part_total,
}],
trial: None,
+ events_emitted: 0,
};
let state = Mutex::new(part_state.clone());
let model = Mutex::new(dummy_model(part_state));
@@ -2938,6 +3003,7 @@ mod tests {
},
],
trial: None,
+ events_emitted: 0,
};
let state = Mutex::new(poisoned.clone());
let model = Mutex::new(dummy_model(poisoned));
diff --git a/rust/batch-import-worker/src/job/model.rs b/rust/batch-import-worker/src/job/model.rs
index feea3478dae0..cfda97751818 100644
--- a/rust/batch-import-worker/src/job/model.rs
+++ b/rust/batch-import-worker/src/job/model.rs
@@ -60,6 +60,10 @@ pub struct JobState {
// and running summary instead of starting over.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub trial: Option<crate::trial::TrialProgress>,
+ // Events committed to the sink across all parts. Old states have no value,
+ // so it defaults to 0.
+ #[serde(default)]
+ pub events_emitted: u64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -99,6 +103,26 @@ impl JobState {
part.current_offset = part.current_offset.saturating_sub(consumed);
Some(part.current_offset)
}
+
+ /// Return why the job must not complete, or `None` when it can. The read loop
+ /// decides completion from its in-memory parts, so this checks the stored parts
+ /// as well: a job must never report success for a part it did not finish, or
+ /// for a source that listed no parts at all.
+ pub fn completion_blocker(&self) -> Option<String> {
+ if self.parts.is_empty() {
+ return Some("job has no parts to import".to_string());
+ }
+ let unfinished: Vec<&PartState> = self.parts.iter().filter(|p| !p.is_done()).collect();
+ let first = unfinished.first()?;
+ Some(format!(
+ "{} of {} parts are not finished in the stored state, first is {} at offset {} of {:?}",
+ unfinished.len(),
+ self.parts.len(),
+ first.key,
+ first.current_offset,
+ first.total_size
+ ))
+ }
}
impl JobModel {
@@ -421,6 +445,7 @@ impl TryFrom<(JobRow, &[String], String)> for JobModel {
None => JobState {
parts: vec![],
trial: None,
+ events_emitted: 0,
},
};
@@ -522,6 +547,7 @@ mod tests {
})
.collect(),
trial: None,
+ events_emitted: 0,
}
}
@@ -587,6 +613,42 @@ mod tests {
}
}
+ #[test]
+ fn test_completion_blocker() {
+ let part = |offset: u64, total: Option<u64>| PartState {
+ key: "k".to_string(),
+ current_offset: offset,
+ total_size: total,
+ };
+ let cases: [(&str, Vec<PartState>, bool); 5] = [
+ ("no parts", vec![], true),
+ (
+ "all done",
+ vec![part(10, Some(10)), part(0, Some(0))],
+ false,
+ ),
+ ("overshot counts as done", vec![part(11, Some(10))], false),
+ (
+ "one part behind",
+ vec![part(10, Some(10)), part(4, Some(10))],
+ true,
+ ),
+ ("unknown size", vec![part(10, None)], true),
+ ];
+ for (name, parts, blocked) in cases {
+ let state = JobState {
+ parts,
+ trial: None,
+ events_emitted: 0,
+ };
+ assert_eq!(
+ state.completion_blocker().is_some(),
+ blocked,
+ "case '{name}'"
+ );
+ }
+ }
+
#[test]
fn test_advance_part_offset_increments_and_returns_new_offset() {
let mut state = make_job_state(&[("a", 100), ("b", 0)]);
diff --git a/rust/batch-import-worker/src/metrics.rs b/rust/batch-import-worker/src/metrics.rs
index 009ddefe7a2f..68e896e1180b 100644
--- a/rust/batch-import-worker/src/metrics.rs
+++ b/rust/batch-import-worker/src/metrics.rs
@@ -13,6 +13,9 @@ pub const TEMP_BUCKET_READ_DURATION_SECONDS: &str =
pub const STAGED_PLAINTEXT_CEILING_TRIPPED: &str =
"batch_import_staged_plaintext_ceiling_tripped_total";
pub const PART_CLEANUP_TOTAL: &str = "batch_import_part_cleanup_total";
+pub const JOB_COMPLETION_TOTAL: &str = "batch_import_job_completion_total";
+pub const COMPLETED_JOB_EVENTS: &str = "batch_import_completed_job_events";
+pub const COMPLETED_JOB_PARTS: &str = "batch_import_completed_job_parts";
use metrics::{counter, gauge, histogram};
@@ -65,3 +68,15 @@ pub fn staged_plaintext_ceiling_tripped() {
pub fn part_cleanup(outcome: &'static str) {
counter!(PART_CLEANUP_TOTAL, "outcome" => outcome).increment(1);
}
+
+/// Count a job that reached the end of its parts, by outcome: "completed",
+/// "completed_empty" (no events emitted) or "blocked" (paused instead of completed).
+pub fn job_completion(outcome: &'static str) {
+ counter!(JOB_COMPLETION_TOTAL, "outcome" => outcome).increment(1);
+}
+
+/// Record the size of a completed job, so silent partial imports show up as outliers.
+pub fn completed_job_size(parts: usize, events: u64) {
+ histogram!(COMPLETED_JOB_PARTS).record(parts as f64);
+ histogram!(COMPLETED_JOB_EVENTS).record(events as f64);
+}
diff --git a/rust/batch-import-worker/tests/common/harness.rs b/rust/batch-import-worker/tests/common/harness.rs
index ef598bed1cbb..4f7ba6eddae2 100644
--- a/rust/batch-import-worker/tests/common/harness.rs
+++ b/rust/batch-import-worker/tests/common/harness.rs
@@ -90,7 +90,11 @@ impl Harness {
total_size: None,
})
.collect();
- let job_state = JobState { parts, trial: None };
+ let job_state = JobState {
+ parts,
+ trial: None,
+ events_emitted: 0,
+ };
let model = JobModel {
id: job_id,