Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(personhog): resume parked harness deletes under their own op id (#124911)

Zach Brown1de595fdfc0b
diff --git a/rust/personhog-test-harness/src/scenarios/traffic.rs b/rust/personhog-test-harness/src/scenarios/traffic.rs
index d16802f5f4b7..04014eba27c7 100644
--- a/rust/personhog-test-harness/src/scenarios/traffic.rs
+++ b/rust/personhog-test-harness/src/scenarios/traffic.rs
@@ -17,8 +17,8 @@
//! expected behavior legitimately differs across stack versions as
//! admission hardening lands.
//!
-//! Direct database access is seeding and verification only — confined to
-//! the configured target table (the writer's validation table), never
+//! Direct database access is seeding, verification, and finding parked
+//! deletes — confined to the configured target table set, never
//! posthog_person, and never a DELETE: every row removal goes through the
//! lifecycle delete saga, so leftovers from crashed runs and the saga's
//! tombstones are external cleanup's business. A startup sentinel
@@ -165,6 +165,9 @@ pub async fn run(args: TrafficArgs) -> Result<()> {
)
.await?;
+ let owned_teams: Vec<i64> = team_ids.iter().copied().chain([hostile_team_id]).collect();
+ resume_parked_deletes(&lifecycle, &pool, &args.pg_target_table, &owned_teams).await?;
+
// Hostile targets live for the process lifetime: their documents stay
// small (fixed keys, no journal growth) and their outcomes are only
// observed, never verified.
@@ -607,11 +610,13 @@ async fn delete_pool(lifecycle: &LifecycleClient, team_id: i64, person_ids: &[i6
// The lifecycle service caps batches at 250 person ids.
for chunk in person_ids.chunks(200) {
- let op_id = uuid::Uuid::new_v4();
let started = std::time::Instant::now();
- let outcomes = lifecycle
- .delete_persons(team_id, chunk.to_vec(), &op_id)
- .await?;
+ let outcomes = with_op_retry(DELETE_ATTEMPTS, DELETE_BACKOFF, |op_id| async move {
+ lifecycle
+ .delete_persons(team_id, chunk.to_vec(), &op_id)
+ .await
+ })
+ .await?;
histogram!("personhog_traffic_pool_delete_duration_ms")
.record(started.elapsed().as_secs_f64() * 1000.0);
for (person_id, outcome) in outcomes {
@@ -635,6 +640,70 @@ async fn delete_pool(lifecycle: &LifecycleClient, team_id: i64, person_ids: &[i6
Ok(())
}
+/// The sweeper never resumes a parked op, so the bed re-sends its own.
+async fn resume_parked_deletes(
+ lifecycle: &LifecycleClient,
+ pool: &PgPool,
+ person_table: &str,
+ team_ids: &[i64],
+) -> Result<()> {
+ let (op_table, _) = seed::lifecycle_tables_for(person_table);
+ let teams: Vec<i32> = team_ids.iter().map(|&team| team as i32).collect();
+ let parked: Vec<(Uuid, i32, Vec<i64>)> = sqlx::query_as(&format!(
+ "SELECT op_id, team_id, \
+ ARRAY(SELECT jsonb_array_elements_text(request->'person_ids')::bigint) \
+ FROM {op_table} \
+ WHERE op_type = 'delete' AND completed_at IS NULL AND parked_at IS NOT NULL \
+ AND team_id = ANY($1)"
+ ))
+ .bind(&teams)
+ .fetch_all(pool)
+ .await
+ .context("listing parked deletes")?;
+
+ for (op_id, team_id, person_ids) in parked {
+ let result = match lifecycle
+ .delete_persons(team_id as i64, person_ids, &op_id)
+ .await
+ {
+ Ok(_) => "completed",
+ Err(error) => {
+ tracing::warn!(%op_id, team_id, error = format!("{error:#}"), "parked delete resume failed");
+ "failed"
+ }
+ };
+ counter!("personhog_traffic_parked_delete_resumes_total", "result" => result).increment(1);
+ }
+ Ok(())
+}
+
+const DELETE_ATTEMPTS: u32 = 5;
+const DELETE_BACKOFF: Duration = Duration::from_secs(1);
+
+/// Retry under one op id: a parked op resumes only on a retry with its own id.
+async fn with_op_retry<T, F, Fut>(attempts: u32, backoff: Duration, mut call: F) -> Result<T>
+where
+ F: FnMut(Uuid) -> Fut,
+ Fut: std::future::Future<Output = Result<T>>,
+{
+ let op_id = Uuid::new_v4();
+ let mut delay = backoff;
+ let mut attempt = 0;
+ loop {
+ attempt += 1;
+ match call(op_id).await {
+ Ok(value) => return Ok(value),
+ Err(error) if attempt < attempts => {
+ counter!("personhog_traffic_lifecycle_retries_total").increment(1);
+ tracing::warn!(%op_id, attempt, error = format!("{error:#}"), "lifecycle call failed; retrying");
+ tokio::time::sleep(delay).await;
+ delay *= 2;
+ }
+ Err(error) => return Err(error.context(format!("op {op_id} failed {attempts} times"))),
+ }
+ }
+}
+
fn chaos_config(args: &TrafficArgs) -> ChaosConfig {
let target = |kind: TargetKind, namespace: &str| TargetSpec {
kind,
@@ -916,6 +985,35 @@ mod tests {
assert_eq!(cfg.etcd_target.map(|t| t.kind), Some(TargetKind::Etcd));
}
+ #[tokio::test]
+ async fn lifecycle_retries_reuse_one_op_id_and_give_up_after_the_last_attempt() {
+ let seen = std::sync::Mutex::new(Vec::new());
+ let result = with_op_retry(3, Duration::ZERO, |op_id| {
+ let mut seen = seen.lock().unwrap();
+ seen.push(op_id);
+ let attempt = seen.len();
+ async move {
+ if attempt < 3 {
+ bail!("refused")
+ }
+ Ok(attempt)
+ }
+ })
+ .await;
+ assert_eq!(result.unwrap(), 3);
+ let ids = seen.lock().unwrap().clone();
+ assert!(ids.iter().all(|id| *id == ids[0]));
+
+ let calls = std::sync::atomic::AtomicU32::new(0);
+ let result: Result<()> = with_op_retry(3, Duration::ZERO, |_| {
+ calls.fetch_add(1, Ordering::SeqCst);
+ async { bail!("refused") }
+ })
+ .await;
+ assert!(result.is_err());
+ assert_eq!(calls.load(Ordering::SeqCst), 3);
+ }
+
#[test]
fn vacuous_or_panicking_configurations_are_rejected() {
let valid = valid_args();