:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! Engine tests against a dummy op type: the engine must be generic over op
//! types, so these drive a two-step fake op and assert the
//! create-or-attach, lease, CAS advance, recorded-outcome, and sweeper
//! behaviors that every op type shares.
//!
//! Isolation against the other test binaries sharing this database: the
//! dummy registers under 'merge' (the CHECK constraint allows nothing
//! else), which is safe because this file never sweeps. The suite's
//! single sweep test lives in lifecycle_merge_tests (where the sweeper
//! drives a real merge), and its one-hour-lease scan window cannot reach
//! this file's rows: every op this file inserts directly is live-leased,
//! fresh with a NULL lease, parked, or completed.
mod common;
use std::sync::atomic::{AtomicUsize, Ordering};
use async_trait::async_trait;
use common::TestContext;
use serde_json::json;
use sqlx::postgres::PgPool;
use uuid::Uuid;
use personhog_common::grpc::semantic_refusal;
use personhog_identity::config::OutboxWrites;
use personhog_identity::lifecycle::detach::DetachedDrives;
use personhog_identity::lifecycle::engine::{
abort_op_in_tx, advance_step_in_tx, complete_op_in_tx, OpDriver, OpRow, SagaError,
STEP_ABORTED, STEP_COMPLETED,
};
use personhog_identity::pools::IdentityPools;
/// Two-step dummy op: `started → half → <final step>`. Counts step
/// executions so tests can prove what did (or did not) re-run.
struct DummyDriver {
steps_run: AtomicUsize,
pause: std::time::Duration,
final_step: &'static str,
outbox_writes: OutboxWrites,
}
impl DummyDriver {
fn new() -> Self {
Self::with(
std::time::Duration::ZERO,
STEP_COMPLETED,
OutboxWrites::Disabled,
)
}
fn paced(pause: std::time::Duration) -> Self {
Self::with(pause, STEP_COMPLETED, OutboxWrites::Disabled)
}
fn settling(final_step: &'static str, outbox_writes: OutboxWrites) -> Self {
Self::with(std::time::Duration::ZERO, final_step, outbox_writes)
}
fn with(
pause: std::time::Duration,
final_step: &'static str,
outbox_writes: OutboxWrites,
) -> Self {
Self {
steps_run: AtomicUsize::new(0),
pause,
final_step,
outbox_writes,
}
}
}
#[async_trait]
impl OpDriver for DummyDriver {
fn op_type(&self) -> &'static str {
"merge"
}
fn initial_step(&self) -> &'static str {
"started"
}
async fn run_step(&self, pools: &IdentityPools, op: &OpRow) -> Result<(), SagaError> {
self.steps_run.fetch_add(1, Ordering::SeqCst);
tokio::time::sleep(self.pause).await;
let mut tx = pools.heavy().begin().await.map_err(SagaError::Db)?;
let advanced = match op.step.as_str() {
"started" => {
advance_step_in_tx(
&mut tx,
&common::default_tables(),
op.op_id,
"started",
"half",
)
.await?
}
"half" if self.final_step == STEP_COMPLETED => {
complete_op_in_tx(
&mut tx,
&common::default_tables(),
op.op_id,
"half",
&json!({"ok": true}),
self.outbox_writes,
)
.await?
}
"half" => {
abort_op_in_tx(
&mut tx,
&common::default_tables(),
op.op_id,
"half",
&json!({"ok": true}),
)
.await?
}
other => {
return Err(SagaError::CorruptState(format!("unknown step {other}")));
}
};
if !advanced {
tx.rollback().await.map_err(SagaError::Db)?;
return Ok(());
}
tx.commit().await.map_err(SagaError::Db)?;
Ok(())
}
}
async fn parked_state(pool: &PgPool, op_id: Uuid) -> (bool, Option<String>) {
let row: (Option<chrono::DateTime<chrono::Utc>>, Option<String>) =
sqlx::query_as("SELECT parked_at, parked_reason FROM lifecycle_op WHERE op_id = $1")
.bind(op_id)
.fetch_one(pool)
.await
.expect("op row exists");
(row.0.is_some(), row.1)
}
async fn op_row(ctx: &TestContext, op_id: Uuid) -> (String, i32, Option<serde_json::Value>, bool) {
let row: (
String,
i32,
Option<serde_json::Value>,
Option<chrono::DateTime<chrono::Utc>>,
) = sqlx::query_as(
"SELECT step, attempt, outcome, completed_at FROM lifecycle_op WHERE op_id = $1",
)
.bind(op_id)
.fetch_one(&ctx.pool)
.await
.expect("op row exists");
(row.0, row.1, row.2, row.3.is_some())
}
#[tokio::test]
async fn a_new_op_is_driven_to_completion_and_records_its_outcome() {
let ctx = TestContext::new().await;
let engine = ctx.engine();
let driver = DummyDriver::new();
let op_id = Uuid::now_v7();
let row = engine
.execute(&driver, op_id, ctx.team_id, &json!({"work": 1}))
.await
.expect("op completes");
assert_eq!(row.step, STEP_COMPLETED);
assert_eq!(row.outcome, Some(json!({"ok": true})));
assert!(row.completed_at.is_some());
assert_eq!(driver.steps_run.load(Ordering::SeqCst), 2);
let (step, attempt, _, completed) = op_row(&ctx, op_id).await;
assert_eq!(step, STEP_COMPLETED);
assert_eq!(attempt, 1, "one claim by the driving call");
assert!(completed);
ctx.cleanup().await.expect("cleanup");
}
#[tokio::test]
async fn a_retry_with_the_same_op_id_returns_the_recorded_outcome_without_rerunning() {
let ctx = TestContext::new().await;
let engine = ctx.engine();
let driver = DummyDriver::new();
let op_id = Uuid::now_v7();
let request = json!({"work": 2});
engine
.execute(&driver, op_id, ctx.team_id, &request)
.await
.expect("op completes");
let steps_after_first = driver.steps_run.load(Ordering::SeqCst);
let row = engine
.execute(&driver, op_id, ctx.team_id, &request)
.await
.expect("retry attaches");
assert_eq!(row.outcome, Some(json!({"ok": true})));
assert_eq!(
driver.steps_run.load(Ordering::SeqCst),
steps_after_first,
"a completed op never re-runs"
);
ctx.cleanup().await.expect("cleanup");
}
#[tokio::test]
async fn an_op_id_reused_with_a_different_request_is_rejected() {
let ctx = TestContext::new().await;
let engine = ctx.engine();
let driver = DummyDriver::new();
let op_id = Uuid::now_v7();
engine
.execute(&driver, op_id, ctx.team_id, &json!({"work": 3}))
.await
.expect("op completes");
let err = engine
.execute(&driver, op_id, ctx.team_id, &json!({"work": 999}))
.await
.expect_err("different request must not attach");
assert!(matches!(err, SagaError::RequestMismatch(_)));
let err = engine
.execute(&driver, op_id, ctx.team_id + 1, &json!({"work": 3}))
.await
.expect_err("different team must not attach");
assert!(matches!(err, SagaError::RequestMismatch(_)));
ctx.cleanup().await.expect("cleanup");
}
#[tokio::test]
async fn a_first_call_is_not_compared_against_its_own_normalized_insert() {
let ctx = TestContext::new().await;
let engine = ctx.engine();
let driver = DummyDriver::new();
let op_id = Uuid::now_v7();
// Postgres renders 1e17 back as an integer, so the reloaded row never
// equals this request value-for-value. The call that won the insert
// owns the row by construction and must not verify against its own
// jsonb-normalized copy.
let row = engine
.execute(&driver, op_id, ctx.team_id, &json!({"count": 1e17}))
.await
.expect("the inserting call is never a request mismatch");
assert_eq!(row.step, STEP_COMPLETED);
ctx.cleanup().await.expect("cleanup");
}
#[tokio::test]
async fn resume_picks_up_an_op_from_its_saved_step() {
let ctx = TestContext::new().await;
let engine = ctx.engine();
let driver = DummyDriver::new();
let op_id = Uuid::now_v7();
// An op abandoned mid-flight: past its first step, lease released by
// the crashed driver's error path (NULL, not expired — an expired
// lease would make this row claimable by the sweep test's engine).
sqlx::query(
r#"
INSERT INTO lifecycle_op (op_id, op_type, team_id, step, request)
VALUES ($1, 'merge', $2, 'half', '{}'::jsonb)
"#,
)
.bind(op_id)
.bind(ctx.team_id as i32)
.execute(&ctx.pool)
.await
.expect("insert abandoned op");
let row = engine
.resume(&driver, op_id)
.await
.expect("resume completes");
assert_eq!(row.step, STEP_COMPLETED);
assert_eq!(
driver.steps_run.load(Ordering::SeqCst),
1,
"only the remaining step runs"
);
ctx.cleanup().await.expect("cleanup");
}
async fn pointer_kinds(ctx: &TestContext, op_id: Uuid) -> Vec<String> {
sqlx::query_scalar("SELECT kind FROM person_publish_outbox WHERE op_id = $1")
.bind(op_id)
.fetch_all(&ctx.pool)
.await
.expect("query runs")
}
#[tokio::test]
async fn only_a_completed_op_with_outbox_writes_enabled_enqueues_a_publish_pointer() {
let ctx = TestContext::new().await;
// The pointers this test asserts on are pending rows a draining test
// in another process would otherwise claim and delete.
let _drain_lock = ctx.hold_drain_lock().await;
let engine = ctx.engine();
let cases = [
(
OutboxWrites::Enabled,
STEP_COMPLETED,
vec!["merge_op".to_string()],
),
(OutboxWrites::Enabled, STEP_ABORTED, vec![]),
(OutboxWrites::Disabled, STEP_COMPLETED, vec![]),
];
for (outbox_writes, final_step, expected) in cases {
let driver = DummyDriver::settling(final_step, outbox_writes);
let op_id = Uuid::now_v7();
engine
.execute(&driver, op_id, ctx.team_id, &json!({}))
.await
.expect("op settles");
assert_eq!(
pointer_kinds(&ctx, op_id).await,
expected,
"{outbox_writes:?} {final_step}"
);
}
// The pointer rides the compare-and-swap: a completion that loses it
// (the row is not on `from`) commits nothing, pointer included.
let driver = DummyDriver::settling(STEP_COMPLETED, OutboxWrites::Enabled);
let op_id = Uuid::now_v7();
engine
.create_or_attach(&driver, op_id, ctx.team_id, &json!({}))
.await
.expect("op created");
let mut tx = ctx.pool.begin().await.expect("begin");
let advanced = complete_op_in_tx(
&mut tx,
&common::default_tables(),
op_id,
"half",
&json!({}),
OutboxWrites::Enabled,
)
.await
.expect("statement runs");
tx.commit().await.expect("commit");
assert!(!advanced, "the row is on 'started', not 'half'");
assert!(pointer_kinds(&ctx, op_id).await.is_empty());
ctx.cleanup().await.expect("cleanup");
}
async fn remaining_ops(ctx: &TestContext, op_ids: &[Uuid]) -> Vec<Uuid> {
sqlx::query_scalar("SELECT op_id FROM lifecycle_op WHERE op_id = ANY($1) ORDER BY op_id")
.bind(op_ids)
.fetch_all(&ctx.pool)
.await
.expect("query runs")
}
#[tokio::test]
async fn gc_reaps_completed_ops_past_retention_unless_a_publish_pointer_remains() {
let ctx = TestContext::new().await;
// The pending pointer must survive until GC has seen it, so no
// draining test in another process may claim it meanwhile.
let _drain_lock = ctx.hold_drain_lock().await;
let engine = ctx.engine();
let driver = DummyDriver::new();
// Three completed ops: no outbox row, a pending pointer, a given-up pointer.
let plain = Uuid::now_v7();
let pending = Uuid::now_v7();
let given_up = Uuid::now_v7();
let all = [plain, pending, given_up];
for op_id in all {
engine
.execute(&driver, op_id, ctx.team_id, &json!({}))
.await
.expect("op completes");
}
sqlx::query(
r#"
INSERT INTO person_publish_outbox (team_id, kind, op_id, given_up_at, last_error)
VALUES ($1, 'merge_op', $2, NULL, NULL), ($1, 'merge_op', $3, now(), 'poison')
"#,
)
.bind(ctx.team_id as i32)
.bind(pending)
.bind(given_up)
.execute(&ctx.pool)
.await
.expect("queue pointers");
// Backdate the completions past retention; GC reaps only the op nobody
// still has to publish, and cascades its per-person rows, without
// touching other tests' fresh rows.
sqlx::query(
"UPDATE lifecycle_op SET completed_at = now() - interval '2 hours' WHERE op_id = ANY($1)",
)
.bind(&all[..])
.execute(&ctx.pool)
.await
.expect("backdate completions");
engine
.gc(std::time::Duration::from_secs(3600))
.await
.expect("gc runs");
assert_eq!(
remaining_ops(&ctx, &all).await,
vec![pending, given_up],
"ops with a pointer survive, the plain op is deleted"
);
// Once the pointers are gone (published, or cleared by an operator), the
// next pass reaps the ops.
sqlx::query("DELETE FROM person_publish_outbox WHERE op_id = ANY($1)")
.bind(&all[..])
.execute(&ctx.pool)
.await
.expect("clear pointers");
engine
.gc(std::time::Duration::from_secs(3600))
.await
.expect("gc runs");
assert!(remaining_ops(&ctx, &all).await.is_empty());
ctx.cleanup().await.expect("cleanup");
}
#[tokio::test]
async fn a_live_lease_blocks_a_second_driver_until_it_lapses() {
let ctx = TestContext::new().await;
let driver = DummyDriver::new();
let op_id = Uuid::now_v7();
// Held by "another instance" for longer than the test engine's execute
// timeout would wait — but we assert Busy well before that.
sqlx::query(
r#"
INSERT INTO lifecycle_op (op_id, op_type, team_id, step, request, lease_expires_at)
VALUES ($1, 'merge', $2, 'started', '{}'::jsonb, now() + interval '1 hour')
"#,
)
.bind(op_id)
.bind(ctx.team_id as i32)
.execute(&ctx.pool)
.await
.expect("insert leased op");
let short_engine = personhog_identity::lifecycle::engine::Engine::new(
ctx.pools.clone(),
personhog_identity::lifecycle::engine::EngineConfig {
lease: std::time::Duration::from_secs(5),
execute_timeout: std::time::Duration::from_millis(200),
poll_interval: std::time::Duration::from_millis(25),
attempt_alert_threshold: 5,
gc_batch_limit: 10_000,
},
ctx.tables.clone(),
);
let err = short_engine
.execute(&driver, op_id, ctx.team_id, &json!({}))
.await
.expect_err("cannot steal a live lease");
assert!(matches!(err, SagaError::Busy));
assert_eq!(
driver.steps_run.load(Ordering::SeqCst),
0,
"no step ran while another instance held the lease"
);
ctx.cleanup().await.expect("cleanup");
}
/// Sleeps through the execute deadline without ever advancing, modeling
/// a wedged step under a held claim.
struct SlowDriver;
#[async_trait]
impl OpDriver for SlowDriver {
fn op_type(&self) -> &'static str {
"merge"
}
fn initial_step(&self) -> &'static str {
"started"
}
async fn run_step(&self, _pools: &IdentityPools, _op: &OpRow) -> Result<(), SagaError> {
tokio::time::sleep(std::time::Duration::from_millis(400)).await;
Ok(())
}
}
#[tokio::test]
async fn a_drive_that_runs_out_its_own_deadline_says_so_and_releases_the_lease() {
let ctx = TestContext::new().await;
let op_id = Uuid::now_v7();
let short_engine = personhog_identity::lifecycle::engine::Engine::new(
ctx.pools.clone(),
personhog_identity::lifecycle::engine::EngineConfig {
lease: std::time::Duration::from_secs(5),
execute_timeout: std::time::Duration::from_millis(200),
poll_interval: std::time::Duration::from_millis(25),
attempt_alert_threshold: 5,
gc_batch_limit: 10_000,
},
ctx.tables.clone(),
);
// The claim was ours the whole time, so the answer must not blame a
// contending instance that does not exist; and the lease releases so
// the retry is not locked out for a lease term on top.
let err = short_engine
.execute(&SlowDriver, op_id, ctx.team_id, &json!({}))
.await
.expect_err("the slow drive runs out its deadline");
assert!(matches!(err, SagaError::DeadlineElapsed));
let lease_live: bool = sqlx::query_scalar(
"SELECT lease_expires_at IS NOT NULL AND lease_expires_at >= now() FROM lifecycle_op WHERE op_id = $1",
)
.bind(op_id)
.fetch_one(&ctx.pool)
.await
.expect("op row exists");
assert!(!lease_live, "the deadline return released the lease");
ctx.cleanup().await.expect("cleanup");
}
#[tokio::test]
async fn resume_bails_immediately_when_another_driver_holds_the_lease() {
let ctx = TestContext::new().await;
// The default test engine's 10s execute timeout: a resume that waits
// out a live lease instead of bailing would blow the elapsed assertion.
let engine = ctx.engine();
let driver = DummyDriver::new();
let op_id = Uuid::now_v7();
sqlx::query(
r#"
INSERT INTO lifecycle_op (op_id, op_type, team_id, step, request, lease_expires_at)
VALUES ($1, 'merge', $2, 'started', '{}'::jsonb, now() + interval '1 hour')
"#,
)
.bind(op_id)
.bind(ctx.team_id as i32)
.execute(&ctx.pool)
.await
.expect("insert leased op");
let started = std::time::Instant::now();
let err = engine
.resume(&driver, op_id)
.await
.expect_err("live lease means not abandoned");
assert!(matches!(err, SagaError::Busy));
assert_eq!(driver.steps_run.load(Ordering::SeqCst), 0);
assert!(
started.elapsed() < std::time::Duration::from_secs(5),
"resume must skip a live lease, not poll out the execute timeout"
);
ctx.cleanup().await.expect("cleanup");
}
/// Simulates another instance stealing the op mid-step: bumps `attempt` and
/// takes a one-hour lease, exactly what a concurrent `try_claim` does.
async fn steal_lease(pool: &PgPool, op_id: Uuid) {
sqlx::query(
"UPDATE lifecycle_op SET attempt = attempt + 1, lease_expires_at = now() + interval '1 hour' WHERE op_id = $1",
)
.bind(op_id)
.execute(pool)
.await
.expect("steal lease");
}
async fn lease_is_live(pool: &PgPool, op_id: Uuid) -> bool {
sqlx::query_scalar("SELECT lease_expires_at > now() FROM lifecycle_op WHERE op_id = $1")
.bind(op_id)
.fetch_one(pool)
.await
.ok()
.flatten()
.unwrap_or(false)
}
/// Driver whose first step gets its lease stolen mid-run. `fail_after_steal`
/// picks the exit: Err exercises the engine's release path, Ok without
/// advancing (a lost CAS) exercises the attempt check on the reloaded row.
struct StolenLeaseDriver {
steps_run: AtomicUsize,
fail_after_steal: bool,
}
#[async_trait]
impl OpDriver for StolenLeaseDriver {
fn op_type(&self) -> &'static str {
"merge"
}
fn initial_step(&self) -> &'static str {
"started"
}
async fn run_step(&self, pools: &IdentityPools, op: &OpRow) -> Result<(), SagaError> {
self.steps_run.fetch_add(1, Ordering::SeqCst);
steal_lease(pools.fast(), op.op_id).await;
if self.fail_after_steal {
Err(SagaError::CorruptState("simulated step failure".into()))
} else {
Ok(())
}
}
}
#[tokio::test]
async fn a_driver_whose_lease_was_stolen_stops_running_steps_instead_of_renewing() {
let ctx = TestContext::new().await;
let driver = StolenLeaseDriver {
steps_run: AtomicUsize::new(0),
fail_after_steal: false,
};
let op_id = Uuid::now_v7();
let short_engine = personhog_identity::lifecycle::engine::Engine::new(
ctx.pools.clone(),
personhog_identity::lifecycle::engine::EngineConfig {
lease: std::time::Duration::from_secs(5),
execute_timeout: std::time::Duration::from_millis(200),
poll_interval: std::time::Duration::from_millis(25),
attempt_alert_threshold: 5,
gc_batch_limit: 10_000,
},
ctx.tables.clone(),
);
let err = short_engine
.execute(&driver, op_id, ctx.team_id, &json!({}))
.await
.expect_err("waits on the stealer's lease until the deadline");
assert!(matches!(err, SagaError::Busy));
assert_eq!(
driver.steps_run.load(Ordering::SeqCst),
1,
"no step ran after the lease was stolen"
);
assert!(
lease_is_live(&ctx.pool, op_id).await,
"the displaced driver must not have renewed or cleared the stealer's lease"
);
ctx.cleanup().await.expect("cleanup");
}
/// The two-step dummy op, recording the op's lease expiry as each step
/// starts and pausing its first step for `first_step_pause`.
struct LeaseRecordingDriver {
inner: DummyDriver,
first_step_pause: std::time::Duration,
expiries: std::sync::Mutex<Vec<chrono::DateTime<chrono::Utc>>>,
}
#[async_trait]
impl OpDriver for LeaseRecordingDriver {
fn op_type(&self) -> &'static str {
"merge"
}
fn initial_step(&self) -> &'static str {
"started"
}
async fn run_step(&self, pools: &IdentityPools, op: &OpRow) -> Result<(), SagaError> {
let expiry: chrono::DateTime<chrono::Utc> =
sqlx::query_scalar("SELECT lease_expires_at FROM lifecycle_op WHERE op_id = $1")
.bind(op.op_id)
.fetch_one(pools.fast())
.await
.map_err(SagaError::Db)?;
self.expiries.lock().unwrap().push(expiry);
if op.step == "started" {
tokio::time::sleep(self.first_step_pause).await;
}
self.inner.run_step(pools, op).await
}
}
/// Lease expiries seen by the dummy op's two steps under a `lease`-long lease.
async fn step_lease_expiries(
ctx: &TestContext,
lease: std::time::Duration,
first_step_pause: std::time::Duration,
) -> Vec<chrono::DateTime<chrono::Utc>> {
let driver = LeaseRecordingDriver {
inner: DummyDriver::new(),
first_step_pause,
expiries: std::sync::Mutex::new(Vec::new()),
};
let engine = personhog_identity::lifecycle::engine::Engine::new(
ctx.pools.clone(),
personhog_identity::lifecycle::engine::EngineConfig {
lease,
execute_timeout: std::time::Duration::from_secs(10),
poll_interval: std::time::Duration::from_millis(25),
attempt_alert_threshold: 5,
gc_batch_limit: 10_000,
},
ctx.tables.clone(),
);
engine
.execute(&driver, Uuid::now_v7(), ctx.team_id, &json!({}))
.await
.expect("the op completes");
assert_eq!(driver.inner.steps_run.load(Ordering::SeqCst), 2);
driver.expiries.into_inner().unwrap()
}
#[tokio::test]
async fn steps_that_finish_inside_a_third_of_the_lease_run_on_the_claim_without_renewing() {
let ctx = TestContext::new().await;
let expiries = step_lease_expiries(
&ctx,
std::time::Duration::from_secs(30),
std::time::Duration::ZERO,
)
.await;
assert_eq!(
expiries[0], expiries[1],
"the second step ran on the claim's lease"
);
ctx.cleanup().await.expect("cleanup");
}
#[tokio::test]
async fn a_step_that_starts_after_a_third_of_the_lease_renews_it_first() {
let ctx = TestContext::new().await;
let expiries = step_lease_expiries(
&ctx,
std::time::Duration::from_secs(3),
std::time::Duration::from_millis(1_200),
)
.await;
assert!(
expiries[1] > expiries[0],
"the second step ran on a renewed lease"
);
ctx.cleanup().await.expect("cleanup");
}
#[tokio::test]
async fn a_failing_driver_whose_lease_was_stolen_does_not_release_the_stealers_lease() {
let ctx = TestContext::new().await;
let engine = ctx.engine();
let driver = StolenLeaseDriver {
steps_run: AtomicUsize::new(0),
fail_after_steal: true,
};
let op_id = Uuid::now_v7();
let err = engine
.execute(&driver, op_id, ctx.team_id, &json!({}))
.await
.expect_err("step failure propagates");
assert!(matches!(err, SagaError::CorruptState(_)));
assert!(
lease_is_live(&ctx.pool, op_id).await,
"the stale driver's error-path release must not clear the stealer's lease"
);
ctx.cleanup().await.expect("cleanup");
}
/// Refuses its first `refusals` steps with a definitive leader refusal
/// (a semantic refusal, as classified by `SagaError::leader`), then
/// completes in one step.
struct RefusingDriver {
refusals_left: AtomicUsize,
steps_run: AtomicUsize,
}
impl RefusingDriver {
fn new(refusals: usize) -> Self {
Self {
refusals_left: AtomicUsize::new(refusals),
steps_run: AtomicUsize::new(0),
}
}
}
#[async_trait]
impl OpDriver for RefusingDriver {
fn op_type(&self) -> &'static str {
"merge"
}
fn initial_step(&self) -> &'static str {
"started"
}
async fn run_step(&self, pools: &IdentityPools, op: &OpRow) -> Result<(), SagaError> {
self.steps_run.fetch_add(1, Ordering::SeqCst);
if self
.refusals_left
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1))
.is_ok()
{
return Err(SagaError::leader(semantic_refusal(
"injected refusal",
"test-refusal",
)));
}
let mut tx = pools.heavy().begin().await.map_err(SagaError::Db)?;
let advanced = complete_op_in_tx(
&mut tx,
&common::default_tables(),
op.op_id,
"started",
&json!({}),
OutboxWrites::Disabled,
)
.await?;
if !advanced {
tx.rollback().await.map_err(SagaError::Db)?;
return Ok(());
}
tx.commit().await.map_err(SagaError::Db)?;
Ok(())
}
}
#[tokio::test]
async fn a_definitive_leader_refusal_parks_the_op() {
let ctx = TestContext::new().await;
let engine = ctx.engine();
let driver = RefusingDriver::new(1);
let op_id = Uuid::now_v7();
let err = engine
.execute(&driver, op_id, ctx.team_id, &json!({}))
.await
.expect_err("the refusal surfaces");
assert!(matches!(err, SagaError::LeaderRefused(_)));
let (parked, reason) = parked_state(&ctx.pool, op_id).await;
assert!(parked, "a refused op is parked, not left for the sweeper");
assert_eq!(reason.as_deref(), Some("test-refusal"));
assert!(
!lease_is_live(&ctx.pool, op_id).await,
"parking drops the lease so an explicit retry need not wait it out"
);
let (step, _, _, completed) = op_row(&ctx, op_id).await;
assert_eq!(step, "started", "the refused step made no progress");
assert!(!completed);
ctx.cleanup().await.expect("cleanup");
}
#[tokio::test]
async fn an_explicit_retry_with_the_same_op_id_unparks_and_redrives() {
let ctx = TestContext::new().await;
let engine = ctx.engine();
let driver = RefusingDriver::new(1);
let op_id = Uuid::now_v7();
engine
.execute(&driver, op_id, ctx.team_id, &json!({}))
.await
.expect_err("first attempt is refused and parks");
// The driver no longer refuses: an explicit retry with the same
// op_id claims, un-parks, and completes.
let row = engine
.execute(&driver, op_id, ctx.team_id, &json!({}))
.await
.expect("explicit retry completes the parked op");
assert_eq!(row.step, STEP_COMPLETED);
let (parked, reason) = parked_state(&ctx.pool, op_id).await;
assert!(!parked, "claiming un-parks");
assert_eq!(reason, None);
assert_eq!(driver.steps_run.load(Ordering::SeqCst), 2);
ctx.cleanup().await.expect("cleanup");
}
/// Steals the lease mid-step, then returns a definitive refusal: the
/// displaced driver's park must lose the `attempt` compare-and-swap.
struct StolenLeaseRefusingDriver;
#[async_trait]
impl OpDriver for StolenLeaseRefusingDriver {
fn op_type(&self) -> &'static str {
"merge"
}
fn initial_step(&self) -> &'static str {
"started"
}
async fn run_step(&self, pools: &IdentityPools, op: &OpRow) -> Result<(), SagaError> {
steal_lease(pools.fast(), op.op_id).await;
Err(SagaError::leader(semantic_refusal(
"injected refusal",
"test-refusal",
)))
}
}
#[tokio::test]
async fn a_displaced_drivers_refusal_does_not_park_the_stealers_op() {
let ctx = TestContext::new().await;
let engine = ctx.engine();
let op_id = Uuid::now_v7();
let err = engine
.execute(&StolenLeaseRefusingDriver, op_id, ctx.team_id, &json!({}))
.await
.expect_err("the displaced driver cannot answer for the op");
// Not the refusal: the stealer may re-drive the step against fresh
// state and complete the op, so a displaced driver's refusal is
// stale; the caller gets the retriable answer.
assert!(matches!(err, SagaError::Busy));
let (parked, _) = parked_state(&ctx.pool, op_id).await;
assert!(
!parked,
"a displaced driver must not park an op another driver is actively driving"
);
assert!(
lease_is_live(&ctx.pool, op_id).await,
"the stealer's lease survives the displaced driver's park attempt"
);
ctx.cleanup().await.expect("cleanup");
}
#[tokio::test]
async fn step_once_rejects_a_foreign_teams_op() {
let ctx = TestContext::new().await;
let engine = ctx.engine();
let driver = DummyDriver::new();
let op_id = Uuid::now_v7();
engine
.create_or_attach(&driver, op_id, ctx.team_id, &json!({}))
.await
.expect("create op");
let err = engine
.step_once(&driver, op_id, ctx.team_id + 1)
.await
.expect_err("a mismatched team cannot drive the op");
assert!(matches!(err, SagaError::RequestMismatch(_)));
assert_eq!(driver.steps_run.load(Ordering::SeqCst), 0);
ctx.cleanup().await.expect("cleanup");
}
#[tokio::test]
async fn resume_does_not_unpark() {
let ctx = TestContext::new().await;
let engine = ctx.engine();
let driver = RefusingDriver::new(1);
let op_id = Uuid::now_v7();
engine
.execute(&driver, op_id, ctx.team_id, &json!({}))
.await
.expect_err("first attempt is refused and parks");
// The sweeper's entry point must not claim and un-park a parked op,
// even reaching it directly as a scan-then-park race would; that
// signal is reserved for an explicit retry.
let err = engine
.resume(&driver, op_id)
.await
.expect_err("resume cannot claim a parked op");
assert!(matches!(err, SagaError::Busy));
let (parked, _) = parked_state(&ctx.pool, op_id).await;
assert!(parked, "the op stays parked");
assert_eq!(
driver.steps_run.load(Ordering::SeqCst),
1,
"resume ran no step"
);
ctx.cleanup().await.expect("cleanup");
}
/// A concocted Postgres error carrying just a SQLSTATE, so tests can hand
/// the engine the exact errors Postgres emits for lock conflicts without
/// manufacturing a real deadlock.
#[derive(Debug)]
struct FakePgError(&'static str);
impl std::fmt::Display for FakePgError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "fake database error ({})", self.0)
}
}
impl std::error::Error for FakePgError {}
impl sqlx::error::DatabaseError for FakePgError {
fn message(&self) -> &str {
"fake database error"
}
fn code(&self) -> Option<std::borrow::Cow<'_, str>> {
Some(std::borrow::Cow::Borrowed(self.0))
}
fn as_error(&self) -> &(dyn std::error::Error + Send + Sync + 'static) {
self
}
fn as_error_mut(&mut self) -> &mut (dyn std::error::Error + Send + Sync + 'static) {
self
}
fn into_error(self: Box<Self>) -> Box<dyn std::error::Error + Send + Sync + 'static> {
self
}
fn kind(&self) -> sqlx::error::ErrorKind {
sqlx::error::ErrorKind::Other
}
}
fn db_error(code: &'static str) -> SagaError {
SagaError::Db(sqlx::Error::Database(Box::new(FakePgError(code))))
}
#[test]
fn database_conflicts_classify_as_retriable() {
for code in ["40P01", "40001", "57014"] {
assert!(db_error(code).is_db_conflict(), "{code} must be a conflict");
}
assert!(
!db_error("23505").is_db_conflict(),
"a unique violation is not a conflict"
);
assert!(!SagaError::Busy.is_db_conflict());
}
fn connection_lost() -> SagaError {
SagaError::Db(sqlx::Error::Io(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"expected to read 5 bytes, got 0 bytes at EOF",
)))
}
#[test]
fn lost_connections_classify_as_retriable_and_unavailable() {
for err in [
connection_lost(),
db_error("08006"),
db_error("57P01"),
db_error("57P02"),
] {
assert!(
err.is_db_connection_lost(),
"{err} must be a lost connection"
);
assert!(!err.is_db_conflict());
assert_eq!(tonic::Status::from(err).code(), tonic::Code::Unavailable);
}
for err in [db_error("23505"), db_error("40P01"), SagaError::Busy] {
assert!(
!err.is_db_connection_lost(),
"{err} is not a lost connection"
);
}
}
type FailStep = fn() -> SagaError;
/// Driver whose first attempts fail with `error`; every later attempt
/// behaves like [`DummyDriver`]. Failing more than once exercises the
/// repeated backoff passes of the retry loop, not just the first.
struct DeadlockingDriver {
inner: DummyDriver,
fail_first: usize,
attempts: AtomicUsize,
error: FailStep,
}
#[async_trait]
impl OpDriver for DeadlockingDriver {
fn op_type(&self) -> &'static str {
"merge"
}
fn initial_step(&self) -> &'static str {
"started"
}
async fn run_step(&self, pools: &IdentityPools, op: &OpRow) -> Result<(), SagaError> {
if self.attempts.fetch_add(1, Ordering::SeqCst) < self.fail_first {
return Err((self.error)());
}
self.inner.run_step(pools, op).await
}
}
#[tokio::test]
async fn a_step_that_loses_a_database_conflict_or_its_connection_is_retried_not_surfaced() {
let ctx = TestContext::new().await;
let engine = ctx.engine();
let failures: [(&str, FailStep); 2] = [
("deadlock", || db_error("40P01")),
("lost connection", connection_lost),
];
for (name, error) in failures {
let driver = DeadlockingDriver {
inner: DummyDriver::new(),
fail_first: 3,
attempts: AtomicUsize::new(0),
error,
};
let op_id = Uuid::now_v7();
let row = engine
.execute(&driver, op_id, ctx.team_id, &json!({"work": 1}))
.await
.unwrap_or_else(|err| panic!("{name}: retried inside the engine, not surfaced: {err}"));
assert_eq!(row.step, STEP_COMPLETED, "{name}");
assert_eq!(
driver.inner.steps_run.load(Ordering::SeqCst),
2,
"{name}: both real steps ran after the failed attempts"
);
let (_, attempt, _, completed) = op_row(&ctx, op_id).await;
assert!(completed, "{name}");
assert_eq!(
attempt, 1,
"{name}: the retry re-drives under the original claim"
);
}
ctx.cleanup().await.expect("cleanup");
}
#[tokio::test]
async fn a_claim_skips_a_row_a_concurrent_writer_holds_instead_of_queueing() {
let ctx = TestContext::new().await;
let engine = ctx.engine();
let driver = DummyDriver::new();
let op_id = Uuid::now_v7();
// Created but not driven: the lease is free, so only the row lock below
// stands between resume and a successful claim.
engine
.create_or_attach(&driver, op_id, ctx.team_id, &json!({"work": 7}))
.await
.expect("create");
// Hold the row lock the way any concurrent writer would (a renew, a
// step advance, a rival claim mid-flight).
let mut tx = ctx.pool.begin().await.expect("begin");
sqlx::query("SELECT 1 FROM lifecycle_op WHERE op_id = $1 FOR UPDATE")
.bind(op_id)
.execute(&mut *tx)
.await
.expect("lock the op row");
// The claim must answer Busy promptly rather than queueing on the tuple
// until the lock holder commits.
let result = tokio::time::timeout(
std::time::Duration::from_secs(5),
engine.resume(&driver, op_id),
)
.await
.expect("resume must not wait out a concurrent writer's row lock");
assert!(matches!(result, Err(SagaError::Busy)));
assert_eq!(driver.steps_run.load(Ordering::SeqCst), 0);
tx.rollback().await.expect("rollback");
ctx.cleanup().await.expect("cleanup");
}
async fn wait_until_completed(ctx: &TestContext, op_id: Uuid) -> bool {
for _ in 0..200 {
if let Ok(row) = sqlx::query_scalar::<_, Option<chrono::DateTime<chrono::Utc>>>(
"SELECT completed_at FROM lifecycle_op WHERE op_id = $1",
)
.bind(op_id)
.fetch_one(&ctx.pool)
.await
{
if row.is_some() {
return true;
}
}
tokio::time::sleep(std::time::Duration::from_millis(25)).await;
}
false
}
#[tokio::test]
async fn a_retry_attaches_to_a_detached_drive_instead_of_waiting_out_the_lease() {
let ctx = TestContext::new().await;
let engine = std::sync::Arc::new(ctx.engine());
let driver = std::sync::Arc::new(DummyDriver::paced(std::time::Duration::from_millis(150)));
let drives = DetachedDrives::new(4);
let op_id = Uuid::now_v7();
let detached_engine = engine.clone();
let detached_driver = driver.clone();
let hung_up = tokio::time::timeout(
std::time::Duration::from_millis(1),
drives.run(async move {
detached_engine
.execute(detached_driver.as_ref(), op_id, ctx.team_id, &json!({}))
.await
}),
)
.await;
assert!(
hung_up.is_err(),
"the caller gave up before the drive finished"
);
assert_eq!(drives.in_flight(), 1);
let started = std::time::Instant::now();
let row = engine
.execute(driver.as_ref(), op_id, ctx.team_id, &json!({}))
.await
.expect("the retry attaches and returns the terminal row");
assert_eq!(row.step, STEP_COMPLETED);
assert!(
started.elapsed() < std::time::Duration::from_secs(5),
"the retry waited on the live driver, not on the lease"
);
assert_eq!(
driver.steps_run.load(Ordering::SeqCst),
2,
"the retry polled behind the detached drive instead of re-running steps"
);
ctx.cleanup().await.expect("cleanup");
}
#[tokio::test]
async fn with_detaching_disabled_a_dropped_caller_abandons_the_op() {
let ctx = TestContext::new().await;
let engine = std::sync::Arc::new(ctx.engine());
let driver = std::sync::Arc::new(DummyDriver::paced(std::time::Duration::from_millis(50)));
let drives = DetachedDrives::new(0);
let op_id = Uuid::now_v7();
let created = std::sync::Arc::new(tokio::sync::Notify::new());
let inline_engine = engine.clone();
let inline_driver = driver.clone();
let inline_created = created.clone();
{
let call = drives.run(async move {
inline_engine
.create_or_attach(inline_driver.as_ref(), op_id, ctx.team_id, &json!({}))
.await
.expect("create the op");
inline_created.notify_one();
inline_engine
.execute(inline_driver.as_ref(), op_id, ctx.team_id, &json!({}))
.await
});
tokio::pin!(call);
tokio::select! {
_ = &mut call => panic!("the drive finished before the caller hung up"),
_ = created.notified() => {}
}
}
tokio::time::sleep(std::time::Duration::from_millis(300)).await;
let (step, _, _, completed) = op_row(&ctx, op_id).await;
assert_eq!(
step, "started",
"an inline drive stops where its caller dropped it"
);
assert!(!completed);
assert_eq!(drives.in_flight(), 0);
ctx.cleanup().await.expect("cleanup");
}
#[tokio::test]
async fn a_detached_drive_finishes_after_its_caller_is_gone() {
let ctx = TestContext::new().await;
let engine = std::sync::Arc::new(ctx.engine());
let driver = std::sync::Arc::new(DummyDriver::paced(std::time::Duration::from_millis(50)));
let drives = DetachedDrives::new(4);
let op_id = Uuid::now_v7();
let hung_up = tokio::time::timeout(
std::time::Duration::from_millis(1),
drives.run(async move {
engine
.execute(driver.as_ref(), op_id, ctx.team_id, &json!({}))
.await
}),
)
.await;
assert!(hung_up.is_err());
assert!(
wait_until_completed(&ctx, op_id).await,
"the detached drive completed the op without its caller"
);
let (step, _, outcome, completed) = op_row(&ctx, op_id).await;
assert_eq!(step, STEP_COMPLETED);
assert!(completed);
assert_eq!(outcome, Some(json!({"ok": true})));
drives.shutdown().await;
assert_eq!(drives.in_flight(), 0);
ctx.cleanup().await.expect("cleanup");
}
async fn insert_leased_op(ctx: &TestContext, op_id: Uuid, lease: &str) {
sqlx::query(&format!(
"INSERT INTO lifecycle_op (op_id, op_type, team_id, step, request, lease_expires_at) VALUES ($1, 'merge', $2, 'started', '{{}}'::jsonb, {lease})"
))
.bind(op_id)
.bind(ctx.team_id as i32)
.execute(&ctx.pool)
.await
.expect("insert leased op");
}
#[tokio::test]
async fn waiting_for_a_lease_lapse_returns_at_once_for_a_missing_or_finished_op() {
let ctx = TestContext::new().await;
let engine = ctx.engine();
let driver = DummyDriver::new();
let missing = Uuid::now_v7();
assert!(!engine
.wait_for_lease_lapse(missing, engine.execute_deadline())
.await
.expect("a missing op is claimable"));
let finished = Uuid::now_v7();
engine
.execute(&driver, finished, ctx.team_id, &json!({}))
.await
.expect("drive to completion");
assert!(engine
.wait_for_lease_lapse(finished, engine.execute_deadline())
.await
.expect("a finished op is reported terminal"));
ctx.cleanup().await.expect("cleanup");
}
#[tokio::test]
async fn waiting_for_a_lease_lapse_polls_until_it_lapses_and_gives_up_at_the_deadline() {
let ctx = TestContext::new().await;
let short_engine = personhog_identity::lifecycle::engine::Engine::new(
ctx.pools.clone(),
personhog_identity::lifecycle::engine::EngineConfig {
lease: std::time::Duration::from_secs(5),
execute_timeout: std::time::Duration::from_millis(300),
poll_interval: std::time::Duration::from_millis(25),
attempt_alert_threshold: 5,
gc_batch_limit: 10_000,
},
ctx.tables.clone(),
);
let lapsing = Uuid::now_v7();
insert_leased_op(&ctx, lapsing, "now() + interval '150 milliseconds'").await;
let started = std::time::Instant::now();
assert!(!short_engine
.wait_for_lease_lapse(lapsing, short_engine.execute_deadline())
.await
.expect("the lease lapses"));
assert!(started.elapsed() >= std::time::Duration::from_millis(100));
let held = Uuid::now_v7();
insert_leased_op(&ctx, held, "now() + interval '1 hour'").await;
let err = short_engine
.wait_for_lease_lapse(held, short_engine.execute_deadline())
.await
.expect_err("a lease held past the deadline is reported as busy");
assert!(matches!(err, SagaError::Busy));
ctx.cleanup().await.expect("cleanup");
}
#[tokio::test]
async fn a_shared_deadline_bounds_the_lease_wait_and_the_drive_together() {
let ctx = TestContext::new().await;
let engine = personhog_identity::lifecycle::engine::Engine::new(
ctx.pools.clone(),
personhog_identity::lifecycle::engine::EngineConfig {
lease: std::time::Duration::from_secs(5),
execute_timeout: std::time::Duration::from_millis(200),
poll_interval: std::time::Duration::from_millis(10),
attempt_alert_threshold: 5,
gc_batch_limit: 10_000,
},
ctx.tables.clone(),
);
let driver = DummyDriver::paced(std::time::Duration::from_millis(100));
let op_id = Uuid::now_v7();
insert_leased_op(&ctx, op_id, "now() + interval '150 milliseconds'").await;
let started = std::time::Instant::now();
let deadline = engine.execute_deadline();
assert!(!engine
.wait_for_lease_lapse(op_id, deadline)
.await
.expect("the lease lapses inside the budget"));
let err = engine
.execute_by(&driver, op_id, ctx.team_id, &json!({}), deadline)
.await
.expect_err("the drive inherits what is left of the budget");
assert!(matches!(err, SagaError::DeadlineElapsed));
assert!(
started.elapsed() < std::time::Duration::from_millis(400),
"the two phases shared one budget instead of taking one each"
);
let (_, _, _, completed) = op_row(&ctx, op_id).await;
assert!(!completed);
ctx.cleanup().await.expect("cleanup");
}