:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
// Shared by every integration-test binary; each binary compiles its own copy
// and none of them uses every helper.
#![allow(dead_code)]
pub mod sim_leader;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use rand::Rng;
use sqlx::postgres::PgPool;
use uuid::Uuid;
use personhog_coordination::changelog::LiveChangelog;
use personhog_coordination::types::ChangelogConfig;
use personhog_identity::config::{IdentityTables, OutboxWrites};
use personhog_identity::pools::IdentityPools;
use personhog_identity::publish::config::MergeEventsAllowlist;
use personhog_identity::publish::messages::{DistinctIdRow, PersonMergeEvent};
use personhog_identity::publish::sink::{PublishBatch, PublishError, PublishSink};
use personhog_identity::publish::{DrainConfig, Publisher};
use personhog_identity::storage::postgres::PostgresIdentityStorage;
/// Serializes every test that drains the publish outbox, across the
/// processes nextest runs tests in: the drain claims the oldest pending
/// rows of every team, so two concurrent drains would take each other's
/// rows. See [`TestContext::hold_drain_lock`].
const DRAIN_LOCK_KEY: i64 = 0x5048_5055_424c_4953;
/// The production table set. Most tests run here; the raw-SQL assertion
/// helpers in the test binaries assume it.
/// Leader fan-out width the suites run with; production reads it from config.
pub const FAN_OUT_CONCURRENCY: usize = 8;
/// Partition count the delete driver groups its fence batches by.
pub const NUM_PARTITIONS: u32 = 4;
pub fn changelog() -> LiveChangelog {
LiveChangelog::new(ChangelogConfig::new("personhog_updates", NUM_PARTITIONS))
}
pub fn default_tables() -> IdentityTables {
IdentityTables::real()
}
/// The validation (shadow) table set — the service's config default.
pub fn tmp_tables() -> IdentityTables {
IdentityTables::validation()
}
pub struct TestContext {
pub pool: PgPool,
pub pools: IdentityPools,
pub storage: Arc<PostgresIdentityStorage>,
pub team_id: i64,
pub tables: IdentityTables,
}
impl TestContext {
pub async fn new() -> Self {
Self::new_with_tables(default_tables()).await
}
pub async fn new_with_tables(tables: IdentityTables) -> Self {
let database_url = std::env::var("DATABASE_URL").unwrap_or_else(|_| {
"postgres://posthog:posthog@localhost:5432/posthog_persons".to_string()
});
let pool = PgPool::connect(&database_url)
.await
.expect("Failed to connect to test database");
let pools = IdentityPools::shared(pool.clone());
let storage = Arc::new(PostgresIdentityStorage::new(
pools.clone(),
tables.clone(),
OutboxWrites::Disabled,
));
let team_id = rand::thread_rng().gen_range(1_000_000..100_000_000);
Self {
pool,
pools,
storage,
team_id,
tables,
}
}
/// Inserts a person with a random (non-deterministic) uuid plus a distinct
/// id row pointing at it. Returns the person id.
pub async fn insert_person_with_distinct_id(&self, distinct_id: &str) -> i64 {
let insert_sql = format!(
r#"
INSERT INTO {}
(created_at, properties, properties_last_updated_at, properties_last_operation,
team_id, is_identified, uuid, version)
VALUES (now(), '{{}}'::jsonb, '{{}}'::jsonb, '{{}}'::jsonb, $1, false, gen_random_uuid(), 0)
RETURNING id
"#,
self.tables.person
);
let person_id: i64 = sqlx::query_scalar(&insert_sql)
.bind(self.team_id as i32)
.fetch_one(&self.pool)
.await
.expect("Failed to insert person");
let pdi_sql = format!(
r#"
INSERT INTO {} (distinct_id, person_id, team_id, version)
VALUES ($1, $2, $3, 0)
"#,
self.tables.person_distinct_id
);
sqlx::query(&pdi_sql)
.bind(distinct_id)
.bind(person_id)
.bind(self.team_id as i32)
.execute(&self.pool)
.await
.expect("Failed to insert distinct id");
person_id
}
/// An engine over this context's pool, tuned for tests: a short poll
/// so contention tests run in milliseconds, and a lease long enough
/// that a live test's op never looks expired to the suite's single
/// sweep test (in lifecycle_merge_tests), which would otherwise claim
/// and drive it with the wrong harness. Lease-expiry tests seed
/// explicit lease_expires_at values instead of waiting this out.
pub fn engine(&self) -> personhog_identity::lifecycle::engine::Engine {
personhog_identity::lifecycle::engine::Engine::new(
self.pools.clone(),
personhog_identity::lifecycle::engine::EngineConfig {
lease: std::time::Duration::from_secs(300),
execute_timeout: std::time::Duration::from_secs(10),
poll_interval: std::time::Duration::from_millis(25),
attempt_alert_threshold: 5,
gc_batch_limit: 10_000,
},
self.tables.clone(),
)
}
/// Parks the tmp person sequence in a random high range. Collision
/// tests insert a real posthog_person row with a tmp person's numeric
/// id; without this the explicit-id insert could race another test's
/// sequence-assigned real id.
pub async fn park_tmp_person_sequence(&self) {
let base: i64 = rand::thread_rng().gen_range(1_000_000_000_000..2_000_000_000_000);
sqlx::query("SELECT setval('personhog_person_tmp_id_seq', $1)")
.bind(base)
.execute(&self.pool)
.await
.expect("park tmp person sequence");
}
/// Seeds this team's rows in the *real* namespace keyed by the given
/// numeric person id: a posthog_person row plus a mapping row and a
/// hash-key override referencing it. Validation-set tests plant these
/// so a leftover hardcoded real table in a saga query would visibly
/// destroy or move them.
pub async fn seed_real_namespace_collision(&self, person_id: i64, distinct_id: &str) {
sqlx::query(
r#"
INSERT INTO posthog_person
(id, created_at, properties, properties_last_updated_at,
properties_last_operation, team_id, is_identified, uuid, version)
VALUES ($1, now(), '{}'::jsonb, '{}'::jsonb, '{}'::jsonb, $2, false,
gen_random_uuid(), 0)
"#,
)
.bind(person_id)
.bind(self.team_id as i32)
.execute(&self.pool)
.await
.expect("insert colliding real person");
sqlx::query(
r#"
INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id, version)
VALUES ($1, $2, $3, 0)
"#,
)
.bind(distinct_id)
.bind(person_id)
.bind(self.team_id as i32)
.execute(&self.pool)
.await
.expect("insert colliding real mapping");
sqlx::query(
r#"
INSERT INTO posthog_featureflaghashkeyoverride
(feature_flag_key, hash_key, person_id, team_id)
VALUES ('flag', 'real-hash', $1, $2)
"#,
)
.bind(person_id)
.bind(self.team_id as i32)
.execute(&self.pool)
.await
.expect("insert colliding real override");
}
/// Removes this team's rows from the real namespace tables — the
/// counterpart of [`seed_real_namespace_collision`] for validation-set
/// tests, whose [`cleanup`](Self::cleanup) only touches the configured
/// tables.
pub async fn cleanup_real_namespace(&self) -> Result<(), sqlx::Error> {
sqlx::query("DELETE FROM posthog_featureflaghashkeyoverride WHERE team_id = $1")
.bind(self.team_id as i32)
.execute(&self.pool)
.await?;
sqlx::query("DELETE FROM posthog_persondistinctid WHERE team_id = $1")
.bind(self.team_id as i32)
.execute(&self.pool)
.await?;
sqlx::query("DELETE FROM posthog_person WHERE team_id = $1")
.bind(self.team_id as i32)
.execute(&self.pool)
.await?;
Ok(())
}
/// Storage that writes the publish outbox, for tests that assert on the
/// rows a create or attach leaves there; `storage` never writes it.
/// Take [`hold_drain_lock`](Self::hold_drain_lock) before the first
/// write through it.
pub fn storage_with(&self, outbox_writes: OutboxWrites) -> PostgresIdentityStorage {
PostgresIdentityStorage::new(self.pools.clone(), self.tables.clone(), outbox_writes)
}
/// A transaction-level advisory lock that every test writing outbox
/// rows takes before it writes any row it will assert on, and holds
/// until its assertions are done. Dropping the returned transaction
/// releases it, on a panic too.
///
/// Once the lock is held, every other team's outbox row is a leftover
/// of a test that panicked before its cleanup, because every writer
/// takes this lock first. They are deleted here: the claim takes the
/// oldest rows across all teams, and leftovers would spend this test's
/// pass budget and change what its own rows resolve to.
pub async fn hold_drain_lock(&self) -> sqlx::Transaction<'static, sqlx::Postgres> {
let mut tx = self.pool.begin().await.expect("begin lock transaction");
sqlx::query("SELECT pg_advisory_xact_lock($1)")
.bind(DRAIN_LOCK_KEY)
.execute(&mut *tx)
.await
.expect("take the drain lock");
sqlx::query(&format!(
"DELETE FROM {} WHERE team_id <> $1",
self.tables.person_publish_outbox
))
.bind(self.team_id as i32)
.execute(&self.pool)
.await
.expect("clear leftover outbox rows");
tx
}
/// A publisher over this context's pool and table set. The batch is
/// large so rows another test left behind when it crashed before its
/// cleanup cannot crowd this test's rows out of a pass; they drain into
/// the sink, where [`RecordingSink::rows_for_team`] ignores them. The
/// row budget is the caller's, so a test can make it small enough that
/// its own pointer drains in chunks.
pub fn publisher<S: PublishSink>(&self, sink: S, max_rows_per_pass: i64) -> Publisher<S> {
self.build_publisher(sink, max_rows_per_pass, None)
}
pub fn publisher_with_merge_events<S: PublishSink>(
&self,
sink: S,
max_rows_per_pass: i64,
allowlist: MergeEventsAllowlist,
) -> Publisher<S> {
self.build_publisher(sink, max_rows_per_pass, Some(allowlist))
}
fn build_publisher<S: PublishSink>(
&self,
sink: S,
max_rows_per_pass: i64,
merge_events: Option<MergeEventsAllowlist>,
) -> Publisher<S> {
Publisher::new(
self.pool.clone(),
self.tables.clone(),
sink,
DrainConfig {
batch_size: 10_000,
max_rows_per_pass,
interval: Duration::from_millis(10),
lag_alert: Duration::from_secs(300),
merge_events,
},
)
}
pub async fn outbox_rows(&self) -> Vec<OutboxRow> {
sqlx::query_as(&format!(
r#"
SELECT id, kind, op_id, cursor_person_id, cursor_mapping_ordinal, attempts,
given_up_at IS NOT NULL AS given_up, last_error
FROM {} WHERE team_id = $1 ORDER BY id
"#,
self.tables.person_publish_outbox
))
.bind(self.team_id as i32)
.fetch_all(&self.pool)
.await
.expect("read outbox rows")
}
pub async fn cleanup(&self) -> Result<(), sqlx::Error> {
sqlx::query(&format!(
"DELETE FROM {} WHERE team_id = $1",
self.tables.person_publish_outbox
))
.bind(self.team_id as i32)
.execute(&self.pool)
.await?;
sqlx::query(&format!(
"DELETE FROM {} WHERE team_id = $1",
self.tables.lifecycle_op
))
.bind(self.team_id as i32)
.execute(&self.pool)
.await?;
sqlx::query(&format!(
"DELETE FROM {} WHERE team_id = $1",
self.tables.person_distinct_id
))
.bind(self.team_id as i32)
.execute(&self.pool)
.await?;
sqlx::query(&format!(
"DELETE FROM {} WHERE team_id = $1",
self.tables.person
))
.bind(self.team_id as i32)
.execute(&self.pool)
.await?;
Ok(())
}
}
#[derive(Debug, Clone, PartialEq, Eq, sqlx::FromRow)]
pub struct OutboxRow {
pub id: i64,
pub kind: String,
pub op_id: Option<Uuid>,
pub cursor_person_id: i64,
pub cursor_mapping_ordinal: i64,
pub attempts: i32,
pub given_up: bool,
pub last_error: Option<String>,
}
/// A sink that keeps every batch it acked. Clones share the record, so a
/// test hands one clone to the publisher and reads the other.
#[derive(Default, Clone)]
pub struct RecordingSink {
batches: Arc<Mutex<Vec<PublishBatch>>>,
}
impl RecordingSink {
pub fn batches(&self) -> Vec<PublishBatch> {
self.batches.lock().unwrap().clone()
}
/// Every distinct id row published for `team_id`, in publish order.
/// Rows of other teams are leftovers of other tests and are ignored.
pub fn rows_for_team(&self, team_id: i64) -> Vec<DistinctIdRow> {
self.batches()
.into_iter()
.flat_map(|b| b.distinct_id_rows)
.filter(|r| r.team_id == team_id)
.collect()
}
pub fn merge_events_for_team(&self, team_id: i64) -> Vec<PersonMergeEvent> {
self.batches()
.into_iter()
.flat_map(|b| b.merge_events)
.filter(|e| i64::from(e.team_id) == team_id)
.collect()
}
}
#[async_trait::async_trait]
impl PublishSink for RecordingSink {
async fn send(&self, batch: &PublishBatch) -> Result<(), PublishError> {
self.batches.lock().unwrap().push(batch.clone());
Ok(())
}
}
pub struct FailingSink;
#[async_trait::async_trait]
impl PublishSink for FailingSink {
async fn send(&self, _batch: &PublishBatch) -> Result<(), PublishError> {
Err(PublishError::Kafka("brokers unreachable".to_string()))
}
}
/// Leader-surface stubs for test binaries whose flows never reach the
/// leader: an accidental call surfaces as a clean unimplemented error
/// instead of a hang or a panic mid-transaction.
pub struct UnusedLeader;
#[async_trait::async_trait]
impl personhog_identity::leader::PersonStateReader for UnusedLeader {
async fn is_identified(
&self,
_team_id: i64,
_person_id: i64,
) -> Result<Option<bool>, tonic::Status> {
Err(tonic::Status::unimplemented("not exercised by this test"))
}
}
#[async_trait::async_trait]
impl personhog_identity::leader::PropertyWriter for UnusedLeader {
async fn update_person_properties(
&self,
_request: personhog_proto::personhog::types::v1::UpdatePersonPropertiesRequest,
) -> Result<personhog_proto::personhog::types::v1::UpdatePersonPropertiesResponse, tonic::Status>
{
Err(tonic::Status::unimplemented("not exercised by this test"))
}
}
#[async_trait::async_trait]
impl personhog_identity::leader::LifecycleLeader for UnusedLeader {
async fn fence_person(
&self,
_request: personhog_proto::personhog::types::v1::FencePersonRequest,
) -> Result<personhog_proto::personhog::types::v1::FencePersonResponse, tonic::Status> {
Err(tonic::Status::unimplemented("not exercised by this test"))
}
async fn fence_persons(
&self,
_request: personhog_proto::personhog::types::v1::FencePersonsRequest,
) -> Result<personhog_proto::personhog::types::v1::FencePersonsResponse, tonic::Status> {
Err(tonic::Status::unimplemented("not exercised by this test"))
}
async fn release_fence(
&self,
_request: personhog_proto::personhog::types::v1::ReleaseFenceRequest,
) -> Result<personhog_proto::personhog::types::v1::ReleaseFenceResponse, tonic::Status> {
Err(tonic::Status::unimplemented("not exercised by this test"))
}
async fn release_fences(
&self,
_request: personhog_proto::personhog::types::v1::ReleaseFencesRequest,
) -> Result<personhog_proto::personhog::types::v1::ReleaseFencesResponse, tonic::Status> {
Err(tonic::Status::unimplemented("not exercised by this test"))
}
async fn fold_person_document(
&self,
_request: personhog_proto::personhog::types::v1::FoldPersonDocumentRequest,
) -> Result<personhog_proto::personhog::types::v1::FoldPersonDocumentResponse, tonic::Status>
{
Err(tonic::Status::unimplemented("not exercised by this test"))
}
}
/// IdentityStorage stub for validation-only test binaries: any call means
/// validation failed to reject first.
pub struct UnusedStorage;
#[async_trait::async_trait]
impl personhog_identity::storage::IdentityStorage for UnusedStorage {
async fn resolve_distinct_ids(
&self,
_keys: &[(i64, String)],
) -> personhog_identity::storage::StorageResult<
std::collections::HashMap<(i64, String), personhog_identity::storage::Person>,
> {
panic!("not exercised by this test")
}
async fn resolve_person_ids(
&self,
_keys: &[(i64, String)],
) -> personhog_identity::storage::StorageResult<std::collections::HashMap<(i64, String), i64>>
{
panic!("not exercised by this test")
}
async fn get_distinct_ids_for_persons(
&self,
_team_id: i64,
_person_ids: &[i64],
_limit_per_person: Option<i64>,
) -> personhog_identity::storage::StorageResult<
Vec<personhog_identity::storage::DistinctIdMapping>,
> {
panic!("not exercised by this test")
}
async fn create_person_stubs(
&self,
_stubs: &[personhog_identity::storage::PersonStub],
) -> personhog_identity::storage::StorageResult<Vec<personhog_identity::storage::StubOutcome>>
{
panic!("not exercised by this test")
}
async fn attach_distinct_ids(
&self,
_team_id: i64,
_person_id: i64,
_distinct_ids: &[String],
) -> personhog_identity::storage::StorageResult<
std::collections::HashMap<String, personhog_identity::storage::AttachOutcome>,
> {
panic!("not exercised by this test")
}
}
/// Wraps the real storage to inject the target-establishment races that
/// cannot be produced deterministically otherwise. Each injection fires
/// once, and the "concurrent winner" side effect is committed through the
/// real storage so the entrance's re-resolution finds what a genuine race
/// would have left behind.
pub struct RacingStorage {
inner: Arc<PostgresIdentityStorage>,
/// attach_distinct_ids: map the ids to THIS person instead (the
/// concurrent winner) and answer AlreadyMapped.
pub hijack_attach_to: std::sync::Mutex<Option<i64>>,
/// attach_distinct_ids: attach nothing (the person vanished under a
/// racing lifecycle op).
pub vanish_attach: std::sync::Mutex<bool>,
/// create_person_stubs: commit the stub as a concurrent winner and
/// answer LostRace.
pub lose_create_race: std::sync::Mutex<bool>,
/// Every key passed to resolve_distinct_ids, for tests that pin what
/// does and does not reach the resolution query.
pub resolved_keys: std::sync::Mutex<Vec<(i64, String)>>,
}
impl RacingStorage {
pub fn new(inner: Arc<PostgresIdentityStorage>) -> Self {
Self {
inner,
hijack_attach_to: std::sync::Mutex::new(None),
vanish_attach: std::sync::Mutex::new(false),
lose_create_race: std::sync::Mutex::new(false),
resolved_keys: std::sync::Mutex::new(Vec::new()),
}
}
}
#[async_trait::async_trait]
impl personhog_identity::storage::IdentityStorage for RacingStorage {
async fn resolve_distinct_ids(
&self,
keys: &[(i64, String)],
) -> personhog_identity::storage::StorageResult<
std::collections::HashMap<(i64, String), personhog_identity::storage::Person>,
> {
self.resolved_keys
.lock()
.unwrap()
.extend(keys.iter().cloned());
self.inner.resolve_distinct_ids(keys).await
}
async fn resolve_person_ids(
&self,
keys: &[(i64, String)],
) -> personhog_identity::storage::StorageResult<std::collections::HashMap<(i64, String), i64>>
{
self.inner.resolve_person_ids(keys).await
}
async fn get_distinct_ids_for_persons(
&self,
team_id: i64,
person_ids: &[i64],
limit_per_person: Option<i64>,
) -> personhog_identity::storage::StorageResult<
Vec<personhog_identity::storage::DistinctIdMapping>,
> {
self.inner
.get_distinct_ids_for_persons(team_id, person_ids, limit_per_person)
.await
}
async fn create_person_stubs(
&self,
stubs: &[personhog_identity::storage::PersonStub],
) -> personhog_identity::storage::StorageResult<Vec<personhog_identity::storage::StubOutcome>>
{
let lose = std::mem::take(&mut *self.lose_create_race.lock().unwrap());
if lose {
self.inner.create_person_stubs(stubs).await?;
return Ok(stubs
.iter()
.map(|_| personhog_identity::storage::StubOutcome::LostRace)
.collect());
}
self.inner.create_person_stubs(stubs).await
}
async fn attach_distinct_ids(
&self,
team_id: i64,
person_id: i64,
distinct_ids: &[String],
) -> personhog_identity::storage::StorageResult<
std::collections::HashMap<String, personhog_identity::storage::AttachOutcome>,
> {
let vanish = std::mem::take(&mut *self.vanish_attach.lock().unwrap());
if vanish {
return Ok(std::collections::HashMap::new());
}
let winner = self.hijack_attach_to.lock().unwrap().take();
if let Some(winner) = winner {
self.inner
.attach_distinct_ids(team_id, winner, distinct_ids)
.await?;
return Ok(distinct_ids
.iter()
.map(|d| {
(
d.clone(),
personhog_identity::storage::AttachOutcome::AlreadyMapped {
person_id: winner,
},
)
})
.collect());
}
self.inner
.attach_distinct_ids(team_id, person_id, distinct_ids)
.await
}
}