:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! The migration saga's store operations: requests, the saga record, seal acks and the flip.
use etcd_client::{Compare, CompareOp, DeleteOptions, Txn, TxnOp, WatchStream};
use super::{count_call, PersonhogStore, StoreKey};
use crate::error::Result;
use crate::types::{
ChangelogConfig, ClusterEndpoint, DrainReport, Generation, MigrationPhase, MigrationRequest,
MigrationState, SealAck,
};
impl PersonhogStore {
pub async fn get_migration(&self) -> Result<Option<MigrationState>> {
count_call("get_migration");
let key = self.key(StoreKey::Migration);
Ok(self.inner.get(&key).await?)
}
pub async fn get_migration_with_mod_revision(&self) -> Result<Option<(MigrationState, i64)>> {
count_call("get_migration_with_mod_revision");
let key = self.key(StoreKey::Migration);
Ok(self.inner.get_with_mod_revision(&key).await?)
}
/// Writes the saga record directly, skipping the request and its quiet-fleet start.
#[cfg(any(test, feature = "test-support"))]
pub async fn create_migration(&self, migration: &MigrationState) -> Result<bool> {
count_call("create_migration");
let key = self.key(StoreKey::Migration);
let txn = Txn::new()
.when(vec![Compare::create_revision(
key.clone(),
CompareOp::Equal,
0,
)])
.and_then(vec![TxnOp::put(key, serde_json::to_vec(migration)?, None)]);
let resp = self.inner.txn(txn).await?;
Ok(resp.succeeded())
}
pub async fn cas_migration_phase(
&self,
expected_id: &str,
expected_mod_revision: i64,
next: MigrationPhase,
) -> Result<bool> {
count_call("cas_migration_phase");
self.cas_phase(expected_id, expected_mod_revision, next, None)
.await
}
/// Move the saga to Cancelling, recording why.
pub async fn cancel_migration(
&self,
expected_id: &str,
expected_mod_revision: i64,
reason: &str,
) -> Result<bool> {
count_call("cancel_migration");
self.cas_phase(
expected_id,
expected_mod_revision,
MigrationPhase::Cancelling,
Some(reason),
)
.await
}
async fn cas_phase(
&self,
expected_id: &str,
expected_mod_revision: i64,
next: MigrationPhase,
cancel_reason: Option<&str>,
) -> Result<bool> {
let key = self.key(StoreKey::Migration);
let Some((mut migration, mod_revision)) = self
.inner
.get_with_mod_revision::<MigrationState>(&key)
.await?
else {
return Ok(false);
};
if migration.id != expected_id
|| mod_revision != expected_mod_revision
|| !migration.phase.may_transition_to(next)
{
return Ok(false);
}
migration.phase = next;
migration.phase_entered_at_ms = assignment_coordination::util::now_millis();
migration.blocking.clear();
if let Some(reason) = cancel_reason {
migration.cancel_reason = Some(reason.to_string());
}
let txn = Txn::new()
.when(vec![Compare::mod_revision(
key.clone(),
CompareOp::Equal,
mod_revision,
)])
.and_then(vec![TxnOp::put(key, serde_json::to_vec(&migration)?, None)]);
Ok(self.inner.txn(txn).await?.succeeded())
}
pub async fn set_migration_blocking(
&self,
expected_mod_revision: i64,
blocking: Vec<String>,
) -> Result<bool> {
count_call("set_migration_blocking");
let key = self.key(StoreKey::Migration);
let Some((mut migration, mod_revision)) = self
.inner
.get_with_mod_revision::<MigrationState>(&key)
.await?
else {
return Ok(false);
};
if mod_revision != expected_mod_revision {
return Ok(false);
}
migration.blocking = blocking;
let txn = Txn::new()
.when(vec![Compare::mod_revision(
key.clone(),
CompareOp::Equal,
mod_revision,
)])
.and_then(vec![TxnOp::put(key, serde_json::to_vec(&migration)?, None)]);
Ok(self.inner.txn(txn).await?.succeeded())
}
pub async fn flip_changelog(
&self,
migration: &MigrationState,
migration_mod_revision: i64,
config: &ChangelogConfig,
config_mod_revision: i64,
) -> Result<bool> {
count_call("flip_changelog");
let migration_key = self.key(StoreKey::Migration);
let config_key = self.key(StoreKey::ChangelogConfig);
let mut draining = config.draining.clone();
draining.push(Generation {
topic: migration.from.topic.clone(),
partitions: migration.from.partitions,
epoch: config.epoch,
cluster: migration.from.cluster.clone(),
});
let flipped = ChangelogConfig {
topic: migration.to.topic.clone(),
partitions: migration.to.partitions,
cluster: migration.to.cluster.clone(),
epoch: config.epoch + 1,
writer_group: config.writer_group.clone(),
retire_groups: config.retire_groups.clone(),
draining,
};
let mut record = migration.clone();
record.phase = MigrationPhase::Resuming;
record.phase_entered_at_ms = assignment_coordination::util::now_millis();
record.blocking.clear();
let txn = Txn::new()
.when(vec![
Compare::mod_revision(
migration_key.clone(),
CompareOp::Equal,
migration_mod_revision,
),
Compare::mod_revision(config_key.clone(), CompareOp::Equal, config_mod_revision),
])
.and_then(vec![
TxnOp::put(config_key, serde_json::to_vec(&flipped)?, None),
TxnOp::put(migration_key, serde_json::to_vec(&record)?, None),
]);
Ok(self.inner.txn(txn).await?.succeeded())
}
pub async fn archive_migration(
&self,
migration: &MigrationState,
mod_revision: i64,
final_phase: MigrationPhase,
) -> Result<bool> {
count_call("archive_migration");
let key = self.key(StoreKey::Migration);
let mut record = migration.clone();
record.phase = final_phase;
record.phase_entered_at_ms = assignment_coordination::util::now_millis();
let txn = Txn::new()
.when(vec![Compare::mod_revision(
key.clone(),
CompareOp::Equal,
mod_revision,
)])
.and_then(vec![
TxnOp::put(
self.key(StoreKey::MigrationHistory(&migration.id)),
serde_json::to_vec(&record)?,
None,
),
TxnOp::delete(
self.key(StoreKey::SealAcksPrefix),
Some(DeleteOptions::new().with_prefix()),
),
TxnOp::delete(key, None),
]);
Ok(self.inner.txn(txn).await?.succeeded())
}
pub async fn list_migration_history(&self) -> Result<Vec<MigrationState>> {
count_call("list_migration_history");
let key = self.key(StoreKey::MigrationHistoryPrefix);
Ok(self.inner.list(&key).await?)
}
pub async fn put_seal_ack(&self, ack: &SealAck) -> Result<()> {
count_call("put_seal_ack");
let key = self.key(StoreKey::SealAck(&ack.pod_name));
Ok(self.inner.put(&key, ack, None).await?)
}
/// The mod revision of one pod's seal ack, to tell a rewrite from no write.
#[cfg(any(test, feature = "test-support"))]
pub async fn seal_ack_mod_revision(&self, pod_name: &str) -> Result<Option<i64>> {
let key = self.key(StoreKey::SealAck(pod_name));
Ok(self
.inner
.get_with_mod_revision::<SealAck>(&key)
.await?
.map(|(_, rev)| rev))
}
pub async fn list_seal_acks(&self) -> Result<Vec<SealAck>> {
count_call("list_seal_acks");
let key = self.key(StoreKey::SealAcksPrefix);
Ok(self.inner.list(&key).await?)
}
/// Add a consumer group that retire waits on: false when the config is not
/// seeded or moved. A group the config already gates on is a no-op.
pub async fn add_changelog_retire_group(&self, group: &str) -> Result<bool> {
count_call("add_changelog_retire_group");
let Some((mut config, mod_revision)) =
self.get_changelog_config_with_mod_revision().await?
else {
return Ok(false);
};
if config.gated_groups().any(|g| g == group) {
return Ok(true);
}
config.retire_groups.push(group.to_string());
self.put_changelog_config_if_unchanged(&config, mod_revision)
.await
}
/// Stop retire waiting on a group: false when the config is not seeded or
/// moved. A group the list does not hold is a no-op.
pub async fn remove_changelog_retire_group(&self, group: &str) -> Result<bool> {
count_call("remove_changelog_retire_group");
let Some((mut config, mod_revision)) =
self.get_changelog_config_with_mod_revision().await?
else {
return Ok(false);
};
let before = config.retire_groups.len();
config.retire_groups.retain(|g| g != group);
if config.retire_groups.len() == before {
return Ok(true);
}
self.put_changelog_config_if_unchanged(&config, mod_revision)
.await
}
async fn put_changelog_config_if_unchanged(
&self,
config: &ChangelogConfig,
mod_revision: i64,
) -> Result<bool> {
let key = self.key(StoreKey::ChangelogConfig);
let txn = Txn::new()
.when(vec![Compare::mod_revision(
key.clone(),
CompareOp::Equal,
mod_revision,
)])
.and_then(vec![TxnOp::put(key, serde_json::to_vec(config)?, None)]);
Ok(self.inner.txn(txn).await?.succeeded())
}
/// Drop a drained generation, and its drain reports, from the config the
/// caller's drain checks read: only while that config is still current and
/// no saga exists. A saga completes only once every partition it created
/// has an owner, which routers rely on once nothing drains.
pub async fn retire_changelog_topic(
&self,
config: &ChangelogConfig,
mod_revision: i64,
topic: &str,
) -> Result<bool> {
count_call("retire_changelog_topic");
let mut retired = config.clone();
retired.draining.retain(|g| g.topic != topic);
if retired.draining.len() == config.draining.len() {
return Ok(false);
}
let key = self.key(StoreKey::ChangelogConfig);
let txn = Txn::new()
.when(vec![
Compare::mod_revision(key.clone(), CompareOp::Equal, mod_revision),
Compare::create_revision(self.key(StoreKey::Migration), CompareOp::Equal, 0),
])
.and_then(vec![
TxnOp::put(key, serde_json::to_vec(&retired)?, None),
TxnOp::delete(
self.key(StoreKey::DrainReportsForTopic(topic)),
Some(DeleteOptions::new().with_prefix()),
),
]);
Ok(self.inner.txn(txn).await?.succeeded())
}
/// Records the cluster of a live config that has none, at its epoch. Refused while a
/// migration is requested or running, since a request names the live config as it was.
pub async fn set_changelog_cluster(
&self,
config: &ChangelogConfig,
mod_revision: i64,
cluster: &ClusterEndpoint,
) -> Result<bool> {
count_call("set_changelog_cluster");
if config.cluster.is_some() {
return Ok(false);
}
let named = ChangelogConfig {
cluster: Some(cluster.clone()),
..config.clone()
};
let key = self.key(StoreKey::ChangelogConfig);
let txn = Txn::new()
.when(vec![
Compare::mod_revision(key.clone(), CompareOp::Equal, mod_revision),
Compare::create_revision(self.key(StoreKey::Migration), CompareOp::Equal, 0),
Compare::create_revision(self.key(StoreKey::MigrationRequest), CompareOp::Equal, 0),
])
.and_then(vec![TxnOp::put(key, serde_json::to_vec(&named)?, None)]);
Ok(self.inner.txn(txn).await?.succeeded())
}
pub async fn put_drain_report(&self, report: &DrainReport) -> Result<()> {
count_call("put_drain_report");
let key = self.key(StoreKey::DrainReport {
topic: &report.topic,
group: &report.group,
});
Ok(self.inner.put(&key, report, None).await?)
}
pub async fn list_drain_reports(&self) -> Result<Vec<DrainReport>> {
count_call("list_drain_reports");
let key = self.key(StoreKey::DrainReportsPrefix);
Ok(self.inner.list(&key).await?)
}
pub async fn delete_drain_reports_for(&self, topic: &str) -> Result<()> {
count_call("delete_drain_reports_for");
let key = self.key(StoreKey::DrainReportsForTopic(topic));
self.inner.delete_prefix(&key).await?;
Ok(())
}
pub async fn watch_drain_reports_from(&self, start_revision: i64) -> Result<WatchStream> {
count_call("watch_drain_reports_from");
let key = self.key(StoreKey::DrainReportsPrefix);
Ok(self.inner.watch_from(&key, start_revision).await?)
}
/// Write a request, refused while another request or a saga exists.
pub async fn create_migration_request(&self, request: &MigrationRequest) -> Result<bool> {
count_call("create_migration_request");
let key = self.key(StoreKey::MigrationRequest);
let txn = Txn::new()
.when(vec![
Compare::create_revision(key.clone(), CompareOp::Equal, 0),
Compare::create_revision(self.key(StoreKey::Migration), CompareOp::Equal, 0),
])
.and_then(vec![TxnOp::put(key, serde_json::to_vec(request)?, None)]);
Ok(self.inner.txn(txn).await?.succeeded())
}
pub async fn get_migration_request_with_mod_revision(
&self,
) -> Result<Option<(MigrationRequest, i64)>> {
count_call("get_migration_request_with_mod_revision");
let key = self.key(StoreKey::MigrationRequest);
Ok(self.inner.get_with_mod_revision(&key).await?)
}
/// Rewrite a waiting request (its first sighting or what it waits on), guarded on its revision.
pub async fn update_migration_request(
&self,
record: &MigrationRequest,
mod_revision: i64,
) -> Result<bool> {
count_call("update_migration_request");
let key = self.key(StoreKey::MigrationRequest);
let txn = Txn::new()
.when(vec![Compare::mod_revision(
key.clone(),
CompareOp::Equal,
mod_revision,
)])
.and_then(vec![TxnOp::put(key, serde_json::to_vec(record)?, None)]);
Ok(self.inner.txn(txn).await?.succeeded())
}
/// Start the saga from a request, only if no handoff record exists that was
/// written after `handoffs_revision`; from an empty read, that is none at all.
pub async fn start_migration(
&self,
request_mod_revision: i64,
handoffs_revision: i64,
migration: &MigrationState,
) -> Result<bool> {
count_call("start_migration");
let request_key = self.key(StoreKey::MigrationRequest);
let migration_key = self.key(StoreKey::Migration);
let txn = Txn::new()
.when(vec![
Compare::mod_revision(request_key.clone(), CompareOp::Equal, request_mod_revision),
Compare::create_revision(migration_key.clone(), CompareOp::Equal, 0),
Compare::mod_revision(
self.key(StoreKey::HandoffsPrefix),
CompareOp::Less,
handoffs_revision + 1,
)
.with_prefix(),
])
.and_then(vec![
TxnOp::put(migration_key, serde_json::to_vec(migration)?, None),
TxnOp::delete(request_key, None),
]);
Ok(self.inner.txn(txn).await?.succeeded())
}
/// Archive a request that will not start, with its reason, guarded on its revision.
pub async fn reject_migration_request(
&self,
request: &MigrationRequest,
mod_revision: i64,
reason: &str,
) -> Result<bool> {
count_call("reject_migration_request");
let key = self.key(StoreKey::MigrationRequest);
let record = request.rejected(reason, assignment_coordination::util::now_millis());
let txn = Txn::new()
.when(vec![Compare::mod_revision(
key.clone(),
CompareOp::Equal,
mod_revision,
)])
.and_then(vec![
TxnOp::put(
self.key(StoreKey::MigrationHistory(&request.id)),
serde_json::to_vec(&record)?,
None,
),
TxnOp::delete(key, None),
]);
Ok(self.inner.txn(txn).await?.succeeded())
}
pub async fn watch_migration_request_from(&self, start_revision: i64) -> Result<WatchStream> {
count_call("watch_migration_request_from");
let key = self.key(StoreKey::MigrationRequest);
Ok(self.inner.watch_key_from(&key, start_revision).await?)
}
pub async fn watch_migration_from(&self, start_revision: i64) -> Result<WatchStream> {
count_call("watch_migration_from");
let key = self.key(StoreKey::Migration);
Ok(self.inner.watch_key_from(&key, start_revision).await?)
}
pub async fn watch_seal_acks_from(&self, start_revision: i64) -> Result<WatchStream> {
count_call("watch_seal_acks_from");
let key = self.key(StoreKey::SealAcksPrefix);
Ok(self.inner.watch_from(&key, start_revision).await?)
}
/// A shrunk partition's hold, acks and assignment go in one guarded transaction.
pub async fn dissolve_partition(
&self,
partition: u32,
handoff_mod_revision: i64,
) -> Result<bool> {
count_call("dissolve_partition");
let handoff_key = self.key(StoreKey::Handoff(partition));
let prefix_delete = || Some(DeleteOptions::new().with_prefix());
let txn = Txn::new()
.when(vec![Compare::mod_revision(
handoff_key.clone(),
CompareOp::Equal,
handoff_mod_revision,
)])
.and_then(vec![
TxnOp::delete(
self.key(StoreKey::FreezeAcksForPartition(partition)),
prefix_delete(),
),
TxnOp::delete(
self.key(StoreKey::DrainedAcksForPartition(partition)),
prefix_delete(),
),
TxnOp::delete(
self.key(StoreKey::WarmedAcksForPartition(partition)),
prefix_delete(),
),
TxnOp::delete(self.key(StoreKey::Assignment(partition)), None),
TxnOp::delete(handoff_key, None),
]);
Ok(self.inner.txn(txn).await?.succeeded())
}
}