:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
//! The leader's share of a changelog migration: seal the old generation and ack it.
use std::collections::{hash_map, BTreeSet, HashMap, HashSet};
use std::sync::Arc;
use std::time::Duration;
use assignment_coordination::store::parse_watch_value;
use metrics::{counter, histogram};
use personhog_coordination::protocol::migration::{seal_overdue, seal_targets};
use personhog_coordination::store::PersonhogStore;
use personhog_coordination::types::{MigrationPhase, MigrationState, SealAck};
use tokio::sync::Semaphore;
use tokio::task::{JoinHandle, JoinSet};
use tokio_util::sync::CancellationToken;
use tracing::{info, warn};
use crate::fencing::FencedChangelogProducers;
pub const ACTIVE_INTERVAL: Duration = Duration::from_millis(250);
pub const IDLE_INTERVAL: Duration = Duration::from_secs(2);
/// Each seal spends one blocking thread per lane.
const SEAL_CONCURRENCY: usize = 8;
pub struct SealDuty {
pub store: Arc<PersonhogStore>,
pub pod_name: String,
pub fenced: Arc<FencedChangelogProducers>,
}
/// Registers the seal metrics at zero. Migrations are rare and short, so a
/// series born at its first seal would hide that seal from `increase()`.
fn register_seal_metrics() {
for outcome in ["ok", "error"] {
counter!("personhog_leader_fence_seal_total", "outcome" => outcome).increment(0);
}
for path in ["prepared", "cold"] {
drop(histogram!("personhog_leader_fence_seal_ms", "path" => path));
}
counter!("personhog_leader_seal_duty_failures_total").increment(0);
counter!("personhog_leader_seal_overdue_passes_total").increment(0);
}
impl SealDuty {
/// Wakes when the saga or its phase changes, and on an interval for retries.
pub async fn run(self, cancel: CancellationToken) {
register_seal_metrics();
let mut sealed: HashSet<(String, u32)> = HashSet::new();
let mut preconnecting: HashMap<(String, u32), JoinHandle<()>> = HashMap::new();
let mut acked: Option<(String, BTreeSet<u32>)> = None;
let mut seen: Option<(String, MigrationPhase)> = None;
let preconnects = Arc::new(Semaphore::new(SEAL_CONCURRENCY));
let mut watch = None;
loop {
if watch.is_none() {
watch = match self.store.current_revision().await {
Ok(revision) => self.store.watch_migration_from(revision + 1).await.ok(),
Err(_) => None,
};
}
let interval = match self.store.get_migration().await {
Ok(Some(migration)) => {
seen = Some((migration.id.clone(), migration.phase));
let pass = match migration.phase {
MigrationPhase::Freezing | MigrationPhase::Draining => {
self.prepare(&migration, &mut preconnecting, &preconnects)
.await
}
MigrationPhase::Sealing => {
self.tick(&migration, &mut sealed, &mut acked).await
}
_ => Ok(()),
};
if let Err(e) = pass {
warn!(migration = %migration.id, error = %e, "seal duty pass failed");
counter!("personhog_leader_seal_duty_failures_total").increment(1);
}
// Only Sealing retries on a clock; the watch wakes every other phase.
if migration.phase == MigrationPhase::Sealing {
ACTIVE_INTERVAL
} else {
IDLE_INTERVAL
}
}
Ok(None) => {
seen = None;
sealed.clear();
preconnecting.clear();
acked = None;
IDLE_INTERVAL
}
Err(e) => {
warn!(error = %e, "seal duty could not read the migration");
IDLE_INTERVAL
}
};
let woke = async {
let Some(stream) = watch.as_mut() else {
return std::future::pending().await;
};
loop {
match stream.message().await {
// The coordinator rewrites the record on every seal ack; only a
// new saga or phase is worth a pass.
Ok(Some(resp)) if !resp.canceled() => {
// A deletion has no value to parse, so it reads as no saga.
let changed = resp.events().iter().any(|e| {
parse_watch_value::<MigrationState>(e)
.ok()
.map(|m| (m.id, m.phase))
!= seen
});
if changed {
return true;
}
}
// A cancelled response ends the stream; rebuild it like a dead one.
_ => return false,
}
}
};
tokio::select! {
_ = cancel.cancelled() => return,
_ = tokio::time::sleep(interval) => {}
live = woke => {
if !live {
watch = None;
}
}
}
}
}
async fn targets(&self, migration: &MigrationState, overdue: bool) -> Result<Vec<u32>, String> {
let pods = self
.store
.list_pods()
.await
.map_err(|e| format!("list pods: {e}"))?;
let registered: HashSet<&str> = pods.iter().map(|p| p.pod_name.as_str()).collect();
let assignments = self
.store
.list_assignments()
.await
.map_err(|e| format!("list assignments: {e}"))?;
// Past the grace the acks matter: what no ack reports sealed is a target too.
let acks = if overdue {
self.store
.list_seal_acks()
.await
.map_err(|e| format!("list seal acks: {e}"))?
} else {
Vec::new()
};
Ok(seal_targets(
&self.pod_name,
&migration.id,
migration.from.partitions,
&assignments,
®istered,
&acks,
overdue,
))
}
/// Asks for each target's old-topic connection unless a request for it is still
/// running: one the sweep discarded is dialed again on the next pass.
async fn prepare(
&self,
migration: &MigrationState,
preconnecting: &mut HashMap<(String, u32), JoinHandle<()>>,
preconnects: &Arc<Semaphore>,
) -> Result<(), String> {
preconnecting.retain(|_, task| !task.is_finished());
for partition in self.targets(migration, false).await? {
let hash_map::Entry::Vacant(slot) =
preconnecting.entry((migration.id.clone(), partition))
else {
continue;
};
let fenced = Arc::clone(&self.fenced);
let from = migration.from.clone();
let permits = Arc::clone(preconnects);
slot.insert(tokio::spawn(async move {
let _permit = permits.acquire().await;
fenced.preconnect(partition, &from).await
}));
}
Ok(())
}
/// One Sealing pass: seal every target not yet sealed, then ack the state of all of them.
pub async fn tick(
&self,
migration: &MigrationState,
sealed: &mut HashSet<(String, u32)>,
acked: &mut Option<(String, BTreeSet<u32>)>,
) -> Result<(), String> {
let overdue = seal_overdue(migration, assignment_coordination::util::now_millis());
let targets = self.targets(migration, overdue).await?;
let topic = migration.from.topic.as_str();
let cluster = migration.from.cluster.as_ref().map(|c| c.name.as_str());
let pending: Vec<u32> = targets
.iter()
.copied()
.filter(|p| !sealed.contains(&(migration.id.clone(), *p)))
.collect();
if overdue && !pending.is_empty() {
counter!("personhog_leader_seal_overdue_passes_total").increment(1);
warn!(migration = %migration.id, partitions = ?pending, "partitions still unsealed past the seal grace; sealing every one this pod can reach");
}
if !pending.is_empty() {
let permits = Arc::new(Semaphore::new(SEAL_CONCURRENCY));
let mut seals = JoinSet::new();
for partition in pending {
let fenced = Arc::clone(&self.fenced);
let from = migration.from.clone();
let permits = Arc::clone(&permits);
seals.spawn(async move {
let _permit = permits.acquire().await;
(partition, fenced.seal(partition, &from).await)
});
}
while let Some(joined) = seals.join_next().await {
let (partition, result) = joined.map_err(|e| format!("seal join: {e}"))?;
match result {
Ok(path) => {
info!(partition, %topic, ?cluster, path, "sealed the old changelog partition");
sealed.insert((migration.id.clone(), partition));
// Acked as it lands; a failed write is retried by the next ack.
if let Err(e) = self.ack(migration, sealed, acked).await {
warn!(partition, error = %e, "seal ack failed; the next ack carries it");
}
}
Err(e) => {
warn!(partition, %topic, ?cluster, error = %e, "seal failed; retrying")
}
}
}
}
self.ack(migration, sealed, acked).await
}
/// Reports every partition this process has sealed, so an overdue target
/// drops out of the unsealed set once acked; a restarted leader reseals.
async fn ack(
&self,
migration: &MigrationState,
sealed: &HashSet<(String, u32)>,
acked: &mut Option<(String, BTreeSet<u32>)>,
) -> Result<(), String> {
let report: BTreeSet<u32> = sealed
.iter()
.filter(|(id, _)| *id == migration.id)
.map(|(_, p)| *p)
.collect();
// An unchanged ack wakes the coordinator for nothing.
if acked.as_ref() == Some(&(migration.id.clone(), report.clone())) {
return Ok(());
}
self.store
.put_seal_ack(&SealAck {
migration_id: migration.id.clone(),
pod_name: self.pod_name.clone(),
sealed: report.clone(),
})
.await
.map_err(|e| format!("write seal ack: {e}"))?;
*acked = Some((migration.id.clone(), report));
Ok(())
}
}