Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(personhog): register the seal and migration transition metrics at zero (#121326)

Zach Brown918789fb8968
diff --git a/rust/personhog-coordination/src/coordinator.rs b/rust/personhog-coordination/src/coordinator.rs
index 87cb4f5ed228..a1469639ccce 100644
--- a/rust/personhog-coordination/src/coordinator.rs
+++ b/rust/personhog-coordination/src/coordinator.rs
@@ -881,6 +881,7 @@ impl Coordinator {
mut request_stream: WatchStream,
) -> Result<()> {
let _gauge = migration::PhaseGauge;
+ migration::register_transition_metrics();
loop {
tokio::select! {
_ = cancel.cancelled() => return Ok(()),
diff --git a/rust/personhog-coordination/src/migration.rs b/rust/personhog-coordination/src/migration.rs
index a89a1d94c187..cc24a8e5fc14 100644
--- a/rust/personhog-coordination/src/migration.rs
+++ b/rust/personhog-coordination/src/migration.rs
@@ -50,6 +50,14 @@ impl Drop for PhaseGauge {
}
}
+/// Registers each phase's transition counter at zero, so the first saga a
+/// coordinator drives shows up under `increase()`.
+pub fn register_transition_metrics() {
+ for phase in PHASES {
+ counter!("personhog_migration_transitions_total", "to" => phase_label(phase)).increment(0);
+ }
+}
+
pub fn record_phase(current: Option<MigrationPhase>) {
for phase in PHASES {
gauge!("personhog_migration_phase", "phase" => phase_label(phase))
diff --git a/rust/personhog-leader/src/migration.rs b/rust/personhog-leader/src/migration.rs
index 07f59f6a0354..631b65331721 100644
--- a/rust/personhog-leader/src/migration.rs
+++ b/rust/personhog-leader/src/migration.rs
@@ -6,7 +6,7 @@ use std::time::Duration;
use assignment_coordination::store::parse_watch_value;
-use metrics::counter;
+use metrics::{counter, histogram};
use personhog_coordination::protocol::migration::{seal_overdue, seal_targets};
use personhog_coordination::store::PersonhogStore;
use personhog_coordination::types::{MigrationPhase, MigrationState, SealAck};
@@ -28,9 +28,23 @@ pub struct SealDuty {
pub fenced: Arc<FencedChangelogProducers>,
}
+/// Registers the seal metrics at zero. Migrations are rare and short, so a
+/// series born at its first seal would hide that seal from `increase()`.
+fn register_seal_metrics() {
+ for outcome in ["ok", "error"] {
+ counter!("personhog_leader_fence_seal_total", "outcome" => outcome).increment(0);
+ }
+ for path in ["prepared", "cold"] {
+ drop(histogram!("personhog_leader_fence_seal_ms", "path" => path));
+ }
+ counter!("personhog_leader_seal_duty_failures_total").increment(0);
+ counter!("personhog_leader_seal_overdue_passes_total").increment(0);
+}
+
impl SealDuty {
/// Wakes when the saga or its phase changes, and on an interval for retries.
pub async fn run(self, cancel: CancellationToken) {
+ register_seal_metrics();
let mut sealed: HashSet<(String, u32)> = HashSet::new();
let mut preconnecting: HashMap<(String, u32), JoinHandle<()>> = HashMap::new();
let mut acked: Option<(String, BTreeSet<u32>)> = None;