:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(personhog): follow a repartition in the parity monitor (#119407)
José Sequeiraf0d203f6b3a5
diff --git a/rust/personhog-parity-monitor/README.md b/rust/personhog-parity-monitor/README.mdindex 06300cc2822f..1861d0d096f2 100644--- a/rust/personhog-parity-monitor/README.md+++ b/rust/personhog-parity-monitor/README.md@@ -30,8 +30,8 @@ All names start with `personhog_parity_`. Labels carry outcomes only, never a te| `merge_partition_total` | `outcome`: `same`, `different` | Merges present on both feeds, by partition number; only counted while both merge topics have the same partition count || `merge_placement_total` | `feed`, `outcome`: `expected`, `unexpected` | Arrival partition against the key's murmur2 partition || `version_ties_total`, `merge_conflicts_total` | `feed` | A feed contradicting itself |-| `messages_total`, `decode_failures_total`, `read_errors_total`, `filtered_total` | `feed`, `topic` | Reader counters |-| `topic_partitions` | `feed`, `topic` | Partition counts from metadata, read once at startup |+| `messages_total`, `decode_failures_total`, `read_errors_total`, `metadata_errors_total`, `filtered_total` | `feed`, `topic` | Reader counters |+| `topic_partitions` | `feed`, `topic` | Partition counts from metadata, re-read every `PARITY_METADATA_POLL_SECONDS` || `topic_pair_partitions_equal` | `topic` | Whether the two feeds' topics of that kind have the same count || `pending_entries` | `kind` | Entries waiting to settle || `watermark_age_seconds` | `feed` | Wall clock minus the feed's watermark; a stalled feed shows here |@@ -54,6 +54,7 @@ All names start with `personhog_parity_`. Labels carry outcomes only, never a te| `PARITY_TEAMS` | empty, every team | Comma separated team ids; set it to the merge events allowlist when that is a subset || `PARITY_SAMPLE_RING_SIZE` | `500` | Disagreements kept for `/debug/mismatches` || `PARITY_SWEEP_INTERVAL_MS` | `1000` | How often entries are settled |+| `PARITY_METADATA_POLL_SECONDS` | `30` | How often each topic's partition list is re-read; partitions a repartition added are assigned from their beginning, and the placement check uses the new count || `DEBUG_API_SECRET` | unset, endpoint off | Required in the `x-debug-api-secret` header to read `/debug/mismatches`, which carries distinct ids || `BIND_HOST`, `BIND_PORT` | `0.0.0.0`, `3301` | `/metrics`, `/_readiness`, `/_liveness`, `/debug/mismatches` |diff --git a/rust/personhog-parity-monitor/src/config.rs b/rust/personhog-parity-monitor/src/config.rsindex 2bc43c98faff..c8e24646c875 100644--- a/rust/personhog-parity-monitor/src/config.rs+++ b/rust/personhog-parity-monitor/src/config.rs@@ -47,6 +47,10 @@ pub struct Config {pub parity_sample_ring_size: usize,#[envconfig(default = "1000")]pub parity_sweep_interval_ms: u64,+ /// Half the default grace, so the partitions a repartition adds are+ /// read before their counterparts on the other feed settle one-sided.+ #[envconfig(default = "30")]+ pub parity_metadata_poll_seconds: u64,/// Unset leaves `/debug/mismatches` off; its samples carry distinct ids.pub debug_api_secret: Option<String>,@@ -123,6 +127,10 @@ impl Config {Duration::from_millis(self.parity_sweep_interval_ms.max(100))}+ pub fn metadata_poll(&self) -> Duration {+ Duration::from_secs(self.parity_metadata_poll_seconds.max(1))+ }+pub fn bind(&self) -> String {format!("{}:{}", self.bind_host, self.bind_port)}diff --git a/rust/personhog-parity-monitor/src/feed.rs b/rust/personhog-parity-monitor/src/feed.rsindex 4bddf1005a41..64ade4f9f732 100644--- a/rust/personhog-parity-monitor/src/feed.rs+++ b/rust/personhog-parity-monitor/src/feed.rs@@ -2,6 +2,7 @@//! nothing committed: a consumer group would split the partitions between//! replicas and break the cross-feed join, and a restart resumes nothing.+use std::sync::Arc;use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};use anyhow::{bail, Context, Result};@@ -12,7 +13,7 @@ use rdkafka::error::KafkaError;use rdkafka::message::BorrowedMessage;use rdkafka::{ClientConfig, Message, Offset, TopicPartitionList};use tokio::sync::mpsc;-use tracing::{debug, error, warn};+use tracing::{debug, error, info, warn};use crate::config::FeedConfig;use crate::join::{Event, Kind, Record, Topic};@@ -28,7 +29,8 @@ const PREFETCH_KBYTES: &str = "16384";pub struct TopicReader {topic: Topic,name: String,- consumer: StreamConsumer,+ consumer: Arc<StreamConsumer>,+ partitions: Vec<i32>,}impl TopicReader {@@ -61,18 +63,8 @@ impl TopicReader {let consumer: StreamConsumer = client.create().with_context(|| format!("creating the {} reader for {name}", feed.side.label()))?;- let metadata = consumer- .fetch_metadata(Some(&name), METADATA_TIMEOUT)- .with_context(|| format!("fetching metadata of {name}"))?;- let partitions: Vec<i32> = metadata- .topics()- .iter()- .find(|entry| entry.name() == name)- .map(|entry| entry.partitions().iter().map(|p| p.id()).collect())- .unwrap_or_default();- if partitions.is_empty() {- bail!("topic {name} has no partitions on {}", feed.kafka_hosts);- }+ let partitions = fetch_partitions(&consumer, &name)+ .with_context(|| format!("on {}", feed.kafka_hosts))?;let mut assignment = TopicPartitionList::new();for partition in &partitions {// The end offset is read here, before anyone learns the reader is@@ -86,13 +78,15 @@ impl TopicReader {consumer.assign(&assignment).with_context(|| format!("assigning {name}"))?;+ let count = partitions.len() as u32;Ok((Self {topic,name,- consumer,+ consumer: Arc::new(consumer),+ partitions,},- partitions.len() as u32,+ count,))}@@ -101,7 +95,12 @@ impl TopicReader {}/// The first decode failure per topic is logged in full; the counter is the signal.- pub async fn run(self, events: mpsc::Sender<Event>, health: HealthHandle) {+ pub async fn run(+ mut self,+ events: mpsc::Sender<Event>,+ health: HealthHandle,+ metadata_poll: Duration,+ ) {let feed = self.topic.side.label();let kind = self.topic.kind.label();let messages =@@ -112,13 +111,40 @@ impl TopicReader {"feed" => feed,"topic" => kind);+ let metadata_errors = metrics::counter!(+ "personhog_parity_metadata_errors_total",+ "feed" => feed,+ "topic" => kind+ );let mut last_health_report = Instant::now() - HEALTH_REPORT_INTERVAL;+ let mut next_metadata_poll = Instant::now() + metadata_poll;let mut decode_failure_logged = false;loop {if last_health_report.elapsed() >= HEALTH_REPORT_INTERVAL {health.report_healthy().await;last_health_report = Instant::now();}+ if Instant::now() >= next_metadata_poll {+ match self.assign_new_partitions().await {+ Ok(Some(partitions)) => {+ let assigned = Event::Assigned {+ topic: self.topic,+ partitions,+ };+ if events.send(assigned).await.is_err() {+ return;+ }+ }+ Ok(None) => {}+ Err(error) => {+ warn!(topic = %self.name, %error, "refreshing the partition list failed");+ metadata_errors.increment(1);+ }+ }+ // Armed after the request, so a slow one cannot leave the+ // next poll already due and starve the reads between polls.+ next_metadata_poll = Instant::now() + metadata_poll;+ }let message = match tokio::time::timeout(RECV_TIMEOUT, self.consumer.recv()).await {Err(_) => continue,// The consumer cannot recover; the reader stops reporting, so@@ -162,6 +188,33 @@ impl TopicReader {}}+ /// New partitions are read from their beginning: they were empty at+ /// creation, so the end would skip rows that landed before this poll.+ async fn assign_new_partitions(&mut self) -> Result<Option<u32>> {+ let consumer = Arc::clone(&self.consumer);+ let name = self.name.clone();+ let current = tokio::task::spawn_blocking(move || fetch_partitions(&consumer, &name))+ .await+ .context("the metadata task panicked")??;+ let added: Vec<i32> = current+ .into_iter()+ .filter(|partition| !self.partitions.contains(partition))+ .collect();+ if added.is_empty() {+ return Ok(None);+ }+ let mut assignment = TopicPartitionList::new();+ for partition in &added {+ assignment.add_partition_offset(&self.name, *partition, Offset::Beginning)?;+ }+ self.consumer+ .incremental_assign(&assignment)+ .with_context(|| format!("assigning the new partitions of {}", self.name))?;+ info!(topic = %self.name, ?added, "assigned new partitions from their beginning");+ self.partitions.extend(added);+ Ok(Some(self.partitions.len() as u32))+ }+fn decode(&self, message: &BorrowedMessage<'_>) -> Result<Record, DecodeError> {let payload = message.payload().unwrap_or_default();match self.topic.kind {@@ -171,6 +224,22 @@ impl TopicReader {}}+fn fetch_partitions(consumer: &StreamConsumer, name: &str) -> Result<Vec<i32>> {+ let metadata = consumer+ .fetch_metadata(Some(name), METADATA_TIMEOUT)+ .with_context(|| format!("fetching metadata of {name}"))?;+ let partitions: Vec<i32> = metadata+ .topics()+ .iter()+ .find(|entry| entry.name() == name)+ .map(|entry| entry.partitions().iter().map(|p| p.id()).collect())+ .unwrap_or_default();+ if partitions.is_empty() {+ bail!("topic {name} has no partitions");+ }+ Ok(partitions)+}+pub fn now_ms() -> i64 {SystemTime::now().duration_since(UNIX_EPOCH)diff --git a/rust/personhog-parity-monitor/src/join.rs b/rust/personhog-parity-monitor/src/join.rsindex 43f01c67935a..7f207a8167a0 100644--- a/rust/personhog-parity-monitor/src/join.rs+++ b/rust/personhog-parity-monitor/src/join.rs@@ -981,5 +981,25 @@ mod tests {let report = join.observe(merge(Side::Personhog, expected + 1, 1_000, 7, 9), t0);assert_eq!(report.placement, Some((Side::Personhog, false)));assert_eq!(report.merge_conflict, Some(Side::Personhog));++ let old = (10..).find(|n| {+ let key = format!("{TEAM}:{}", uuid(*n));+ partition_for_key(key.as_bytes(), 64) != partition_for_key(key.as_bytes(), 128)+ });+ let old = old.expect("some key lands differently on 64 and 128 partitions");+ let key = format!("{TEAM}:{}", uuid(old));+ let expected = partition_for_key(key.as_bytes(), 128) as i32;+ join.observe(+ Event::Assigned {+ topic: topic(Side::Personhog, Kind::MergeEvents),+ partitions: 128,+ },+ t0,+ );+ assert_eq!(+ join.observe(merge(Side::Personhog, expected, 1_000, old, 11), t0)+ .placement,+ Some((Side::Personhog, true))+ );}}diff --git a/rust/personhog-parity-monitor/src/main.rs b/rust/personhog-parity-monitor/src/main.rsindex cf5417be33c4..0a65479aae5d 100644--- a/rust/personhog-parity-monitor/src/main.rs+++ b/rust/personhog-parity-monitor/src/main.rs@@ -95,7 +95,7 @@ async fn main() -> Result<()> {LIVENESS_DEADLINE,).await;- tokio::spawn(reader.run(events.clone(), handle));+ tokio::spawn(reader.run(events.clone(), handle, config.metadata_poll()));}}drop(events);