Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(personhog): follow a repartition in the parity monitor (#119407)

José Sequeiraf0d203f6b3a5
diff --git a/rust/personhog-parity-monitor/README.md b/rust/personhog-parity-monitor/README.md
index 06300cc2822f..1861d0d096f2 100644
--- a/rust/personhog-parity-monitor/README.md
+++ b/rust/personhog-parity-monitor/README.md
@@ -30,8 +30,8 @@ All names start with `personhog_parity_`. Labels carry outcomes only, never a te
| `merge_partition_total` | `outcome`: `same`, `different` | Merges present on both feeds, by partition number; only counted while both merge topics have the same partition count |
| `merge_placement_total` | `feed`, `outcome`: `expected`, `unexpected` | Arrival partition against the key's murmur2 partition |
| `version_ties_total`, `merge_conflicts_total` | `feed` | A feed contradicting itself |
-| `messages_total`, `decode_failures_total`, `read_errors_total`, `filtered_total` | `feed`, `topic` | Reader counters |
-| `topic_partitions` | `feed`, `topic` | Partition counts from metadata, read once at startup |
+| `messages_total`, `decode_failures_total`, `read_errors_total`, `metadata_errors_total`, `filtered_total` | `feed`, `topic` | Reader counters |
+| `topic_partitions` | `feed`, `topic` | Partition counts from metadata, re-read every `PARITY_METADATA_POLL_SECONDS` |
| `topic_pair_partitions_equal` | `topic` | Whether the two feeds' topics of that kind have the same count |
| `pending_entries` | `kind` | Entries waiting to settle |
| `watermark_age_seconds` | `feed` | Wall clock minus the feed's watermark; a stalled feed shows here |
@@ -54,6 +54,7 @@ All names start with `personhog_parity_`. Labels carry outcomes only, never a te
| `PARITY_TEAMS` | empty, every team | Comma separated team ids; set it to the merge events allowlist when that is a subset |
| `PARITY_SAMPLE_RING_SIZE` | `500` | Disagreements kept for `/debug/mismatches` |
| `PARITY_SWEEP_INTERVAL_MS` | `1000` | How often entries are settled |
+| `PARITY_METADATA_POLL_SECONDS` | `30` | How often each topic's partition list is re-read; partitions a repartition added are assigned from their beginning, and the placement check uses the new count |
| `DEBUG_API_SECRET` | unset, endpoint off | Required in the `x-debug-api-secret` header to read `/debug/mismatches`, which carries distinct ids |
| `BIND_HOST`, `BIND_PORT` | `0.0.0.0`, `3301` | `/metrics`, `/_readiness`, `/_liveness`, `/debug/mismatches` |
diff --git a/rust/personhog-parity-monitor/src/config.rs b/rust/personhog-parity-monitor/src/config.rs
index 2bc43c98faff..c8e24646c875 100644
--- a/rust/personhog-parity-monitor/src/config.rs
+++ b/rust/personhog-parity-monitor/src/config.rs
@@ -47,6 +47,10 @@ pub struct Config {
pub parity_sample_ring_size: usize,
#[envconfig(default = "1000")]
pub parity_sweep_interval_ms: u64,
+ /// Half the default grace, so the partitions a repartition adds are
+ /// read before their counterparts on the other feed settle one-sided.
+ #[envconfig(default = "30")]
+ pub parity_metadata_poll_seconds: u64,
/// Unset leaves `/debug/mismatches` off; its samples carry distinct ids.
pub debug_api_secret: Option<String>,
@@ -123,6 +127,10 @@ impl Config {
Duration::from_millis(self.parity_sweep_interval_ms.max(100))
}
+ pub fn metadata_poll(&self) -> Duration {
+ Duration::from_secs(self.parity_metadata_poll_seconds.max(1))
+ }
+
pub fn bind(&self) -> String {
format!("{}:{}", self.bind_host, self.bind_port)
}
diff --git a/rust/personhog-parity-monitor/src/feed.rs b/rust/personhog-parity-monitor/src/feed.rs
index 4bddf1005a41..64ade4f9f732 100644
--- a/rust/personhog-parity-monitor/src/feed.rs
+++ b/rust/personhog-parity-monitor/src/feed.rs
@@ -2,6 +2,7 @@
//! nothing committed: a consumer group would split the partitions between
//! replicas and break the cross-feed join, and a restart resumes nothing.
+use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use anyhow::{bail, Context, Result};
@@ -12,7 +13,7 @@ use rdkafka::error::KafkaError;
use rdkafka::message::BorrowedMessage;
use rdkafka::{ClientConfig, Message, Offset, TopicPartitionList};
use tokio::sync::mpsc;
-use tracing::{debug, error, warn};
+use tracing::{debug, error, info, warn};
use crate::config::FeedConfig;
use crate::join::{Event, Kind, Record, Topic};
@@ -28,7 +29,8 @@ const PREFETCH_KBYTES: &str = "16384";
pub struct TopicReader {
topic: Topic,
name: String,
- consumer: StreamConsumer,
+ consumer: Arc<StreamConsumer>,
+ partitions: Vec<i32>,
}
impl TopicReader {
@@ -61,18 +63,8 @@ impl TopicReader {
let consumer: StreamConsumer = client
.create()
.with_context(|| format!("creating the {} reader for {name}", feed.side.label()))?;
- let metadata = consumer
- .fetch_metadata(Some(&name), METADATA_TIMEOUT)
- .with_context(|| format!("fetching metadata of {name}"))?;
- let partitions: Vec<i32> = metadata
- .topics()
- .iter()
- .find(|entry| entry.name() == name)
- .map(|entry| entry.partitions().iter().map(|p| p.id()).collect())
- .unwrap_or_default();
- if partitions.is_empty() {
- bail!("topic {name} has no partitions on {}", feed.kafka_hosts);
- }
+ let partitions = fetch_partitions(&consumer, &name)
+ .with_context(|| format!("on {}", feed.kafka_hosts))?;
let mut assignment = TopicPartitionList::new();
for partition in &partitions {
// The end offset is read here, before anyone learns the reader is
@@ -86,13 +78,15 @@ impl TopicReader {
consumer
.assign(&assignment)
.with_context(|| format!("assigning {name}"))?;
+ let count = partitions.len() as u32;
Ok((
Self {
topic,
name,
- consumer,
+ consumer: Arc::new(consumer),
+ partitions,
},
- partitions.len() as u32,
+ count,
))
}
@@ -101,7 +95,12 @@ impl TopicReader {
}
/// The first decode failure per topic is logged in full; the counter is the signal.
- pub async fn run(self, events: mpsc::Sender<Event>, health: HealthHandle) {
+ pub async fn run(
+ mut self,
+ events: mpsc::Sender<Event>,
+ health: HealthHandle,
+ metadata_poll: Duration,
+ ) {
let feed = self.topic.side.label();
let kind = self.topic.kind.label();
let messages =
@@ -112,13 +111,40 @@ impl TopicReader {
"feed" => feed,
"topic" => kind
);
+ let metadata_errors = metrics::counter!(
+ "personhog_parity_metadata_errors_total",
+ "feed" => feed,
+ "topic" => kind
+ );
let mut last_health_report = Instant::now() - HEALTH_REPORT_INTERVAL;
+ let mut next_metadata_poll = Instant::now() + metadata_poll;
let mut decode_failure_logged = false;
loop {
if last_health_report.elapsed() >= HEALTH_REPORT_INTERVAL {
health.report_healthy().await;
last_health_report = Instant::now();
}
+ if Instant::now() >= next_metadata_poll {
+ match self.assign_new_partitions().await {
+ Ok(Some(partitions)) => {
+ let assigned = Event::Assigned {
+ topic: self.topic,
+ partitions,
+ };
+ if events.send(assigned).await.is_err() {
+ return;
+ }
+ }
+ Ok(None) => {}
+ Err(error) => {
+ warn!(topic = %self.name, %error, "refreshing the partition list failed");
+ metadata_errors.increment(1);
+ }
+ }
+ // Armed after the request, so a slow one cannot leave the
+ // next poll already due and starve the reads between polls.
+ next_metadata_poll = Instant::now() + metadata_poll;
+ }
let message = match tokio::time::timeout(RECV_TIMEOUT, self.consumer.recv()).await {
Err(_) => continue,
// The consumer cannot recover; the reader stops reporting, so
@@ -162,6 +188,33 @@ impl TopicReader {
}
}
+ /// New partitions are read from their beginning: they were empty at
+ /// creation, so the end would skip rows that landed before this poll.
+ async fn assign_new_partitions(&mut self) -> Result<Option<u32>> {
+ let consumer = Arc::clone(&self.consumer);
+ let name = self.name.clone();
+ let current = tokio::task::spawn_blocking(move || fetch_partitions(&consumer, &name))
+ .await
+ .context("the metadata task panicked")??;
+ let added: Vec<i32> = current
+ .into_iter()
+ .filter(|partition| !self.partitions.contains(partition))
+ .collect();
+ if added.is_empty() {
+ return Ok(None);
+ }
+ let mut assignment = TopicPartitionList::new();
+ for partition in &added {
+ assignment.add_partition_offset(&self.name, *partition, Offset::Beginning)?;
+ }
+ self.consumer
+ .incremental_assign(&assignment)
+ .with_context(|| format!("assigning the new partitions of {}", self.name))?;
+ info!(topic = %self.name, ?added, "assigned new partitions from their beginning");
+ self.partitions.extend(added);
+ Ok(Some(self.partitions.len() as u32))
+ }
+
fn decode(&self, message: &BorrowedMessage<'_>) -> Result<Record, DecodeError> {
let payload = message.payload().unwrap_or_default();
match self.topic.kind {
@@ -171,6 +224,22 @@ impl TopicReader {
}
}
+fn fetch_partitions(consumer: &StreamConsumer, name: &str) -> Result<Vec<i32>> {
+ let metadata = consumer
+ .fetch_metadata(Some(name), METADATA_TIMEOUT)
+ .with_context(|| format!("fetching metadata of {name}"))?;
+ let partitions: Vec<i32> = metadata
+ .topics()
+ .iter()
+ .find(|entry| entry.name() == name)
+ .map(|entry| entry.partitions().iter().map(|p| p.id()).collect())
+ .unwrap_or_default();
+ if partitions.is_empty() {
+ bail!("topic {name} has no partitions");
+ }
+ Ok(partitions)
+}
+
pub fn now_ms() -> i64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
diff --git a/rust/personhog-parity-monitor/src/join.rs b/rust/personhog-parity-monitor/src/join.rs
index 43f01c67935a..7f207a8167a0 100644
--- a/rust/personhog-parity-monitor/src/join.rs
+++ b/rust/personhog-parity-monitor/src/join.rs
@@ -981,5 +981,25 @@ mod tests {
let report = join.observe(merge(Side::Personhog, expected + 1, 1_000, 7, 9), t0);
assert_eq!(report.placement, Some((Side::Personhog, false)));
assert_eq!(report.merge_conflict, Some(Side::Personhog));
+
+ let old = (10..).find(|n| {
+ let key = format!("{TEAM}:{}", uuid(*n));
+ partition_for_key(key.as_bytes(), 64) != partition_for_key(key.as_bytes(), 128)
+ });
+ let old = old.expect("some key lands differently on 64 and 128 partitions");
+ let key = format!("{TEAM}:{}", uuid(old));
+ let expected = partition_for_key(key.as_bytes(), 128) as i32;
+ join.observe(
+ Event::Assigned {
+ topic: topic(Side::Personhog, Kind::MergeEvents),
+ partitions: 128,
+ },
+ t0,
+ );
+ assert_eq!(
+ join.observe(merge(Side::Personhog, expected, 1_000, old, 11), t0)
+ .placement,
+ Some((Side::Personhog, true))
+ );
}
}
diff --git a/rust/personhog-parity-monitor/src/main.rs b/rust/personhog-parity-monitor/src/main.rs
index cf5417be33c4..0a65479aae5d 100644
--- a/rust/personhog-parity-monitor/src/main.rs
+++ b/rust/personhog-parity-monitor/src/main.rs
@@ -95,7 +95,7 @@ async fn main() -> Result<()> {
LIVENESS_DEADLINE,
)
.await;
- tokio::spawn(reader.run(events.clone(), handle));
+ tokio::spawn(reader.run(events.clone(), handle, config.metadata_poll()));
}
}
drop(events);