:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(personhog): run the parity monitor from the harness stack (#119398)
José Sequeiraa635e728177e
diff --git a/.github/workflows/ci-nodejs.yml b/.github/workflows/ci-nodejs.ymlindex 1953b02f0eed..fadaba5f4f07 100644--- a/.github/workflows/ci-nodejs.yml+++ b/.github/workflows/ci-nodejs.yml@@ -199,8 +199,8 @@ jobs:RUST_REBUILD_ALL: ${{ steps.rust-affected.outputs.rebuild-all }}RUST_AFFECTED_CRATES: ${{ steps.rust-affected.outputs.crates }}RUST_NODE_E2E_CRATES: '["ingestion-consumer"]'- # The Kafka parity e2e spawns identity and the publisher through the harness.- RUST_PERSONHOG_E2E_CRATES: '["personhog-identity", "personhog-test-harness"]'+ # The Kafka parity e2e spawns identity, the publisher and the parity monitor through the harness.+ RUST_PERSONHOG_E2E_CRATES: '["personhog-identity", "personhog-test-harness", "personhog-parity-monitor"]'run: |# `schedule` alongside `push`: the paths filter is skipped on both, so# NODEJS_CHANGED is empty there. Without this the hourly run would decide@@ -954,7 +954,12 @@ jobs:- name: Pre-build the personhog stack and harnessrun: |if [ -f nodejs/tests/service-e2e/personhog-kafka-parity.serial.test.ts ]; then- cd rust && cargo build -p personhog-replica -p personhog-router -p personhog-leader -p personhog-writer -p personhog-identity -p personhog-test-harness+ packages=(-p personhog-replica -p personhog-router -p personhog-leader -p personhog-writer -p personhog-identity -p personhog-test-harness)+ # The parity test spawns the monitor through the stack; a branch without the crate runs the test without it.+ if [ -d rust/personhog-parity-monitor ]; then+ packages+=(-p personhog-parity-monitor)+ fi+ cd rust && cargo build "${packages[@]}"elseecho "Kafka parity e2e test is not present on this branch; skipping the stack build."fidiff --git a/nodejs/tests/service-e2e/personhog-kafka-parity.serial.test.ts b/nodejs/tests/service-e2e/personhog-kafka-parity.serial.test.tsindex 60969a01840a..dbfeba4293f4 100644--- a/nodejs/tests/service-e2e/personhog-kafka-parity.serial.test.ts+++ b/nodejs/tests/service-e2e/personhog-kafka-parity.serial.test.ts@@ -69,6 +69,17 @@ const STACK_STOP_GRACE_MS = 60_000const SETTLE_TIMEOUT_MS = 30_000const FEED_CONVERGENCE_TIMEOUT_MS = 30_000const TOPIC_READ_TIMEOUT_MS = 30_000+// The harness runs the monitor with a 15 s grace; the rest is margin.+const MONITOR_SETTLE_TIMEOUT_MS = 60_000+/** Outcomes that are not a disagreement: a stub is one feed's own path to the result. */+const MONITOR_AGREEMENTS = new Set(['agree', 'node_stub', 'personhog_stub', 'same', 'expected'])+const MONITOR_FAULTS = new Set([+ 'personhog_parity_version_ties_total',+ 'personhog_parity_merge_conflicts_total',+ 'personhog_parity_decode_failures_total',+ 'personhog_parity_overflow_evictions_total',+ 'personhog_parity_warmup_evictions_total',+])/** The ready line the harness's `stack` subcommand writes. */interface StackReady {@@ -77,9 +88,16 @@ interface StackReady {changelog_topic: stringdistinct_ids_topic: stringmerge_events_topic: string+ monitor_addr: stringlog_dir: string}+interface MetricSample {+ name: string+ labels: Record<string, string>+ value: number+}+interface DistinctIdRow {person_id: stringteam_id: number@@ -121,6 +139,7 @@ describe('personhog kafka parity (e2e)', () => {let stack: ServiceProcess | undefinedlet workers: ServiceProcess[] = []let workerUrl: string+ let monitorUrl: stringlet readyDir: stringlet inputTopic: string/** Every topic this run created, the stack's included, deleted at the end. */@@ -257,11 +276,12 @@ describe('personhog kafka parity (e2e)', () => {// Each service registers as soon as it is spawned, so one that never// becomes ready is still stopped in afterAll.readyDir = mkdtempSync(path.join(tmpdir(), 'personhog-kafka-parity-'))- const ready = await startPersonhogStack(hub, path.join(readyDir, 'ready.json'), (service) => {+ const ready = await startPersonhogStack(hub, path.join(readyDir, 'ready.json'), nodeTopics, (service) => {stack = service})ownTopics.push(ready.changelog_topic, ready.distinct_ids_topic, ready.merge_events_topic)personhogTopics = { distinctIds: ready.distinct_ids_topic, mergeEvents: ready.merge_events_topic }+ monitorUrl = `http://${ready.monitor_addr}`workerUrl = await startNodeIngestionApiWorker(hub,@@ -467,8 +487,69 @@ describe('personhog kafka parity (e2e)', () => {.reduce((sum, value) => sum + value, 0)expect(errors).toBe(0)})++ it('the parity monitor settled every entry of both feeds as agreement', async () => {+ // A key settles at least once, so agreement below the fold's size+ // means the monitor read a different topic than the fold did. The+ // counts are polled with the pending gauges: a reader still behind+ // the last rows reports nothing pending before they arrive.+ const node = await feedState(nodeTopics)+ const personhog = await feedState(personhogTopics)+ const sharedMerges = Object.entries(node.survivors).filter(+ ([person, survivor]) => survivor !== person && person in personhog.survivors+ ).length+ let samples!: MetricSample[]+ await pollUntil(async () => {+ const response = await stack!.request(`${monitorUrl}/metrics`, { method: 'GET' })+ expect(response.statusCode).toBe(200)+ samples = parsePrometheus(response.body)+ expect(metric(samples, 'personhog_parity_pending_entries', { kind: 'mappings' })).toBe(0)+ expect(metric(samples, 'personhog_parity_pending_entries', { kind: 'merges' })).toBe(0)+ expect(+ metric(samples, 'personhog_parity_settled_mappings_total', { outcome: 'agree' })+ ).toBeGreaterThanOrEqual(Object.keys(node.mappings).length)+ expect(+ metric(samples, 'personhog_parity_settled_merges_total', { outcome: 'agree' })+ ).toBeGreaterThanOrEqual(sharedMerges)+ }, MONITOR_SETTLE_TIMEOUT_MS)++ expect(monitorDisagreements(samples)).toEqual([])+ })})+function parsePrometheus(body: string): MetricSample[] {+ const samples: MetricSample[] = []+ for (const line of body.split('\n')) {+ const match = /^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+(\S+)/.exec(line)+ if (!match) {+ continue+ }+ const labels: Record<string, string> = {}+ for (const pair of (match[2] ?? '').matchAll(/([a-zA-Z_][a-zA-Z0-9_]*)="([^"]*)"/g)) {+ labels[pair[1]] = pair[2]+ }+ samples.push({ name: match[1], labels, value: Number(match[3]) })+ }+ return samples+}++function metric(samples: MetricSample[], name: string, labels: Record<string, string>): number | undefined {+ return samples.find(+ (sample) => sample.name === name && Object.entries(labels).every(([key, value]) => sample.labels[key] === value)+ )?.value+}++function monitorDisagreements(samples: MetricSample[]): string[] {+ return samples+ .filter(+ ({ name, labels, value }) =>+ value > 0 &&+ name.startsWith('personhog_parity_') &&+ (MONITOR_FAULTS.has(name) || ('outcome' in labels && !MONITOR_AGREEMENTS.has(labels.outcome)))+ )+ .map(({ name, labels, value }) => `${name}${JSON.stringify(labels)} ${value}`)+}+/*** The same end state on both feeds. A person only one feed ever named is* that feed's own path to the result, a stub the other backend never made@@ -504,7 +585,12 @@ function expectEveryPersonAccountedFor(feed: FeedState): void {expect(stale).toEqual([])}-async function startPersonhogStack(hub: Hub, readyFile: string, register: Register): Promise<StackReady> {+async function startPersonhogStack(+ hub: Hub,+ readyFile: string,+ nodeTopics: FeedTopics,+ register: Register+): Promise<StackReady> {if (!existsSync(HARNESS_BINARY)) {throw new Error(`${HARNESS_BINARY} is missing; build the personhog stack and harness first`)}@@ -514,6 +600,13 @@ async function startPersonhogStack(hub: Hub, readyFile: string, register: Regist['stack','--publish',+ // The stack assigns the Node topics (created above) before it+ // writes the ready file, so no event of this run predates them.+ '--monitor',+ '--monitor-node-distinct-ids-topic',+ nodeTopics.distinctIds,+ '--monitor-node-merge-events-topic',+ nodeTopics.mergeEvents,'--bin-dir',RUST_BIN_DIR,'--pg-target-table',@@ -542,7 +635,11 @@ async function startPersonhogStack(hub: Hub, readyFile: string, register: Registthrow new Error(`personhog stack not ready within ${STACK_READY_TIMEOUT_MS}ms:\n${service.getOutput()}`)}}- return parseJSON(readFileSync(readyFile, 'utf8')) as StackReady+ const ready = parseJSON(readFileSync(readyFile, 'utf8')) as StackReady+ if (!ready.monitor_addr) {+ throw new Error(`the stack reported no monitor address; is ${HARNESS_BINARY} built from this branch?`)+ }+ return ready}async function startNodeIngestionApiWorker(diff --git a/rust/personhog-test-harness/README.md b/rust/personhog-test-harness/README.mdindex ad4eb454f0ed..bc9d43e71ac0 100644--- a/rust/personhog-test-harness/README.md+++ b/rust/personhog-test-harness/README.md@@ -17,7 +17,7 @@ For `gate` without `--external-router-url`, build the service binaries first:cargo build -p personhog-replica -p personhog-router -p personhog-leader -p personhog-writer -p personhog-identity -p personhog-test-harness```-The identity crate also builds `personhog-publisher`, which `--publish` spawns.+The identity crate also builds `personhog-publisher`, which `--publish` spawns; `stack --monitor` needs `cargo build -p personhog-parity-monitor` as well.## `gate` — the e2e correctness gate@@ -286,6 +286,15 @@ Once the stack is up it prints one line, `PERSONHOG_STACK_READY {…}`, and writA parent process should wait for the file rather than parse stdout, which the logs share.A spawned service that exits ends the command with that service's last log lines, so a fixture never stays up over a dead stack.+`--monitor` also spawns `personhog-parity-monitor` (build it too) against the publish topics and the Node path's topics named by `--monitor-node-distinct-ids-topic` and `--monitor-node-merge-events-topic`, which must exist before the stack starts; the ready line carries its metrics address as `monitor_addr`.+The monitor assigns every partition before the ready line is written and runs with a 15 s grace and no warmup, so a test can read its `/metrics` once its traffic stops and assert that every entry settled as agreement.++```bash+target/debug/personhog-test-harness stack --publish --ready-file /tmp/personhog-stack.json \+ --monitor --monitor-node-distinct-ids-topic clickhouse_person_distinct_id_test \+ --monitor-node-merge-events-topic person_merge_events_test+```+## `seed` / `cleanup` — manage traffic targets```bashdiff --git a/rust/personhog-test-harness/src/cli.rs b/rust/personhog-test-harness/src/cli.rsindex 54a410f9898f..96b7e3bd0985 100644--- a/rust/personhog-test-harness/src/cli.rs+++ b/rust/personhog-test-harness/src/cli.rs@@ -467,6 +467,21 @@ pub struct StackArgs {#[arg(long, default_value_t = 100)]pub publish_rows_per_pass: i64,+ /// Also run `personhog-parity-monitor` against the publish topics and+ /// the Node path's topics named below, and put its metrics address in+ /// the ready line. Requires --publish; the Node topics must exist.+ #[arg(long, default_value_t = false)]+ pub monitor: bool,++ /// The Node path's `clickhouse_person_distinct_id` topic the monitor+ /// compares the publish topic against.+ #[arg(long)]+ pub monitor_node_distinct_ids_topic: Option<String>,++ /// The Node path's `person_merge_events` topic.+ #[arg(long)]+ pub monitor_node_merge_events_topic: Option<String>,+/// Also write the ready line's JSON to this file once the stack is up./// A parent process polls for the file instead of parsing stdout,/// which the logs share.diff --git a/rust/personhog-test-harness/src/scenarios/gate.rs b/rust/personhog-test-harness/src/scenarios/gate.rsindex ea483e271c67..982212d57c92 100644--- a/rust/personhog-test-harness/src/scenarios/gate.rs+++ b/rust/personhog-test-harness/src/scenarios/gate.rs@@ -298,6 +298,7 @@ pub async fn run(args: GateArgs) -> Result<()> {publishers: args.publishers,rows_per_pass: args.publish_rows_per_pass,}),+ monitor: None,}).await?,)diff --git a/rust/personhog-test-harness/src/scenarios/stack_cmd.rs b/rust/personhog-test-harness/src/scenarios/stack_cmd.rsindex 438e135f2ee0..ad3f052ec78d 100644--- a/rust/personhog-test-harness/src/scenarios/stack_cmd.rs+++ b/rust/personhog-test-harness/src/scenarios/stack_cmd.rs@@ -16,7 +16,7 @@ use crate::cli::StackArgs;use crate::publish;use crate::scenarios::gate::default_bin_dir;use crate::seed;-use crate::stack::{PublishConfig, Stack, StackConfig};+use crate::stack::{MonitorConfig, PublishConfig, Stack, StackConfig};/// Prefix of the one stdout line a parent process can wait for. The logs/// share stdout, so the marker is what makes the JSON findable.@@ -34,6 +34,23 @@ pub async fn run(args: StackArgs) -> Result<()> {if args.publish && args.publish_rows_per_pass <= 0 {bail!("--publish-rows-per-pass must be positive");}+ let monitor = match (+ args.monitor,+ &args.monitor_node_distinct_ids_topic,+ &args.monitor_node_merge_events_topic,+ ) {+ (false, _, _) => None,+ (true, _, _) if !args.publish => {+ bail!("--monitor needs --publish: the monitor compares the publish topics")+ }+ (true, Some(distinct_ids), Some(merge_events)) => Some(MonitorConfig {+ node_distinct_ids_topic: distinct_ids.clone(),+ node_merge_events_topic: merge_events.clone(),+ }),+ (true, _, _) => bail!(+ "--monitor needs --monitor-node-distinct-ids-topic and --monitor-node-merge-events-topic"+ ),+ };if let Some(path) = &args.ready_file {refuse_stale_ready_file(path)?;}@@ -80,6 +97,7 @@ pub async fn run(args: StackArgs) -> Result<()> {publishers: args.publishers,rows_per_pass: args.publish_rows_per_pass,}),+ monitor,}).await?;@@ -94,6 +112,7 @@ pub async fn run(args: StackArgs) -> Result<()> {"changelog_topic": stack.changelog_topic(),"distinct_ids_topic": topics.map(|t| t.distinct_ids.clone()),"merge_events_topic": topics.map(|t| t.merge_events.clone()),+ "monitor_addr": stack.monitor_url.as_deref().map(strip_scheme),"log_dir": stack.log_dir.display().to_string(),});println!("{READY_MARKER}{ready}");diff --git a/rust/personhog-test-harness/src/stack/mod.rs b/rust/personhog-test-harness/src/stack/mod.rsindex b19109563990..bc8518a81722 100644--- a/rust/personhog-test-harness/src/stack/mod.rs+++ b/rust/personhog-test-harness/src/stack/mod.rs@@ -35,6 +35,12 @@ const ROUTER_METRICS_BASE_PORT: u16 = 24154;const LEADER_METRICS_BASE_PORT: u16 = 24160;/// Above the leader range, which grows with every scale-up.const PUBLISHER_METRICS_BASE_PORT: u16 = 24200;+/// Above the publishers, whose count is a flag.+const MONITOR_PORT: u16 = 24300;+/// Short, because a fixture's run ends with a wait for the last entries to+/// settle; long enough to cover the publisher drain on a loaded CI runner.+const MONITOR_GRACE_SECONDS: u64 = 15;+const MONITOR_IDLE_PARTITION_SECONDS: u64 = 3;const MAX_ROUTERS: u32 = 4;/// Partitions of the two per-run publish topics. More than one, so the/// keyed merge events spread the way they do in production.@@ -76,6 +82,8 @@ pub struct StackConfig {/// identity service writes, to two per-run topics. Needs/// `spawn_identity`: nothing else writes the outbox.pub publish: Option<PublishConfig>,+ /// Also spawn the parity monitor against the publish topics; needs `publish`.+ pub monitor: Option<MonitorConfig>,}pub struct PublishConfig {@@ -83,6 +91,11 @@ pub struct PublishConfig {pub rows_per_pass: i64,}+pub struct MonitorConfig {+ pub node_distinct_ids_topic: String,+ pub node_merge_events_topic: String,+}+#[derive(Clone)]pub struct PublishTopics {pub distinct_ids: String,@@ -124,6 +137,7 @@ pub struct Stack {/// router's URL, which proxies identity RPCs to it.pub identity_url: Option<String>,publish_topics: Option<PublishTopics>,+ pub monitor_url: Option<String>,pub log_dir: PathBuf,}@@ -163,6 +177,9 @@ impl Stack {if config.publish.is_some() && !config.spawn_identity {bail!("publishing needs the identity service: nothing else writes the outbox");}+ if config.monitor.is_some() && config.publish.is_none() {+ bail!("the parity monitor needs the publish topics: pass --publish");+ }let run_id = chrono::Utc::now().format("%Y%m%d_%H%M%S").to_string();let topic = format!("personhog_test_harness_{run_id}");@@ -182,13 +199,21 @@ impl Stack {if config.publish.is_some() {binaries.push("personhog-publisher");}+ if config.monitor.is_some() {+ binaries.push("personhog-parity-monitor");+ }for bin in &binaries {let path = config.bin_dir.join(bin);if !path.exists() {bail!("{} not found — build the stack first:\n cargo build -p personhog-replica \- -p personhog-router -p personhog-leader -p personhog-writer -p personhog-identity",- path.display()+ -p personhog-router -p personhog-leader -p personhog-writer -p personhog-identity{}",+ path.display(),+ if config.monitor.is_some() {+ " -p personhog-parity-monitor"+ } else {+ ""+ });}}@@ -399,6 +424,43 @@ impl Stack {None};+ let monitor_url = match (&config.monitor, &publish_topics) {+ (Some(monitor), Some(topics)) => {+ infra.push(ServiceProcess::spawn(+ "parity-monitor",+ &config.bin_dir.join("personhog-parity-monitor"),+ &[+ ("NODE_KAFKA_HOSTS", config.kafka_hosts.clone()),+ (+ "NODE_DISTINCT_ID_TOPIC",+ monitor.node_distinct_ids_topic.clone(),+ ),+ (+ "NODE_MERGE_EVENTS_TOPIC",+ monitor.node_merge_events_topic.clone(),+ ),+ ("PERSONHOG_KAFKA_HOSTS", config.kafka_hosts.clone()),+ ("PERSONHOG_DISTINCT_ID_TOPIC", topics.distinct_ids.clone()),+ ("PERSONHOG_MERGE_EVENTS_TOPIC", topics.merge_events.clone()),+ ("PARITY_GRACE_SECONDS", MONITOR_GRACE_SECONDS.to_string()),+ (+ "PARITY_IDLE_PARTITION_SECONDS",+ MONITOR_IDLE_PARTITION_SECONDS.to_string(),+ ),+ // The readers assign before the ready line, so no+ // counterpart predates them and every entry is judged.+ ("PARITY_WARMUP_SECONDS", "0".to_string()),+ ("PARITY_SWEEP_INTERVAL_MS", "500".to_string()),+ ("BIND_HOST", "127.0.0.1".to_string()),+ ("BIND_PORT", MONITOR_PORT.to_string()),+ ],+ &log_dir,+ )?);+ Some(format!("http://127.0.0.1:{MONITOR_PORT}"))+ }+ _ => None,+ };+let mut stack = Self {config,infra,@@ -412,6 +474,7 @@ impl Stack {router_url,identity_url,publish_topics,+ monitor_url,log_dir,};@@ -868,6 +931,15 @@ impl Stack {).await?;}+ if self.monitor_url.is_some() {+ // The monitor binds after it assigned every partition at the end+ // of the log, so a listening port means no later row is missed.+ wait_tcp(+ &format!("127.0.0.1:{MONITOR_PORT}"),+ Duration::from_secs(30),+ )+ .await?;+ }tracing::info!(router = %self.router_url,elapsed_ms = start.elapsed().as_millis() as u64,