Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(personhog): run the parity monitor from the harness stack (#119398)

José Sequeiraa635e728177e
diff --git a/.github/workflows/ci-nodejs.yml b/.github/workflows/ci-nodejs.yml
index 1953b02f0eed..fadaba5f4f07 100644
--- a/.github/workflows/ci-nodejs.yml
+++ b/.github/workflows/ci-nodejs.yml
@@ -199,8 +199,8 @@ jobs:
RUST_REBUILD_ALL: ${{ steps.rust-affected.outputs.rebuild-all }}
RUST_AFFECTED_CRATES: ${{ steps.rust-affected.outputs.crates }}
RUST_NODE_E2E_CRATES: '["ingestion-consumer"]'
- # The Kafka parity e2e spawns identity and the publisher through the harness.
- RUST_PERSONHOG_E2E_CRATES: '["personhog-identity", "personhog-test-harness"]'
+ # The Kafka parity e2e spawns identity, the publisher and the parity monitor through the harness.
+ RUST_PERSONHOG_E2E_CRATES: '["personhog-identity", "personhog-test-harness", "personhog-parity-monitor"]'
run: |
# `schedule` alongside `push`: the paths filter is skipped on both, so
# NODEJS_CHANGED is empty there. Without this the hourly run would decide
@@ -954,7 +954,12 @@ jobs:
- name: Pre-build the personhog stack and harness
run: |
if [ -f nodejs/tests/service-e2e/personhog-kafka-parity.serial.test.ts ]; then
- cd rust && cargo build -p personhog-replica -p personhog-router -p personhog-leader -p personhog-writer -p personhog-identity -p personhog-test-harness
+ packages=(-p personhog-replica -p personhog-router -p personhog-leader -p personhog-writer -p personhog-identity -p personhog-test-harness)
+ # The parity test spawns the monitor through the stack; a branch without the crate runs the test without it.
+ if [ -d rust/personhog-parity-monitor ]; then
+ packages+=(-p personhog-parity-monitor)
+ fi
+ cd rust && cargo build "${packages[@]}"
else
echo "Kafka parity e2e test is not present on this branch; skipping the stack build."
fi
diff --git a/nodejs/tests/service-e2e/personhog-kafka-parity.serial.test.ts b/nodejs/tests/service-e2e/personhog-kafka-parity.serial.test.ts
index 60969a01840a..dbfeba4293f4 100644
--- a/nodejs/tests/service-e2e/personhog-kafka-parity.serial.test.ts
+++ b/nodejs/tests/service-e2e/personhog-kafka-parity.serial.test.ts
@@ -69,6 +69,17 @@ const STACK_STOP_GRACE_MS = 60_000
const SETTLE_TIMEOUT_MS = 30_000
const FEED_CONVERGENCE_TIMEOUT_MS = 30_000
const TOPIC_READ_TIMEOUT_MS = 30_000
+// The harness runs the monitor with a 15 s grace; the rest is margin.
+const MONITOR_SETTLE_TIMEOUT_MS = 60_000
+/** Outcomes that are not a disagreement: a stub is one feed's own path to the result. */
+const MONITOR_AGREEMENTS = new Set(['agree', 'node_stub', 'personhog_stub', 'same', 'expected'])
+const MONITOR_FAULTS = new Set([
+ 'personhog_parity_version_ties_total',
+ 'personhog_parity_merge_conflicts_total',
+ 'personhog_parity_decode_failures_total',
+ 'personhog_parity_overflow_evictions_total',
+ 'personhog_parity_warmup_evictions_total',
+])
/** The ready line the harness's `stack` subcommand writes. */
interface StackReady {
@@ -77,9 +88,16 @@ interface StackReady {
changelog_topic: string
distinct_ids_topic: string
merge_events_topic: string
+ monitor_addr: string
log_dir: string
}
+interface MetricSample {
+ name: string
+ labels: Record<string, string>
+ value: number
+}
+
interface DistinctIdRow {
person_id: string
team_id: number
@@ -121,6 +139,7 @@ describe('personhog kafka parity (e2e)', () => {
let stack: ServiceProcess | undefined
let workers: ServiceProcess[] = []
let workerUrl: string
+ let monitorUrl: string
let readyDir: string
let inputTopic: string
/** Every topic this run created, the stack's included, deleted at the end. */
@@ -257,11 +276,12 @@ describe('personhog kafka parity (e2e)', () => {
// Each service registers as soon as it is spawned, so one that never
// becomes ready is still stopped in afterAll.
readyDir = mkdtempSync(path.join(tmpdir(), 'personhog-kafka-parity-'))
- const ready = await startPersonhogStack(hub, path.join(readyDir, 'ready.json'), (service) => {
+ const ready = await startPersonhogStack(hub, path.join(readyDir, 'ready.json'), nodeTopics, (service) => {
stack = service
})
ownTopics.push(ready.changelog_topic, ready.distinct_ids_topic, ready.merge_events_topic)
personhogTopics = { distinctIds: ready.distinct_ids_topic, mergeEvents: ready.merge_events_topic }
+ monitorUrl = `http://${ready.monitor_addr}`
workerUrl = await startNodeIngestionApiWorker(
hub,
@@ -467,8 +487,69 @@ describe('personhog kafka parity (e2e)', () => {
.reduce((sum, value) => sum + value, 0)
expect(errors).toBe(0)
})
+
+ it('the parity monitor settled every entry of both feeds as agreement', async () => {
+ // A key settles at least once, so agreement below the fold's size
+ // means the monitor read a different topic than the fold did. The
+ // counts are polled with the pending gauges: a reader still behind
+ // the last rows reports nothing pending before they arrive.
+ const node = await feedState(nodeTopics)
+ const personhog = await feedState(personhogTopics)
+ const sharedMerges = Object.entries(node.survivors).filter(
+ ([person, survivor]) => survivor !== person && person in personhog.survivors
+ ).length
+ let samples!: MetricSample[]
+ await pollUntil(async () => {
+ const response = await stack!.request(`${monitorUrl}/metrics`, { method: 'GET' })
+ expect(response.statusCode).toBe(200)
+ samples = parsePrometheus(response.body)
+ expect(metric(samples, 'personhog_parity_pending_entries', { kind: 'mappings' })).toBe(0)
+ expect(metric(samples, 'personhog_parity_pending_entries', { kind: 'merges' })).toBe(0)
+ expect(
+ metric(samples, 'personhog_parity_settled_mappings_total', { outcome: 'agree' })
+ ).toBeGreaterThanOrEqual(Object.keys(node.mappings).length)
+ expect(
+ metric(samples, 'personhog_parity_settled_merges_total', { outcome: 'agree' })
+ ).toBeGreaterThanOrEqual(sharedMerges)
+ }, MONITOR_SETTLE_TIMEOUT_MS)
+
+ expect(monitorDisagreements(samples)).toEqual([])
+ })
})
+function parsePrometheus(body: string): MetricSample[] {
+ const samples: MetricSample[] = []
+ for (const line of body.split('\n')) {
+ const match = /^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+(\S+)/.exec(line)
+ if (!match) {
+ continue
+ }
+ const labels: Record<string, string> = {}
+ for (const pair of (match[2] ?? '').matchAll(/([a-zA-Z_][a-zA-Z0-9_]*)="([^"]*)"/g)) {
+ labels[pair[1]] = pair[2]
+ }
+ samples.push({ name: match[1], labels, value: Number(match[3]) })
+ }
+ return samples
+}
+
+function metric(samples: MetricSample[], name: string, labels: Record<string, string>): number | undefined {
+ return samples.find(
+ (sample) => sample.name === name && Object.entries(labels).every(([key, value]) => sample.labels[key] === value)
+ )?.value
+}
+
+function monitorDisagreements(samples: MetricSample[]): string[] {
+ return samples
+ .filter(
+ ({ name, labels, value }) =>
+ value > 0 &&
+ name.startsWith('personhog_parity_') &&
+ (MONITOR_FAULTS.has(name) || ('outcome' in labels && !MONITOR_AGREEMENTS.has(labels.outcome)))
+ )
+ .map(({ name, labels, value }) => `${name}${JSON.stringify(labels)} ${value}`)
+}
+
/**
* The same end state on both feeds. A person only one feed ever named is
* that feed's own path to the result, a stub the other backend never made
@@ -504,7 +585,12 @@ function expectEveryPersonAccountedFor(feed: FeedState): void {
expect(stale).toEqual([])
}
-async function startPersonhogStack(hub: Hub, readyFile: string, register: Register): Promise<StackReady> {
+async function startPersonhogStack(
+ hub: Hub,
+ readyFile: string,
+ nodeTopics: FeedTopics,
+ register: Register
+): Promise<StackReady> {
if (!existsSync(HARNESS_BINARY)) {
throw new Error(`${HARNESS_BINARY} is missing; build the personhog stack and harness first`)
}
@@ -514,6 +600,13 @@ async function startPersonhogStack(hub: Hub, readyFile: string, register: Regist
[
'stack',
'--publish',
+ // The stack assigns the Node topics (created above) before it
+ // writes the ready file, so no event of this run predates them.
+ '--monitor',
+ '--monitor-node-distinct-ids-topic',
+ nodeTopics.distinctIds,
+ '--monitor-node-merge-events-topic',
+ nodeTopics.mergeEvents,
'--bin-dir',
RUST_BIN_DIR,
'--pg-target-table',
@@ -542,7 +635,11 @@ async function startPersonhogStack(hub: Hub, readyFile: string, register: Regist
throw new Error(`personhog stack not ready within ${STACK_READY_TIMEOUT_MS}ms:\n${service.getOutput()}`)
}
}
- return parseJSON(readFileSync(readyFile, 'utf8')) as StackReady
+ const ready = parseJSON(readFileSync(readyFile, 'utf8')) as StackReady
+ if (!ready.monitor_addr) {
+ throw new Error(`the stack reported no monitor address; is ${HARNESS_BINARY} built from this branch?`)
+ }
+ return ready
}
async function startNodeIngestionApiWorker(
diff --git a/rust/personhog-test-harness/README.md b/rust/personhog-test-harness/README.md
index ad4eb454f0ed..bc9d43e71ac0 100644
--- a/rust/personhog-test-harness/README.md
+++ b/rust/personhog-test-harness/README.md
@@ -17,7 +17,7 @@ For `gate` without `--external-router-url`, build the service binaries first:
cargo build -p personhog-replica -p personhog-router -p personhog-leader -p personhog-writer -p personhog-identity -p personhog-test-harness
```
-The identity crate also builds `personhog-publisher`, which `--publish` spawns.
+The identity crate also builds `personhog-publisher`, which `--publish` spawns; `stack --monitor` needs `cargo build -p personhog-parity-monitor` as well.
## `gate` — the e2e correctness gate
@@ -286,6 +286,15 @@ Once the stack is up it prints one line, `PERSONHOG_STACK_READY {…}`, and writ
A parent process should wait for the file rather than parse stdout, which the logs share.
A spawned service that exits ends the command with that service's last log lines, so a fixture never stays up over a dead stack.
+`--monitor` also spawns `personhog-parity-monitor` (build it too) against the publish topics and the Node path's topics named by `--monitor-node-distinct-ids-topic` and `--monitor-node-merge-events-topic`, which must exist before the stack starts; the ready line carries its metrics address as `monitor_addr`.
+The monitor assigns every partition before the ready line is written and runs with a 15 s grace and no warmup, so a test can read its `/metrics` once its traffic stops and assert that every entry settled as agreement.
+
+```bash
+target/debug/personhog-test-harness stack --publish --ready-file /tmp/personhog-stack.json \
+ --monitor --monitor-node-distinct-ids-topic clickhouse_person_distinct_id_test \
+ --monitor-node-merge-events-topic person_merge_events_test
+```
+
## `seed` / `cleanup` — manage traffic targets
```bash
diff --git a/rust/personhog-test-harness/src/cli.rs b/rust/personhog-test-harness/src/cli.rs
index 54a410f9898f..96b7e3bd0985 100644
--- a/rust/personhog-test-harness/src/cli.rs
+++ b/rust/personhog-test-harness/src/cli.rs
@@ -467,6 +467,21 @@ pub struct StackArgs {
#[arg(long, default_value_t = 100)]
pub publish_rows_per_pass: i64,
+ /// Also run `personhog-parity-monitor` against the publish topics and
+ /// the Node path's topics named below, and put its metrics address in
+ /// the ready line. Requires --publish; the Node topics must exist.
+ #[arg(long, default_value_t = false)]
+ pub monitor: bool,
+
+ /// The Node path's `clickhouse_person_distinct_id` topic the monitor
+ /// compares the publish topic against.
+ #[arg(long)]
+ pub monitor_node_distinct_ids_topic: Option<String>,
+
+ /// The Node path's `person_merge_events` topic.
+ #[arg(long)]
+ pub monitor_node_merge_events_topic: Option<String>,
+
/// Also write the ready line's JSON to this file once the stack is up.
/// A parent process polls for the file instead of parsing stdout,
/// which the logs share.
diff --git a/rust/personhog-test-harness/src/scenarios/gate.rs b/rust/personhog-test-harness/src/scenarios/gate.rs
index ea483e271c67..982212d57c92 100644
--- a/rust/personhog-test-harness/src/scenarios/gate.rs
+++ b/rust/personhog-test-harness/src/scenarios/gate.rs
@@ -298,6 +298,7 @@ pub async fn run(args: GateArgs) -> Result<()> {
publishers: args.publishers,
rows_per_pass: args.publish_rows_per_pass,
}),
+ monitor: None,
})
.await?,
)
diff --git a/rust/personhog-test-harness/src/scenarios/stack_cmd.rs b/rust/personhog-test-harness/src/scenarios/stack_cmd.rs
index 438e135f2ee0..ad3f052ec78d 100644
--- a/rust/personhog-test-harness/src/scenarios/stack_cmd.rs
+++ b/rust/personhog-test-harness/src/scenarios/stack_cmd.rs
@@ -16,7 +16,7 @@ use crate::cli::StackArgs;
use crate::publish;
use crate::scenarios::gate::default_bin_dir;
use crate::seed;
-use crate::stack::{PublishConfig, Stack, StackConfig};
+use crate::stack::{MonitorConfig, PublishConfig, Stack, StackConfig};
/// Prefix of the one stdout line a parent process can wait for. The logs
/// share stdout, so the marker is what makes the JSON findable.
@@ -34,6 +34,23 @@ pub async fn run(args: StackArgs) -> Result<()> {
if args.publish && args.publish_rows_per_pass <= 0 {
bail!("--publish-rows-per-pass must be positive");
}
+ let monitor = match (
+ args.monitor,
+ &args.monitor_node_distinct_ids_topic,
+ &args.monitor_node_merge_events_topic,
+ ) {
+ (false, _, _) => None,
+ (true, _, _) if !args.publish => {
+ bail!("--monitor needs --publish: the monitor compares the publish topics")
+ }
+ (true, Some(distinct_ids), Some(merge_events)) => Some(MonitorConfig {
+ node_distinct_ids_topic: distinct_ids.clone(),
+ node_merge_events_topic: merge_events.clone(),
+ }),
+ (true, _, _) => bail!(
+ "--monitor needs --monitor-node-distinct-ids-topic and --monitor-node-merge-events-topic"
+ ),
+ };
if let Some(path) = &args.ready_file {
refuse_stale_ready_file(path)?;
}
@@ -80,6 +97,7 @@ pub async fn run(args: StackArgs) -> Result<()> {
publishers: args.publishers,
rows_per_pass: args.publish_rows_per_pass,
}),
+ monitor,
})
.await?;
@@ -94,6 +112,7 @@ pub async fn run(args: StackArgs) -> Result<()> {
"changelog_topic": stack.changelog_topic(),
"distinct_ids_topic": topics.map(|t| t.distinct_ids.clone()),
"merge_events_topic": topics.map(|t| t.merge_events.clone()),
+ "monitor_addr": stack.monitor_url.as_deref().map(strip_scheme),
"log_dir": stack.log_dir.display().to_string(),
});
println!("{READY_MARKER}{ready}");
diff --git a/rust/personhog-test-harness/src/stack/mod.rs b/rust/personhog-test-harness/src/stack/mod.rs
index b19109563990..bc8518a81722 100644
--- a/rust/personhog-test-harness/src/stack/mod.rs
+++ b/rust/personhog-test-harness/src/stack/mod.rs
@@ -35,6 +35,12 @@ const ROUTER_METRICS_BASE_PORT: u16 = 24154;
const LEADER_METRICS_BASE_PORT: u16 = 24160;
/// Above the leader range, which grows with every scale-up.
const PUBLISHER_METRICS_BASE_PORT: u16 = 24200;
+/// Above the publishers, whose count is a flag.
+const MONITOR_PORT: u16 = 24300;
+/// Short, because a fixture's run ends with a wait for the last entries to
+/// settle; long enough to cover the publisher drain on a loaded CI runner.
+const MONITOR_GRACE_SECONDS: u64 = 15;
+const MONITOR_IDLE_PARTITION_SECONDS: u64 = 3;
const MAX_ROUTERS: u32 = 4;
/// Partitions of the two per-run publish topics. More than one, so the
/// keyed merge events spread the way they do in production.
@@ -76,6 +82,8 @@ pub struct StackConfig {
/// identity service writes, to two per-run topics. Needs
/// `spawn_identity`: nothing else writes the outbox.
pub publish: Option<PublishConfig>,
+ /// Also spawn the parity monitor against the publish topics; needs `publish`.
+ pub monitor: Option<MonitorConfig>,
}
pub struct PublishConfig {
@@ -83,6 +91,11 @@ pub struct PublishConfig {
pub rows_per_pass: i64,
}
+pub struct MonitorConfig {
+ pub node_distinct_ids_topic: String,
+ pub node_merge_events_topic: String,
+}
+
#[derive(Clone)]
pub struct PublishTopics {
pub distinct_ids: String,
@@ -124,6 +137,7 @@ pub struct Stack {
/// router's URL, which proxies identity RPCs to it.
pub identity_url: Option<String>,
publish_topics: Option<PublishTopics>,
+ pub monitor_url: Option<String>,
pub log_dir: PathBuf,
}
@@ -163,6 +177,9 @@ impl Stack {
if config.publish.is_some() && !config.spawn_identity {
bail!("publishing needs the identity service: nothing else writes the outbox");
}
+ if config.monitor.is_some() && config.publish.is_none() {
+ bail!("the parity monitor needs the publish topics: pass --publish");
+ }
let run_id = chrono::Utc::now().format("%Y%m%d_%H%M%S").to_string();
let topic = format!("personhog_test_harness_{run_id}");
@@ -182,13 +199,21 @@ impl Stack {
if config.publish.is_some() {
binaries.push("personhog-publisher");
}
+ if config.monitor.is_some() {
+ binaries.push("personhog-parity-monitor");
+ }
for bin in &binaries {
let path = config.bin_dir.join(bin);
if !path.exists() {
bail!(
"{} not found — build the stack first:\n cargo build -p personhog-replica \
- -p personhog-router -p personhog-leader -p personhog-writer -p personhog-identity",
- path.display()
+ -p personhog-router -p personhog-leader -p personhog-writer -p personhog-identity{}",
+ path.display(),
+ if config.monitor.is_some() {
+ " -p personhog-parity-monitor"
+ } else {
+ ""
+ }
);
}
}
@@ -399,6 +424,43 @@ impl Stack {
None
};
+ let monitor_url = match (&config.monitor, &publish_topics) {
+ (Some(monitor), Some(topics)) => {
+ infra.push(ServiceProcess::spawn(
+ "parity-monitor",
+ &config.bin_dir.join("personhog-parity-monitor"),
+ &[
+ ("NODE_KAFKA_HOSTS", config.kafka_hosts.clone()),
+ (
+ "NODE_DISTINCT_ID_TOPIC",
+ monitor.node_distinct_ids_topic.clone(),
+ ),
+ (
+ "NODE_MERGE_EVENTS_TOPIC",
+ monitor.node_merge_events_topic.clone(),
+ ),
+ ("PERSONHOG_KAFKA_HOSTS", config.kafka_hosts.clone()),
+ ("PERSONHOG_DISTINCT_ID_TOPIC", topics.distinct_ids.clone()),
+ ("PERSONHOG_MERGE_EVENTS_TOPIC", topics.merge_events.clone()),
+ ("PARITY_GRACE_SECONDS", MONITOR_GRACE_SECONDS.to_string()),
+ (
+ "PARITY_IDLE_PARTITION_SECONDS",
+ MONITOR_IDLE_PARTITION_SECONDS.to_string(),
+ ),
+ // The readers assign before the ready line, so no
+ // counterpart predates them and every entry is judged.
+ ("PARITY_WARMUP_SECONDS", "0".to_string()),
+ ("PARITY_SWEEP_INTERVAL_MS", "500".to_string()),
+ ("BIND_HOST", "127.0.0.1".to_string()),
+ ("BIND_PORT", MONITOR_PORT.to_string()),
+ ],
+ &log_dir,
+ )?);
+ Some(format!("http://127.0.0.1:{MONITOR_PORT}"))
+ }
+ _ => None,
+ };
+
let mut stack = Self {
config,
infra,
@@ -412,6 +474,7 @@ impl Stack {
router_url,
identity_url,
publish_topics,
+ monitor_url,
log_dir,
};
@@ -868,6 +931,15 @@ impl Stack {
)
.await?;
}
+ if self.monitor_url.is_some() {
+ // The monitor binds after it assigned every partition at the end
+ // of the log, so a listening port means no later row is missed.
+ wait_tcp(
+ &format!("127.0.0.1:{MONITOR_PORT}"),
+ Duration::from_secs(30),
+ )
+ .await?;
+ }
tracing::info!(
router = %self.router_url,
elapsed_ms = start.elapsed().as_millis() as u64,