:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(dagster): remove the personhog shadow lane jobs (#129683)
Nick Bestff73a7d82e9c
diff --git a/posthog/dags/locations/ingestion.py b/posthog/dags/locations/ingestion.pyindex 9f8dba5dcaa8..fd4a03b2294e 100644--- a/posthog/dags/locations/ingestion.py+++ b/posthog/dags/locations/ingestion.py@@ -1,13 +1,6 @@import dagster-from posthog.dags import (- detach_distinct_id,- distinct_id_usage,- ingestion_assets,- person_property_reconciliation,- personhog_shadow_drift,- personhog_shadow_lane,-)+from posthog.dags import detach_distinct_id, distinct_id_usage, ingestion_assets, person_property_reconciliationfrom . import loggers, resources@@ -19,8 +12,6 @@ defs = dagster.Definitions(detach_distinct_id.detach_distinct_id_job,distinct_id_usage.distinct_id_usage_monitoring,person_property_reconciliation.person_property_reconciliation_job,- personhog_shadow_drift.personhog_shadow_lane_stop_and_compare_job,- personhog_shadow_lane.personhog_shadow_lane_start_job,],schedules=[distinct_id_usage.distinct_id_usage_monitoring_schedule,diff --git a/posthog/dags/personhog_shadow_drift.py b/posthog/dags/personhog_shadow_drift.pydeleted file mode 100644index 9b3dd3fdfb25..000000000000--- a/posthog/dags/personhog_shadow_drift.py+++ /dev/null@@ -1,623 +0,0 @@-"""Stop the personhog shadow lane and measure drift between the two write paths.--The job scales the lane's consumers to zero, waits for every in-flight write-to land (the personhog writer keeps draining its changelog backlog after the-consumers stop), then compares the legacy tables against the personhog tmp-tables in the shadow persons database and reports drift per category.--It never resets the database: after the run an operator can connect directly-and inspect the exact rows behind every reported number. A fresh validation-run starts with personhog_shadow_lane_start_job and reset_state=true.-"""--import time-from collections.abc import Callable, Mapping-from contextlib import closing-from itertools import groupby-from typing import Literal--import dagster-import psycopg2-from prometheus_client import CollectorRegistry, Gauge-from pydantic import Field--from posthog.dags.common import JobOwners-from posthog.dags.personhog_shadow_lane import (- SHADOW_CONSUMER_DEPLOYMENT,- SHADOW_DB_URL_ENV_VAR,- SHADOW_NAMESPACE,- SHADOW_PROCESSOR_DEPLOYMENT,- apps_api,- deployment_pod_count,- read_shadow_write_counter,- scale_deployment,- shadow_db_connection,- wait_for_deployments,- wait_for_quiescence,-)-from posthog.dataclasses import frozen-from posthog.metrics import pushed_metrics_registry--SETTLE_METRICS_JOB = "personhog_shadow_lane_settle"-DRIFT_METRICS_JOB = "personhog_shadow_lane_drift"---@frozen-class DriftCategoryReport:- category: str- legacy_total: int- personhog_total: int- missing_in_personhog: int- missing_in_legacy: int- # Matched keys where at least one compared field differs. Field-level- # counts overlap on a row, so this is counted separately in SQL.- mismatched_rows: int- field_mismatches: dict[str, int]- samples: list[str]-- @property- def compared_keys(self) -> int:- # Legacy keys plus the keys only personhog has: the union of both sides.- return self.legacy_total + self.missing_in_legacy-- @property- def drifted_rows(self) -> int:- return self.missing_in_personhog + self.missing_in_legacy + self.mismatched_rows-- @property- def drift_pct(self) -> float:- if self.compared_keys == 0:- return 0.0- return 100.0 * self.drifted_rows / self.compared_keys---PropertyDriftKind = Literal["only_in_legacy", "only_in_personhog", "value_differs"]--_PROPERTY_DRIFT_MAX_ROWS = 100---@frozen-class PropertyKeyDrift:- key: str- kind: PropertyDriftKind- persons: int---@frozen-class PropertyDriftSample:- sampled_persons: int- key_drifts: list[PropertyKeyDrift]- person_details: list[str]---# Each category is a pair of queries over the same two CTEs: one aggregates-# the counts, the other lists sample rows. They share the CTE prefix and the-# mismatch condition so the samples always explain the counts.-#-# Both paths see the same deletions but record them differently: the legacy-# path deletes rows, personhog tombstones them with is_deleted. Every-# comparison filters is_deleted on both sides so a tombstone against a-# deleted row does not read as drift.-# The join hashes one whole side, so carrying properties through it costs-# memory proportional to the documents. jsonb text output is canonical-# (sorted keys, no duplicates, fixed whitespace), so hashing it shrinks-# each row to 32 bytes. This is stricter than jsonb equality: numeric-# formatting differences (1 vs 1.0) hash differently but compare equal-# as jsonb, so they count as drift here.-_PERSON_SIDES = """-WITH legacy AS (- SELECT team_id, uuid, md5(properties::text) AS properties_hash, is_identified, created_at, version- FROM posthog_person WHERE NOT is_deleted-), personhog AS (- SELECT team_id, uuid, md5(properties::text) AS properties_hash, is_identified, created_at, version- FROM personhog_person_tmp WHERE NOT is_deleted-)-"""--# version differences are reported per field but excluded from-# mismatched_rows: version counts how many writes a row took, so a benign-# difference in update batching shifts it without any end-state divergence.-_PERSON_ROW_DIFFERS = """(- l.properties_hash IS DISTINCT FROM p.properties_hash- OR l.is_identified IS DISTINCT FROM p.is_identified- OR l.created_at IS DISTINCT FROM p.created_at)"""--_PERSON_DRIFT_SQL = f"""-{_PERSON_SIDES}-SELECT- (SELECT count(*) FROM legacy) AS legacy_total,- (SELECT count(*) FROM personhog) AS personhog_total,- count(*) FILTER (WHERE p.uuid IS NULL) AS missing_in_personhog,- count(*) FILTER (WHERE l.uuid IS NULL) AS missing_in_legacy,- count(*) FILTER (WHERE l.uuid IS NOT NULL AND p.uuid IS NOT NULL- AND l.properties_hash IS DISTINCT FROM p.properties_hash) AS properties,- count(*) FILTER (WHERE l.uuid IS NOT NULL AND p.uuid IS NOT NULL- AND l.is_identified IS DISTINCT FROM p.is_identified) AS is_identified,- count(*) FILTER (WHERE l.uuid IS NOT NULL AND p.uuid IS NOT NULL- AND l.created_at IS DISTINCT FROM p.created_at) AS created_at,- count(*) FILTER (WHERE l.uuid IS NOT NULL AND p.uuid IS NOT NULL- AND l.version IS DISTINCT FROM p.version) AS version,- count(*) FILTER (WHERE l.uuid IS NOT NULL AND p.uuid IS NOT NULL- AND {_PERSON_ROW_DIFFERS}) AS mismatched_rows-FROM legacy l-FULL OUTER JOIN personhog p USING (team_id, uuid)-"""--_PERSON_SAMPLE_SQL = f"""-{_PERSON_SIDES}-SELECT team_id, uuid,- CASE WHEN p.uuid IS NULL THEN 'missing_in_personhog'- WHEN l.uuid IS NULL THEN 'missing_in_legacy'- ELSE 'field_mismatch' END AS drift-FROM legacy l-FULL OUTER JOIN personhog p USING (team_id, uuid)-WHERE p.uuid IS NULL OR l.uuid IS NULL OR {_PERSON_ROW_DIFFERS}-LIMIT %(limit)s-"""--# These sides join a child table to persons before the full outer join.-# MATERIALIZED stops the planner from serving the sample LIMIT through-# index-ordered merge joins: inlined, it merges the child table with persons-# on team_id alone and checks person_id as a join filter, so every row is-# compared against every person of its team. The count queries reference-# each side twice and are materialized either way. The person sides stay-# inline because their full outer join reads both (team_id, uuid) indexes-# in order with no intermediate join.-_DISTINCT_ID_SIDES = """-WITH legacy AS MATERIALIZED (- SELECT d.team_id, d.distinct_id, p.uuid AS person_uuid, d.version- FROM posthog_persondistinctid d- JOIN posthog_person p ON p.id = d.person_id AND p.team_id = d.team_id- WHERE NOT d.is_deleted AND NOT p.is_deleted-), personhog AS MATERIALIZED (- SELECT d.team_id, d.distinct_id, p.uuid AS person_uuid, d.version- FROM personhog_persondistinctid_tmp d- JOIN personhog_person_tmp p ON p.team_id = d.team_id AND p.id = d.person_id- WHERE NOT d.is_deleted AND NOT p.is_deleted-)-"""--_DISTINCT_ID_DRIFT_SQL = f"""-{_DISTINCT_ID_SIDES}-SELECT- (SELECT count(*) FROM legacy) AS legacy_total,- (SELECT count(*) FROM personhog) AS personhog_total,- count(*) FILTER (WHERE p.person_uuid IS NULL) AS missing_in_personhog,- count(*) FILTER (WHERE l.person_uuid IS NULL) AS missing_in_legacy,- count(*) FILTER (WHERE l.person_uuid IS NOT NULL AND p.person_uuid IS NOT NULL- AND l.person_uuid <> p.person_uuid) AS mismatched_rows,- count(*) FILTER (WHERE l.person_uuid IS NOT NULL AND p.person_uuid IS NOT NULL- AND l.version IS DISTINCT FROM p.version) AS version-FROM legacy l-FULL OUTER JOIN personhog p USING (team_id, distinct_id)-"""--_DISTINCT_ID_SAMPLE_SQL = f"""-{_DISTINCT_ID_SIDES}-SELECT team_id, distinct_id, l.person_uuid AS legacy_person, p.person_uuid AS personhog_person,- CASE WHEN p.person_uuid IS NULL THEN 'missing_in_personhog'- WHEN l.person_uuid IS NULL THEN 'missing_in_legacy'- ELSE 'person_mismatch' END AS drift-FROM legacy l-FULL OUTER JOIN personhog p USING (team_id, distinct_id)-WHERE p.person_uuid IS NULL OR l.person_uuid IS NULL OR l.person_uuid <> p.person_uuid-LIMIT %(limit)s-"""--# MATERIALIZED for the same reason as _DISTINCT_ID_SIDES.-_HASH_KEY_SIDES = """-WITH legacy AS MATERIALIZED (- SELECT h.team_id, p.uuid AS person_uuid, h.feature_flag_key, h.hash_key- FROM posthog_featureflaghashkeyoverride h- JOIN posthog_person p ON p.id = h.person_id AND p.team_id = h.team_id- WHERE NOT p.is_deleted-), personhog AS MATERIALIZED (- SELECT h.team_id, p.uuid AS person_uuid, h.feature_flag_key, h.hash_key- FROM personhog_featureflaghashkeyoverride_tmp h- JOIN personhog_person_tmp p ON p.team_id = h.team_id AND p.id = h.person_id- WHERE NOT p.is_deleted-)-"""--_HASH_KEY_DRIFT_SQL = f"""-{_HASH_KEY_SIDES}-SELECT- (SELECT count(*) FROM legacy) AS legacy_total,- (SELECT count(*) FROM personhog) AS personhog_total,- count(*) FILTER (WHERE p.hash_key IS NULL) AS missing_in_personhog,- count(*) FILTER (WHERE l.hash_key IS NULL) AS missing_in_legacy,- count(*) FILTER (WHERE l.hash_key IS NOT NULL AND p.hash_key IS NOT NULL- AND l.hash_key <> p.hash_key) AS mismatched_rows-FROM legacy l-FULL OUTER JOIN personhog p USING (team_id, person_uuid, feature_flag_key)-"""--_HASH_KEY_SAMPLE_SQL = f"""-{_HASH_KEY_SIDES}-SELECT team_id, person_uuid, feature_flag_key,- CASE WHEN p.hash_key IS NULL THEN 'missing_in_personhog'- WHEN l.hash_key IS NULL THEN 'missing_in_legacy'- ELSE 'hash_key_mismatch' END AS drift-FROM legacy l-FULL OUTER JOIN personhog p USING (team_id, person_uuid, feature_flag_key)-WHERE p.hash_key IS NULL OR l.hash_key IS NULL OR l.hash_key <> p.hash_key-LIMIT %(limit)s-"""--_PROPERTY_DIFF_ROWS = f"""-{_PERSON_SIDES}, mismatched AS (- SELECT team_id, uuid- FROM legacy l- JOIN personhog p USING (team_id, uuid)- WHERE l.properties_hash IS DISTINCT FROM p.properties_hash- ORDER BY team_id, uuid- LIMIT %(limit)s-), pairs AS (- SELECT m.team_id, m.uuid, l.properties AS legacy_props, p.properties AS personhog_props- FROM mismatched m- JOIN posthog_person l USING (team_id, uuid)- JOIN personhog_person_tmp p USING (team_id, uuid)-), keys AS (- SELECT team_id, uuid, legacy_props, personhog_props, k.key- FROM pairs, LATERAL (- SELECT jsonb_object_keys(legacy_props) AS key- UNION SELECT jsonb_object_keys(personhog_props)- ) k-), diff AS (- SELECT team_id, uuid, key,- CASE WHEN NOT legacy_props ? key THEN 'only_in_personhog'- WHEN NOT personhog_props ? key THEN 'only_in_legacy'- ELSE 'value_differs' END AS kind,- jsonb_typeof(legacy_props -> key) AS legacy_type,- jsonb_typeof(personhog_props -> key) AS personhog_type,- length((legacy_props -> key)::text) AS legacy_length,- length((personhog_props -> key)::text) AS personhog_length- FROM keys- WHERE (legacy_props -> key)::text IS DISTINCT FROM (personhog_props -> key)::text-)-"""--_PROPERTY_KEY_DRIFT_SQL = f"""-{_PROPERTY_DIFF_ROWS}-SELECT key, kind, count(*) AS persons, (SELECT count(*) FROM mismatched) AS sampled_persons-FROM diff-GROUP BY key, kind-ORDER BY persons DESC, key-"""--_PROPERTY_DETAIL_SQL = f"""-{_PROPERTY_DIFF_ROWS}-SELECT * FROM diff ORDER BY team_id, uuid, key-"""--# Columns every *_DRIFT_SQL returns. _run_category reports any other column-# as a per-field mismatch count, so a new summary column must be added here.-_TOTAL_COLUMNS = frozenset(- {"legacy_total", "personhog_total", "missing_in_personhog", "missing_in_legacy", "mismatched_rows"}-)---def _person_samples(row: Mapping[str, object]) -> str:- return f"team={row['team_id']} uuid={row['uuid']} {row['drift']}"---def _distinct_id_samples(row: Mapping[str, object]) -> str:- return (- f"team={row['team_id']} distinct_id={row['distinct_id']!r} "- f"legacy_person={row['legacy_person']} personhog_person={row['personhog_person']} {row['drift']}"- )---def _hash_key_samples(row: Mapping[str, object]) -> str:- return f"team={row['team_id']} person={row['person_uuid']} flag={row['feature_flag_key']} {row['drift']}"---def _run_category(- cursor: psycopg2.extensions.cursor,- category: str,- drift_sql: str,- sample_sql: str,- format_sample: Callable[[Mapping[str, object]], str],- sample_size: int,-) -> DriftCategoryReport:- cursor.execute(drift_sql)- row = cursor.fetchone()- totals = {key: int(row[key]) for key in _TOTAL_COLUMNS}- field_mismatches = {key: int(value) for key, value in row.items() if key not in _TOTAL_COLUMNS}- drifted = totals["missing_in_personhog"] + totals["missing_in_legacy"] + totals["mismatched_rows"]- samples: list[str] = []- # The sample query repeats the full join, so skip it when the counts already say it returns nothing.- if sample_size > 0 and drifted > 0:- cursor.execute(sample_sql, {"limit": sample_size})- samples = [format_sample(sample) for sample in cursor.fetchall()]- return DriftCategoryReport(category=category, field_mismatches=field_mismatches, samples=samples, **totals)---def _configure_session(cursor: psycopg2.extensions.cursor, statement_timeout_minutes: int) -> None:- cursor.execute("SET application_name = 'dagster_personhog_shadow_drift'")- cursor.execute("SET statement_timeout = %s", (f"{statement_timeout_minutes}min",))- # work_mem applies per hash node and per parallel worker, and hash joins- # get hash_mem_multiplier times it, so a three-join query can reserve- # many times this value at once.- cursor.execute("SET work_mem = '256MB'")---def compute_shadow_drift(- connection: psycopg2.extensions.connection, sample_size: int, statement_timeout_minutes: int = 30-) -> list[DriftCategoryReport]:- with connection.cursor() as cursor:- _configure_session(cursor, statement_timeout_minutes)- return [- _run_category(cursor, "persons", _PERSON_DRIFT_SQL, _PERSON_SAMPLE_SQL, _person_samples, sample_size),- _run_category(- cursor,- "distinct_ids",- _DISTINCT_ID_DRIFT_SQL,- _DISTINCT_ID_SAMPLE_SQL,- _distinct_id_samples,- sample_size,- ),- _run_category(- cursor,- "hash_key_overrides",- _HASH_KEY_DRIFT_SQL,- _HASH_KEY_SAMPLE_SQL,- _hash_key_samples,- sample_size,- ),- ]---def _format_property_side(jsonb_type: object, text_length: object) -> str:- if jsonb_type is None:- return "absent"- return f"{jsonb_type}({text_length})"---def _format_person_property_detail(team_id: object, person_uuid: object, rows: list[Mapping[str, object]]) -> str:- keys = "; ".join(- f"{row['key']!r} {row['kind']} "- f"legacy={_format_property_side(row['legacy_type'], row['legacy_length'])} "- f"personhog={_format_property_side(row['personhog_type'], row['personhog_length'])}"- for row in rows[:_PROPERTY_DRIFT_MAX_ROWS]- )- if len(rows) > _PROPERTY_DRIFT_MAX_ROWS:- keys += "; ..."- return f"team={team_id} uuid={person_uuid} differing_keys={len(rows)}: {keys}"---def sample_property_drift(- connection: psycopg2.extensions.connection,- persons_limit: int,- detail_limit: int,- statement_timeout_minutes: int = 30,-) -> PropertyDriftSample:- detail_rows: list[Mapping[str, object]] = []- with connection.cursor() as cursor:- _configure_session(cursor, statement_timeout_minutes)- cursor.execute(_PROPERTY_KEY_DRIFT_SQL, {"limit": persons_limit})- key_rows = cursor.fetchall()- if detail_limit > 0:- cursor.execute(_PROPERTY_DETAIL_SQL, {"limit": min(detail_limit, persons_limit)})- detail_rows = cursor.fetchall()-- key_drifts = [PropertyKeyDrift(key=row["key"], kind=row["kind"], persons=int(row["persons"])) for row in key_rows]- sampled_persons = int(key_rows[0]["sampled_persons"]) if key_rows else 0- person_details = [- _format_person_property_detail(team_id, person_uuid, list(person_rows))- for (team_id, person_uuid), person_rows in groupby(detail_rows, key=lambda row: (row["team_id"], row["uuid"]))- ]- return PropertyDriftSample(sampled_persons=sampled_persons, key_drifts=key_drifts, person_details=person_details)---def _property_drift_table(sample: PropertyDriftSample) -> str:- lines = ["| key | kind | persons |", "|---|---|---|"]- for drift in sample.key_drifts[:_PROPERTY_DRIFT_MAX_ROWS]:- key = drift.key.replace("|", "\\|").replace("\n", "\\n").replace("\r", "\\r")- lines.append(f"| {key} | {drift.kind} | {drift.persons} |")- return "\n".join(lines)---def record_settle_gauges(- registry: CollectorRegistry, database: str, settle_seconds: float, completed_at: float-) -> None:- Gauge(- "posthog_personhog_shadow_lane_settle_seconds",- "Time the shadow persons database took to stop receiving writes after the lane stopped",- ["database"],- registry=registry,- ).labels(database=database).set(settle_seconds)- Gauge(- "posthog_personhog_shadow_lane_settle_last_success_timestamp_seconds",- "Unix time when the shadow persons database last settled after a stop",- ["database"],- registry=registry,- ).labels(database=database).set(completed_at)---def record_drift_gauges(- registry: CollectorRegistry, database: str, reports: list[DriftCategoryReport], completed_at: float-) -> None:- def per_category(name: str, help_text: str) -> Gauge:- return Gauge(- f"posthog_personhog_shadow_lane_drift_{name}", help_text, ["database", "category"], registry=registry- )-- legacy_rows = per_category("legacy_rows", "Live rows the legacy path holds")- personhog_rows = per_category("personhog_rows", "Live rows the personhog path holds")- missing_in_personhog = per_category("missing_in_personhog_rows", "Keys only the legacy path holds")- missing_in_legacy = per_category("missing_in_legacy_rows", "Keys only the personhog path holds")- mismatched = per_category("mismatched_rows", "Keys both paths hold with different content; excludes version")- ratio = per_category("ratio", "Drifted keys over all compared keys, from 0 to 1")- field_mismatched = Gauge(- "posthog_personhog_shadow_lane_drift_field_mismatched_rows",- "Keys both paths hold where one field differs. A key with two different fields counts in both",- ["database", "category", "field"],- registry=registry,- )-- for report in reports:- labels = {"database": database, "category": report.category}- legacy_rows.labels(**labels).set(report.legacy_total)- personhog_rows.labels(**labels).set(report.personhog_total)- missing_in_personhog.labels(**labels).set(report.missing_in_personhog)- missing_in_legacy.labels(**labels).set(report.missing_in_legacy)- mismatched.labels(**labels).set(report.mismatched_rows)- ratio.labels(**labels).set(report.drift_pct / 100)- for field_name, count in report.field_mismatches.items():- field_mismatched.labels(field=field_name, **labels).set(count)-- Gauge(- "posthog_personhog_shadow_lane_drift_last_success_timestamp_seconds",- "Unix time when the last drift report finished",- ["database"],- registry=registry,- ).labels(database=database).set(completed_at)---class ShadowLaneStopConfig(dagster.Config):- namespace: str = SHADOW_NAMESPACE- consumer_deployment: str = SHADOW_CONSUMER_DEPLOYMENT- processor_deployment: str = SHADOW_PROCESSOR_DEPLOYMENT- # Per deployment: the consumers get the full budget to drain before the processors scale.- stop_timeout_seconds: int = 600---class ShadowSettleConfig(dagster.Config):- shadow_db_env_var: str = SHADOW_DB_URL_ENV_VAR- poll_seconds: int = 30- stable_checks: int = 4- timeout_seconds: int = 3600---class ShadowDriftConfig(dagster.Config):- shadow_db_env_var: str = SHADOW_DB_URL_ENV_VAR- sample_size: int = 10- property_diff_sample_size: int = 500- statement_timeout_minutes: int = Field(default=30, gt=0)---def stop_shadow_lane(context: dagster.OpExecutionContext, config: ShadowLaneStopConfig) -> None:- apps = apps_api()-- def is_stopped(deployment: str) -> bool:- if deployment_pod_count(apps, config.namespace, deployment) > 0:- return False- context.log.info(f"{deployment} has no pods left")- return True-- # Consumers first, and processors only once the consumers' pods are gone. A consumer drains its in-flight- # batches before it exits, so the processors then hold no un-acked work. A processor scaled down while a- # consumer still waits on it gets only its drain window to finish those batches, and fails what is left.- for deployment in (config.consumer_deployment, config.processor_deployment):- context.log.info(f"Scaling {config.namespace}/{deployment} to 0 replicas")- scale_deployment(apps, config.namespace, deployment, 0)- pending = wait_for_deployments([deployment], is_stopped, timeout_seconds=config.stop_timeout_seconds)- if pending:- raise dagster.Failure(- description=f"Pods still running after {config.stop_timeout_seconds}s: {', '.join(sorted(pending))}"- )--[email protected](ins={"lane_stopped": dagster.In(dagster.Nothing)})-def wait_for_shadow_settle(context: dagster.OpExecutionContext, config: ShadowSettleConfig) -> None:- """Wait until nothing writes to the shadow persons database any more.-- The consumers are down, but the personhog writer keeps applying its- changelog backlog. A quiet database cannot be told apart from a stalled- writer here: if the writer crashed with backlog uncommitted, the counter- also holds still. Identity creates the person row immediately with- empty properties, and the writer fills them in later, so a stalled- writer mostly shows as properties mismatches, not missing_in_personhog.- Check the writer's health before trusting an unexpectedly large- properties count.- """- with closing(shadow_db_connection(config.shadow_db_env_var)) as connection:- connection.autocommit = True-- started = time.monotonic()- try:- polls = wait_for_quiescence(- lambda: read_shadow_write_counter(connection),- poll_seconds=config.poll_seconds,- stable_checks=config.stable_checks,- timeout_seconds=config.timeout_seconds,- on_progress=context.log.info,- )- except TimeoutError as timeout:- raise dagster.Failure(- description=(- f"The shadow persons database is still receiving writes after {config.timeout_seconds}s. "- "The personhog writer may still be draining a large backlog, or something outside the lane "- "is writing to this database. The consumers stay down; re-run this job to keep waiting."- )- ) from timeout-- waited = time.monotonic() - started- context.log.info(f"Shadow persons database settled after {waited:.0f}s ({polls} polls)")- with pushed_metrics_registry(f"{SETTLE_METRICS_JOB}_{config.shadow_db_env_var}") as registry:- record_settle_gauges(registry, config.shadow_db_env_var, waited, time.time())- context.add_output_metadata({"settle_seconds": dagster.MetadataValue.float(round(waited, 1))})--[email protected](ins={"settled": dagster.In(dagster.Nothing)})-def report_shadow_drift(context: dagster.OpExecutionContext, config: ShadowDriftConfig) -> None:- with closing(shadow_db_connection(config.shadow_db_env_var)) as connection:- reports = compute_shadow_drift(connection, config.sample_size, config.statement_timeout_minutes)- persons_report = next(report for report in reports if report.category == "persons")- property_drift: PropertyDriftSample | None = None- if config.property_diff_sample_size > 0 and persons_report.field_mismatches.get("properties", 0) > 0:- property_drift = sample_property_drift(- connection,- persons_limit=config.property_diff_sample_size,- detail_limit=config.sample_size,- statement_timeout_minutes=config.statement_timeout_minutes,- )-- with pushed_metrics_registry(f"{DRIFT_METRICS_JOB}_{config.shadow_db_env_var}") as registry:- record_drift_gauges(registry, config.shadow_db_env_var, reports, time.time())-- header = "| category | legacy | personhog | missing in personhog | missing in legacy | mismatched | drift % |"- separator = "|---|---|---|---|---|---|---|"- lines = [header, separator]- metadata: dict[str, dagster.MetadataValue] = {}- for report in reports:- lines.append(- f"| {report.category} | {report.legacy_total} | {report.personhog_total} "- f"| {report.missing_in_personhog} | {report.missing_in_legacy} "- f"| {report.mismatched_rows} | {report.drift_pct:.4f} |"- )- metadata[f"{report.category}_drift_pct"] = dagster.MetadataValue.float(round(report.drift_pct, 4))- context.log.info(- f"[{report.category}] legacy={report.legacy_total} personhog={report.personhog_total} "- f"missing_in_personhog={report.missing_in_personhog} missing_in_legacy={report.missing_in_legacy} "- f"mismatched_rows={report.mismatched_rows} drift={report.drift_pct:.4f}% "- f"field_mismatches={report.field_mismatches}"- )- for sample in report.samples:- context.log.info(f"[{report.category}] drift sample: {sample}")-- metadata["summary"] = dagster.MetadataValue.md("\n".join(lines))- if property_drift is not None:- context.log.info(- f"[persons] property drift by key over {property_drift.sampled_persons} sampled persons "- f"({len(property_drift.key_drifts)} key rows, up to {_PROPERTY_DRIFT_MAX_ROWS} shown):\n"- f"{_property_drift_table(property_drift)}"- )- for detail in property_drift.person_details:- context.log.info(f"[persons] property drift detail: {detail}")- metadata["persons_property_drift_sampled_persons"] = dagster.MetadataValue.int(property_drift.sampled_persons)- metadata["persons_property_drift_key_rows"] = dagster.MetadataValue.int(len(property_drift.key_drifts))- metadata["persons_property_drift_keys"] = dagster.MetadataValue.md(_property_drift_table(property_drift))- context.add_output_metadata(metadata)--[email protected](tags={"owner": JobOwners.TEAM_INGESTION.value})-def personhog_shadow_lane_stop_and_compare_job() -> None:- """Stop the shadow lane, wait for writes to drain, and report drift between the two paths.-- Leaves the shadow persons database untouched so the drift behind the- report can be inspected over a direct connection afterwards.- """- report_shadow_drift(wait_for_shadow_settle(stop_shadow_lane()))diff --git a/posthog/dags/personhog_shadow_lane.py b/posthog/dags/personhog_shadow_lane.pydeleted file mode 100644index 56f59908c2a4..000000000000--- a/posthog/dags/personhog_shadow_lane.py+++ /dev/null@@ -1,520 +0,0 @@-"""Operate the personhog shadow validation lane.--The shadow lane is a second consumer group on a production ingestion topic,-set in the lane's deployment, whose worker runs with PERSONS_STORE_MODE=shadow:-every person write goes through both the legacy direct-to-Postgres path and-the personhog path, into a dedicated persons-shadow database. Validation runs compare the two paths'-tables afterwards, so each run must start from identical (empty) state on-both sides.--This module holds the shared plumbing (Kubernetes scaling, shadow DB access,-the table inventory) and the start job. The stop/drain/compare job lives in-personhog_shadow_drift.py.--The Dagster service account needs get/patch on deployments and-deployments/scale plus list on pods in the lane namespace, and the lane's-ArgoCD Applications-must ignore Deployment spec.replicas, or ArgoCD self-heal reverts every scale-these jobs apply. Both are charts-side prerequisites. Resetting the lane's-offsets also needs the Kafka bootstrap servers of the lane's cluster in-PERSONHOG_SHADOW_KAFKA_BOOTSTRAP_SERVERS on the Dagster deployment.-"""--import os-import time-from collections.abc import Callable, Iterable-from concurrent.futures import Future-from contextlib import closing-from typing import Any, Protocol-from urllib.parse import parse_qs, urlparse--import dagster-import psycopg2-import psycopg2.extras-from confluent_kafka import KafkaError, KafkaException-from confluent_kafka.admin import AdminClient-from kubernetes import (- client as k8s_client,- config as k8s_config,-)-from prometheus_client import CollectorRegistry, Gauge-from pydantic import Field--from posthog.dags.common import JobOwners-from posthog.metrics import pushed_metrics_registry--SHADOW_NAMESPACE = "ingestion-analytics-team2-shadow"-SHADOW_CONSUMER_DEPLOYMENT = "ingestion-analytics-team2-shadow-consumer"-SHADOW_PROCESSOR_DEPLOYMENT = "ingestion-analytics-team2-shadow-processor"-SHADOW_DB_URL_ENV_VAR = "PERSONS_SHADOW_DB_URL"-SHADOW_CONSUMER_GROUP = "ingestion-analytics-team2-shadow"-SHADOW_KAFKA_BOOTSTRAP_ENV_VAR = "PERSONHOG_SHADOW_KAFKA_BOOTSTRAP_SERVERS"-SHADOW_KAFKA_SECURITY_PROTOCOL_ENV_VAR = "PERSONHOG_SHADOW_KAFKA_SECURITY_PROTOCOL"-# The client library defines these error codes at runtime, but its type stubs omit them.-GROUP_ID_NOT_FOUND: int = KafkaError.GROUP_ID_NOT_FOUND # type: ignore[attr-defined]-NON_EMPTY_GROUP: int = KafkaError.NON_EMPTY_GROUP # type: ignore[attr-defined]--START_METRICS_JOB = "personhog_shadow_lane_start"--# Every table the lane writes, per path. The reset truncates both lists in one-# statement so a validation run starts from state where "row missing on one-# side" can only mean drift. Kept in sync with rust/persons_migrations/.-LEGACY_STATE_TABLES = [- "posthog_person",- "posthog_persondistinctid",- "posthog_personlessdistinctid",- "posthog_featureflaghashkeyoverride",- "posthog_personoverridemapping",- "posthog_personoverride",- "posthog_pendingpersonoverride",- "posthog_flatpersonoverride",- "posthog_group",- "posthog_grouptypemapping",-]-# Saga bookkeeping the identity sweeper rewrites on every pass: it garbage-collects-# completed ops past retention and re-claims failing ones. Excluded from the drain-# counter because that activity never stops, and any saga step that changes person-# state also lands in the person, distinct id or override tables.-LIFECYCLE_OP_TABLES = [- "lifecycle_op",- "lifecycle_op_person",- "lifecycle_op_tmp",- "lifecycle_op_person_tmp",-]-PERSONHOG_STATE_TABLES = [- "personhog_person_tmp",- "personhog_persondistinctid_tmp",- "personhog_featureflaghashkeyoverride_tmp",- *LIFECYCLE_OP_TABLES,- "person_pg_cleanup_queue",- "person_tombstone_publish_queue",- "person_publish_outbox",- "person_publish_outbox_tmp",-]---def require_shadow_dsn(connection_url: str) -> None:- """Refuse a DSN that does not look like the persons-shadow cluster.-- The reset truncates person tables. The only structural difference between- the shadow DSN and the production persons DSN is the cluster name, so this- guard is what stands between a misconfigured env var and truncating- production state.- """- parsed = urlparse(connection_url)- # psycopg2 also accepts libpq key/value strings ("host=... dbname=..."),- # which urlparse leaves whole in .path, so every check below would see an- # empty authority and pass on "shadow" appearing in any field.- if parsed.scheme not in ("postgres", "postgresql"):- raise dagster.Failure(- description=(- "Connection URL must be a postgres:// or postgresql:// URI. "- "Refusing a key/value DSN whose target the guard cannot check."- )- )- # libpq resolves the connection target from more than the URL authority:- # host, hostaddr, dbname, and service query parameters override it, and a- # comma in the host names several servers. Any of those lets a URL carry- # a shadow-looking hostname while connecting elsewhere, so reject them- # instead of trying to validate what libpq would do.- overrides = {"host", "hostaddr", "dbname", "service"} & set(parse_qs(parsed.query))- if overrides:- raise dagster.Failure(- description=(- f"Connection URL overrides its target through query parameters ({', '.join(sorted(overrides))}). "- "Refusing a DSN whose effective target can differ from its hostname."- )- )- hostname = parsed.hostname or ""- if "," in (parsed.netloc or ""):- raise dagster.Failure(- description="Connection URL names several hosts. Refusing a DSN with more than one target."- )- identity = f"{hostname}/{parsed.path.lstrip('/')}"- if "shadow" not in identity:- raise dagster.Failure(- description=(- f"Connection target {identity!r} does not contain 'shadow'. "- "Refusing to operate on a database that is not the persons-shadow cluster."- )- )---def shadow_db_connection(env_var: str) -> psycopg2.extensions.connection:- connection_url = os.environ.get(env_var)- if not connection_url:- raise dagster.Failure(- description=(- f"Environment variable {env_var} is not set on the Dagster deployment. "- "It must hold the persons-shadow database connection URL."- )- )- require_shadow_dsn(connection_url)- return psycopg2.connect(- connection_url,- cursor_factory=psycopg2.extras.RealDictCursor,- connect_timeout=10,- options="-c application_name=dagster_personhog_shadow_lane",- )---def _load_k8s_config() -> None:- try:- k8s_config.load_incluster_config()- except k8s_config.ConfigException:- k8s_config.load_kube_config()---def apps_api() -> k8s_client.AppsV1Api:- _load_k8s_config()- return k8s_client.AppsV1Api()---def deployment_pod_count(apps: k8s_client.AppsV1Api, namespace: str, name: str) -> int:- """Count the deployment's pods straight from the pod list.-- status.replicas lags a fresh scale and leaves out terminating pods, which- can still write during their grace period, so the reset guard cannot rely- on it.- """- deployment = apps.read_namespaced_deployment(name=name, namespace=namespace)- if deployment.spec.replicas:- return deployment.spec.replicas- selector = ",".join(f"{key}={value}" for key, value in deployment.spec.selector.match_labels.items())- pods = k8s_client.CoreV1Api().list_namespaced_pod(namespace=namespace, label_selector=selector)- terminal_phases = {"Succeeded", "Failed"}- return sum(1 for pod in pods.items if pod.status.phase not in terminal_phases)---def scale_deployment(apps: k8s_client.AppsV1Api, namespace: str, name: str, replicas: int) -> None:- apps.patch_namespaced_deployment_scale(name=name, namespace=namespace, body={"spec": {"replicas": replicas}})---def deployment_ready_replicas(apps: k8s_client.AppsV1Api, namespace: str, name: str) -> int:- deployment = apps.read_namespaced_deployment(name=name, namespace=namespace)- return deployment.status.ready_replicas or 0---def wait_for_deployments(- deployments: Iterable[str],- is_settled: Callable[[str], bool],- *,- timeout_seconds: float,- poll_seconds: float = 10,- sleep: Callable[[float], None] = time.sleep,-) -> set[str]:- """Poll each deployment until is_settled accepts it or the deadline passes.-- Returns the deployments still pending at the deadline, so the caller- decides how to report them.- """- deadline = time.monotonic() + timeout_seconds- pending = set(deployments)- while pending and time.monotonic() < deadline:- for deployment in sorted(pending):- if is_settled(deployment):- pending.discard(deployment)- if pending:- sleep(poll_seconds)- return pending---def wait_for_quiescence(- read_write_counter: Callable[[], int],- *,- poll_seconds: float,- stable_checks: int,- timeout_seconds: float,- sleep: Callable[[float], None] = time.sleep,- on_progress: Callable[[str], None] | None = None,-) -> int:- """Poll a monotonic write counter until it holds still for stable_checks consecutive polls.-- Returns the number of polls taken. Raises TimeoutError when the counter is- still moving at the deadline.- """- deadline = time.monotonic() + timeout_seconds- last = read_write_counter()- stable = 0- polls = 1- while stable < stable_checks:- if time.monotonic() >= deadline:- raise TimeoutError(f"write counter still moving after {timeout_seconds}s (last value {last})")- sleep(poll_seconds)- current = read_write_counter()- polls += 1- if current == last:- stable += 1- else:- if on_progress is not None:- on_progress(f"writes still landing: counter moved {last} -> {current}")- stable = 0- last = current- return polls---def read_shadow_write_counter(connection: psycopg2.extensions.connection) -> int:- """Sum the database's tuple-write counters, minus the sweeper's bookkeeping tables.-- The shadow database serves only the lane, so a stable sum means every- writer has drained; a table allowlist would silently go stale when the- lane gains a table. LIFECYCLE_OP_TABLES are the one exception, because- the identity sweeper keeps them moving while the lane is stopped. The- connection must be in autocommit so each poll is its own transaction and- reads a fresh pg_stat snapshot instead of the first transaction's cached one.- """- with connection.cursor() as cursor:- cursor.execute(- "SELECT COALESCE(SUM(n_tup_ins + n_tup_upd + n_tup_del), 0) AS writes "- "FROM pg_stat_user_tables WHERE relname != ALL(%(excluded)s)",- {"excluded": LIFECYCLE_OP_TABLES},- )- return int(cursor.fetchone()["writes"])---class ShadowLaneStartConfig(dagster.Config):- reset_state: bool = False- # Deletes the lane's consumer group, so the lane starts at the end of its topic instead of its committed offsets.- reset_offsets: bool = False- consumer_replicas: int = Field(default=4, gt=0, le=512)- processor_replicas: int = Field(default=8, gt=0, le=512)- processor_step_replicas: int = Field(default=64, gt=0, le=512)- namespace: str = SHADOW_NAMESPACE- consumer_deployment: str = SHADOW_CONSUMER_DEPLOYMENT- processor_deployment: str = SHADOW_PROCESSOR_DEPLOYMENT- shadow_db_env_var: str = SHADOW_DB_URL_ENV_VAR- # Applies to each processor step and to the consumers separately.- ready_timeout_seconds: int = 600- # How long the reset waits for in-flight writes to stop before truncating.- reset_settle_poll_seconds: int = 10- reset_settle_stable_checks: int = Field(default=3, ge=1)- reset_settle_timeout_seconds: int = 600---def _require_lane_stopped(config: ShadowLaneStartConfig, apps: k8s_client.AppsV1Api, option: str, reason: str) -> None:- for deployment in (config.consumer_deployment, config.processor_deployment):- pods = deployment_pod_count(apps, config.namespace, deployment)- if pods > 0:- raise dagster.Failure(- description=(- f"Deployment {deployment} still wants or has {pods} pod(s). "- f"{reason} "- f"Run the stop-and-compare job first, then start with {option}."- )- )---def _reset_shadow_state(- context: dagster.OpExecutionContext, config: ShadowLaneStartConfig, apps: k8s_client.AppsV1Api-) -> None:- _require_lane_stopped(- config, apps, "reset_state", "A reset while consumers run would leave the two paths inconsistent."- )-- tables = LEGACY_STATE_TABLES + PERSONHOG_STATE_TABLES- with closing(shadow_db_connection(config.shadow_db_env_var)) as connection:- connection.autocommit = True-- # The lane deployments are down, but the personhog writer drains its- # changelog into these tables from outside the lane namespace. A- # truncate that races it leaves old-run rows behind, so wait for the- # whole database to go quiet first.- try:- wait_for_quiescence(- lambda: read_shadow_write_counter(connection),- poll_seconds=config.reset_settle_poll_seconds,- stable_checks=config.reset_settle_stable_checks,- timeout_seconds=config.reset_settle_timeout_seconds,- on_progress=context.log.info,- )- except TimeoutError as timeout:- raise dagster.Failure(- description=(- f"The shadow persons database is still receiving writes after "- f"{config.reset_settle_timeout_seconds}s, most likely the personhog writer draining its "- "backlog. Nothing was truncated; re-run once it settles."- )- ) from timeout-- context.log.info(f"Truncating {len(tables)} tables in the shadow persons database: {', '.join(tables)}")- with connection.cursor() as cursor:- cursor.execute("SET lock_timeout = '30s'")- cursor.execute("SET statement_timeout = '10min'")- # One statement so the reset is atomic; CASCADE covers the FKs- # between the person and distinct id tables on both sides. No- # RESTART IDENTITY: the shadow personhog leader stays up across- # resets and caches persons by (team_id, person_id) without- # re-reading Postgres, so restarted sequences would map new rows- # to stale cached persons from the previous run.- cursor.execute(f"TRUNCATE {', '.join(tables)} CASCADE")- context.log.info("Shadow persons database reset complete")---class ConsumerGroupAdmin(Protocol):- def delete_consumer_groups(self, group_ids: list[str], **kwargs: Any) -> dict[str, Future]: ...---def shadow_kafka_admin() -> AdminClient:- bootstrap_servers = os.environ.get(SHADOW_KAFKA_BOOTSTRAP_ENV_VAR)- if not bootstrap_servers:- raise dagster.Failure(- description=(- f"Environment variable {SHADOW_KAFKA_BOOTSTRAP_ENV_VAR} is not set on the Dagster deployment. "- "It must hold the bootstrap servers of the Kafka cluster the shadow lane consumes from."- )- )- return AdminClient(- {- "bootstrap.servers": bootstrap_servers,- "security.protocol": os.environ.get(SHADOW_KAFKA_SECURITY_PROTOCOL_ENV_VAR, "SSL"),- # The wheel's bundled OpenSSL does not read the system CA store unless told to probe for it.- "ssl.ca.location": "probe",- }- )---def _reset_consumer_offsets(- context: dagster.OpExecutionContext,- config: ShadowLaneStartConfig,- apps: k8s_client.AppsV1Api,- admin_factory: Callable[[], ConsumerGroupAdmin] = shadow_kafka_admin,-) -> bool:- """Delete the lane's consumer group, so the next start falls back to the consumer's offset reset policy.-- The ingestion consumer resets to the latest offset when its group has no committed offset, so the lane- starts at the end of its topic. Returns False when the group did not exist.- """- _require_lane_stopped(config, apps, "reset_offsets", "Kafka refuses to delete a consumer group with members.")- admin = admin_factory()- future = admin.delete_consumer_groups([SHADOW_CONSUMER_GROUP], request_timeout=30)[SHADOW_CONSUMER_GROUP]- try:- future.result()- except KafkaException as exception:- error = exception.args[0]- if error.code() == GROUP_ID_NOT_FOUND:- context.log.info(f"Consumer group {SHADOW_CONSUMER_GROUP} does not exist, nothing to reset")- return False- if error.code() == NON_EMPTY_GROUP:- raise dagster.Failure(- description=(- f"Consumer group {SHADOW_CONSUMER_GROUP} still has members, so Kafka refused to delete it. "- "Wait for the lane's consumers to leave the group, then retry."- )- ) from exception- raise- context.log.info(f"Deleted consumer group {SHADOW_CONSUMER_GROUP}; the lane starts from the latest offsets")- return True---def processor_ramp(current: int, target: int, step: int) -> list[int]:- """Replica counts to scale the processors through, ending at target.-- A partly scaled deployment repeats its current count first, so a retried run waits for- pods still pending from the last run before it adds more. A scale down goes straight to- target, since removing pods needs no new nodes.- """- if target <= current:- return [target]- first = current if current > 0 else step- return [*range(first, target, step), target]---def _scale_and_wait(- context: dagster.OpExecutionContext,- config: ShadowLaneStartConfig,- apps: k8s_client.AppsV1Api,- deployment: str,- replicas: int,-) -> None:- context.log.info(f"Scaling {config.namespace}/{deployment} to {replicas} replicas")- scale_deployment(apps, config.namespace, deployment, replicas)-- def is_ready(name: str) -> bool:- return deployment_ready_replicas(apps, config.namespace, name) >= replicas-- if wait_for_deployments([deployment], is_ready, timeout_seconds=config.ready_timeout_seconds):- raise dagster.Failure(- description=(- f"{deployment} did not reach {replicas} ready replicas within {config.ready_timeout_seconds}s. "- "The scale was applied; check the pods in the lane namespace."- )- )- context.log.info(f"{deployment} is ready with {replicas} replica(s)")---def record_start_gauges(registry: CollectorRegistry, config: ShadowLaneStartConfig, completed_at: float) -> None:- Gauge(- "posthog_personhog_shadow_lane_start_last_success_timestamp_seconds",- "Unix time when the shadow lane last started with every replica ready",- ["namespace"],- registry=registry,- ).labels(namespace=config.namespace).set(completed_at)- Gauge(- "posthog_personhog_shadow_lane_start_reset_state",- "1 when the last start truncated the shadow persons database first, 0 when it resumed from existing state",- ["namespace"],- registry=registry,- ).labels(namespace=config.namespace).set(1 if config.reset_state else 0)- Gauge(- "posthog_personhog_shadow_lane_start_reset_offsets",- "1 when the last start deleted the lane's consumer group first, 0 when it resumed from committed offsets",- ["namespace"],- registry=registry,- ).labels(namespace=config.namespace).set(1 if config.reset_offsets else 0)- replicas = Gauge(- "posthog_personhog_shadow_lane_start_replicas",- "Replicas the last start requested, by deployment",- ["namespace", "deployment"],- registry=registry,- )- replicas.labels(namespace=config.namespace, deployment=config.consumer_deployment).set(config.consumer_replicas)- replicas.labels(namespace=config.namespace, deployment=config.processor_deployment).set(config.processor_replicas)---def start_shadow_lane(context: dagster.OpExecutionContext, config: ShadowLaneStartConfig) -> None:- apps = apps_api()- if config.reset_state:- _reset_shadow_state(context, config, apps)- else:- context.log.info("reset_state is false, leaving the shadow persons database untouched")- if config.reset_offsets:- _reset_consumer_offsets(context, config, apps)- else:- context.log.info("reset_offsets is false, the lane resumes from its committed offsets")-- # Every processor is ready before any consumer starts. A consumer that starts first pins its keys to the- # few processors already up, and those keys stay on them while they have work in flight.- current = apps.read_namespaced_deployment(name=config.processor_deployment, namespace=config.namespace)- for replicas in processor_ramp(- current.spec.replicas or 0, config.processor_replicas, config.processor_step_replicas- ):- _scale_and_wait(context, config, apps, config.processor_deployment, replicas)- _scale_and_wait(context, config, apps, config.consumer_deployment, config.consumer_replicas)-- with pushed_metrics_registry(f"{START_METRICS_JOB}_{config.namespace}") as registry:- record_start_gauges(registry, config, time.time())-- context.add_output_metadata(- {- "reset_state": dagster.MetadataValue.bool(config.reset_state),- "reset_offsets": dagster.MetadataValue.bool(config.reset_offsets),- "consumer_replicas": dagster.MetadataValue.int(config.consumer_replicas),- "processor_replicas": dagster.MetadataValue.int(config.processor_replicas),- }- )--[email protected](tags={"owner": JobOwners.TEAM_INGESTION.value})-def personhog_shadow_lane_start_job():- """Start the shadow lane consumers, optionally resetting the shadow persons database first.-- Run with default config to resume consuming from the committed offsets.- Set ops.start_shadow_lane.config.reset_state to true to truncate both- paths' tables before starting, which every fresh validation run needs.- Set reset_offsets to true to start from the end of the topic instead.- Both resets refuse to run while the lane has pods. Processors scale up in- steps of processor_step_replicas, and the consumers start only once every- processor is ready.- """- start_shadow_lane()diff --git a/posthog/dags/tests/test_personhog_shadow_drift.py b/posthog/dags/tests/test_personhog_shadow_drift.pydeleted file mode 100644index bcb54f3a1411..000000000000--- a/posthog/dags/tests/test_personhog_shadow_drift.py+++ /dev/null@@ -1,197 +0,0 @@-import uuid--import pytest--import psycopg2-import psycopg2.extras-from prometheus_client import CollectorRegistry--from posthog.dags.personhog_shadow_drift import (- DriftCategoryReport,- PropertyKeyDrift,- compute_shadow_drift,- record_drift_gauges,- sample_property_drift,-)-from posthog.persons_db import persons_db_url--TEAM_ID = 990000123-CREATED_AT = "2026-01-01T00:00:00+00:00"---def _insert_person(- cursor, table: str, person_uuid: uuid.UUID, properties: dict, is_deleted: bool = False, version: int = 1-) -> int:- cursor.execute(- f"INSERT INTO {table} (created_at, properties, is_identified, uuid, version, team_id, is_deleted) "- "VALUES (%s, %s, false, %s, %s, %s, %s) RETURNING id",- (CREATED_AT, psycopg2.extras.Json(properties), str(person_uuid), version, TEAM_ID, is_deleted),- )- return cursor.fetchone()["id"]--[email protected]_db_direct[email protected]_db(transaction=True)-def _session_statement_timeout(connection: psycopg2.extensions.connection) -> str:- with connection.cursor() as cursor:- cursor.execute("SHOW statement_timeout")- return cursor.fetchone()["statement_timeout"]---def test_compute_shadow_drift_counts_each_category() -> None:- connection = psycopg2.connect(persons_db_url(writer=True), cursor_factory=psycopg2.extras.RealDictCursor)- connection.autocommit = True- matched = uuid.uuid4()- legacy_only = uuid.uuid4()- personhog_only = uuid.uuid4()- props_differ = uuid.uuid4()- tombstoned = uuid.uuid4()-- try:- with connection.cursor() as cursor:- legacy_matched = _insert_person(cursor, "posthog_person", matched, {"a": 1})- _insert_person(cursor, "posthog_person", legacy_only, {})- _insert_person(cursor, "posthog_person", props_differ, {"b": 1, "legacy_key": "xyz", "same": True})-- ph_matched = _insert_person(cursor, "personhog_person_tmp", matched, {"a": 1}, version=2)- _insert_person(cursor, "personhog_person_tmp", personhog_only, {})- ph_props = _insert_person(- cursor, "personhog_person_tmp", props_differ, {"b": 2, "personhog_key": [1, 2], "same": True}- )- ph_tombstoned = _insert_person(cursor, "personhog_person_tmp", tombstoned, {}, is_deleted=True)-- cursor.execute(- "INSERT INTO posthog_persondistinctid (distinct_id, version, person_id, team_id) VALUES "- "('shadow-drift-did-1', 1, %s, %s), ('shadow-drift-did-2', 1, %s, %s)",- (legacy_matched, TEAM_ID, legacy_matched, TEAM_ID),- )- cursor.execute(- "INSERT INTO personhog_persondistinctid_tmp (distinct_id, version, person_id, team_id) VALUES "- "('shadow-drift-did-1', 1, %s, %s), ('shadow-drift-did-2', 1, %s, %s), "- "('shadow-drift-did-3', 1, %s, %s)",- (ph_matched, TEAM_ID, ph_props, TEAM_ID, ph_tombstoned, TEAM_ID),- )-- cursor.execute(- "INSERT INTO posthog_featureflaghashkeyoverride (feature_flag_key, hash_key, person_id, team_id) "- "VALUES ('flag-1', 'hash-same', %s, %s), ('flag-2', 'hash-legacy', %s, %s)",- (legacy_matched, TEAM_ID, legacy_matched, TEAM_ID),- )- cursor.execute(- "INSERT INTO personhog_featureflaghashkeyoverride_tmp (feature_flag_key, hash_key, person_id, team_id) "- "VALUES ('flag-1', 'hash-same', %s, %s), ('flag-2', 'hash-personhog', %s, %s), "- "('flag-3', 'hash-tombstoned', %s, %s)",- (ph_matched, TEAM_ID, ph_matched, TEAM_ID, ph_tombstoned, TEAM_ID),- )-- reports = {- report.category: report- for report in compute_shadow_drift(connection, sample_size=10, statement_timeout_minutes=239)- }- compute_timeout = _session_statement_timeout(connection)- property_drift = sample_property_drift(- connection, persons_limit=500, detail_limit=10, statement_timeout_minutes=241- )- sample_timeout = _session_statement_timeout(connection)- finally:- connection.close()-- assert (compute_timeout, sample_timeout) == ("239min", "241min")-- persons = reports["persons"]- assert (persons.legacy_total, persons.personhog_total) == (3, 3)- assert (persons.missing_in_personhog, persons.missing_in_legacy, persons.mismatched_rows) == (1, 1, 1)- assert persons.field_mismatches == {"properties": 1, "is_identified": 0, "created_at": 0, "version": 1}- assert persons.drift_pct == 75.0- assert sorted(sample.rsplit(" ", 1)[1] for sample in persons.samples) == [- "field_mismatch",- "missing_in_legacy",- "missing_in_personhog",- ]-- assert property_drift.sampled_persons == 1- assert property_drift.key_drifts == [- PropertyKeyDrift(key="b", kind="value_differs", persons=1),- PropertyKeyDrift(key="legacy_key", kind="only_in_legacy", persons=1),- PropertyKeyDrift(key="personhog_key", kind="only_in_personhog", persons=1),- ]- assert property_drift.person_details == [- f"team={TEAM_ID} uuid={props_differ} differing_keys=3: "- "'b' value_differs legacy=number(1) personhog=number(1); "- "'legacy_key' only_in_legacy legacy=string(5) personhog=absent; "- "'personhog_key' only_in_personhog legacy=absent personhog=array(6)"- ]-- distinct_ids = reports["distinct_ids"]- assert (distinct_ids.legacy_total, distinct_ids.personhog_total) == (2, 2)- assert (distinct_ids.missing_in_personhog, distinct_ids.missing_in_legacy, distinct_ids.mismatched_rows) == (- 0,- 0,- 1,- )- assert distinct_ids.field_mismatches == {"version": 0}- assert distinct_ids.samples == [- f"team={TEAM_ID} distinct_id='shadow-drift-did-2' "- f"legacy_person={matched} personhog_person={props_differ} person_mismatch"- ]-- hash_keys = reports["hash_key_overrides"]- assert (hash_keys.legacy_total, hash_keys.personhog_total) == (2, 2)- assert (hash_keys.missing_in_personhog, hash_keys.missing_in_legacy, hash_keys.mismatched_rows) == (0, 0, 1)- assert hash_keys.samples == [f"team={TEAM_ID} person={matched} flag=flag-2 hash_key_mismatch"]---def test_drift_gauges_keep_each_count_under_its_own_name_and_category() -> None:- reports = [- DriftCategoryReport(- category="persons",- legacy_total=90,- personhog_total=95,- missing_in_personhog=5,- missing_in_legacy=10,- mismatched_rows=10,- field_mismatches={"properties": 7, "version": 40},- samples=[],- ),- DriftCategoryReport(- category="distinct_ids",- legacy_total=3,- personhog_total=3,- missing_in_personhog=0,- missing_in_legacy=0,- mismatched_rows=0,- field_mismatches={"version": 0},- samples=[],- ),- ]- registry = CollectorRegistry()-- record_drift_gauges(registry, "PERSONS_SHADOW_DB_URL", reports, completed_at=1_700_000_000.0)-- def sample(name: str, **labels: str) -> float | None:- return registry.get_sample_value(- f"posthog_personhog_shadow_lane_drift_{name}", {"database": "PERSONS_SHADOW_DB_URL", **labels}- )-- assert {- name: sample(name, category="persons")- for name in (- "legacy_rows",- "personhog_rows",- "missing_in_personhog_rows",- "missing_in_legacy_rows",- "mismatched_rows",- "ratio",- )- } == {- "legacy_rows": 90,- "personhog_rows": 95,- "missing_in_personhog_rows": 5,- "missing_in_legacy_rows": 10,- "mismatched_rows": 10,- "ratio": 0.25,- }- assert sample("field_mismatched_rows", category="persons", field="version") == 40- assert sample("field_mismatched_rows", category="persons", field="properties") == 7- assert sample("ratio", category="distinct_ids") == 0- assert sample("last_success_timestamp_seconds") == 1_700_000_000.0diff --git a/posthog/dags/tests/test_personhog_shadow_lane.py b/posthog/dags/tests/test_personhog_shadow_lane.pydeleted file mode 100644index f32b8b6bbe0f..000000000000--- a/posthog/dags/tests/test_personhog_shadow_lane.py+++ /dev/null@@ -1,410 +0,0 @@-import uuid-from collections.abc import Callable-from concurrent.futures import Future-from contextlib import nullcontext-from functools import partial-from types import SimpleNamespace--import pytest--import dagster-import psycopg2-import psycopg2.extras-from confluent_kafka import KafkaError, KafkaException-from parameterized import parameterized--from posthog.dags.personhog_shadow_drift import ShadowLaneStopConfig, stop_shadow_lane-from posthog.dags.personhog_shadow_lane import (- GROUP_ID_NOT_FOUND,- NON_EMPTY_GROUP,- SHADOW_CONSUMER_DEPLOYMENT,- SHADOW_CONSUMER_GROUP,- SHADOW_KAFKA_BOOTSTRAP_ENV_VAR,- SHADOW_PROCESSOR_DEPLOYMENT,- ShadowLaneStartConfig,- _reset_consumer_offsets,- _reset_shadow_state,- processor_ramp,- read_shadow_write_counter,- require_shadow_dsn,- shadow_kafka_admin,- start_shadow_lane,- wait_for_deployments,- wait_for_quiescence,-)-from posthog.persons_db import persons_db_url---class TestRequireShadowDsn:- @parameterized.expand(- [- ("postgres://user:[email protected]:5432/posthog",),- ("postgres://user@localhost:5432/persons_shadow_test",),- ]- )- def test_accepts_shadow_targets(self, url: str) -> None:- require_shadow_dsn(url)-- @parameterized.expand(- [- (- "postgres://user:pass@posthog-cloud-persons-prod-us-east-1.cluster-abc.us-east-1.rds.amazonaws.com:5432/posthog",- ),- ("postgres://user@localhost:5432/posthog_persons",),- ("postgres://user:shadow@localhost:5432/posthog_persons",),- # libpq lets query parameters and multi-host authorities point the- # connection somewhere other than the URL's hostname.- ("postgres://[email protected]:5432/posthog?hostaddr=10.0.0.1",),- ("postgres://[email protected]:5432/posthog?host=prod.example",),- ("postgres://[email protected]:5432/posthog?dbname=production",),- ("postgres://[email protected]:5432/posthog?service=prod",),- ("postgres://[email protected],prod.example:5432/posthog",),- # A libpq key/value DSN has no URL authority for the guard to check.- ("host=prod.example dbname=posthog user=dagster_shadow",),- ]- )- def test_rejects_non_shadow_targets(self, url: str) -> None:- with pytest.raises(dagster.Failure):- require_shadow_dsn(url)---class TestWaitForQuiescence:- @parameterized.expand(- [- ("flat_counter", [5, 5, 5, 5], 3, 4),- ("moving_then_flat", [5, 6, 6, 7, 7, 7], 2, 6),- ]- )- def test_returns_once_counter_is_stable(- self, _name: str, readings: list[int], stable_checks: int, expected_polls: int- ) -> None:- values = iter(readings)- polls = wait_for_quiescence(- lambda: next(values),- poll_seconds=0,- stable_checks=stable_checks,- timeout_seconds=60,- sleep=lambda _seconds: None,- )- assert polls == expected_polls-- def test_raises_when_counter_never_settles(self) -> None:- counter = iter(range(1000))- with pytest.raises(TimeoutError):- wait_for_quiescence(- lambda: next(counter),- poll_seconds=0,- stable_checks=2,- timeout_seconds=0,- sleep=lambda _seconds: None,- )---class TestWaitForDeployments:- @parameterized.expand(- [- ("all_settle", {"consumer": 2, "processor": 1}, 60, set()),- ("deadline_passed", {"consumer": 0, "processor": 0}, 0, {"consumer", "processor"}),- ]- )- def test_returns_what_is_still_pending(- self, _name: str, settles_on_check: dict[str, int], timeout_seconds: int, expected_pending: set[str]- ) -> None:- checks: dict[str, int] = dict.fromkeys(settles_on_check, 0)-- def is_settled(deployment: str) -> bool:- checks[deployment] += 1- return 0 < settles_on_check[deployment] <= checks[deployment]-- pending = wait_for_deployments(- settles_on_check, is_settled, timeout_seconds=timeout_seconds, sleep=lambda _seconds: None- )- assert pending == expected_pending---class TestProcessorRamp:- @parameterized.expand(- [- ("from_zero", 0, 256, 64, [64, 128, 192, 256]),- ("partial_last_step", 0, 150, 64, [64, 128, 150]),- ("holds_at_current_before_stepping", 200, 512, 128, [200, 328, 456, 512]),- ("target_below_one_step", 0, 8, 64, [8]),- ("scale_down_is_direct", 512, 256, 64, [256]),- ]- )- def test_steps(self, _name: str, current: int, target: int, step: int, expected: list[int]) -> None:- assert processor_ramp(current, target, step) == expected---class _ScalingAppsApi:- def __init__(self, ready_cap: int | None = None, processor_replicas: int = 0) -> None:- self.replicas = {SHADOW_PROCESSOR_DEPLOYMENT: processor_replicas, SHADOW_CONSUMER_DEPLOYMENT: 0}- self.ready_cap = ready_cap- self.scales: list[tuple[str, int]] = []-- def read_namespaced_deployment(self, name: str, namespace: str) -> SimpleNamespace:- desired = self.replicas[name]- ready = desired if self.ready_cap is None else min(desired, self.ready_cap)- return SimpleNamespace(spec=SimpleNamespace(replicas=desired), status=SimpleNamespace(ready_replicas=ready))-- def patch_namespaced_deployment_scale(self, name: str, namespace: str, body: dict) -> None:- self.replicas[name] = body["spec"]["replicas"]- self.scales.append((name, self.replicas[name]))---class _FakeClock:- def __init__(self) -> None:- self.now = 0.0-- def monotonic(self) -> float:- return self.now-- def time(self) -> float:- return self.now-- def sleep(self, seconds: float) -> None:- self.now += seconds---def scaling_apps(monkeypatch: pytest.MonkeyPatch) -> Callable[..., _ScalingAppsApi]:- def install(**kwargs: int) -> _ScalingAppsApi:- apps = _ScalingAppsApi(**kwargs)- clock = _FakeClock()- monkeypatch.setattr("posthog.dags.personhog_shadow_lane.apps_api", lambda: apps)- monkeypatch.setattr("posthog.dags.personhog_shadow_lane.pushed_metrics_registry", lambda _job: nullcontext())- monkeypatch.setattr("posthog.dags.personhog_shadow_lane.time", clock)- monkeypatch.setattr(- "posthog.dags.personhog_shadow_lane.wait_for_deployments",- partial(wait_for_deployments, sleep=clock.sleep),- )- return apps-- return install---def test_start_steps_processors_up_before_any_consumer(scaling_apps: Callable[..., _ScalingAppsApi]) -> None:- apps = scaling_apps()- config = ShadowLaneStartConfig(processor_replicas=150, processor_step_replicas=64, consumer_replicas=4)-- start_shadow_lane(dagster.build_op_context(), config)-- assert apps.scales == [- (SHADOW_PROCESSOR_DEPLOYMENT, 64),- (SHADOW_PROCESSOR_DEPLOYMENT, 128),- (SHADOW_PROCESSOR_DEPLOYMENT, 150),- (SHADOW_CONSUMER_DEPLOYMENT, 4),- ]--[email protected]("processor_replicas", [0, 64], ids=["fresh_start", "retry_with_pods_still_pending"])-def test_start_leaves_consumers_down_while_a_processor_step_is_unready(- scaling_apps: Callable[..., _ScalingAppsApi], processor_replicas: int-) -> None:- apps = scaling_apps(ready_cap=0, processor_replicas=processor_replicas)- config = ShadowLaneStartConfig(processor_replicas=150, consumer_replicas=4, ready_timeout_seconds=60)-- with pytest.raises(dagster.Failure, match="did not reach 64 ready replicas"):- start_shadow_lane(dagster.build_op_context(), config)-- assert apps.scales == [(SHADOW_PROCESSOR_DEPLOYMENT, 64)]---class _StoppingAppsApi:- """A deployment scaled to zero keeps one pod for `linger_polls` pod-count polls, as a draining pod does."""-- def __init__(self, linger_polls: int) -> None:- self.replicas = {SHADOW_CONSUMER_DEPLOYMENT: 4, SHADOW_PROCESSOR_DEPLOYMENT: 150}- self.linger = dict.fromkeys(self.replicas, 0)- self.linger_polls = linger_polls- self.scales: list[tuple[str, int]] = []- self.consumer_pods_at_scale: list[int] = []-- def pods(self, name: str) -> int:- if self.replicas[name] > 0:- return self.replicas[name]- return 1 if self.linger[name] > 0 else 0-- def pod_count(self, name: str) -> int:- count = self.pods(name)- if self.replicas[name] == 0 and self.linger[name] > 0:- self.linger[name] -= 1- return count-- def patch_namespaced_deployment_scale(self, name: str, namespace: str, body: dict) -> None:- self.consumer_pods_at_scale.append(self.pods(SHADOW_CONSUMER_DEPLOYMENT))- self.replicas[name] = body["spec"]["replicas"]- self.linger[name] = self.linger_polls- self.scales.append((name, self.replicas[name]))---def stopping_apps(monkeypatch: pytest.MonkeyPatch) -> Callable[..., _StoppingAppsApi]:- def install(linger_polls: int) -> _StoppingAppsApi:- apps = _StoppingAppsApi(linger_polls)- clock = _FakeClock()- monkeypatch.setattr("posthog.dags.personhog_shadow_lane.time", clock)- monkeypatch.setattr("posthog.dags.personhog_shadow_drift.apps_api", lambda: apps)- monkeypatch.setattr(- "posthog.dags.personhog_shadow_drift.deployment_pod_count",- lambda _apps, _namespace, name: apps.pod_count(name),- )- monkeypatch.setattr(- "posthog.dags.personhog_shadow_drift.wait_for_deployments",- partial(wait_for_deployments, sleep=clock.sleep),- )- return apps-- return install---def test_stop_scales_processors_only_after_the_consumer_pods_are_gone(- stopping_apps: Callable[..., _StoppingAppsApi],-) -> None:- apps = stopping_apps(linger_polls=2)-- # Each deployment drains in 20 s of polls, within its own 30 s budget; together they would exceed one.- stop_shadow_lane(dagster.build_op_context(), ShadowLaneStopConfig(stop_timeout_seconds=30))-- assert apps.scales == [(SHADOW_CONSUMER_DEPLOYMENT, 0), (SHADOW_PROCESSOR_DEPLOYMENT, 0)]- assert apps.consumer_pods_at_scale == [4, 0]---def test_stop_leaves_processors_up_while_a_consumer_pod_lingers(- stopping_apps: Callable[..., _StoppingAppsApi],-) -> None:- apps = stopping_apps(linger_polls=100)-- with pytest.raises(dagster.Failure, match="Pods still running"):- stop_shadow_lane(dagster.build_op_context(), ShadowLaneStopConfig(stop_timeout_seconds=30))-- assert apps.scales == [(SHADOW_CONSUMER_DEPLOYMENT, 0)]---class _FakeAppsApi:- def __init__(self, desired_replicas: int) -> None:- self.desired_replicas = desired_replicas-- def read_namespaced_deployment(self, name: str, namespace: str) -> SimpleNamespace:- return SimpleNamespace(- spec=SimpleNamespace(replicas=self.desired_replicas, selector=SimpleNamespace(match_labels={"app": name}))- )---class _NoPodsCoreApi:- def list_namespaced_pod(self, namespace: str, label_selector: str) -> SimpleNamespace:- return SimpleNamespace(items=[])---def stopped_lane(monkeypatch: pytest.MonkeyPatch) -> _FakeAppsApi:- monkeypatch.setattr("posthog.dags.personhog_shadow_lane.k8s_client.CoreV1Api", _NoPodsCoreApi)- return _FakeAppsApi(desired_replicas=0)---class _FakeKafkaAdmin:- def __init__(self, error_code: int | None = None) -> None:- self.error_code = error_code- self.deleted: list[str] = []-- def delete_consumer_groups(self, group_ids: list[str], **_kwargs: object) -> dict[str, Future]:- futures: dict[str, Future] = {}- for group_id in group_ids:- future: Future = Future()- if self.error_code is None:- self.deleted.append(group_id)- future.set_result(None)- else:- future.set_exception(KafkaException(KafkaError(self.error_code)))- futures[group_id] = future- return futures---def _no_admin() -> _FakeKafkaAdmin:- raise AssertionError("the admin client must not be reached while the lane has pods")--- [- (- "reset_state",- lambda context, apps: _reset_shadow_state(context, ShadowLaneStartConfig(reset_state=True), apps),- ),- (- "reset_offsets",- lambda context, apps: _reset_consumer_offsets(- context, ShadowLaneStartConfig(reset_offsets=True), apps, _no_admin- ),- ),- ]-)-def test_reset_refuses_while_lane_wants_pods(- _name: str, reset: Callable[[dagster.OpExecutionContext, _FakeAppsApi], object]-) -> None:- with pytest.raises(dagster.Failure, match="stop-and-compare"):- reset(dagster.build_op_context(), _FakeAppsApi(desired_replicas=2))--- "error_code,expected",- [- pytest.param(None, True, id="deleted"),- pytest.param(GROUP_ID_NOT_FOUND, False, id="missing_group"),- ],-)-def test_offset_reset_deletes_the_consumer_group(- stopped_lane: _FakeAppsApi, error_code: int | None, expected: bool-) -> None:- admin = _FakeKafkaAdmin(error_code)- config = ShadowLaneStartConfig(reset_offsets=True)- deleted = _reset_consumer_offsets(dagster.build_op_context(), config, stopped_lane, lambda: admin)- assert deleted == expected- assert admin.deleted == ([SHADOW_CONSUMER_GROUP] if expected else [])---def test_offset_reset_fails_while_the_group_has_members(stopped_lane: _FakeAppsApi) -> None:- admin = _FakeKafkaAdmin(NON_EMPTY_GROUP)- with pytest.raises(dagster.Failure, match="still has members"):- _reset_consumer_offsets(- dagster.build_op_context(), ShadowLaneStartConfig(reset_offsets=True), stopped_lane, lambda: admin- )---def test_offset_reset_needs_the_bootstrap_servers(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.delenv(SHADOW_KAFKA_BOOTSTRAP_ENV_VAR, raising=False)- with pytest.raises(dagster.Failure, match=SHADOW_KAFKA_BOOTSTRAP_ENV_VAR):- shadow_kafka_admin()--[email protected]_db_direct[email protected]_db(transaction=True)-def test_write_counter_ignores_lifecycle_op_tables() -> None:- connection = psycopg2.connect(persons_db_url(writer=True), cursor_factory=psycopg2.extras.RealDictCursor)- connection.autocommit = True- op_id = str(uuid.uuid4())- person_uuid = str(uuid.uuid4())- try:- before = read_shadow_write_counter(connection)-- with connection.cursor() as cursor:- cursor.execute(- "INSERT INTO lifecycle_op_tmp (op_id, op_type, team_id, step, request) "- "VALUES (%s, 'merge', 2, 'start', '{}')",- (op_id,),- )- cursor.execute("DELETE FROM lifecycle_op_tmp WHERE op_id = %s", (op_id,))- cursor.execute("SELECT pg_stat_force_next_flush()")- assert read_shadow_write_counter(connection) == before-- with connection.cursor() as cursor:- cursor.execute(- "INSERT INTO posthog_person (created_at, properties, is_identified, uuid, version, team_id, is_deleted) "- "VALUES (now(), '{}', false, %s, 1, 2, false)",- (person_uuid,),- )- cursor.execute("SELECT pg_stat_force_next_flush()")- assert read_shadow_write_counter(connection) == before + 1- finally:- with connection.cursor() as cursor:- cursor.execute("DELETE FROM posthog_person WHERE uuid = %s", (person_uuid,))- connection.close()