Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(dagster): remove the personhog shadow lane jobs (#129683)

Nick Bestff73a7d82e9c
diff --git a/posthog/dags/locations/ingestion.py b/posthog/dags/locations/ingestion.py
index 9f8dba5dcaa8..fd4a03b2294e 100644
--- a/posthog/dags/locations/ingestion.py
+++ b/posthog/dags/locations/ingestion.py
@@ -1,13 +1,6 @@
import dagster
-from posthog.dags import (
- detach_distinct_id,
- distinct_id_usage,
- ingestion_assets,
- person_property_reconciliation,
- personhog_shadow_drift,
- personhog_shadow_lane,
-)
+from posthog.dags import detach_distinct_id, distinct_id_usage, ingestion_assets, person_property_reconciliation
from . import loggers, resources
@@ -19,8 +12,6 @@ defs = dagster.Definitions(
detach_distinct_id.detach_distinct_id_job,
distinct_id_usage.distinct_id_usage_monitoring,
person_property_reconciliation.person_property_reconciliation_job,
- personhog_shadow_drift.personhog_shadow_lane_stop_and_compare_job,
- personhog_shadow_lane.personhog_shadow_lane_start_job,
],
schedules=[
distinct_id_usage.distinct_id_usage_monitoring_schedule,
diff --git a/posthog/dags/personhog_shadow_drift.py b/posthog/dags/personhog_shadow_drift.py
deleted file mode 100644
index 9b3dd3fdfb25..000000000000
--- a/posthog/dags/personhog_shadow_drift.py
+++ /dev/null
@@ -1,623 +0,0 @@
-"""Stop the personhog shadow lane and measure drift between the two write paths.
-
-The job scales the lane's consumers to zero, waits for every in-flight write
-to land (the personhog writer keeps draining its changelog backlog after the
-consumers stop), then compares the legacy tables against the personhog tmp
-tables in the shadow persons database and reports drift per category.
-
-It never resets the database: after the run an operator can connect directly
-and inspect the exact rows behind every reported number. A fresh validation
-run starts with personhog_shadow_lane_start_job and reset_state=true.
-"""
-
-import time
-from collections.abc import Callable, Mapping
-from contextlib import closing
-from itertools import groupby
-from typing import Literal
-
-import dagster
-import psycopg2
-from prometheus_client import CollectorRegistry, Gauge
-from pydantic import Field
-
-from posthog.dags.common import JobOwners
-from posthog.dags.personhog_shadow_lane import (
- SHADOW_CONSUMER_DEPLOYMENT,
- SHADOW_DB_URL_ENV_VAR,
- SHADOW_NAMESPACE,
- SHADOW_PROCESSOR_DEPLOYMENT,
- apps_api,
- deployment_pod_count,
- read_shadow_write_counter,
- scale_deployment,
- shadow_db_connection,
- wait_for_deployments,
- wait_for_quiescence,
-)
-from posthog.dataclasses import frozen
-from posthog.metrics import pushed_metrics_registry
-
-SETTLE_METRICS_JOB = "personhog_shadow_lane_settle"
-DRIFT_METRICS_JOB = "personhog_shadow_lane_drift"
-
-
-@frozen
-class DriftCategoryReport:
- category: str
- legacy_total: int
- personhog_total: int
- missing_in_personhog: int
- missing_in_legacy: int
- # Matched keys where at least one compared field differs. Field-level
- # counts overlap on a row, so this is counted separately in SQL.
- mismatched_rows: int
- field_mismatches: dict[str, int]
- samples: list[str]
-
- @property
- def compared_keys(self) -> int:
- # Legacy keys plus the keys only personhog has: the union of both sides.
- return self.legacy_total + self.missing_in_legacy
-
- @property
- def drifted_rows(self) -> int:
- return self.missing_in_personhog + self.missing_in_legacy + self.mismatched_rows
-
- @property
- def drift_pct(self) -> float:
- if self.compared_keys == 0:
- return 0.0
- return 100.0 * self.drifted_rows / self.compared_keys
-
-
-PropertyDriftKind = Literal["only_in_legacy", "only_in_personhog", "value_differs"]
-
-_PROPERTY_DRIFT_MAX_ROWS = 100
-
-
-@frozen
-class PropertyKeyDrift:
- key: str
- kind: PropertyDriftKind
- persons: int
-
-
-@frozen
-class PropertyDriftSample:
- sampled_persons: int
- key_drifts: list[PropertyKeyDrift]
- person_details: list[str]
-
-
-# Each category is a pair of queries over the same two CTEs: one aggregates
-# the counts, the other lists sample rows. They share the CTE prefix and the
-# mismatch condition so the samples always explain the counts.
-#
-# Both paths see the same deletions but record them differently: the legacy
-# path deletes rows, personhog tombstones them with is_deleted. Every
-# comparison filters is_deleted on both sides so a tombstone against a
-# deleted row does not read as drift.
-# The join hashes one whole side, so carrying properties through it costs
-# memory proportional to the documents. jsonb text output is canonical
-# (sorted keys, no duplicates, fixed whitespace), so hashing it shrinks
-# each row to 32 bytes. This is stricter than jsonb equality: numeric
-# formatting differences (1 vs 1.0) hash differently but compare equal
-# as jsonb, so they count as drift here.
-_PERSON_SIDES = """
-WITH legacy AS (
- SELECT team_id, uuid, md5(properties::text) AS properties_hash, is_identified, created_at, version
- FROM posthog_person WHERE NOT is_deleted
-), personhog AS (
- SELECT team_id, uuid, md5(properties::text) AS properties_hash, is_identified, created_at, version
- FROM personhog_person_tmp WHERE NOT is_deleted
-)
-"""
-
-# version differences are reported per field but excluded from
-# mismatched_rows: version counts how many writes a row took, so a benign
-# difference in update batching shifts it without any end-state divergence.
-_PERSON_ROW_DIFFERS = """(
- l.properties_hash IS DISTINCT FROM p.properties_hash
- OR l.is_identified IS DISTINCT FROM p.is_identified
- OR l.created_at IS DISTINCT FROM p.created_at)"""
-
-_PERSON_DRIFT_SQL = f"""
-{_PERSON_SIDES}
-SELECT
- (SELECT count(*) FROM legacy) AS legacy_total,
- (SELECT count(*) FROM personhog) AS personhog_total,
- count(*) FILTER (WHERE p.uuid IS NULL) AS missing_in_personhog,
- count(*) FILTER (WHERE l.uuid IS NULL) AS missing_in_legacy,
- count(*) FILTER (WHERE l.uuid IS NOT NULL AND p.uuid IS NOT NULL
- AND l.properties_hash IS DISTINCT FROM p.properties_hash) AS properties,
- count(*) FILTER (WHERE l.uuid IS NOT NULL AND p.uuid IS NOT NULL
- AND l.is_identified IS DISTINCT FROM p.is_identified) AS is_identified,
- count(*) FILTER (WHERE l.uuid IS NOT NULL AND p.uuid IS NOT NULL
- AND l.created_at IS DISTINCT FROM p.created_at) AS created_at,
- count(*) FILTER (WHERE l.uuid IS NOT NULL AND p.uuid IS NOT NULL
- AND l.version IS DISTINCT FROM p.version) AS version,
- count(*) FILTER (WHERE l.uuid IS NOT NULL AND p.uuid IS NOT NULL
- AND {_PERSON_ROW_DIFFERS}) AS mismatched_rows
-FROM legacy l
-FULL OUTER JOIN personhog p USING (team_id, uuid)
-"""
-
-_PERSON_SAMPLE_SQL = f"""
-{_PERSON_SIDES}
-SELECT team_id, uuid,
- CASE WHEN p.uuid IS NULL THEN 'missing_in_personhog'
- WHEN l.uuid IS NULL THEN 'missing_in_legacy'
- ELSE 'field_mismatch' END AS drift
-FROM legacy l
-FULL OUTER JOIN personhog p USING (team_id, uuid)
-WHERE p.uuid IS NULL OR l.uuid IS NULL OR {_PERSON_ROW_DIFFERS}
-LIMIT %(limit)s
-"""
-
-# These sides join a child table to persons before the full outer join.
-# MATERIALIZED stops the planner from serving the sample LIMIT through
-# index-ordered merge joins: inlined, it merges the child table with persons
-# on team_id alone and checks person_id as a join filter, so every row is
-# compared against every person of its team. The count queries reference
-# each side twice and are materialized either way. The person sides stay
-# inline because their full outer join reads both (team_id, uuid) indexes
-# in order with no intermediate join.
-_DISTINCT_ID_SIDES = """
-WITH legacy AS MATERIALIZED (
- SELECT d.team_id, d.distinct_id, p.uuid AS person_uuid, d.version
- FROM posthog_persondistinctid d
- JOIN posthog_person p ON p.id = d.person_id AND p.team_id = d.team_id
- WHERE NOT d.is_deleted AND NOT p.is_deleted
-), personhog AS MATERIALIZED (
- SELECT d.team_id, d.distinct_id, p.uuid AS person_uuid, d.version
- FROM personhog_persondistinctid_tmp d
- JOIN personhog_person_tmp p ON p.team_id = d.team_id AND p.id = d.person_id
- WHERE NOT d.is_deleted AND NOT p.is_deleted
-)
-"""
-
-_DISTINCT_ID_DRIFT_SQL = f"""
-{_DISTINCT_ID_SIDES}
-SELECT
- (SELECT count(*) FROM legacy) AS legacy_total,
- (SELECT count(*) FROM personhog) AS personhog_total,
- count(*) FILTER (WHERE p.person_uuid IS NULL) AS missing_in_personhog,
- count(*) FILTER (WHERE l.person_uuid IS NULL) AS missing_in_legacy,
- count(*) FILTER (WHERE l.person_uuid IS NOT NULL AND p.person_uuid IS NOT NULL
- AND l.person_uuid <> p.person_uuid) AS mismatched_rows,
- count(*) FILTER (WHERE l.person_uuid IS NOT NULL AND p.person_uuid IS NOT NULL
- AND l.version IS DISTINCT FROM p.version) AS version
-FROM legacy l
-FULL OUTER JOIN personhog p USING (team_id, distinct_id)
-"""
-
-_DISTINCT_ID_SAMPLE_SQL = f"""
-{_DISTINCT_ID_SIDES}
-SELECT team_id, distinct_id, l.person_uuid AS legacy_person, p.person_uuid AS personhog_person,
- CASE WHEN p.person_uuid IS NULL THEN 'missing_in_personhog'
- WHEN l.person_uuid IS NULL THEN 'missing_in_legacy'
- ELSE 'person_mismatch' END AS drift
-FROM legacy l
-FULL OUTER JOIN personhog p USING (team_id, distinct_id)
-WHERE p.person_uuid IS NULL OR l.person_uuid IS NULL OR l.person_uuid <> p.person_uuid
-LIMIT %(limit)s
-"""
-
-# MATERIALIZED for the same reason as _DISTINCT_ID_SIDES.
-_HASH_KEY_SIDES = """
-WITH legacy AS MATERIALIZED (
- SELECT h.team_id, p.uuid AS person_uuid, h.feature_flag_key, h.hash_key
- FROM posthog_featureflaghashkeyoverride h
- JOIN posthog_person p ON p.id = h.person_id AND p.team_id = h.team_id
- WHERE NOT p.is_deleted
-), personhog AS MATERIALIZED (
- SELECT h.team_id, p.uuid AS person_uuid, h.feature_flag_key, h.hash_key
- FROM personhog_featureflaghashkeyoverride_tmp h
- JOIN personhog_person_tmp p ON p.team_id = h.team_id AND p.id = h.person_id
- WHERE NOT p.is_deleted
-)
-"""
-
-_HASH_KEY_DRIFT_SQL = f"""
-{_HASH_KEY_SIDES}
-SELECT
- (SELECT count(*) FROM legacy) AS legacy_total,
- (SELECT count(*) FROM personhog) AS personhog_total,
- count(*) FILTER (WHERE p.hash_key IS NULL) AS missing_in_personhog,
- count(*) FILTER (WHERE l.hash_key IS NULL) AS missing_in_legacy,
- count(*) FILTER (WHERE l.hash_key IS NOT NULL AND p.hash_key IS NOT NULL
- AND l.hash_key <> p.hash_key) AS mismatched_rows
-FROM legacy l
-FULL OUTER JOIN personhog p USING (team_id, person_uuid, feature_flag_key)
-"""
-
-_HASH_KEY_SAMPLE_SQL = f"""
-{_HASH_KEY_SIDES}
-SELECT team_id, person_uuid, feature_flag_key,
- CASE WHEN p.hash_key IS NULL THEN 'missing_in_personhog'
- WHEN l.hash_key IS NULL THEN 'missing_in_legacy'
- ELSE 'hash_key_mismatch' END AS drift
-FROM legacy l
-FULL OUTER JOIN personhog p USING (team_id, person_uuid, feature_flag_key)
-WHERE p.hash_key IS NULL OR l.hash_key IS NULL OR l.hash_key <> p.hash_key
-LIMIT %(limit)s
-"""
-
-_PROPERTY_DIFF_ROWS = f"""
-{_PERSON_SIDES}, mismatched AS (
- SELECT team_id, uuid
- FROM legacy l
- JOIN personhog p USING (team_id, uuid)
- WHERE l.properties_hash IS DISTINCT FROM p.properties_hash
- ORDER BY team_id, uuid
- LIMIT %(limit)s
-), pairs AS (
- SELECT m.team_id, m.uuid, l.properties AS legacy_props, p.properties AS personhog_props
- FROM mismatched m
- JOIN posthog_person l USING (team_id, uuid)
- JOIN personhog_person_tmp p USING (team_id, uuid)
-), keys AS (
- SELECT team_id, uuid, legacy_props, personhog_props, k.key
- FROM pairs, LATERAL (
- SELECT jsonb_object_keys(legacy_props) AS key
- UNION SELECT jsonb_object_keys(personhog_props)
- ) k
-), diff AS (
- SELECT team_id, uuid, key,
- CASE WHEN NOT legacy_props ? key THEN 'only_in_personhog'
- WHEN NOT personhog_props ? key THEN 'only_in_legacy'
- ELSE 'value_differs' END AS kind,
- jsonb_typeof(legacy_props -> key) AS legacy_type,
- jsonb_typeof(personhog_props -> key) AS personhog_type,
- length((legacy_props -> key)::text) AS legacy_length,
- length((personhog_props -> key)::text) AS personhog_length
- FROM keys
- WHERE (legacy_props -> key)::text IS DISTINCT FROM (personhog_props -> key)::text
-)
-"""
-
-_PROPERTY_KEY_DRIFT_SQL = f"""
-{_PROPERTY_DIFF_ROWS}
-SELECT key, kind, count(*) AS persons, (SELECT count(*) FROM mismatched) AS sampled_persons
-FROM diff
-GROUP BY key, kind
-ORDER BY persons DESC, key
-"""
-
-_PROPERTY_DETAIL_SQL = f"""
-{_PROPERTY_DIFF_ROWS}
-SELECT * FROM diff ORDER BY team_id, uuid, key
-"""
-
-# Columns every *_DRIFT_SQL returns. _run_category reports any other column
-# as a per-field mismatch count, so a new summary column must be added here.
-_TOTAL_COLUMNS = frozenset(
- {"legacy_total", "personhog_total", "missing_in_personhog", "missing_in_legacy", "mismatched_rows"}
-)
-
-
-def _person_samples(row: Mapping[str, object]) -> str:
- return f"team={row['team_id']} uuid={row['uuid']} {row['drift']}"
-
-
-def _distinct_id_samples(row: Mapping[str, object]) -> str:
- return (
- f"team={row['team_id']} distinct_id={row['distinct_id']!r} "
- f"legacy_person={row['legacy_person']} personhog_person={row['personhog_person']} {row['drift']}"
- )
-
-
-def _hash_key_samples(row: Mapping[str, object]) -> str:
- return f"team={row['team_id']} person={row['person_uuid']} flag={row['feature_flag_key']} {row['drift']}"
-
-
-def _run_category(
- cursor: psycopg2.extensions.cursor,
- category: str,
- drift_sql: str,
- sample_sql: str,
- format_sample: Callable[[Mapping[str, object]], str],
- sample_size: int,
-) -> DriftCategoryReport:
- cursor.execute(drift_sql)
- row = cursor.fetchone()
- totals = {key: int(row[key]) for key in _TOTAL_COLUMNS}
- field_mismatches = {key: int(value) for key, value in row.items() if key not in _TOTAL_COLUMNS}
- drifted = totals["missing_in_personhog"] + totals["missing_in_legacy"] + totals["mismatched_rows"]
- samples: list[str] = []
- # The sample query repeats the full join, so skip it when the counts already say it returns nothing.
- if sample_size > 0 and drifted > 0:
- cursor.execute(sample_sql, {"limit": sample_size})
- samples = [format_sample(sample) for sample in cursor.fetchall()]
- return DriftCategoryReport(category=category, field_mismatches=field_mismatches, samples=samples, **totals)
-
-
-def _configure_session(cursor: psycopg2.extensions.cursor, statement_timeout_minutes: int) -> None:
- cursor.execute("SET application_name = 'dagster_personhog_shadow_drift'")
- cursor.execute("SET statement_timeout = %s", (f"{statement_timeout_minutes}min",))
- # work_mem applies per hash node and per parallel worker, and hash joins
- # get hash_mem_multiplier times it, so a three-join query can reserve
- # many times this value at once.
- cursor.execute("SET work_mem = '256MB'")
-
-
-def compute_shadow_drift(
- connection: psycopg2.extensions.connection, sample_size: int, statement_timeout_minutes: int = 30
-) -> list[DriftCategoryReport]:
- with connection.cursor() as cursor:
- _configure_session(cursor, statement_timeout_minutes)
- return [
- _run_category(cursor, "persons", _PERSON_DRIFT_SQL, _PERSON_SAMPLE_SQL, _person_samples, sample_size),
- _run_category(
- cursor,
- "distinct_ids",
- _DISTINCT_ID_DRIFT_SQL,
- _DISTINCT_ID_SAMPLE_SQL,
- _distinct_id_samples,
- sample_size,
- ),
- _run_category(
- cursor,
- "hash_key_overrides",
- _HASH_KEY_DRIFT_SQL,
- _HASH_KEY_SAMPLE_SQL,
- _hash_key_samples,
- sample_size,
- ),
- ]
-
-
-def _format_property_side(jsonb_type: object, text_length: object) -> str:
- if jsonb_type is None:
- return "absent"
- return f"{jsonb_type}({text_length})"
-
-
-def _format_person_property_detail(team_id: object, person_uuid: object, rows: list[Mapping[str, object]]) -> str:
- keys = "; ".join(
- f"{row['key']!r} {row['kind']} "
- f"legacy={_format_property_side(row['legacy_type'], row['legacy_length'])} "
- f"personhog={_format_property_side(row['personhog_type'], row['personhog_length'])}"
- for row in rows[:_PROPERTY_DRIFT_MAX_ROWS]
- )
- if len(rows) > _PROPERTY_DRIFT_MAX_ROWS:
- keys += "; ..."
- return f"team={team_id} uuid={person_uuid} differing_keys={len(rows)}: {keys}"
-
-
-def sample_property_drift(
- connection: psycopg2.extensions.connection,
- persons_limit: int,
- detail_limit: int,
- statement_timeout_minutes: int = 30,
-) -> PropertyDriftSample:
- detail_rows: list[Mapping[str, object]] = []
- with connection.cursor() as cursor:
- _configure_session(cursor, statement_timeout_minutes)
- cursor.execute(_PROPERTY_KEY_DRIFT_SQL, {"limit": persons_limit})
- key_rows = cursor.fetchall()
- if detail_limit > 0:
- cursor.execute(_PROPERTY_DETAIL_SQL, {"limit": min(detail_limit, persons_limit)})
- detail_rows = cursor.fetchall()
-
- key_drifts = [PropertyKeyDrift(key=row["key"], kind=row["kind"], persons=int(row["persons"])) for row in key_rows]
- sampled_persons = int(key_rows[0]["sampled_persons"]) if key_rows else 0
- person_details = [
- _format_person_property_detail(team_id, person_uuid, list(person_rows))
- for (team_id, person_uuid), person_rows in groupby(detail_rows, key=lambda row: (row["team_id"], row["uuid"]))
- ]
- return PropertyDriftSample(sampled_persons=sampled_persons, key_drifts=key_drifts, person_details=person_details)
-
-
-def _property_drift_table(sample: PropertyDriftSample) -> str:
- lines = ["| key | kind | persons |", "|---|---|---|"]
- for drift in sample.key_drifts[:_PROPERTY_DRIFT_MAX_ROWS]:
- key = drift.key.replace("|", "\\|").replace("\n", "\\n").replace("\r", "\\r")
- lines.append(f"| {key} | {drift.kind} | {drift.persons} |")
- return "\n".join(lines)
-
-
-def record_settle_gauges(
- registry: CollectorRegistry, database: str, settle_seconds: float, completed_at: float
-) -> None:
- Gauge(
- "posthog_personhog_shadow_lane_settle_seconds",
- "Time the shadow persons database took to stop receiving writes after the lane stopped",
- ["database"],
- registry=registry,
- ).labels(database=database).set(settle_seconds)
- Gauge(
- "posthog_personhog_shadow_lane_settle_last_success_timestamp_seconds",
- "Unix time when the shadow persons database last settled after a stop",
- ["database"],
- registry=registry,
- ).labels(database=database).set(completed_at)
-
-
-def record_drift_gauges(
- registry: CollectorRegistry, database: str, reports: list[DriftCategoryReport], completed_at: float
-) -> None:
- def per_category(name: str, help_text: str) -> Gauge:
- return Gauge(
- f"posthog_personhog_shadow_lane_drift_{name}", help_text, ["database", "category"], registry=registry
- )
-
- legacy_rows = per_category("legacy_rows", "Live rows the legacy path holds")
- personhog_rows = per_category("personhog_rows", "Live rows the personhog path holds")
- missing_in_personhog = per_category("missing_in_personhog_rows", "Keys only the legacy path holds")
- missing_in_legacy = per_category("missing_in_legacy_rows", "Keys only the personhog path holds")
- mismatched = per_category("mismatched_rows", "Keys both paths hold with different content; excludes version")
- ratio = per_category("ratio", "Drifted keys over all compared keys, from 0 to 1")
- field_mismatched = Gauge(
- "posthog_personhog_shadow_lane_drift_field_mismatched_rows",
- "Keys both paths hold where one field differs. A key with two different fields counts in both",
- ["database", "category", "field"],
- registry=registry,
- )
-
- for report in reports:
- labels = {"database": database, "category": report.category}
- legacy_rows.labels(**labels).set(report.legacy_total)
- personhog_rows.labels(**labels).set(report.personhog_total)
- missing_in_personhog.labels(**labels).set(report.missing_in_personhog)
- missing_in_legacy.labels(**labels).set(report.missing_in_legacy)
- mismatched.labels(**labels).set(report.mismatched_rows)
- ratio.labels(**labels).set(report.drift_pct / 100)
- for field_name, count in report.field_mismatches.items():
- field_mismatched.labels(field=field_name, **labels).set(count)
-
- Gauge(
- "posthog_personhog_shadow_lane_drift_last_success_timestamp_seconds",
- "Unix time when the last drift report finished",
- ["database"],
- registry=registry,
- ).labels(database=database).set(completed_at)
-
-
-class ShadowLaneStopConfig(dagster.Config):
- namespace: str = SHADOW_NAMESPACE
- consumer_deployment: str = SHADOW_CONSUMER_DEPLOYMENT
- processor_deployment: str = SHADOW_PROCESSOR_DEPLOYMENT
- # Per deployment: the consumers get the full budget to drain before the processors scale.
- stop_timeout_seconds: int = 600
-
-
-class ShadowSettleConfig(dagster.Config):
- shadow_db_env_var: str = SHADOW_DB_URL_ENV_VAR
- poll_seconds: int = 30
- stable_checks: int = 4
- timeout_seconds: int = 3600
-
-
-class ShadowDriftConfig(dagster.Config):
- shadow_db_env_var: str = SHADOW_DB_URL_ENV_VAR
- sample_size: int = 10
- property_diff_sample_size: int = 500
- statement_timeout_minutes: int = Field(default=30, gt=0)
-
-
-def stop_shadow_lane(context: dagster.OpExecutionContext, config: ShadowLaneStopConfig) -> None:
- apps = apps_api()
-
- def is_stopped(deployment: str) -> bool:
- if deployment_pod_count(apps, config.namespace, deployment) > 0:
- return False
- context.log.info(f"{deployment} has no pods left")
- return True
-
- # Consumers first, and processors only once the consumers' pods are gone. A consumer drains its in-flight
- # batches before it exits, so the processors then hold no un-acked work. A processor scaled down while a
- # consumer still waits on it gets only its drain window to finish those batches, and fails what is left.
- for deployment in (config.consumer_deployment, config.processor_deployment):
- context.log.info(f"Scaling {config.namespace}/{deployment} to 0 replicas")
- scale_deployment(apps, config.namespace, deployment, 0)
- pending = wait_for_deployments([deployment], is_stopped, timeout_seconds=config.stop_timeout_seconds)
- if pending:
- raise dagster.Failure(
- description=f"Pods still running after {config.stop_timeout_seconds}s: {', '.join(sorted(pending))}"
- )
-
-
[email protected](ins={"lane_stopped": dagster.In(dagster.Nothing)})
-def wait_for_shadow_settle(context: dagster.OpExecutionContext, config: ShadowSettleConfig) -> None:
- """Wait until nothing writes to the shadow persons database any more.
-
- The consumers are down, but the personhog writer keeps applying its
- changelog backlog. A quiet database cannot be told apart from a stalled
- writer here: if the writer crashed with backlog uncommitted, the counter
- also holds still. Identity creates the person row immediately with
- empty properties, and the writer fills them in later, so a stalled
- writer mostly shows as properties mismatches, not missing_in_personhog.
- Check the writer's health before trusting an unexpectedly large
- properties count.
- """
- with closing(shadow_db_connection(config.shadow_db_env_var)) as connection:
- connection.autocommit = True
-
- started = time.monotonic()
- try:
- polls = wait_for_quiescence(
- lambda: read_shadow_write_counter(connection),
- poll_seconds=config.poll_seconds,
- stable_checks=config.stable_checks,
- timeout_seconds=config.timeout_seconds,
- on_progress=context.log.info,
- )
- except TimeoutError as timeout:
- raise dagster.Failure(
- description=(
- f"The shadow persons database is still receiving writes after {config.timeout_seconds}s. "
- "The personhog writer may still be draining a large backlog, or something outside the lane "
- "is writing to this database. The consumers stay down; re-run this job to keep waiting."
- )
- ) from timeout
-
- waited = time.monotonic() - started
- context.log.info(f"Shadow persons database settled after {waited:.0f}s ({polls} polls)")
- with pushed_metrics_registry(f"{SETTLE_METRICS_JOB}_{config.shadow_db_env_var}") as registry:
- record_settle_gauges(registry, config.shadow_db_env_var, waited, time.time())
- context.add_output_metadata({"settle_seconds": dagster.MetadataValue.float(round(waited, 1))})
-
-
[email protected](ins={"settled": dagster.In(dagster.Nothing)})
-def report_shadow_drift(context: dagster.OpExecutionContext, config: ShadowDriftConfig) -> None:
- with closing(shadow_db_connection(config.shadow_db_env_var)) as connection:
- reports = compute_shadow_drift(connection, config.sample_size, config.statement_timeout_minutes)
- persons_report = next(report for report in reports if report.category == "persons")
- property_drift: PropertyDriftSample | None = None
- if config.property_diff_sample_size > 0 and persons_report.field_mismatches.get("properties", 0) > 0:
- property_drift = sample_property_drift(
- connection,
- persons_limit=config.property_diff_sample_size,
- detail_limit=config.sample_size,
- statement_timeout_minutes=config.statement_timeout_minutes,
- )
-
- with pushed_metrics_registry(f"{DRIFT_METRICS_JOB}_{config.shadow_db_env_var}") as registry:
- record_drift_gauges(registry, config.shadow_db_env_var, reports, time.time())
-
- header = "| category | legacy | personhog | missing in personhog | missing in legacy | mismatched | drift % |"
- separator = "|---|---|---|---|---|---|---|"
- lines = [header, separator]
- metadata: dict[str, dagster.MetadataValue] = {}
- for report in reports:
- lines.append(
- f"| {report.category} | {report.legacy_total} | {report.personhog_total} "
- f"| {report.missing_in_personhog} | {report.missing_in_legacy} "
- f"| {report.mismatched_rows} | {report.drift_pct:.4f} |"
- )
- metadata[f"{report.category}_drift_pct"] = dagster.MetadataValue.float(round(report.drift_pct, 4))
- context.log.info(
- f"[{report.category}] legacy={report.legacy_total} personhog={report.personhog_total} "
- f"missing_in_personhog={report.missing_in_personhog} missing_in_legacy={report.missing_in_legacy} "
- f"mismatched_rows={report.mismatched_rows} drift={report.drift_pct:.4f}% "
- f"field_mismatches={report.field_mismatches}"
- )
- for sample in report.samples:
- context.log.info(f"[{report.category}] drift sample: {sample}")
-
- metadata["summary"] = dagster.MetadataValue.md("\n".join(lines))
- if property_drift is not None:
- context.log.info(
- f"[persons] property drift by key over {property_drift.sampled_persons} sampled persons "
- f"({len(property_drift.key_drifts)} key rows, up to {_PROPERTY_DRIFT_MAX_ROWS} shown):\n"
- f"{_property_drift_table(property_drift)}"
- )
- for detail in property_drift.person_details:
- context.log.info(f"[persons] property drift detail: {detail}")
- metadata["persons_property_drift_sampled_persons"] = dagster.MetadataValue.int(property_drift.sampled_persons)
- metadata["persons_property_drift_key_rows"] = dagster.MetadataValue.int(len(property_drift.key_drifts))
- metadata["persons_property_drift_keys"] = dagster.MetadataValue.md(_property_drift_table(property_drift))
- context.add_output_metadata(metadata)
-
-
[email protected](tags={"owner": JobOwners.TEAM_INGESTION.value})
-def personhog_shadow_lane_stop_and_compare_job() -> None:
- """Stop the shadow lane, wait for writes to drain, and report drift between the two paths.
-
- Leaves the shadow persons database untouched so the drift behind the
- report can be inspected over a direct connection afterwards.
- """
- report_shadow_drift(wait_for_shadow_settle(stop_shadow_lane()))
diff --git a/posthog/dags/personhog_shadow_lane.py b/posthog/dags/personhog_shadow_lane.py
deleted file mode 100644
index 56f59908c2a4..000000000000
--- a/posthog/dags/personhog_shadow_lane.py
+++ /dev/null
@@ -1,520 +0,0 @@
-"""Operate the personhog shadow validation lane.
-
-The shadow lane is a second consumer group on a production ingestion topic,
-set in the lane's deployment, whose worker runs with PERSONS_STORE_MODE=shadow:
-every person write goes through both the legacy direct-to-Postgres path and
-the personhog path, into a dedicated persons-shadow database. Validation runs compare the two paths'
-tables afterwards, so each run must start from identical (empty) state on
-both sides.
-
-This module holds the shared plumbing (Kubernetes scaling, shadow DB access,
-the table inventory) and the start job. The stop/drain/compare job lives in
-personhog_shadow_drift.py.
-
-The Dagster service account needs get/patch on deployments and
-deployments/scale plus list on pods in the lane namespace, and the lane's
-ArgoCD Applications
-must ignore Deployment spec.replicas, or ArgoCD self-heal reverts every scale
-these jobs apply. Both are charts-side prerequisites. Resetting the lane's
-offsets also needs the Kafka bootstrap servers of the lane's cluster in
-PERSONHOG_SHADOW_KAFKA_BOOTSTRAP_SERVERS on the Dagster deployment.
-"""
-
-import os
-import time
-from collections.abc import Callable, Iterable
-from concurrent.futures import Future
-from contextlib import closing
-from typing import Any, Protocol
-from urllib.parse import parse_qs, urlparse
-
-import dagster
-import psycopg2
-import psycopg2.extras
-from confluent_kafka import KafkaError, KafkaException
-from confluent_kafka.admin import AdminClient
-from kubernetes import (
- client as k8s_client,
- config as k8s_config,
-)
-from prometheus_client import CollectorRegistry, Gauge
-from pydantic import Field
-
-from posthog.dags.common import JobOwners
-from posthog.metrics import pushed_metrics_registry
-
-SHADOW_NAMESPACE = "ingestion-analytics-team2-shadow"
-SHADOW_CONSUMER_DEPLOYMENT = "ingestion-analytics-team2-shadow-consumer"
-SHADOW_PROCESSOR_DEPLOYMENT = "ingestion-analytics-team2-shadow-processor"
-SHADOW_DB_URL_ENV_VAR = "PERSONS_SHADOW_DB_URL"
-SHADOW_CONSUMER_GROUP = "ingestion-analytics-team2-shadow"
-SHADOW_KAFKA_BOOTSTRAP_ENV_VAR = "PERSONHOG_SHADOW_KAFKA_BOOTSTRAP_SERVERS"
-SHADOW_KAFKA_SECURITY_PROTOCOL_ENV_VAR = "PERSONHOG_SHADOW_KAFKA_SECURITY_PROTOCOL"
-# The client library defines these error codes at runtime, but its type stubs omit them.
-GROUP_ID_NOT_FOUND: int = KafkaError.GROUP_ID_NOT_FOUND # type: ignore[attr-defined]
-NON_EMPTY_GROUP: int = KafkaError.NON_EMPTY_GROUP # type: ignore[attr-defined]
-
-START_METRICS_JOB = "personhog_shadow_lane_start"
-
-# Every table the lane writes, per path. The reset truncates both lists in one
-# statement so a validation run starts from state where "row missing on one
-# side" can only mean drift. Kept in sync with rust/persons_migrations/.
-LEGACY_STATE_TABLES = [
- "posthog_person",
- "posthog_persondistinctid",
- "posthog_personlessdistinctid",
- "posthog_featureflaghashkeyoverride",
- "posthog_personoverridemapping",
- "posthog_personoverride",
- "posthog_pendingpersonoverride",
- "posthog_flatpersonoverride",
- "posthog_group",
- "posthog_grouptypemapping",
-]
-# Saga bookkeeping the identity sweeper rewrites on every pass: it garbage-collects
-# completed ops past retention and re-claims failing ones. Excluded from the drain
-# counter because that activity never stops, and any saga step that changes person
-# state also lands in the person, distinct id or override tables.
-LIFECYCLE_OP_TABLES = [
- "lifecycle_op",
- "lifecycle_op_person",
- "lifecycle_op_tmp",
- "lifecycle_op_person_tmp",
-]
-PERSONHOG_STATE_TABLES = [
- "personhog_person_tmp",
- "personhog_persondistinctid_tmp",
- "personhog_featureflaghashkeyoverride_tmp",
- *LIFECYCLE_OP_TABLES,
- "person_pg_cleanup_queue",
- "person_tombstone_publish_queue",
- "person_publish_outbox",
- "person_publish_outbox_tmp",
-]
-
-
-def require_shadow_dsn(connection_url: str) -> None:
- """Refuse a DSN that does not look like the persons-shadow cluster.
-
- The reset truncates person tables. The only structural difference between
- the shadow DSN and the production persons DSN is the cluster name, so this
- guard is what stands between a misconfigured env var and truncating
- production state.
- """
- parsed = urlparse(connection_url)
- # psycopg2 also accepts libpq key/value strings ("host=... dbname=..."),
- # which urlparse leaves whole in .path, so every check below would see an
- # empty authority and pass on "shadow" appearing in any field.
- if parsed.scheme not in ("postgres", "postgresql"):
- raise dagster.Failure(
- description=(
- "Connection URL must be a postgres:// or postgresql:// URI. "
- "Refusing a key/value DSN whose target the guard cannot check."
- )
- )
- # libpq resolves the connection target from more than the URL authority:
- # host, hostaddr, dbname, and service query parameters override it, and a
- # comma in the host names several servers. Any of those lets a URL carry
- # a shadow-looking hostname while connecting elsewhere, so reject them
- # instead of trying to validate what libpq would do.
- overrides = {"host", "hostaddr", "dbname", "service"} & set(parse_qs(parsed.query))
- if overrides:
- raise dagster.Failure(
- description=(
- f"Connection URL overrides its target through query parameters ({', '.join(sorted(overrides))}). "
- "Refusing a DSN whose effective target can differ from its hostname."
- )
- )
- hostname = parsed.hostname or ""
- if "," in (parsed.netloc or ""):
- raise dagster.Failure(
- description="Connection URL names several hosts. Refusing a DSN with more than one target."
- )
- identity = f"{hostname}/{parsed.path.lstrip('/')}"
- if "shadow" not in identity:
- raise dagster.Failure(
- description=(
- f"Connection target {identity!r} does not contain 'shadow'. "
- "Refusing to operate on a database that is not the persons-shadow cluster."
- )
- )
-
-
-def shadow_db_connection(env_var: str) -> psycopg2.extensions.connection:
- connection_url = os.environ.get(env_var)
- if not connection_url:
- raise dagster.Failure(
- description=(
- f"Environment variable {env_var} is not set on the Dagster deployment. "
- "It must hold the persons-shadow database connection URL."
- )
- )
- require_shadow_dsn(connection_url)
- return psycopg2.connect(
- connection_url,
- cursor_factory=psycopg2.extras.RealDictCursor,
- connect_timeout=10,
- options="-c application_name=dagster_personhog_shadow_lane",
- )
-
-
-def _load_k8s_config() -> None:
- try:
- k8s_config.load_incluster_config()
- except k8s_config.ConfigException:
- k8s_config.load_kube_config()
-
-
-def apps_api() -> k8s_client.AppsV1Api:
- _load_k8s_config()
- return k8s_client.AppsV1Api()
-
-
-def deployment_pod_count(apps: k8s_client.AppsV1Api, namespace: str, name: str) -> int:
- """Count the deployment's pods straight from the pod list.
-
- status.replicas lags a fresh scale and leaves out terminating pods, which
- can still write during their grace period, so the reset guard cannot rely
- on it.
- """
- deployment = apps.read_namespaced_deployment(name=name, namespace=namespace)
- if deployment.spec.replicas:
- return deployment.spec.replicas
- selector = ",".join(f"{key}={value}" for key, value in deployment.spec.selector.match_labels.items())
- pods = k8s_client.CoreV1Api().list_namespaced_pod(namespace=namespace, label_selector=selector)
- terminal_phases = {"Succeeded", "Failed"}
- return sum(1 for pod in pods.items if pod.status.phase not in terminal_phases)
-
-
-def scale_deployment(apps: k8s_client.AppsV1Api, namespace: str, name: str, replicas: int) -> None:
- apps.patch_namespaced_deployment_scale(name=name, namespace=namespace, body={"spec": {"replicas": replicas}})
-
-
-def deployment_ready_replicas(apps: k8s_client.AppsV1Api, namespace: str, name: str) -> int:
- deployment = apps.read_namespaced_deployment(name=name, namespace=namespace)
- return deployment.status.ready_replicas or 0
-
-
-def wait_for_deployments(
- deployments: Iterable[str],
- is_settled: Callable[[str], bool],
- *,
- timeout_seconds: float,
- poll_seconds: float = 10,
- sleep: Callable[[float], None] = time.sleep,
-) -> set[str]:
- """Poll each deployment until is_settled accepts it or the deadline passes.
-
- Returns the deployments still pending at the deadline, so the caller
- decides how to report them.
- """
- deadline = time.monotonic() + timeout_seconds
- pending = set(deployments)
- while pending and time.monotonic() < deadline:
- for deployment in sorted(pending):
- if is_settled(deployment):
- pending.discard(deployment)
- if pending:
- sleep(poll_seconds)
- return pending
-
-
-def wait_for_quiescence(
- read_write_counter: Callable[[], int],
- *,
- poll_seconds: float,
- stable_checks: int,
- timeout_seconds: float,
- sleep: Callable[[float], None] = time.sleep,
- on_progress: Callable[[str], None] | None = None,
-) -> int:
- """Poll a monotonic write counter until it holds still for stable_checks consecutive polls.
-
- Returns the number of polls taken. Raises TimeoutError when the counter is
- still moving at the deadline.
- """
- deadline = time.monotonic() + timeout_seconds
- last = read_write_counter()
- stable = 0
- polls = 1
- while stable < stable_checks:
- if time.monotonic() >= deadline:
- raise TimeoutError(f"write counter still moving after {timeout_seconds}s (last value {last})")
- sleep(poll_seconds)
- current = read_write_counter()
- polls += 1
- if current == last:
- stable += 1
- else:
- if on_progress is not None:
- on_progress(f"writes still landing: counter moved {last} -> {current}")
- stable = 0
- last = current
- return polls
-
-
-def read_shadow_write_counter(connection: psycopg2.extensions.connection) -> int:
- """Sum the database's tuple-write counters, minus the sweeper's bookkeeping tables.
-
- The shadow database serves only the lane, so a stable sum means every
- writer has drained; a table allowlist would silently go stale when the
- lane gains a table. LIFECYCLE_OP_TABLES are the one exception, because
- the identity sweeper keeps them moving while the lane is stopped. The
- connection must be in autocommit so each poll is its own transaction and
- reads a fresh pg_stat snapshot instead of the first transaction's cached one.
- """
- with connection.cursor() as cursor:
- cursor.execute(
- "SELECT COALESCE(SUM(n_tup_ins + n_tup_upd + n_tup_del), 0) AS writes "
- "FROM pg_stat_user_tables WHERE relname != ALL(%(excluded)s)",
- {"excluded": LIFECYCLE_OP_TABLES},
- )
- return int(cursor.fetchone()["writes"])
-
-
-class ShadowLaneStartConfig(dagster.Config):
- reset_state: bool = False
- # Deletes the lane's consumer group, so the lane starts at the end of its topic instead of its committed offsets.
- reset_offsets: bool = False
- consumer_replicas: int = Field(default=4, gt=0, le=512)
- processor_replicas: int = Field(default=8, gt=0, le=512)
- processor_step_replicas: int = Field(default=64, gt=0, le=512)
- namespace: str = SHADOW_NAMESPACE
- consumer_deployment: str = SHADOW_CONSUMER_DEPLOYMENT
- processor_deployment: str = SHADOW_PROCESSOR_DEPLOYMENT
- shadow_db_env_var: str = SHADOW_DB_URL_ENV_VAR
- # Applies to each processor step and to the consumers separately.
- ready_timeout_seconds: int = 600
- # How long the reset waits for in-flight writes to stop before truncating.
- reset_settle_poll_seconds: int = 10
- reset_settle_stable_checks: int = Field(default=3, ge=1)
- reset_settle_timeout_seconds: int = 600
-
-
-def _require_lane_stopped(config: ShadowLaneStartConfig, apps: k8s_client.AppsV1Api, option: str, reason: str) -> None:
- for deployment in (config.consumer_deployment, config.processor_deployment):
- pods = deployment_pod_count(apps, config.namespace, deployment)
- if pods > 0:
- raise dagster.Failure(
- description=(
- f"Deployment {deployment} still wants or has {pods} pod(s). "
- f"{reason} "
- f"Run the stop-and-compare job first, then start with {option}."
- )
- )
-
-
-def _reset_shadow_state(
- context: dagster.OpExecutionContext, config: ShadowLaneStartConfig, apps: k8s_client.AppsV1Api
-) -> None:
- _require_lane_stopped(
- config, apps, "reset_state", "A reset while consumers run would leave the two paths inconsistent."
- )
-
- tables = LEGACY_STATE_TABLES + PERSONHOG_STATE_TABLES
- with closing(shadow_db_connection(config.shadow_db_env_var)) as connection:
- connection.autocommit = True
-
- # The lane deployments are down, but the personhog writer drains its
- # changelog into these tables from outside the lane namespace. A
- # truncate that races it leaves old-run rows behind, so wait for the
- # whole database to go quiet first.
- try:
- wait_for_quiescence(
- lambda: read_shadow_write_counter(connection),
- poll_seconds=config.reset_settle_poll_seconds,
- stable_checks=config.reset_settle_stable_checks,
- timeout_seconds=config.reset_settle_timeout_seconds,
- on_progress=context.log.info,
- )
- except TimeoutError as timeout:
- raise dagster.Failure(
- description=(
- f"The shadow persons database is still receiving writes after "
- f"{config.reset_settle_timeout_seconds}s, most likely the personhog writer draining its "
- "backlog. Nothing was truncated; re-run once it settles."
- )
- ) from timeout
-
- context.log.info(f"Truncating {len(tables)} tables in the shadow persons database: {', '.join(tables)}")
- with connection.cursor() as cursor:
- cursor.execute("SET lock_timeout = '30s'")
- cursor.execute("SET statement_timeout = '10min'")
- # One statement so the reset is atomic; CASCADE covers the FKs
- # between the person and distinct id tables on both sides. No
- # RESTART IDENTITY: the shadow personhog leader stays up across
- # resets and caches persons by (team_id, person_id) without
- # re-reading Postgres, so restarted sequences would map new rows
- # to stale cached persons from the previous run.
- cursor.execute(f"TRUNCATE {', '.join(tables)} CASCADE")
- context.log.info("Shadow persons database reset complete")
-
-
-class ConsumerGroupAdmin(Protocol):
- def delete_consumer_groups(self, group_ids: list[str], **kwargs: Any) -> dict[str, Future]: ...
-
-
-def shadow_kafka_admin() -> AdminClient:
- bootstrap_servers = os.environ.get(SHADOW_KAFKA_BOOTSTRAP_ENV_VAR)
- if not bootstrap_servers:
- raise dagster.Failure(
- description=(
- f"Environment variable {SHADOW_KAFKA_BOOTSTRAP_ENV_VAR} is not set on the Dagster deployment. "
- "It must hold the bootstrap servers of the Kafka cluster the shadow lane consumes from."
- )
- )
- return AdminClient(
- {
- "bootstrap.servers": bootstrap_servers,
- "security.protocol": os.environ.get(SHADOW_KAFKA_SECURITY_PROTOCOL_ENV_VAR, "SSL"),
- # The wheel's bundled OpenSSL does not read the system CA store unless told to probe for it.
- "ssl.ca.location": "probe",
- }
- )
-
-
-def _reset_consumer_offsets(
- context: dagster.OpExecutionContext,
- config: ShadowLaneStartConfig,
- apps: k8s_client.AppsV1Api,
- admin_factory: Callable[[], ConsumerGroupAdmin] = shadow_kafka_admin,
-) -> bool:
- """Delete the lane's consumer group, so the next start falls back to the consumer's offset reset policy.
-
- The ingestion consumer resets to the latest offset when its group has no committed offset, so the lane
- starts at the end of its topic. Returns False when the group did not exist.
- """
- _require_lane_stopped(config, apps, "reset_offsets", "Kafka refuses to delete a consumer group with members.")
- admin = admin_factory()
- future = admin.delete_consumer_groups([SHADOW_CONSUMER_GROUP], request_timeout=30)[SHADOW_CONSUMER_GROUP]
- try:
- future.result()
- except KafkaException as exception:
- error = exception.args[0]
- if error.code() == GROUP_ID_NOT_FOUND:
- context.log.info(f"Consumer group {SHADOW_CONSUMER_GROUP} does not exist, nothing to reset")
- return False
- if error.code() == NON_EMPTY_GROUP:
- raise dagster.Failure(
- description=(
- f"Consumer group {SHADOW_CONSUMER_GROUP} still has members, so Kafka refused to delete it. "
- "Wait for the lane's consumers to leave the group, then retry."
- )
- ) from exception
- raise
- context.log.info(f"Deleted consumer group {SHADOW_CONSUMER_GROUP}; the lane starts from the latest offsets")
- return True
-
-
-def processor_ramp(current: int, target: int, step: int) -> list[int]:
- """Replica counts to scale the processors through, ending at target.
-
- A partly scaled deployment repeats its current count first, so a retried run waits for
- pods still pending from the last run before it adds more. A scale down goes straight to
- target, since removing pods needs no new nodes.
- """
- if target <= current:
- return [target]
- first = current if current > 0 else step
- return [*range(first, target, step), target]
-
-
-def _scale_and_wait(
- context: dagster.OpExecutionContext,
- config: ShadowLaneStartConfig,
- apps: k8s_client.AppsV1Api,
- deployment: str,
- replicas: int,
-) -> None:
- context.log.info(f"Scaling {config.namespace}/{deployment} to {replicas} replicas")
- scale_deployment(apps, config.namespace, deployment, replicas)
-
- def is_ready(name: str) -> bool:
- return deployment_ready_replicas(apps, config.namespace, name) >= replicas
-
- if wait_for_deployments([deployment], is_ready, timeout_seconds=config.ready_timeout_seconds):
- raise dagster.Failure(
- description=(
- f"{deployment} did not reach {replicas} ready replicas within {config.ready_timeout_seconds}s. "
- "The scale was applied; check the pods in the lane namespace."
- )
- )
- context.log.info(f"{deployment} is ready with {replicas} replica(s)")
-
-
-def record_start_gauges(registry: CollectorRegistry, config: ShadowLaneStartConfig, completed_at: float) -> None:
- Gauge(
- "posthog_personhog_shadow_lane_start_last_success_timestamp_seconds",
- "Unix time when the shadow lane last started with every replica ready",
- ["namespace"],
- registry=registry,
- ).labels(namespace=config.namespace).set(completed_at)
- Gauge(
- "posthog_personhog_shadow_lane_start_reset_state",
- "1 when the last start truncated the shadow persons database first, 0 when it resumed from existing state",
- ["namespace"],
- registry=registry,
- ).labels(namespace=config.namespace).set(1 if config.reset_state else 0)
- Gauge(
- "posthog_personhog_shadow_lane_start_reset_offsets",
- "1 when the last start deleted the lane's consumer group first, 0 when it resumed from committed offsets",
- ["namespace"],
- registry=registry,
- ).labels(namespace=config.namespace).set(1 if config.reset_offsets else 0)
- replicas = Gauge(
- "posthog_personhog_shadow_lane_start_replicas",
- "Replicas the last start requested, by deployment",
- ["namespace", "deployment"],
- registry=registry,
- )
- replicas.labels(namespace=config.namespace, deployment=config.consumer_deployment).set(config.consumer_replicas)
- replicas.labels(namespace=config.namespace, deployment=config.processor_deployment).set(config.processor_replicas)
-
-
-def start_shadow_lane(context: dagster.OpExecutionContext, config: ShadowLaneStartConfig) -> None:
- apps = apps_api()
- if config.reset_state:
- _reset_shadow_state(context, config, apps)
- else:
- context.log.info("reset_state is false, leaving the shadow persons database untouched")
- if config.reset_offsets:
- _reset_consumer_offsets(context, config, apps)
- else:
- context.log.info("reset_offsets is false, the lane resumes from its committed offsets")
-
- # Every processor is ready before any consumer starts. A consumer that starts first pins its keys to the
- # few processors already up, and those keys stay on them while they have work in flight.
- current = apps.read_namespaced_deployment(name=config.processor_deployment, namespace=config.namespace)
- for replicas in processor_ramp(
- current.spec.replicas or 0, config.processor_replicas, config.processor_step_replicas
- ):
- _scale_and_wait(context, config, apps, config.processor_deployment, replicas)
- _scale_and_wait(context, config, apps, config.consumer_deployment, config.consumer_replicas)
-
- with pushed_metrics_registry(f"{START_METRICS_JOB}_{config.namespace}") as registry:
- record_start_gauges(registry, config, time.time())
-
- context.add_output_metadata(
- {
- "reset_state": dagster.MetadataValue.bool(config.reset_state),
- "reset_offsets": dagster.MetadataValue.bool(config.reset_offsets),
- "consumer_replicas": dagster.MetadataValue.int(config.consumer_replicas),
- "processor_replicas": dagster.MetadataValue.int(config.processor_replicas),
- }
- )
-
-
[email protected](tags={"owner": JobOwners.TEAM_INGESTION.value})
-def personhog_shadow_lane_start_job():
- """Start the shadow lane consumers, optionally resetting the shadow persons database first.
-
- Run with default config to resume consuming from the committed offsets.
- Set ops.start_shadow_lane.config.reset_state to true to truncate both
- paths' tables before starting, which every fresh validation run needs.
- Set reset_offsets to true to start from the end of the topic instead.
- Both resets refuse to run while the lane has pods. Processors scale up in
- steps of processor_step_replicas, and the consumers start only once every
- processor is ready.
- """
- start_shadow_lane()
diff --git a/posthog/dags/tests/test_personhog_shadow_drift.py b/posthog/dags/tests/test_personhog_shadow_drift.py
deleted file mode 100644
index bcb54f3a1411..000000000000
--- a/posthog/dags/tests/test_personhog_shadow_drift.py
+++ /dev/null
@@ -1,197 +0,0 @@
-import uuid
-
-import pytest
-
-import psycopg2
-import psycopg2.extras
-from prometheus_client import CollectorRegistry
-
-from posthog.dags.personhog_shadow_drift import (
- DriftCategoryReport,
- PropertyKeyDrift,
- compute_shadow_drift,
- record_drift_gauges,
- sample_property_drift,
-)
-from posthog.persons_db import persons_db_url
-
-TEAM_ID = 990000123
-CREATED_AT = "2026-01-01T00:00:00+00:00"
-
-
-def _insert_person(
- cursor, table: str, person_uuid: uuid.UUID, properties: dict, is_deleted: bool = False, version: int = 1
-) -> int:
- cursor.execute(
- f"INSERT INTO {table} (created_at, properties, is_identified, uuid, version, team_id, is_deleted) "
- "VALUES (%s, %s, false, %s, %s, %s, %s) RETURNING id",
- (CREATED_AT, psycopg2.extras.Json(properties), str(person_uuid), version, TEAM_ID, is_deleted),
- )
- return cursor.fetchone()["id"]
-
-
[email protected]_db(transaction=True)
-def _session_statement_timeout(connection: psycopg2.extensions.connection) -> str:
- with connection.cursor() as cursor:
- cursor.execute("SHOW statement_timeout")
- return cursor.fetchone()["statement_timeout"]
-
-
-def test_compute_shadow_drift_counts_each_category() -> None:
- connection = psycopg2.connect(persons_db_url(writer=True), cursor_factory=psycopg2.extras.RealDictCursor)
- connection.autocommit = True
- matched = uuid.uuid4()
- legacy_only = uuid.uuid4()
- personhog_only = uuid.uuid4()
- props_differ = uuid.uuid4()
- tombstoned = uuid.uuid4()
-
- try:
- with connection.cursor() as cursor:
- legacy_matched = _insert_person(cursor, "posthog_person", matched, {"a": 1})
- _insert_person(cursor, "posthog_person", legacy_only, {})
- _insert_person(cursor, "posthog_person", props_differ, {"b": 1, "legacy_key": "xyz", "same": True})
-
- ph_matched = _insert_person(cursor, "personhog_person_tmp", matched, {"a": 1}, version=2)
- _insert_person(cursor, "personhog_person_tmp", personhog_only, {})
- ph_props = _insert_person(
- cursor, "personhog_person_tmp", props_differ, {"b": 2, "personhog_key": [1, 2], "same": True}
- )
- ph_tombstoned = _insert_person(cursor, "personhog_person_tmp", tombstoned, {}, is_deleted=True)
-
- cursor.execute(
- "INSERT INTO posthog_persondistinctid (distinct_id, version, person_id, team_id) VALUES "
- "('shadow-drift-did-1', 1, %s, %s), ('shadow-drift-did-2', 1, %s, %s)",
- (legacy_matched, TEAM_ID, legacy_matched, TEAM_ID),
- )
- cursor.execute(
- "INSERT INTO personhog_persondistinctid_tmp (distinct_id, version, person_id, team_id) VALUES "
- "('shadow-drift-did-1', 1, %s, %s), ('shadow-drift-did-2', 1, %s, %s), "
- "('shadow-drift-did-3', 1, %s, %s)",
- (ph_matched, TEAM_ID, ph_props, TEAM_ID, ph_tombstoned, TEAM_ID),
- )
-
- cursor.execute(
- "INSERT INTO posthog_featureflaghashkeyoverride (feature_flag_key, hash_key, person_id, team_id) "
- "VALUES ('flag-1', 'hash-same', %s, %s), ('flag-2', 'hash-legacy', %s, %s)",
- (legacy_matched, TEAM_ID, legacy_matched, TEAM_ID),
- )
- cursor.execute(
- "INSERT INTO personhog_featureflaghashkeyoverride_tmp (feature_flag_key, hash_key, person_id, team_id) "
- "VALUES ('flag-1', 'hash-same', %s, %s), ('flag-2', 'hash-personhog', %s, %s), "
- "('flag-3', 'hash-tombstoned', %s, %s)",
- (ph_matched, TEAM_ID, ph_matched, TEAM_ID, ph_tombstoned, TEAM_ID),
- )
-
- reports = {
- report.category: report
- for report in compute_shadow_drift(connection, sample_size=10, statement_timeout_minutes=239)
- }
- compute_timeout = _session_statement_timeout(connection)
- property_drift = sample_property_drift(
- connection, persons_limit=500, detail_limit=10, statement_timeout_minutes=241
- )
- sample_timeout = _session_statement_timeout(connection)
- finally:
- connection.close()
-
- assert (compute_timeout, sample_timeout) == ("239min", "241min")
-
- persons = reports["persons"]
- assert (persons.legacy_total, persons.personhog_total) == (3, 3)
- assert (persons.missing_in_personhog, persons.missing_in_legacy, persons.mismatched_rows) == (1, 1, 1)
- assert persons.field_mismatches == {"properties": 1, "is_identified": 0, "created_at": 0, "version": 1}
- assert persons.drift_pct == 75.0
- assert sorted(sample.rsplit(" ", 1)[1] for sample in persons.samples) == [
- "field_mismatch",
- "missing_in_legacy",
- "missing_in_personhog",
- ]
-
- assert property_drift.sampled_persons == 1
- assert property_drift.key_drifts == [
- PropertyKeyDrift(key="b", kind="value_differs", persons=1),
- PropertyKeyDrift(key="legacy_key", kind="only_in_legacy", persons=1),
- PropertyKeyDrift(key="personhog_key", kind="only_in_personhog", persons=1),
- ]
- assert property_drift.person_details == [
- f"team={TEAM_ID} uuid={props_differ} differing_keys=3: "
- "'b' value_differs legacy=number(1) personhog=number(1); "
- "'legacy_key' only_in_legacy legacy=string(5) personhog=absent; "
- "'personhog_key' only_in_personhog legacy=absent personhog=array(6)"
- ]
-
- distinct_ids = reports["distinct_ids"]
- assert (distinct_ids.legacy_total, distinct_ids.personhog_total) == (2, 2)
- assert (distinct_ids.missing_in_personhog, distinct_ids.missing_in_legacy, distinct_ids.mismatched_rows) == (
- 0,
- 0,
- 1,
- )
- assert distinct_ids.field_mismatches == {"version": 0}
- assert distinct_ids.samples == [
- f"team={TEAM_ID} distinct_id='shadow-drift-did-2' "
- f"legacy_person={matched} personhog_person={props_differ} person_mismatch"
- ]
-
- hash_keys = reports["hash_key_overrides"]
- assert (hash_keys.legacy_total, hash_keys.personhog_total) == (2, 2)
- assert (hash_keys.missing_in_personhog, hash_keys.missing_in_legacy, hash_keys.mismatched_rows) == (0, 0, 1)
- assert hash_keys.samples == [f"team={TEAM_ID} person={matched} flag=flag-2 hash_key_mismatch"]
-
-
-def test_drift_gauges_keep_each_count_under_its_own_name_and_category() -> None:
- reports = [
- DriftCategoryReport(
- category="persons",
- legacy_total=90,
- personhog_total=95,
- missing_in_personhog=5,
- missing_in_legacy=10,
- mismatched_rows=10,
- field_mismatches={"properties": 7, "version": 40},
- samples=[],
- ),
- DriftCategoryReport(
- category="distinct_ids",
- legacy_total=3,
- personhog_total=3,
- missing_in_personhog=0,
- missing_in_legacy=0,
- mismatched_rows=0,
- field_mismatches={"version": 0},
- samples=[],
- ),
- ]
- registry = CollectorRegistry()
-
- record_drift_gauges(registry, "PERSONS_SHADOW_DB_URL", reports, completed_at=1_700_000_000.0)
-
- def sample(name: str, **labels: str) -> float | None:
- return registry.get_sample_value(
- f"posthog_personhog_shadow_lane_drift_{name}", {"database": "PERSONS_SHADOW_DB_URL", **labels}
- )
-
- assert {
- name: sample(name, category="persons")
- for name in (
- "legacy_rows",
- "personhog_rows",
- "missing_in_personhog_rows",
- "missing_in_legacy_rows",
- "mismatched_rows",
- "ratio",
- )
- } == {
- "legacy_rows": 90,
- "personhog_rows": 95,
- "missing_in_personhog_rows": 5,
- "missing_in_legacy_rows": 10,
- "mismatched_rows": 10,
- "ratio": 0.25,
- }
- assert sample("field_mismatched_rows", category="persons", field="version") == 40
- assert sample("field_mismatched_rows", category="persons", field="properties") == 7
- assert sample("ratio", category="distinct_ids") == 0
- assert sample("last_success_timestamp_seconds") == 1_700_000_000.0
diff --git a/posthog/dags/tests/test_personhog_shadow_lane.py b/posthog/dags/tests/test_personhog_shadow_lane.py
deleted file mode 100644
index f32b8b6bbe0f..000000000000
--- a/posthog/dags/tests/test_personhog_shadow_lane.py
+++ /dev/null
@@ -1,410 +0,0 @@
-import uuid
-from collections.abc import Callable
-from concurrent.futures import Future
-from contextlib import nullcontext
-from functools import partial
-from types import SimpleNamespace
-
-import pytest
-
-import dagster
-import psycopg2
-import psycopg2.extras
-from confluent_kafka import KafkaError, KafkaException
-from parameterized import parameterized
-
-from posthog.dags.personhog_shadow_drift import ShadowLaneStopConfig, stop_shadow_lane
-from posthog.dags.personhog_shadow_lane import (
- GROUP_ID_NOT_FOUND,
- NON_EMPTY_GROUP,
- SHADOW_CONSUMER_DEPLOYMENT,
- SHADOW_CONSUMER_GROUP,
- SHADOW_KAFKA_BOOTSTRAP_ENV_VAR,
- SHADOW_PROCESSOR_DEPLOYMENT,
- ShadowLaneStartConfig,
- _reset_consumer_offsets,
- _reset_shadow_state,
- processor_ramp,
- read_shadow_write_counter,
- require_shadow_dsn,
- shadow_kafka_admin,
- start_shadow_lane,
- wait_for_deployments,
- wait_for_quiescence,
-)
-from posthog.persons_db import persons_db_url
-
-
-class TestRequireShadowDsn:
- @parameterized.expand(
- [
- ("postgres://user:[email protected]:5432/posthog",),
- ("postgres://user@localhost:5432/persons_shadow_test",),
- ]
- )
- def test_accepts_shadow_targets(self, url: str) -> None:
- require_shadow_dsn(url)
-
- @parameterized.expand(
- [
- (
- "postgres://user:pass@posthog-cloud-persons-prod-us-east-1.cluster-abc.us-east-1.rds.amazonaws.com:5432/posthog",
- ),
- ("postgres://user@localhost:5432/posthog_persons",),
- ("postgres://user:shadow@localhost:5432/posthog_persons",),
- # libpq lets query parameters and multi-host authorities point the
- # connection somewhere other than the URL's hostname.
- ("postgres://[email protected]:5432/posthog?hostaddr=10.0.0.1",),
- ("postgres://[email protected]:5432/posthog?host=prod.example",),
- ("postgres://[email protected]:5432/posthog?dbname=production",),
- ("postgres://[email protected]:5432/posthog?service=prod",),
- ("postgres://[email protected],prod.example:5432/posthog",),
- # A libpq key/value DSN has no URL authority for the guard to check.
- ("host=prod.example dbname=posthog user=dagster_shadow",),
- ]
- )
- def test_rejects_non_shadow_targets(self, url: str) -> None:
- with pytest.raises(dagster.Failure):
- require_shadow_dsn(url)
-
-
-class TestWaitForQuiescence:
- @parameterized.expand(
- [
- ("flat_counter", [5, 5, 5, 5], 3, 4),
- ("moving_then_flat", [5, 6, 6, 7, 7, 7], 2, 6),
- ]
- )
- def test_returns_once_counter_is_stable(
- self, _name: str, readings: list[int], stable_checks: int, expected_polls: int
- ) -> None:
- values = iter(readings)
- polls = wait_for_quiescence(
- lambda: next(values),
- poll_seconds=0,
- stable_checks=stable_checks,
- timeout_seconds=60,
- sleep=lambda _seconds: None,
- )
- assert polls == expected_polls
-
- def test_raises_when_counter_never_settles(self) -> None:
- counter = iter(range(1000))
- with pytest.raises(TimeoutError):
- wait_for_quiescence(
- lambda: next(counter),
- poll_seconds=0,
- stable_checks=2,
- timeout_seconds=0,
- sleep=lambda _seconds: None,
- )
-
-
-class TestWaitForDeployments:
- @parameterized.expand(
- [
- ("all_settle", {"consumer": 2, "processor": 1}, 60, set()),
- ("deadline_passed", {"consumer": 0, "processor": 0}, 0, {"consumer", "processor"}),
- ]
- )
- def test_returns_what_is_still_pending(
- self, _name: str, settles_on_check: dict[str, int], timeout_seconds: int, expected_pending: set[str]
- ) -> None:
- checks: dict[str, int] = dict.fromkeys(settles_on_check, 0)
-
- def is_settled(deployment: str) -> bool:
- checks[deployment] += 1
- return 0 < settles_on_check[deployment] <= checks[deployment]
-
- pending = wait_for_deployments(
- settles_on_check, is_settled, timeout_seconds=timeout_seconds, sleep=lambda _seconds: None
- )
- assert pending == expected_pending
-
-
-class TestProcessorRamp:
- @parameterized.expand(
- [
- ("from_zero", 0, 256, 64, [64, 128, 192, 256]),
- ("partial_last_step", 0, 150, 64, [64, 128, 150]),
- ("holds_at_current_before_stepping", 200, 512, 128, [200, 328, 456, 512]),
- ("target_below_one_step", 0, 8, 64, [8]),
- ("scale_down_is_direct", 512, 256, 64, [256]),
- ]
- )
- def test_steps(self, _name: str, current: int, target: int, step: int, expected: list[int]) -> None:
- assert processor_ramp(current, target, step) == expected
-
-
-class _ScalingAppsApi:
- def __init__(self, ready_cap: int | None = None, processor_replicas: int = 0) -> None:
- self.replicas = {SHADOW_PROCESSOR_DEPLOYMENT: processor_replicas, SHADOW_CONSUMER_DEPLOYMENT: 0}
- self.ready_cap = ready_cap
- self.scales: list[tuple[str, int]] = []
-
- def read_namespaced_deployment(self, name: str, namespace: str) -> SimpleNamespace:
- desired = self.replicas[name]
- ready = desired if self.ready_cap is None else min(desired, self.ready_cap)
- return SimpleNamespace(spec=SimpleNamespace(replicas=desired), status=SimpleNamespace(ready_replicas=ready))
-
- def patch_namespaced_deployment_scale(self, name: str, namespace: str, body: dict) -> None:
- self.replicas[name] = body["spec"]["replicas"]
- self.scales.append((name, self.replicas[name]))
-
-
-class _FakeClock:
- def __init__(self) -> None:
- self.now = 0.0
-
- def monotonic(self) -> float:
- return self.now
-
- def time(self) -> float:
- return self.now
-
- def sleep(self, seconds: float) -> None:
- self.now += seconds
-
-
-def scaling_apps(monkeypatch: pytest.MonkeyPatch) -> Callable[..., _ScalingAppsApi]:
- def install(**kwargs: int) -> _ScalingAppsApi:
- apps = _ScalingAppsApi(**kwargs)
- clock = _FakeClock()
- monkeypatch.setattr("posthog.dags.personhog_shadow_lane.apps_api", lambda: apps)
- monkeypatch.setattr("posthog.dags.personhog_shadow_lane.pushed_metrics_registry", lambda _job: nullcontext())
- monkeypatch.setattr("posthog.dags.personhog_shadow_lane.time", clock)
- monkeypatch.setattr(
- "posthog.dags.personhog_shadow_lane.wait_for_deployments",
- partial(wait_for_deployments, sleep=clock.sleep),
- )
- return apps
-
- return install
-
-
-def test_start_steps_processors_up_before_any_consumer(scaling_apps: Callable[..., _ScalingAppsApi]) -> None:
- apps = scaling_apps()
- config = ShadowLaneStartConfig(processor_replicas=150, processor_step_replicas=64, consumer_replicas=4)
-
- start_shadow_lane(dagster.build_op_context(), config)
-
- assert apps.scales == [
- (SHADOW_PROCESSOR_DEPLOYMENT, 64),
- (SHADOW_PROCESSOR_DEPLOYMENT, 128),
- (SHADOW_PROCESSOR_DEPLOYMENT, 150),
- (SHADOW_CONSUMER_DEPLOYMENT, 4),
- ]
-
-
[email protected]("processor_replicas", [0, 64], ids=["fresh_start", "retry_with_pods_still_pending"])
-def test_start_leaves_consumers_down_while_a_processor_step_is_unready(
- scaling_apps: Callable[..., _ScalingAppsApi], processor_replicas: int
-) -> None:
- apps = scaling_apps(ready_cap=0, processor_replicas=processor_replicas)
- config = ShadowLaneStartConfig(processor_replicas=150, consumer_replicas=4, ready_timeout_seconds=60)
-
- with pytest.raises(dagster.Failure, match="did not reach 64 ready replicas"):
- start_shadow_lane(dagster.build_op_context(), config)
-
- assert apps.scales == [(SHADOW_PROCESSOR_DEPLOYMENT, 64)]
-
-
-class _StoppingAppsApi:
- """A deployment scaled to zero keeps one pod for `linger_polls` pod-count polls, as a draining pod does."""
-
- def __init__(self, linger_polls: int) -> None:
- self.replicas = {SHADOW_CONSUMER_DEPLOYMENT: 4, SHADOW_PROCESSOR_DEPLOYMENT: 150}
- self.linger = dict.fromkeys(self.replicas, 0)
- self.linger_polls = linger_polls
- self.scales: list[tuple[str, int]] = []
- self.consumer_pods_at_scale: list[int] = []
-
- def pods(self, name: str) -> int:
- if self.replicas[name] > 0:
- return self.replicas[name]
- return 1 if self.linger[name] > 0 else 0
-
- def pod_count(self, name: str) -> int:
- count = self.pods(name)
- if self.replicas[name] == 0 and self.linger[name] > 0:
- self.linger[name] -= 1
- return count
-
- def patch_namespaced_deployment_scale(self, name: str, namespace: str, body: dict) -> None:
- self.consumer_pods_at_scale.append(self.pods(SHADOW_CONSUMER_DEPLOYMENT))
- self.replicas[name] = body["spec"]["replicas"]
- self.linger[name] = self.linger_polls
- self.scales.append((name, self.replicas[name]))
-
-
-def stopping_apps(monkeypatch: pytest.MonkeyPatch) -> Callable[..., _StoppingAppsApi]:
- def install(linger_polls: int) -> _StoppingAppsApi:
- apps = _StoppingAppsApi(linger_polls)
- clock = _FakeClock()
- monkeypatch.setattr("posthog.dags.personhog_shadow_lane.time", clock)
- monkeypatch.setattr("posthog.dags.personhog_shadow_drift.apps_api", lambda: apps)
- monkeypatch.setattr(
- "posthog.dags.personhog_shadow_drift.deployment_pod_count",
- lambda _apps, _namespace, name: apps.pod_count(name),
- )
- monkeypatch.setattr(
- "posthog.dags.personhog_shadow_drift.wait_for_deployments",
- partial(wait_for_deployments, sleep=clock.sleep),
- )
- return apps
-
- return install
-
-
-def test_stop_scales_processors_only_after_the_consumer_pods_are_gone(
- stopping_apps: Callable[..., _StoppingAppsApi],
-) -> None:
- apps = stopping_apps(linger_polls=2)
-
- # Each deployment drains in 20 s of polls, within its own 30 s budget; together they would exceed one.
- stop_shadow_lane(dagster.build_op_context(), ShadowLaneStopConfig(stop_timeout_seconds=30))
-
- assert apps.scales == [(SHADOW_CONSUMER_DEPLOYMENT, 0), (SHADOW_PROCESSOR_DEPLOYMENT, 0)]
- assert apps.consumer_pods_at_scale == [4, 0]
-
-
-def test_stop_leaves_processors_up_while_a_consumer_pod_lingers(
- stopping_apps: Callable[..., _StoppingAppsApi],
-) -> None:
- apps = stopping_apps(linger_polls=100)
-
- with pytest.raises(dagster.Failure, match="Pods still running"):
- stop_shadow_lane(dagster.build_op_context(), ShadowLaneStopConfig(stop_timeout_seconds=30))
-
- assert apps.scales == [(SHADOW_CONSUMER_DEPLOYMENT, 0)]
-
-
-class _FakeAppsApi:
- def __init__(self, desired_replicas: int) -> None:
- self.desired_replicas = desired_replicas
-
- def read_namespaced_deployment(self, name: str, namespace: str) -> SimpleNamespace:
- return SimpleNamespace(
- spec=SimpleNamespace(replicas=self.desired_replicas, selector=SimpleNamespace(match_labels={"app": name}))
- )
-
-
-class _NoPodsCoreApi:
- def list_namespaced_pod(self, namespace: str, label_selector: str) -> SimpleNamespace:
- return SimpleNamespace(items=[])
-
-
-def stopped_lane(monkeypatch: pytest.MonkeyPatch) -> _FakeAppsApi:
- monkeypatch.setattr("posthog.dags.personhog_shadow_lane.k8s_client.CoreV1Api", _NoPodsCoreApi)
- return _FakeAppsApi(desired_replicas=0)
-
-
-class _FakeKafkaAdmin:
- def __init__(self, error_code: int | None = None) -> None:
- self.error_code = error_code
- self.deleted: list[str] = []
-
- def delete_consumer_groups(self, group_ids: list[str], **_kwargs: object) -> dict[str, Future]:
- futures: dict[str, Future] = {}
- for group_id in group_ids:
- future: Future = Future()
- if self.error_code is None:
- self.deleted.append(group_id)
- future.set_result(None)
- else:
- future.set_exception(KafkaException(KafkaError(self.error_code)))
- futures[group_id] = future
- return futures
-
-
-def _no_admin() -> _FakeKafkaAdmin:
- raise AssertionError("the admin client must not be reached while the lane has pods")
-
-
- [
- (
- "reset_state",
- lambda context, apps: _reset_shadow_state(context, ShadowLaneStartConfig(reset_state=True), apps),
- ),
- (
- "reset_offsets",
- lambda context, apps: _reset_consumer_offsets(
- context, ShadowLaneStartConfig(reset_offsets=True), apps, _no_admin
- ),
- ),
- ]
-)
-def test_reset_refuses_while_lane_wants_pods(
- _name: str, reset: Callable[[dagster.OpExecutionContext, _FakeAppsApi], object]
-) -> None:
- with pytest.raises(dagster.Failure, match="stop-and-compare"):
- reset(dagster.build_op_context(), _FakeAppsApi(desired_replicas=2))
-
-
- "error_code,expected",
- [
- pytest.param(None, True, id="deleted"),
- pytest.param(GROUP_ID_NOT_FOUND, False, id="missing_group"),
- ],
-)
-def test_offset_reset_deletes_the_consumer_group(
- stopped_lane: _FakeAppsApi, error_code: int | None, expected: bool
-) -> None:
- admin = _FakeKafkaAdmin(error_code)
- config = ShadowLaneStartConfig(reset_offsets=True)
- deleted = _reset_consumer_offsets(dagster.build_op_context(), config, stopped_lane, lambda: admin)
- assert deleted == expected
- assert admin.deleted == ([SHADOW_CONSUMER_GROUP] if expected else [])
-
-
-def test_offset_reset_fails_while_the_group_has_members(stopped_lane: _FakeAppsApi) -> None:
- admin = _FakeKafkaAdmin(NON_EMPTY_GROUP)
- with pytest.raises(dagster.Failure, match="still has members"):
- _reset_consumer_offsets(
- dagster.build_op_context(), ShadowLaneStartConfig(reset_offsets=True), stopped_lane, lambda: admin
- )
-
-
-def test_offset_reset_needs_the_bootstrap_servers(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.delenv(SHADOW_KAFKA_BOOTSTRAP_ENV_VAR, raising=False)
- with pytest.raises(dagster.Failure, match=SHADOW_KAFKA_BOOTSTRAP_ENV_VAR):
- shadow_kafka_admin()
-
-
[email protected]_db(transaction=True)
-def test_write_counter_ignores_lifecycle_op_tables() -> None:
- connection = psycopg2.connect(persons_db_url(writer=True), cursor_factory=psycopg2.extras.RealDictCursor)
- connection.autocommit = True
- op_id = str(uuid.uuid4())
- person_uuid = str(uuid.uuid4())
- try:
- before = read_shadow_write_counter(connection)
-
- with connection.cursor() as cursor:
- cursor.execute(
- "INSERT INTO lifecycle_op_tmp (op_id, op_type, team_id, step, request) "
- "VALUES (%s, 'merge', 2, 'start', '{}')",
- (op_id,),
- )
- cursor.execute("DELETE FROM lifecycle_op_tmp WHERE op_id = %s", (op_id,))
- cursor.execute("SELECT pg_stat_force_next_flush()")
- assert read_shadow_write_counter(connection) == before
-
- with connection.cursor() as cursor:
- cursor.execute(
- "INSERT INTO posthog_person (created_at, properties, is_identified, uuid, version, team_id, is_deleted) "
- "VALUES (now(), '{}', false, %s, 1, 2, false)",
- (person_uuid,),
- )
- cursor.execute("SELECT pg_stat_force_next_flush()")
- assert read_shadow_write_counter(connection) == before + 1
- finally:
- with connection.cursor() as cursor:
- cursor.execute("DELETE FROM posthog_person WHERE uuid = %s", (person_uuid,))
- connection.close()