:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(dags): compute the squash cutoff when the run populates the snapshot (#122064)
Paweł Szczurcda541036b60
diff --git a/posthog/dags/person_overrides.py b/posthog/dags/person_overrides.pyindex c8bfc77e97a4..ca038df45d72 100644--- a/posthog/dags/person_overrides.py+++ b/posthog/dags/person_overrides.py@@ -3,6 +3,8 @@ import datetimefrom dataclasses import dataclassfrom functools import partial+from django.utils import timezone+import dagsterimport pydanticfrom clickhouse_driver import Client@@ -171,17 +173,22 @@ def create_snapshot_table(return table+# Overrides newer than this may not have reached every host yet, so the squash leaves them for the next run.+SQUASH_CUTOFF_MARGIN = datetime.timedelta(days=2)++class PopulateSnapshotTableConfig(dagster.Config):"""Configuration for creating and populating the initial snapshot table."""- timestamp: str = pydantic.Field(+ timestamp: str | None = pydantic.Field(description="The upper bound (non-inclusive) timestamp used when selecting person overrides to be squashed. The ""value can be provided in any format that is can be parsed by ClickHouse. This value should be far enough in ""the past that there is no reasonable likelihood that events or overrides prior to this time have not yet been "- "written to the database and replicated to all hosts in the cluster.",- default=(datetime.datetime.now() - datetime.timedelta(days=2)).strftime("%Y-%m-%d %H:%M:%S"),+ "written to the database and replicated to all hosts in the cluster. If not set, the run uses the time it "+ "populates the snapshot minus two days.",+ default=None,)limit: int | None = pydantic.Field(description="The number of rows to include in the snapshot. If provided, this can be used to limit the total "@@ -189,6 +196,12 @@ class PopulateSnapshotTableConfig(dagster.Config):default=None,)+ def cutoff(self) -> str:+ # Computed per run: a default in the field would be fixed when the code server imports this module.+ if self.timestamp:+ return self.timestamp+ return (timezone.now() - SQUASH_CUTOFF_MARGIN).strftime("%Y-%m-%d %H:%M:%S")+def _parse_cutoff(timestamp: str) -> datetime.datetime | None:# The config takes any format ClickHouse parses, so a value Python cannot read is logged as unknown.@@ -207,13 +220,16 @@ def populate_snapshot_table(config: PopulateSnapshotTableConfig,) -> PersonOverridesSnapshotTable:"""Fill the snapshot data with the selected overrides based on the configuration timestamp."""+ cutoff = config.cutoff()+ context.log.info(f"Squashing person overrides with _timestamp before {cutoff}")+ context.add_output_metadata({"cutoff": dagster.MetadataValue.text(cutoff)})record_run(context,DeletionRunJob.SQUASH_PERSON_OVERRIDES,- squash_cutoff=_parse_cutoff(config.timestamp),+ squash_cutoff=_parse_cutoff(cutoff),squash_limit=config.limit,)- cluster.any_host(partial(table.populate, timestamp=config.timestamp, limit=config.limit)).result()+ cluster.any_host(partial(table.populate, timestamp=cutoff, limit=config.limit)).result()return tablediff --git a/posthog/dags/tests/test_person_overrides.py b/posthog/dags/tests/test_person_overrides.pyindex fdc1629b7c20..e396c7a9338d 100644--- a/posthog/dags/tests/test_person_overrides.py+++ b/posthog/dags/tests/test_person_overrides.py@@ -4,6 +4,7 @@ from functools import partialfrom uuid import UUIDimport pytest+import time_machinefrom unittest.mock import Mock, call, patchfrom django.conf import settings as django_settings@@ -204,6 +205,16 @@ def test_a_person_deletion_after_a_merge_reaches_flag_evaluations(cluster: Clickassert cluster.any_host(surviving_flag_evaluation_person_ids).result() == set()+def test_the_default_cutoff_follows_the_clock_of_the_run() -> None:+ # The code server builds the config schema once, and runs launch days or weeks after that.+ config = PopulateSnapshotTableConfig()++ with time_machine.travel(datetime(2026, 10, 24, 22, 0, tzinfo=UTC), tick=False):+ assert config.cutoff() == "2026-10-22 22:00:00"++ assert PopulateSnapshotTableConfig(timestamp="2025-01-01 00:00:00").cutoff() == "2025-01-01 00:00:00"++@pytest.mark.django_dbdef test_a_failed_squash_run_is_logged_as_failed(cluster: ClickhouseCluster) -> None:result = squash_person_overrides.execute_in_process(