Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(dags): compute the squash cutoff when the run populates the snapshot (#122064)

Paweł Szczurcda541036b60
diff --git a/posthog/dags/person_overrides.py b/posthog/dags/person_overrides.py
index c8bfc77e97a4..ca038df45d72 100644
--- a/posthog/dags/person_overrides.py
+++ b/posthog/dags/person_overrides.py
@@ -3,6 +3,8 @@ import datetime
from dataclasses import dataclass
from functools import partial
+from django.utils import timezone
+
import dagster
import pydantic
from clickhouse_driver import Client
@@ -171,17 +173,22 @@ def create_snapshot_table(
return table
+# Overrides newer than this may not have reached every host yet, so the squash leaves them for the next run.
+SQUASH_CUTOFF_MARGIN = datetime.timedelta(days=2)
+
+
class PopulateSnapshotTableConfig(dagster.Config):
"""
Configuration for creating and populating the initial snapshot table.
"""
- timestamp: str = pydantic.Field(
+ timestamp: str | None = pydantic.Field(
description="The upper bound (non-inclusive) timestamp used when selecting person overrides to be squashed. The "
"value can be provided in any format that is can be parsed by ClickHouse. This value should be far enough in "
"the past that there is no reasonable likelihood that events or overrides prior to this time have not yet been "
- "written to the database and replicated to all hosts in the cluster.",
- default=(datetime.datetime.now() - datetime.timedelta(days=2)).strftime("%Y-%m-%d %H:%M:%S"),
+ "written to the database and replicated to all hosts in the cluster. If not set, the run uses the time it "
+ "populates the snapshot minus two days.",
+ default=None,
)
limit: int | None = pydantic.Field(
description="The number of rows to include in the snapshot. If provided, this can be used to limit the total "
@@ -189,6 +196,12 @@ class PopulateSnapshotTableConfig(dagster.Config):
default=None,
)
+ def cutoff(self) -> str:
+ # Computed per run: a default in the field would be fixed when the code server imports this module.
+ if self.timestamp:
+ return self.timestamp
+ return (timezone.now() - SQUASH_CUTOFF_MARGIN).strftime("%Y-%m-%d %H:%M:%S")
+
def _parse_cutoff(timestamp: str) -> datetime.datetime | None:
# The config takes any format ClickHouse parses, so a value Python cannot read is logged as unknown.
@@ -207,13 +220,16 @@ def populate_snapshot_table(
config: PopulateSnapshotTableConfig,
) -> PersonOverridesSnapshotTable:
"""Fill the snapshot data with the selected overrides based on the configuration timestamp."""
+ cutoff = config.cutoff()
+ context.log.info(f"Squashing person overrides with _timestamp before {cutoff}")
+ context.add_output_metadata({"cutoff": dagster.MetadataValue.text(cutoff)})
record_run(
context,
DeletionRunJob.SQUASH_PERSON_OVERRIDES,
- squash_cutoff=_parse_cutoff(config.timestamp),
+ squash_cutoff=_parse_cutoff(cutoff),
squash_limit=config.limit,
)
- cluster.any_host(partial(table.populate, timestamp=config.timestamp, limit=config.limit)).result()
+ cluster.any_host(partial(table.populate, timestamp=cutoff, limit=config.limit)).result()
return table
diff --git a/posthog/dags/tests/test_person_overrides.py b/posthog/dags/tests/test_person_overrides.py
index fdc1629b7c20..e396c7a9338d 100644
--- a/posthog/dags/tests/test_person_overrides.py
+++ b/posthog/dags/tests/test_person_overrides.py
@@ -4,6 +4,7 @@ from functools import partial
from uuid import UUID
import pytest
+import time_machine
from unittest.mock import Mock, call, patch
from django.conf import settings as django_settings
@@ -204,6 +205,16 @@ def test_a_person_deletion_after_a_merge_reaches_flag_evaluations(cluster: Click
assert cluster.any_host(surviving_flag_evaluation_person_ids).result() == set()
+def test_the_default_cutoff_follows_the_clock_of_the_run() -> None:
+ # The code server builds the config schema once, and runs launch days or weeks after that.
+ config = PopulateSnapshotTableConfig()
+
+ with time_machine.travel(datetime(2026, 10, 24, 22, 0, tzinfo=UTC), tick=False):
+ assert config.cutoff() == "2026-10-22 22:00:00"
+
+ assert PopulateSnapshotTableConfig(timestamp="2025-01-01 00:00:00").cutoff() == "2025-01-01 00:00:00"
+
+
@pytest.mark.django_db
def test_a_failed_squash_run_is_logged_as_failed(cluster: ClickhouseCluster) -> None:
result = squash_person_overrides.execute_in_process(