:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(managed-warehouse): stop backfilling empty teams from 2015 (#119873)
Eric Duongfebcacc7ed66
diff --git a/posthog/dags/events_backfill_to_duckling.py b/posthog/dags/events_backfill_to_duckling.pyindex 308bf98e985b..524764e165b8 100644--- a/posthog/dags/events_backfill_to_duckling.py+++ b/posthog/dags/events_backfill_to_duckling.py@@ -86,7 +86,6 @@ from products.data_warehouse.backend.facade.backfill_status import (from products.data_warehouse.backend.facade.models import ManagedWarehouseBackfillPartitionfrom products.managed_warehouse.backend.facade.api import (DUCKGRES_BUCKET_REGION,- NO_HISTORY_SENTINEL,get_catalog_connection_config,get_org_id_for_team,get_stored_bucket_config,@@ -1049,45 +1048,6 @@ def get_s3_url_for_clickhouse(bucket: str, region: str, path_without_scheme: strreturn f"https://{bucket}.s3.{region}.amazonaws.com/{path_without_scheme}"-def get_earliest_person_date_for_team(team_id: int) -> datetime | None:- """Query ClickHouse to find the earliest person modification date for a team.-- Uses _timestamp (Kafka ingestion time) since persons don't have a natural- event timestamp like events do.-- Returns:- The date of the earliest person modification, or None if no persons exist.- """- cluster = _get_cluster()- workload = Workload.OFFLINE if is_cloud() else Workload.DEFAULT-- def query_earliest(client: Client) -> datetime | None:- # Filter _timestamp >= '1970-01-01' to avoid toDate() overflow on pre-epoch timestamps.- # ClickHouse's Date type is UInt16 (days since 1970-01-01), so negative timestamps- # overflow to the max date (2149-06-06), breaking the backfill sensor logic.- result = client.execute(- """- SELECT toDate(min(_timestamp)) as earliest_date- FROM person- WHERE team_id = %(team_id)s- AND _timestamp >= '1970-01-01'- """,- {"team_id": team_id},- )- if result and result[0][0]:- date_val = result[0][0]- if isinstance(date_val, datetime):- return date_val- return datetime.combine(date_val, datetime.min.time())- return None-- return cluster.any_host_by_role(- fn=query_earliest,- workload=workload,- node_role=NodeRole.DATA,- ).result()--def _validate_identifier(identifier: str) -> None:"""Validate that an identifier is safe for SQL interpolation.@@ -3295,8 +3255,6 @@ def duckling_events_full_backfill_sensor(for bf in unresolved[:EVENTS_BACKFILL_MAX_EARLIEST_LOOKUPS_PER_TICK]:bf = replace(bf, earliest_event_date=resolve_team_earliest_event_date(bf.team_id))updated_backfills[bf.team_id] = bf- if bf.earliest_event_date == NO_HISTORY_SENTINEL:- context.log.info(f"No events for team_id={bf.team_id}; caching no-history sentinel")# This push IS the persistence: the control-plane row is the read source.if not _push_earliest_event_date_to_cp(bf):context.log.warning(@@ -3313,9 +3271,9 @@ def duckling_events_full_backfill_sensor(for bf in backfills:earliest = bf.earliest_event_dateif earliest is None or earliest > last_month_end:- # Unresolved this tick, no-history sentinel, or a team whose earliest event is only- # in the current month — no complete month to full-backfill (the daily sensor owns- # the current month), so skip it.+ # Unresolved this tick, or a team whose earliest event is only in the current+ # month — no complete month to full-backfill (the daily sensor owns the current+ # month), so skip it.continuekeys = [f"{bf.team_id}_{m}" for m in historical_backfill_months(earliest, today=today)]existing_historical_partitions.extend(key for key in keys if key in existing)diff --git a/posthog/dags/test_events_backfill_to_duckling.py b/posthog/dags/test_events_backfill_to_duckling.pyindex 13ac9192c2e0..a180ccce4698 100644--- a/posthog/dags/test_events_backfill_to_duckling.py+++ b/posthog/dags/test_events_backfill_to_duckling.py@@ -62,7 +62,7 @@ from posthog.dags.events_backfill_to_duckling import ()from products.data_warehouse.backend.facade.backfill_status import BackfillOutcome, get_months_in_range-from products.managed_warehouse.backend.facade.api import EARLIEST_BACKFILL_DATE, NO_HISTORY_SENTINEL+from products.managed_warehouse.backend.facade.api import EARLIEST_BACKFILL_DATEfrom products.managed_warehouse.backend.facade.client import ServiceCredential, ServiceCredentialUnavailablefrom products.managed_warehouse.backend.facade.contracts import (DucklingTables,@@ -702,28 +702,6 @@ class TestFullBackfillSensorEarliestDate:# The same tick uses the freshly resolved date (the contract is frozen, so the# sensor replaces its local value rather than mutating the control-plane row).- @patch("posthog.dags.events_backfill_to_duckling.resolve_team_earliest_event_date")- @patch("posthog.dags.events_backfill_to_duckling.list_enabled_backfill_team_memberships")- @patch("posthog.dags.events_backfill_to_duckling.timezone")- @patch("posthog.dags.events_backfill_to_duckling.stale_running_partitions", new=MagicMock(return_value=[]))- def test_no_events_returns_empty(self, mock_tz, mock_list_rows, mock_get_earliest):- from dagster import DagsterInstance, SensorResult, build_sensor_context-- mock_tz.now.return_value = datetime(2025, 2, 10, 12, 0, 0)- mock_get_earliest.return_value = NO_HISTORY_SENTINEL-- backfill = self._bf(1)- mock_list_rows.return_value = [backfill]-- instance = DagsterInstance.ephemeral()- context = build_sensor_context(instance=instance)- result = duckling_events_full_backfill_sensor(context)- assert isinstance(result, SensorResult)- assert result.run_requests is not None-- assert len(result.run_requests) == 0- # No events produces no historical partitions in this tick.-def test_earliest_backfill_date_is_2015(self):assert EARLIEST_BACKFILL_DATE == datetime(2015, 1, 1)@@ -760,18 +738,9 @@ class TestFullBackfillSensorEarliestDate:mock_list_rows.return_value = backfillsmock_projection.objects.unscoped.return_value.filter.return_value.values_list.return_value = []if isinstance(get_earliest, list):- mock_ge.side_effect = [- value.date() if isinstance(value, datetime) else NO_HISTORY_SENTINEL if value is None else value- for value in get_earliest- ]+ mock_ge.side_effect = [value.date() if isinstance(value, datetime) else value for value in get_earliest]else:- mock_ge.return_value = (- get_earliest.date()- if isinstance(get_earliest, datetime)- else NO_HISTORY_SENTINEL- if get_earliest is None- else get_earliest- )+ mock_ge.return_value = get_earliest.date() if isinstance(get_earliest, datetime) else get_earliestinstance = DagsterInstance.ephemeral()if existing:@@ -895,19 +864,13 @@ class TestFullBackfillSensorEarliestDate:runs_filter = mock_get_runs.call_args.kwargs["filters"]assert runs_filter.tags == {"duckling_backfill_type": "full"}- @parameterized.expand(- [- ("resolved_date", datetime(2020, 6, 15), date(2020, 6, 15)),- ("no_events_pushes_sentinel", None, NO_HISTORY_SENTINEL),- ]- )- def test_pushes_freshly_resolved_earliest_date_to_control_plane(self, _name, earliest_dt, expected_pushed):- # The resolved date (or the no-history sentinel) is persisted onto the team's- # duckgres control-plane row — the sensors' read source.+ def test_pushes_freshly_resolved_earliest_date_to_control_plane(self):+ # The resolved date is persisted onto the team's duckgres control-plane row — the+ # sensors' read source.backfill = self._bf(1)with patch("products.managed_warehouse.backend.facade.api.update_team_earliest_event_date") as mock_push:- self._run_full_sensor([backfill], now=datetime(2020, 8, 10, 12, 0, 0), get_earliest=earliest_dt)- mock_push.assert_called_once_with(backfill.organization_id, 1, expected_pushed)+ self._run_full_sensor([backfill], now=datetime(2020, 8, 10, 12, 0, 0), get_earliest=datetime(2020, 6, 15))+ mock_push.assert_called_once_with(backfill.organization_id, 1, date(2020, 6, 15))def test_control_plane_push_failure_does_not_fail_tick(self):# The CP push is the persistence, but it is best-effort within a tick: a push@@ -957,13 +920,7 @@ class TestFullBackfillSensorCpEnumeration:patch("posthog.dags.events_backfill_to_duckling.stale_running_partitions", return_value=[]),patch("posthog.dags.events_backfill_to_duckling.resolve_team_earliest_event_date",- return_value=(- get_earliest.date()- if isinstance(get_earliest, datetime)- else NO_HISTORY_SENTINEL- if get_earliest is None- else get_earliest- ),+ return_value=get_earliest.date() if isinstance(get_earliest, datetime) else get_earliest,),patch("products.managed_warehouse.backend.facade.api.update_team_earliest_event_date") as mock_push,):diff --git a/products/data_warehouse/backend/tasks/tasks.py b/products/data_warehouse/backend/tasks/tasks.pyindex 574d5bd89e7d..61978d8b7ff2 100644--- a/products/data_warehouse/backend/tasks/tasks.py+++ b/products/data_warehouse/backend/tasks/tasks.py@@ -265,17 +265,12 @@ def sync_team_earliest_event_date(team_id: int) -> None:and dropped — the full-backfill sensor resolves and persists the date lazily on alater tick regardless.- A team with no events yet is left unresolved (nothing stored), NOT cached as the- no-history sentinel: this task runs seconds after provision/onboard, when a brand-new- project plausibly hasn't ingested its first events. A cached date is final (nothing- re-resolves a non-NULL value), so persisting the sentinel here would permanently- exclude the team from historical backfill. The full-backfill sensor resolves it- later, when "no events" is a meaningful answer.+ A team with no events yet stores today's date, so its backfill history starts on the+ day it is provisioned (see resolve_team_earliest_event_date)."""# Deferred: ducklake pulls duckdb in via common, and posthog.models must not load# while Celery imports task modules — keep both off this module's import path.from products.managed_warehouse.backend.facade.api import ( # noqa: PLC0415- NO_HISTORY_SENTINEL,get_org_id_for_team,resolve_team_earliest_event_date,update_team_earliest_event_date,@@ -289,9 +284,6 @@ def sync_team_earliest_event_date(team_id: int) -> None:if row.earliest_event_date is not None:returnresolved = resolve_team_earliest_event_date(team_id)- if resolved is None or resolved == NO_HISTORY_SENTINEL:- logger.info("No events for team yet; leaving earliest event date unresolved", team_id=team_id)- returnupdate_team_earliest_event_date(organization_id, team_id, resolved)diff --git a/products/data_warehouse/backend/tests/test_tasks.py b/products/data_warehouse/backend/tests/test_tasks.pyindex f32113ee4d40..125a1c8f5b02 100644--- a/products/data_warehouse/backend/tests/test_tasks.py+++ b/products/data_warehouse/backend/tests/test_tasks.py@@ -62,24 +62,6 @@ def test_sync_task_resolves_and_pushes_to_control_plane(mock_update.assert_called_once_with(ORGANIZATION_ID, TEAM_ID, expected)-@patch("products.managed_warehouse.backend.facade.api.get_org_id_for_team", return_value=ORGANIZATION_ID)-@patch("products.managed_warehouse.backend.facade.api.update_team_earliest_event_date")-@patch("products.managed_warehouse.backend.facade.api.resolve_team_earliest_event_date")-def test_sync_task_leaves_empty_team_unresolved(- mock_get_earliest: MagicMock, mock_update: MagicMock, _mock_org_id: MagicMock-) -> None:- # A just-provisioned project plausibly has no events YET. A cached date is final, so- # storing the no-history sentinel here would permanently exclude the team from- # historical backfill; the task must store nothing and leave the sensor to resolve- # it later.- mock_get_earliest.return_value = None-- with _patch_membership(_membership()):- sync_team_earliest_event_date(TEAM_ID)-- mock_update.assert_not_called()--@patch("products.managed_warehouse.backend.facade.api.get_org_id_for_team", return_value=ORGANIZATION_ID)@patch("products.managed_warehouse.backend.facade.api.update_team_earliest_event_date")@patch("products.managed_warehouse.backend.facade.api.resolve_team_earliest_event_date")diff --git a/products/managed_warehouse/backend/common.py b/products/managed_warehouse/backend/common.pyindex ca30c27b141c..4a6bcc51f264 100644--- a/products/managed_warehouse/backend/common.py+++ b/products/managed_warehouse/backend/common.py@@ -19,7 +19,7 @@ from __future__ import annotationsimport osimport reimport logging-from datetime import date, datetime+from datetime import UTC, date, datetimefrom typing import TYPE_CHECKING, TypedDictfrom uuid import UUID@@ -624,11 +624,6 @@ def get_team_backfill_state(team_id: int) -> ManagedWarehouseBackfillState:# Ignore events before this date — pre-2015 data is typically junk timestamps.EARLIEST_BACKFILL_DATE = datetime(2015, 1, 1)-# Stored on the team's duckgres control-plane row for a team with no events, so callers-# cache "nothing to backfill" instead of re-querying ClickHouse. Far enough in the future-# that any generated backfill months range is always empty.-NO_HISTORY_SENTINEL = date(9999, 12, 31)-@retry(stop=stop_after_attempt(3),@@ -669,9 +664,12 @@ def get_earliest_event_date_for_team(team_id: int) -> datetime | None:# Filter timestamp >= '1970-01-01' to avoid toDate() overflow on pre-epoch timestamps.# ClickHouse's Date type is UInt16 (days since 1970-01-01), so negative timestamps# overflow to the max date (2149-06-06), breaking the backfill sensor logic.+ #+ # Use minOrNull because min() over zero rows returns the type default (1970-01-01),+ # which reads as history and backfills a team with no events from EARLIEST_BACKFILL_DATE.result = client.execute("""- SELECT toDate(min(timestamp)) as earliest_date+ SELECT toDate(minOrNull(timestamp)) as earliest_dateFROM eventsWHERE team_id = %(team_id)sAND timestamp >= '1970-01-01'@@ -696,18 +694,21 @@ def get_earliest_event_date_for_team(team_id: int) -> datetime | None:def resolve_team_earliest_event_date(team_id: int) -> date:"""Resolve the date a team's historical duckling backfill should start from.- Clamps to EARLIEST_BACKFILL_DATE and returns NO_HISTORY_SENTINEL for a team with no- events, so callers can cache the result and never re-query ClickHouse for the team.+ Clamps to EARLIEST_BACKFILL_DATE. A team with no events yet resolves to today, the+ day it is first resolved (usually the day it is provisioned), so the full backfill+ covers each complete month from then on.+ That monthly run is the only backfill of a month's last day, because the daily+ backfill covers the current month up to yesterday. Callers cache the result and never+ re-query ClickHouse for the team."""earliest_dt = get_earliest_event_date_for_team(team_id)if earliest_dt is None:- return NO_HISTORY_SENTINEL+ return datetime.now(UTC).date()return max(earliest_dt, EARLIEST_BACKFILL_DATE).date()__all__ = ["EARLIEST_BACKFILL_DATE",- "NO_HISTORY_SENTINEL","attach_catalog","default_bucket_region","get_earliest_event_date_for_team",diff --git a/products/managed_warehouse/backend/facade/api.py b/products/managed_warehouse/backend/facade/api.pyindex b2bd589df98e..31c9bbea72a4 100644--- a/products/managed_warehouse/backend/facade/api.py+++ b/products/managed_warehouse/backend/facade/api.py@@ -13,11 +13,7 @@ from typing import TYPE_CHECKINGfrom uuid import UUIDfrom products.managed_warehouse.backend import common, storage-from products.managed_warehouse.backend.common import (- DUCKGRES_BUCKET_REGION,- EARLIEST_BACKFILL_DATE,- NO_HISTORY_SENTINEL,-)+from products.managed_warehouse.backend.common import DUCKGRES_BUCKET_REGION, EARLIEST_BACKFILL_DATEfrom products.managed_warehouse.backend.facade.contracts import (DuckgresQueryServerConfig,DuckgresStoredBucketConfig,@@ -36,7 +32,6 @@ if TYPE_CHECKING:__all__ = ["DUCKGRES_BUCKET_REGION","EARLIEST_BACKFILL_DATE",- "NO_HISTORY_SENTINEL","default_bucket_region","deprovision_for_org_deletion","duckgres_data_imports_schema",diff --git a/products/managed_warehouse/backend/presentation/views.py b/products/managed_warehouse/backend/presentation/views.pyindex 1e29048af702..3f911cb5bbfe 100644--- a/products/managed_warehouse/backend/presentation/views.py+++ b/products/managed_warehouse/backend/presentation/views.py@@ -657,8 +657,8 @@ def update_team(def push_team_earliest_event_date(organization_id: UUID | str, team_id: int, earliest: date | None) -> bool:- """Persist a team's resolved earliest event date (incl. the no-history sentinel) onto- its duckgres team row — the read source for the backfill sensors.+ """Persist a team's resolved earliest event date onto its duckgres team row — the+ read source for the backfill sensors.A failure is logged and swallowed; returns True when the control plane accepted thevalue, so callers (the provisioning-time task, the full-backfill sensor) can leavediff --git a/products/managed_warehouse/backend/tests/test_common.py b/products/managed_warehouse/backend/tests/test_common.pyindex 354c8b9d3e73..828be4a59b96 100644--- a/products/managed_warehouse/backend/tests/test_common.py+++ b/products/managed_warehouse/backend/tests/test_common.py@@ -1,4 +1,8 @@+from datetime import date+import pytest+import time_machine+from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_eventsfrom unittest.mock import MagicMock, patchfrom django.test import override_settings@@ -15,6 +19,7 @@ from products.managed_warehouse.backend.common import (initialize_ducklake,is_version_mismatch,reset_ducklake_catalog,+ resolve_team_earliest_event_date,upsert_duckgres_server_for_org,)from products.managed_warehouse.backend.models import DuckgresServer@@ -54,6 +59,23 @@ class TestUpsertDuckgresServerForOrg:assert get_trino_password_for_org(str(org.id)) == "trino-secret"+class TestResolveTeamEarliestEventDate(ClickhouseTestMixin, BaseTest):+ @parameterized.expand(+ [+ ("no_events", None, date(2026, 10, 19)),+ ("has_events", "2026-10-16T12:00:00Z", date(2026, 10, 16)),+ ]+ )+ @time_machine.travel("2026-10-19T12:00:00Z", tick=False)+ def test_resolves_from_the_teams_events(self, _name: str, event_timestamp: str | None, expected: date) -> None:+ if event_timestamp is not None:+ _create_event(team=self.team, event="$pageview", distinct_id="user-1", timestamp=event_timestamp)+ # The cluster client reads ClickHouse directly, so nothing flushes the batched test events for it.+ flush_persons_and_events()++ assert resolve_team_earliest_event_date(self.team.pk) == expected++class TestDefaultBucketRegion:@parameterized.expand([