Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(managed-warehouse): stop backfilling empty teams from 2015 (#119873)

Eric Duongfebcacc7ed66
diff --git a/posthog/dags/events_backfill_to_duckling.py b/posthog/dags/events_backfill_to_duckling.py
index 308bf98e985b..524764e165b8 100644
--- a/posthog/dags/events_backfill_to_duckling.py
+++ b/posthog/dags/events_backfill_to_duckling.py
@@ -86,7 +86,6 @@ from products.data_warehouse.backend.facade.backfill_status import (
from products.data_warehouse.backend.facade.models import ManagedWarehouseBackfillPartition
from products.managed_warehouse.backend.facade.api import (
DUCKGRES_BUCKET_REGION,
- NO_HISTORY_SENTINEL,
get_catalog_connection_config,
get_org_id_for_team,
get_stored_bucket_config,
@@ -1049,45 +1048,6 @@ def get_s3_url_for_clickhouse(bucket: str, region: str, path_without_scheme: str
return f"https://{bucket}.s3.{region}.amazonaws.com/{path_without_scheme}"
-def get_earliest_person_date_for_team(team_id: int) -> datetime | None:
- """Query ClickHouse to find the earliest person modification date for a team.
-
- Uses _timestamp (Kafka ingestion time) since persons don't have a natural
- event timestamp like events do.
-
- Returns:
- The date of the earliest person modification, or None if no persons exist.
- """
- cluster = _get_cluster()
- workload = Workload.OFFLINE if is_cloud() else Workload.DEFAULT
-
- def query_earliest(client: Client) -> datetime | None:
- # Filter _timestamp >= '1970-01-01' to avoid toDate() overflow on pre-epoch timestamps.
- # ClickHouse's Date type is UInt16 (days since 1970-01-01), so negative timestamps
- # overflow to the max date (2149-06-06), breaking the backfill sensor logic.
- result = client.execute(
- """
- SELECT toDate(min(_timestamp)) as earliest_date
- FROM person
- WHERE team_id = %(team_id)s
- AND _timestamp >= '1970-01-01'
- """,
- {"team_id": team_id},
- )
- if result and result[0][0]:
- date_val = result[0][0]
- if isinstance(date_val, datetime):
- return date_val
- return datetime.combine(date_val, datetime.min.time())
- return None
-
- return cluster.any_host_by_role(
- fn=query_earliest,
- workload=workload,
- node_role=NodeRole.DATA,
- ).result()
-
-
def _validate_identifier(identifier: str) -> None:
"""Validate that an identifier is safe for SQL interpolation.
@@ -3295,8 +3255,6 @@ def duckling_events_full_backfill_sensor(
for bf in unresolved[:EVENTS_BACKFILL_MAX_EARLIEST_LOOKUPS_PER_TICK]:
bf = replace(bf, earliest_event_date=resolve_team_earliest_event_date(bf.team_id))
updated_backfills[bf.team_id] = bf
- if bf.earliest_event_date == NO_HISTORY_SENTINEL:
- context.log.info(f"No events for team_id={bf.team_id}; caching no-history sentinel")
# This push IS the persistence: the control-plane row is the read source.
if not _push_earliest_event_date_to_cp(bf):
context.log.warning(
@@ -3313,9 +3271,9 @@ def duckling_events_full_backfill_sensor(
for bf in backfills:
earliest = bf.earliest_event_date
if earliest is None or earliest > last_month_end:
- # Unresolved this tick, no-history sentinel, or a team whose earliest event is only
- # in the current month — no complete month to full-backfill (the daily sensor owns
- # the current month), so skip it.
+ # Unresolved this tick, or a team whose earliest event is only in the current
+ # month — no complete month to full-backfill (the daily sensor owns the current
+ # month), so skip it.
continue
keys = [f"{bf.team_id}_{m}" for m in historical_backfill_months(earliest, today=today)]
existing_historical_partitions.extend(key for key in keys if key in existing)
diff --git a/posthog/dags/test_events_backfill_to_duckling.py b/posthog/dags/test_events_backfill_to_duckling.py
index 13ac9192c2e0..a180ccce4698 100644
--- a/posthog/dags/test_events_backfill_to_duckling.py
+++ b/posthog/dags/test_events_backfill_to_duckling.py
@@ -62,7 +62,7 @@ from posthog.dags.events_backfill_to_duckling import (
)
from products.data_warehouse.backend.facade.backfill_status import BackfillOutcome, get_months_in_range
-from products.managed_warehouse.backend.facade.api import EARLIEST_BACKFILL_DATE, NO_HISTORY_SENTINEL
+from products.managed_warehouse.backend.facade.api import EARLIEST_BACKFILL_DATE
from products.managed_warehouse.backend.facade.client import ServiceCredential, ServiceCredentialUnavailable
from products.managed_warehouse.backend.facade.contracts import (
DucklingTables,
@@ -702,28 +702,6 @@ class TestFullBackfillSensorEarliestDate:
# The same tick uses the freshly resolved date (the contract is frozen, so the
# sensor replaces its local value rather than mutating the control-plane row).
- @patch("posthog.dags.events_backfill_to_duckling.resolve_team_earliest_event_date")
- @patch("posthog.dags.events_backfill_to_duckling.list_enabled_backfill_team_memberships")
- @patch("posthog.dags.events_backfill_to_duckling.timezone")
- @patch("posthog.dags.events_backfill_to_duckling.stale_running_partitions", new=MagicMock(return_value=[]))
- def test_no_events_returns_empty(self, mock_tz, mock_list_rows, mock_get_earliest):
- from dagster import DagsterInstance, SensorResult, build_sensor_context
-
- mock_tz.now.return_value = datetime(2025, 2, 10, 12, 0, 0)
- mock_get_earliest.return_value = NO_HISTORY_SENTINEL
-
- backfill = self._bf(1)
- mock_list_rows.return_value = [backfill]
-
- instance = DagsterInstance.ephemeral()
- context = build_sensor_context(instance=instance)
- result = duckling_events_full_backfill_sensor(context)
- assert isinstance(result, SensorResult)
- assert result.run_requests is not None
-
- assert len(result.run_requests) == 0
- # No events produces no historical partitions in this tick.
-
def test_earliest_backfill_date_is_2015(self):
assert EARLIEST_BACKFILL_DATE == datetime(2015, 1, 1)
@@ -760,18 +738,9 @@ class TestFullBackfillSensorEarliestDate:
mock_list_rows.return_value = backfills
mock_projection.objects.unscoped.return_value.filter.return_value.values_list.return_value = []
if isinstance(get_earliest, list):
- mock_ge.side_effect = [
- value.date() if isinstance(value, datetime) else NO_HISTORY_SENTINEL if value is None else value
- for value in get_earliest
- ]
+ mock_ge.side_effect = [value.date() if isinstance(value, datetime) else value for value in get_earliest]
else:
- mock_ge.return_value = (
- get_earliest.date()
- if isinstance(get_earliest, datetime)
- else NO_HISTORY_SENTINEL
- if get_earliest is None
- else get_earliest
- )
+ mock_ge.return_value = get_earliest.date() if isinstance(get_earliest, datetime) else get_earliest
instance = DagsterInstance.ephemeral()
if existing:
@@ -895,19 +864,13 @@ class TestFullBackfillSensorEarliestDate:
runs_filter = mock_get_runs.call_args.kwargs["filters"]
assert runs_filter.tags == {"duckling_backfill_type": "full"}
- @parameterized.expand(
- [
- ("resolved_date", datetime(2020, 6, 15), date(2020, 6, 15)),
- ("no_events_pushes_sentinel", None, NO_HISTORY_SENTINEL),
- ]
- )
- def test_pushes_freshly_resolved_earliest_date_to_control_plane(self, _name, earliest_dt, expected_pushed):
- # The resolved date (or the no-history sentinel) is persisted onto the team's
- # duckgres control-plane row — the sensors' read source.
+ def test_pushes_freshly_resolved_earliest_date_to_control_plane(self):
+ # The resolved date is persisted onto the team's duckgres control-plane row — the
+ # sensors' read source.
backfill = self._bf(1)
with patch("products.managed_warehouse.backend.facade.api.update_team_earliest_event_date") as mock_push:
- self._run_full_sensor([backfill], now=datetime(2020, 8, 10, 12, 0, 0), get_earliest=earliest_dt)
- mock_push.assert_called_once_with(backfill.organization_id, 1, expected_pushed)
+ self._run_full_sensor([backfill], now=datetime(2020, 8, 10, 12, 0, 0), get_earliest=datetime(2020, 6, 15))
+ mock_push.assert_called_once_with(backfill.organization_id, 1, date(2020, 6, 15))
def test_control_plane_push_failure_does_not_fail_tick(self):
# The CP push is the persistence, but it is best-effort within a tick: a push
@@ -957,13 +920,7 @@ class TestFullBackfillSensorCpEnumeration:
patch("posthog.dags.events_backfill_to_duckling.stale_running_partitions", return_value=[]),
patch(
"posthog.dags.events_backfill_to_duckling.resolve_team_earliest_event_date",
- return_value=(
- get_earliest.date()
- if isinstance(get_earliest, datetime)
- else NO_HISTORY_SENTINEL
- if get_earliest is None
- else get_earliest
- ),
+ return_value=get_earliest.date() if isinstance(get_earliest, datetime) else get_earliest,
),
patch("products.managed_warehouse.backend.facade.api.update_team_earliest_event_date") as mock_push,
):
diff --git a/products/data_warehouse/backend/tasks/tasks.py b/products/data_warehouse/backend/tasks/tasks.py
index 574d5bd89e7d..61978d8b7ff2 100644
--- a/products/data_warehouse/backend/tasks/tasks.py
+++ b/products/data_warehouse/backend/tasks/tasks.py
@@ -265,17 +265,12 @@ def sync_team_earliest_event_date(team_id: int) -> None:
and dropped — the full-backfill sensor resolves and persists the date lazily on a
later tick regardless.
- A team with no events yet is left unresolved (nothing stored), NOT cached as the
- no-history sentinel: this task runs seconds after provision/onboard, when a brand-new
- project plausibly hasn't ingested its first events. A cached date is final (nothing
- re-resolves a non-NULL value), so persisting the sentinel here would permanently
- exclude the team from historical backfill. The full-backfill sensor resolves it
- later, when "no events" is a meaningful answer.
+ A team with no events yet stores today's date, so its backfill history starts on the
+ day it is provisioned (see resolve_team_earliest_event_date).
"""
# Deferred: ducklake pulls duckdb in via common, and posthog.models must not load
# while Celery imports task modules — keep both off this module's import path.
from products.managed_warehouse.backend.facade.api import ( # noqa: PLC0415
- NO_HISTORY_SENTINEL,
get_org_id_for_team,
resolve_team_earliest_event_date,
update_team_earliest_event_date,
@@ -289,9 +284,6 @@ def sync_team_earliest_event_date(team_id: int) -> None:
if row.earliest_event_date is not None:
return
resolved = resolve_team_earliest_event_date(team_id)
- if resolved is None or resolved == NO_HISTORY_SENTINEL:
- logger.info("No events for team yet; leaving earliest event date unresolved", team_id=team_id)
- return
update_team_earliest_event_date(organization_id, team_id, resolved)
diff --git a/products/data_warehouse/backend/tests/test_tasks.py b/products/data_warehouse/backend/tests/test_tasks.py
index f32113ee4d40..125a1c8f5b02 100644
--- a/products/data_warehouse/backend/tests/test_tasks.py
+++ b/products/data_warehouse/backend/tests/test_tasks.py
@@ -62,24 +62,6 @@ def test_sync_task_resolves_and_pushes_to_control_plane(
mock_update.assert_called_once_with(ORGANIZATION_ID, TEAM_ID, expected)
-@patch("products.managed_warehouse.backend.facade.api.get_org_id_for_team", return_value=ORGANIZATION_ID)
-@patch("products.managed_warehouse.backend.facade.api.update_team_earliest_event_date")
-@patch("products.managed_warehouse.backend.facade.api.resolve_team_earliest_event_date")
-def test_sync_task_leaves_empty_team_unresolved(
- mock_get_earliest: MagicMock, mock_update: MagicMock, _mock_org_id: MagicMock
-) -> None:
- # A just-provisioned project plausibly has no events YET. A cached date is final, so
- # storing the no-history sentinel here would permanently exclude the team from
- # historical backfill; the task must store nothing and leave the sensor to resolve
- # it later.
- mock_get_earliest.return_value = None
-
- with _patch_membership(_membership()):
- sync_team_earliest_event_date(TEAM_ID)
-
- mock_update.assert_not_called()
-
-
@patch("products.managed_warehouse.backend.facade.api.get_org_id_for_team", return_value=ORGANIZATION_ID)
@patch("products.managed_warehouse.backend.facade.api.update_team_earliest_event_date")
@patch("products.managed_warehouse.backend.facade.api.resolve_team_earliest_event_date")
diff --git a/products/managed_warehouse/backend/common.py b/products/managed_warehouse/backend/common.py
index ca30c27b141c..4a6bcc51f264 100644
--- a/products/managed_warehouse/backend/common.py
+++ b/products/managed_warehouse/backend/common.py
@@ -19,7 +19,7 @@ from __future__ import annotations
import os
import re
import logging
-from datetime import date, datetime
+from datetime import UTC, date, datetime
from typing import TYPE_CHECKING, TypedDict
from uuid import UUID
@@ -624,11 +624,6 @@ def get_team_backfill_state(team_id: int) -> ManagedWarehouseBackfillState:
# Ignore events before this date — pre-2015 data is typically junk timestamps.
EARLIEST_BACKFILL_DATE = datetime(2015, 1, 1)
-# Stored on the team's duckgres control-plane row for a team with no events, so callers
-# cache "nothing to backfill" instead of re-querying ClickHouse. Far enough in the future
-# that any generated backfill months range is always empty.
-NO_HISTORY_SENTINEL = date(9999, 12, 31)
-
@retry(
stop=stop_after_attempt(3),
@@ -669,9 +664,12 @@ def get_earliest_event_date_for_team(team_id: int) -> datetime | None:
# Filter timestamp >= '1970-01-01' to avoid toDate() overflow on pre-epoch timestamps.
# ClickHouse's Date type is UInt16 (days since 1970-01-01), so negative timestamps
# overflow to the max date (2149-06-06), breaking the backfill sensor logic.
+ #
+ # Use minOrNull because min() over zero rows returns the type default (1970-01-01),
+ # which reads as history and backfills a team with no events from EARLIEST_BACKFILL_DATE.
result = client.execute(
"""
- SELECT toDate(min(timestamp)) as earliest_date
+ SELECT toDate(minOrNull(timestamp)) as earliest_date
FROM events
WHERE team_id = %(team_id)s
AND timestamp >= '1970-01-01'
@@ -696,18 +694,21 @@ def get_earliest_event_date_for_team(team_id: int) -> datetime | None:
def resolve_team_earliest_event_date(team_id: int) -> date:
"""Resolve the date a team's historical duckling backfill should start from.
- Clamps to EARLIEST_BACKFILL_DATE and returns NO_HISTORY_SENTINEL for a team with no
- events, so callers can cache the result and never re-query ClickHouse for the team.
+ Clamps to EARLIEST_BACKFILL_DATE. A team with no events yet resolves to today, the
+ day it is first resolved (usually the day it is provisioned), so the full backfill
+ covers each complete month from then on.
+ That monthly run is the only backfill of a month's last day, because the daily
+ backfill covers the current month up to yesterday. Callers cache the result and never
+ re-query ClickHouse for the team.
"""
earliest_dt = get_earliest_event_date_for_team(team_id)
if earliest_dt is None:
- return NO_HISTORY_SENTINEL
+ return datetime.now(UTC).date()
return max(earliest_dt, EARLIEST_BACKFILL_DATE).date()
__all__ = [
"EARLIEST_BACKFILL_DATE",
- "NO_HISTORY_SENTINEL",
"attach_catalog",
"default_bucket_region",
"get_earliest_event_date_for_team",
diff --git a/products/managed_warehouse/backend/facade/api.py b/products/managed_warehouse/backend/facade/api.py
index b2bd589df98e..31c9bbea72a4 100644
--- a/products/managed_warehouse/backend/facade/api.py
+++ b/products/managed_warehouse/backend/facade/api.py
@@ -13,11 +13,7 @@ from typing import TYPE_CHECKING
from uuid import UUID
from products.managed_warehouse.backend import common, storage
-from products.managed_warehouse.backend.common import (
- DUCKGRES_BUCKET_REGION,
- EARLIEST_BACKFILL_DATE,
- NO_HISTORY_SENTINEL,
-)
+from products.managed_warehouse.backend.common import DUCKGRES_BUCKET_REGION, EARLIEST_BACKFILL_DATE
from products.managed_warehouse.backend.facade.contracts import (
DuckgresQueryServerConfig,
DuckgresStoredBucketConfig,
@@ -36,7 +32,6 @@ if TYPE_CHECKING:
__all__ = [
"DUCKGRES_BUCKET_REGION",
"EARLIEST_BACKFILL_DATE",
- "NO_HISTORY_SENTINEL",
"default_bucket_region",
"deprovision_for_org_deletion",
"duckgres_data_imports_schema",
diff --git a/products/managed_warehouse/backend/presentation/views.py b/products/managed_warehouse/backend/presentation/views.py
index 1e29048af702..3f911cb5bbfe 100644
--- a/products/managed_warehouse/backend/presentation/views.py
+++ b/products/managed_warehouse/backend/presentation/views.py
@@ -657,8 +657,8 @@ def update_team(
def push_team_earliest_event_date(organization_id: UUID | str, team_id: int, earliest: date | None) -> bool:
- """Persist a team's resolved earliest event date (incl. the no-history sentinel) onto
- its duckgres team row — the read source for the backfill sensors.
+ """Persist a team's resolved earliest event date onto its duckgres team row — the
+ read source for the backfill sensors.
A failure is logged and swallowed; returns True when the control plane accepted the
value, so callers (the provisioning-time task, the full-backfill sensor) can leave
diff --git a/products/managed_warehouse/backend/tests/test_common.py b/products/managed_warehouse/backend/tests/test_common.py
index 354c8b9d3e73..828be4a59b96 100644
--- a/products/managed_warehouse/backend/tests/test_common.py
+++ b/products/managed_warehouse/backend/tests/test_common.py
@@ -1,4 +1,8 @@
+from datetime import date
+
import pytest
+import time_machine
+from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events
from unittest.mock import MagicMock, patch
from django.test import override_settings
@@ -15,6 +19,7 @@ from products.managed_warehouse.backend.common import (
initialize_ducklake,
is_version_mismatch,
reset_ducklake_catalog,
+ resolve_team_earliest_event_date,
upsert_duckgres_server_for_org,
)
from products.managed_warehouse.backend.models import DuckgresServer
@@ -54,6 +59,23 @@ class TestUpsertDuckgresServerForOrg:
assert get_trino_password_for_org(str(org.id)) == "trino-secret"
+class TestResolveTeamEarliestEventDate(ClickhouseTestMixin, BaseTest):
+ @parameterized.expand(
+ [
+ ("no_events", None, date(2026, 10, 19)),
+ ("has_events", "2026-10-16T12:00:00Z", date(2026, 10, 16)),
+ ]
+ )
+ @time_machine.travel("2026-10-19T12:00:00Z", tick=False)
+ def test_resolves_from_the_teams_events(self, _name: str, event_timestamp: str | None, expected: date) -> None:
+ if event_timestamp is not None:
+ _create_event(team=self.team, event="$pageview", distinct_id="user-1", timestamp=event_timestamp)
+ # The cluster client reads ClickHouse directly, so nothing flushes the batched test events for it.
+ flush_persons_and_events()
+
+ assert resolve_team_earliest_event_date(self.team.pk) == expected
+
+
class TestDefaultBucketRegion:
@parameterized.expand(
[