:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(marketing-analytics): keep local-day edge hours in precomputed reads (#126291)
Javier Bahamondes965e300712ae
diff --git a/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.py b/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.pyindex 51282443b7b0..5e705942adab 100644--- a/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.py+++ b/products/analytics_platform/backend/lazy_computation/lazy_computation_executor.py@@ -766,6 +766,13 @@ def get_daily_windows(start: datetime, end: datetime) -> list[tuple[datetime, da"""windows = []+ # Windows are UTC days, so a team-local bound must become UTC before its date is taken. Otherwise the+ # edge hours of a local day fall into UTC windows that are never built or served.+ if start.tzinfo is not None:+ start = start.astimezone(UTC)+ if end.tzinfo is not None:+ end = end.astimezone(UTC)+# Normalize to start of day in UTCcurrent = datetime(start.year, start.month, start.day, tzinfo=UTC)end_normalized = datetime(end.year, end.month, end.day, tzinfo=UTC)diff --git a/products/marketing_analytics/backend/hogql_queries/test_conversion_goal_precompute_e2e.py b/products/marketing_analytics/backend/hogql_queries/test_conversion_goal_precompute_e2e.pyindex 60812ccfb0d2..35676f34f1a8 100644--- a/products/marketing_analytics/backend/hogql_queries/test_conversion_goal_precompute_e2e.py+++ b/products/marketing_analytics/backend/hogql_queries/test_conversion_goal_precompute_e2e.py@@ -7,7 +7,13 @@ from django.test import override_settingsfrom parameterized import parameterized-from posthog.schema import AttributionMode, BaseMathType, ConversionGoalFilter1, MarketingAnalyticsDrillDownLevel+from posthog.schema import (+ AttributionMode,+ BaseMathType,+ ConversionGoalFilter1,+ DateRange,+ MarketingAnalyticsDrillDownLevel,+)from posthog.hogql import astfrom posthog.hogql.query import execute_hogql_query@@ -15,6 +21,7 @@ from posthog.hogql.query import execute_hogql_queryfrom posthog.clickhouse.client.execute import sync_executefrom posthog.clickhouse.preaggregation.marketing_conversions_sql import TRUNCATE_MARKETING_CONVERSIONS_TABLE_SQLfrom posthog.clickhouse.preaggregation.marketing_touchpoints_sql import TRUNCATE_MARKETING_TOUCHPOINTS_TABLE_SQL+from posthog.hogql_queries.utils.query_date_range import QueryDateRangefrom products.analytics_platform.backend.models.preaggregation_job import PreaggregationJobfrom products.marketing_analytics.backend.hogql_queries.conversion_goal_processor import ConversionGoalProcessor@@ -310,6 +317,43 @@ class TestConversionGoalPrecomputeEquivalence(ClickhouseTestMixin, APIBaseTest):mediums = {row[1] for row in rows}assert "display" in mediums, f"last touch carried utm_medium=display; the medium array is misaligned: {rows}"+ @parameterized.expand(+ [+ ("ahead_of_utc", "Asia/Jerusalem", "edge_ahead", datetime(2024, 3, 4, 23, 0, tzinfo=UTC)),+ ("behind_utc", "America/Los_Angeles", "edge_behind", datetime(2024, 3, 6, 4, 0, tzinfo=UTC)),+ ]+ )+ def test_precompute_keeps_conversions_at_the_edges_of_a_local_day(+ self, _name: str, timezone: str, campaign: str, converted_at: datetime+ ):+ self.team.timezone = timezone+ self.team.save()+ distinct_id = f"user_{campaign}"+ _create_person(distinct_ids=[distinct_id], team=self.team)+ _create_event(+ team=self.team,+ event="$pageview",+ distinct_id=distinct_id,+ timestamp=datetime(2024, 3, 2, 12, 0, tzinfo=UTC),+ properties={"utm_campaign": campaign, "utm_source": "google", "utm_medium": "cpc"},+ )+ _create_event(team=self.team, event="purchase", distinct_id=distinct_id, timestamp=converted_at)+ flush_persons_and_events()++ local_day = QueryDateRange(+ date_range=DateRange(date_from="2024-03-05", date_to="2024-03-05"),+ team=self.team,+ interval=None,+ now=datetime(2024, 3, 20, tzinfo=UTC),+ )+ rows = self._execute(+ self._make_processor(precompute=True).generate_cte_query(+ additional_conditions=[], date_from=local_day.date_from(), date_to=local_day.date_to()+ )+ )++ assert any(campaign in row for row in rows), f"conversion on the local day was dropped: {rows}"+def test_reused_wide_job_excludes_out_of_range_conversions(self):_create_person(distinct_ids=["user_dec"], team=self.team)_create_event(diff --git a/products/marketing_analytics/dags/marketing_precompute.py b/products/marketing_analytics/dags/marketing_precompute.pyindex 82d2e8b0096b..187ef6b36897 100644--- a/products/marketing_analytics/dags/marketing_precompute.py+++ b/products/marketing_analytics/dags/marketing_precompute.py@@ -40,6 +40,7 @@ from datetime import UTC, datetime, timedeltafrom functools import partialfrom itertools import islicefrom typing import NamedTuple+from zoneinfo import ZoneInfofrom django.db import connections@@ -490,6 +491,7 @@ class _TeamWarmPlan(NamedTuple):attribution_window_days: intfilter_test_accounts: boolwarm_costs: bool+ timezone: ZoneInfodatabase: Database | None = None@property@@ -533,6 +535,7 @@ def _plan_team(team: Team) -> _TeamWarmPlan | None:attribution_window_days=ma_config.attribution_window_days or DEFAULT_ATTRIBUTION_WINDOW_DAYS,filter_test_accounts=bool(ma_config.filter_test_accounts),warm_costs=bool(config.costs_precomputation_enabled and _team_has_cost_sources(team)),+ timezone=team.timezone_info,)except Exception:MARKETING_PRECOMPUTE_TEAM_FAILED.labels(stage="setup").inc()@@ -552,6 +555,16 @@ def _warm_team(context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: da# and schema introspection would be un-attributable in query_log.tag_queries(product=Product.MARKETING_ANALYTICS, feature=Feature.CACHE_WARMUP)team = plan.team+ # Reads span whole team-local days and step back in local wall-clock days, so warm the same bounds. A+ # UTC-based range misses the UTC day that a local day crosses into, and precompute-only reads then report+ # not-ready or fall back to live costs.+ local_now = end.astimezone(plan.timezone)+ local_midnight = local_now.replace(hour=0, minute=0, second=0, microsecond=0)+ local_day_end = local_now.replace(hour=23, minute=59, second=59, microsecond=999999).astimezone(UTC)++ def days_before_today(days: int) -> datetime:+ return (local_midnight - timedelta(days=days)).astimezone(UTC)+conversion_teams = 0costs_teams = 0failures = 0@@ -563,14 +576,20 @@ def _warm_team(context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: datry:# Reach back far enough that a read with up to PRECOMPUTE_WINDOW_DAYS of lookback is fully# covered including its touchpoints attribution backfill ([date_from - attribution_window, date_to]).- tp_start = end - timedelta(days=PRECOMPUTE_WINDOW_DAYS + plan.attribution_window_days)+ tp_start = days_before_today(PRECOMPUTE_WINDOW_DAYS + plan.attribution_window_days)failures += _ensure_touchpoints_for_team(- context, team, plan.filter_test_accounts, tp_start, end, PRECOMPUTE_CHUNK_DAYS, plan.database+ context,+ team,+ plan.filter_test_accounts,+ tp_start,+ local_day_end,+ PRECOMPUTE_CHUNK_DAYS,+ plan.database,)# Conversions need no attribution backfill — the conversion event must fall in the query range.# Goals that aren't precomputable (non-Events/Actions, schema remaps, person/cohort filters) are# skipped inside; a team can warm touchpoints but no conversions if no goal qualifies.- conv_start = end - timedelta(days=PRECOMPUTE_WINDOW_DAYS)+ conv_start = days_before_today(PRECOMPUTE_WINDOW_DAYS)_goals_warmed, conv_failures = _ensure_conversions_for_team(context,team,@@ -578,7 +597,7 @@ def _warm_team(context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: daplan.conversion_goals,plan.filter_test_accounts,conv_start,- end,+ local_day_end,PRECOMPUTE_CHUNK_DAYS,plan.database,)@@ -591,9 +610,9 @@ def _warm_team(context: dagster.OpExecutionContext, plan: _TeamWarmPlan, end: daif plan.warm_costs:try:- costs_start = end - timedelta(days=PRECOMPUTE_WINDOW_DAYS)+ costs_start = days_before_today(PRECOMPUTE_WINDOW_DAYS)_sources_warmed, costs_failures = _ensure_costs_for_team(- context, team, costs_start, end, PRECOMPUTE_CHUNK_DAYS, plan.database+ context, team, costs_start, local_day_end, PRECOMPUTE_CHUNK_DAYS, plan.database)failures += costs_failurescosts_teams += 1 # after the block, mirroring conversion_teams: not counted if it raiseddiff --git a/products/marketing_analytics/dags/tests/test_marketing_precompute.py b/products/marketing_analytics/dags/tests/test_marketing_precompute.pyindex b3f12202473b..80ffbee37331 100644--- a/products/marketing_analytics/dags/tests/test_marketing_precompute.py+++ b/products/marketing_analytics/dags/tests/test_marketing_precompute.py@@ -1,7 +1,8 @@import os-from datetime import UTC, datetime+from datetime import UTC, datetime, timedeltaimport pytest+import time_machinefrom posthog.test.base import APIBaseTestfrom unittest.mock import MagicMock, patch@@ -10,7 +11,7 @@ from django.test import SimpleTestCaseimport dagsterfrom parameterized import parameterized-from posthog.schema import MarketingAnalyticsDrillDownLevel+from posthog.schema import DateRange, MarketingAnalyticsDrillDownLevelfrom posthog.clickhouse.client import sync_executefrom posthog.clickhouse.query_log_archive import (@@ -19,6 +20,7 @@ from posthog.clickhouse.query_log_archive import (SHARDED_QUERY_LOG_ARCHIVE_V3_TABLE_SQL,)from posthog.dags.common import chunk_ranges+from posthog.hogql_queries.utils.query_date_range import QueryDateRangefrom posthog.models import Organization, Teamfrom products.analytics_platform.backend.lazy_computation.lazy_computation_executor import LazyComputationTable@@ -394,15 +396,28 @@ class TestConversionWarming(APIBaseTest):assert ensure_mock.call_count > len(teams)assert database.create_for.call_count == len(teams)+ @parameterized.expand([("utc", "UTC"), ("behind_utc", "America/Los_Angeles"), ("ahead_of_utc", "Asia/Jerusalem")])@patch(_ENSURE, new_callable=_ready_mock)@patch(_SINGLE_CHUNK, _BIG_CHUNK)- def test_touchpoints_window_reaches_back_past_attribution_window(self, ensure_mock):+ def test_touchpoints_window_covers_a_local_read_and_its_attribution_backfill(self, _name, timezone, ensure_mock):team = self._make_team("A", goals=[_INELIGIBLE_GOAL])- expected = PRECOMPUTE_WINDOW_DAYS + team.marketing_analytics_config.attribution_window_days- with patch(_FF, _flag_fn(conversion=True)), patch.dict(os.environ, {SELECTED_TEAM_IDS_ENV_VAR: f"{team.pk}"}):+ team.timezone = timezone+ team.save()+ # The lookback crosses each zone's DST switch (Los Angeles 2024-03-10, Jerusalem 2024-03-29).+ now = datetime(2024, 4, 2, 20, 0, tzinfo=UTC)+ with (+ time_machine.travel(now, tick=False),+ patch(_FF, _flag_fn(conversion=True)),+ patch.dict(os.environ, {SELECTED_TEAM_IDS_ENV_VAR: f"{team.pk}"}),+ ):warm_selected_teams(dagster.build_op_context())+ read = QueryDateRange(+ date_range=DateRange(date_from=f"-{PRECOMPUTE_WINDOW_DAYS}d"), team=team, interval=None, now=now+ )+ attribution = timedelta(days=team.marketing_analytics_config.attribution_window_days)kwargs = ensure_mock.call_args_list[0].kwargs # touchpoints is warmed first- assert (kwargs["time_range_end"] - kwargs["time_range_start"]).days == expected+ assert kwargs["time_range_start"] <= read.date_from() - attribution+ assert kwargs["time_range_end"] >= read.date_to()@patch(_ENSURE, new_callable=_ready_mock)@patch(_SINGLE_CHUNK, _BIG_CHUNK)@@ -433,11 +448,14 @@ class TestConversionWarming(APIBaseTest):for call in ensure_mock.call_args_list:assert (call.kwargs["time_range_end"] - call.kwargs["time_range_start"]).days <= 7+ @parameterized.expand([("warm_error", False, 2), ("invalid_timezone", True, 1)])@patch(_ENSURE, new_callable=_ready_mock)@patch(_SINGLE_CHUNK, _BIG_CHUNK)- def test_one_team_failure_does_not_poison_others(self, ensure_mock):+ def test_one_team_failure_does_not_poison_others(self, _name, invalid_timezone, expected_calls, ensure_mock):t1 = self._make_team("A", goals=[_INELIGIBLE_GOAL])t2 = self._make_team("B", goals=[_INELIGIBLE_GOAL])+ if invalid_timezone:+ Team.objects.filter(pk=t1.pk).update(timezone="Not/AZone")def side_effect(*args, **kwargs):if kwargs["team"].pk == t1.pk:@@ -452,7 +470,7 @@ class TestConversionWarming(APIBaseTest):result = warm_selected_teams(dagster.build_op_context())assert result["teams"] == 2assert result["failures"] == 1- assert ensure_mock.call_count == 2+ assert ensure_mock.call_count == expected_calls@patch(_ENSURE)@patch(_SINGLE_CHUNK, _BIG_CHUNK)