:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(web-analytics): include first all-time aggregate bucket (#124832)
Lucas Ricoy282c0c17e606
diff --git a/docs/internal/web-analytics-query-serving.md b/docs/internal/web-analytics-query-serving.mdindex d6f6ce1601ab..0ece3c60c537 100644--- a/docs/internal/web-analytics-query-serving.md+++ b/docs/internal/web-analytics-query-serving.md@@ -19,6 +19,9 @@ A request walks down this ladder and stops at the first tier whose conditions it| 3. Live fast paths | Session-id-set (filtered) and no-join (unfiltered) shapes that avoid the full events↔sessions join | ~1–20s || 4. Full join | The original events↔sessions join; always works, slowest | ~5–60s |+The preaggregated all-time range includes the hourly bucket containing the first event.+Explicit date ranges keep their requested lower bounds.+## Request flow```textdiff --git a/products/web_analytics/backend/hogql_queries/pre_aggregated/query_builder.py b/products/web_analytics/backend/hogql_queries/pre_aggregated/query_builder.pyindex 381526b0a063..d33aa4d1422f 100644--- a/products/web_analytics/backend/hogql_queries/pre_aggregated/query_builder.py+++ b/products/web_analytics/backend/hogql_queries/pre_aggregated/query_builder.py@@ -1,4 +1,4 @@-from datetime import timedelta+from datetime import datetime, timedeltafrom typing import Optionalfrom posthog.hogql import ast@@ -115,7 +115,7 @@ class WebAnalyticsPreAggregatedQueryBuilder:value=(self.runner.query_compare_to_date_range.date_from()if self.runner.query_compare_to_date_range- else self.runner.query_date_range.date_from()+ else self.current_date_from())),),@@ -153,8 +153,17 @@ class WebAnalyticsPreAggregatedQueryBuilder:return ast.And(exprs=filter_exprs)+ def current_date_from(self) -> datetime:+ date_from = self.runner.query_date_range.date_from()+ if self.runner.query.dateRange and self.runner.query.dateRange.date_from == "all":+ # "all" resolves to the first event's timestamp, which usually falls+ # mid-hour; the hourly tables bucket on the hour, so an untruncated+ # lower bound excludes the bucket holding that first event.+ return date_from.replace(minute=0, second=0, microsecond=0)+ return date_from+def get_date_ranges(self, table_name: Optional[str] = None) -> PeriodFilters:- current_date_from = self.runner.query_date_range.date_from()+ current_date_from = self.current_date_from()current_date_to = self.runner.query_date_range.date_to()if self.runner.query_compare_to_date_range:diff --git a/products/web_analytics/backend/hogql_queries/test/__snapshots__/test_web_overview.ambr b/products/web_analytics/backend/hogql_queries/test/__snapshots__/test_web_overview.ambrindex c4c909cddb1d..11149fc0c06e 100644--- a/products/web_analytics/backend/hogql_queries/test/__snapshots__/test_web_overview.ambr+++ b/products/web_analytics/backend/hogql_queries/test/__snapshots__/test_web_overview.ambr@@ -72,6 +72,26 @@use_hive_partitioning=0'''# ---+# name: TestWebOverviewQueryRunner.test_all_time_preaggregated_range_includes_the_first_hour+ '''+ SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp+ FROM events+ WHERE and(equals(events.team_id, 99999), greater(events.timestamp, toDateTime64('2015-01-01 00:00:00.000000', 6, 'UTC')))+ ORDER BY toDate(events.timestamp) ASC, toTimeZone(events.timestamp, 'UTC') ASC+ LIMIT 1 SETTINGS readonly=2,+ max_execution_time=60,+ allow_experimental_object_type=1,+ max_ast_elements=4000000,+ max_expanded_ast_elements=4000000,+ max_bytes_before_external_group_by=0,+ transform_null_in=1,+ optimize_min_equality_disjunction_chain_length=4294967295,+ optimize_rewrite_aggregate_function_with_if=0,+ optimize_min_inequality_conjunction_chain_length=4294967295,+ allow_experimental_join_condition=1,+ use_hive_partitioning=0+ '''+# ---# name: TestWebOverviewQueryRunner.test_comparison'''SELECT uniqIf(session_person_id, and(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-12-06 00:00:00', 'UTC'))), lessOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-12-13 23:59:59', 'UTC'))))) AS unique_users,diff --git a/products/web_analytics/backend/hogql_queries/test/__snapshots__/test_web_overview.new_events_schema.ambr b/products/web_analytics/backend/hogql_queries/test/__snapshots__/test_web_overview.new_events_schema.ambrindex 4708a5d29a9d..2863bb75c557 100644--- a/products/web_analytics/backend/hogql_queries/test/__snapshots__/test_web_overview.new_events_schema.ambr+++ b/products/web_analytics/backend/hogql_queries/test/__snapshots__/test_web_overview.new_events_schema.ambr@@ -76,6 +76,28 @@output_format_json_escape_forward_slashes=0'''# ---+# name: TestWebOverviewQueryRunner.test_all_time_preaggregated_range_includes_the_first_hour[new_events_schema]+ '''+ SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp+ FROM events_json AS events+ WHERE and(equals(events.team_id, 99999), greater(events.timestamp, toDateTime64('2015-01-01 00:00:00.000000', 6, 'UTC')))+ ORDER BY toDate(events.timestamp) ASC, toTimeZone(events.timestamp, 'UTC') ASC+ LIMIT 1 SETTINGS readonly=2,+ max_execution_time=60,+ allow_experimental_object_type=1,+ max_ast_elements=4000000,+ max_expanded_ast_elements=4000000,+ max_bytes_before_external_group_by=0,+ transform_null_in=1,+ optimize_min_equality_disjunction_chain_length=4294967295,+ optimize_rewrite_aggregate_function_with_if=0,+ optimize_min_inequality_conjunction_chain_length=4294967295,+ allow_experimental_join_condition=1,+ use_hive_partitioning=0,+ json_type_escape_dots_in_keys=1,+ output_format_json_escape_forward_slashes=0+ '''+# ---# name: TestWebOverviewQueryRunner.test_comparison[new_events_schema]'''SELECT uniqIf(session_person_id, and(greaterOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-12-06 00:00:00', 'UTC'))), lessOrEquals(start_timestamp, assumeNotNull(toDateTime('2023-12-13 23:59:59', 'UTC'))))) AS unique_users,diff --git a/products/web_analytics/backend/hogql_queries/test/test_web_overview.py b/products/web_analytics/backend/hogql_queries/test/test_web_overview.pyindex c04387ca472f..18d67c3e8a2b 100644--- a/products/web_analytics/backend/hogql_queries/test/test_web_overview.py+++ b/products/web_analytics/backend/hogql_queries/test/test_web_overview.py@@ -37,6 +37,7 @@ from posthog.schema import (WebStatsTableQuery,)+from posthog.hogql import astfrom posthog.hogql.constants import LimitContextfrom posthog.hogql.context import HogQLContextfrom posthog.hogql.printer import prepare_and_print_ast@@ -864,6 +865,29 @@ class TestWebOverviewQueryRunner(FirstPageviewAttributionTestMixin, ClickhouseTe"Should use pre-aggregated tables when date range includes current date (using UNION ALL with hourly tables)",)+ @time_machine.travel("2023-12-15T12:00:00Z", tick=False)+ def test_all_time_preaggregated_range_includes_the_first_hour(self) -> None:+ s1 = str(uuid7("2023-11-01"))+ self._create_events([("p1", [("2023-11-01T12:34:56", s1)])])+ query = WebOverviewQuery(dateRange=DateRange(date_from="all"), properties=[])+ runner = WebOverviewQueryRunner(team=self.team, query=query)++ builder = WebOverviewPreAggregatedQueryBuilder(runner)+ period_filters = builder.get_date_ranges()++ assert isinstance(period_filters.current_period, ast.And)+ lower_bound = period_filters.current_period.exprs[0]+ assert isinstance(lower_bound, ast.CompareOperation)+ assert lower_bound.right == ast.Constant(value=datetime(2023, 11, 1, 12, tzinfo=UTC))++ generated_query = builder.get_query()+ assert isinstance(generated_query.where, ast.And)+ lower_bound_filter = generated_query.where.exprs[0]+ assert isinstance(lower_bound_filter, ast.CompareOperation)+ assert lower_bound_filter.op == ast.CompareOperationOp.GtEq+ assert lower_bound_filter.left == ast.Field(chain=[builder.bounces_table, "period_bucket"])+ assert lower_bound_filter.right == ast.Constant(value=datetime(2023, 11, 1, 12, tzinfo=UTC))+@time_machine.travel("2023-12-15T12:00:00Z", tick=False)def test_cannot_use_preaggregated_tables_with_unsupported_properties(self):query = WebOverviewQuery(