:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(web-analytics): serve the channel breakdown without the per-event sessions join (#109169)
Lucas Ricoyaef69ec161ec
diff --git a/products/web_analytics/backend/hogql_queries/query_constants/stats_table_queries.py b/products/web_analytics/backend/hogql_queries/query_constants/stats_table_queries.pyindex 0d5bf14810a3..4431ce0a15fb 100644--- a/products/web_analytics/backend/hogql_queries/query_constants/stats_table_queries.py+++ b/products/web_analytics/backend/hogql_queries/query_constants/stats_table_queries.py@@ -288,6 +288,43 @@ WHERE and({inside_periods}, {event_where}, {all_properties})GROUP BY session_id, breakdown_value"""+# Channel-type breakdown in two phases. `$channel_type` is a session field, so the+# sessions side cannot be dropped, but the lazy events↔sessions join re-runs the+# sessions subquery on every events shard and probes it once per event. Here the+# events are first reduced to one row per session with no join (same session key+# as NO_JOIN_MAIN_INNER_QUERY), then joined once to the sessions that start in the+# queried range. The LEFT JOIN keeps sessionless and non-UUIDv7 events under a+# NULL session, as the lazy join does.+CHANNEL_TYPE_TWO_PHASE_INNER_QUERY = """+SELECT+ e.filtered_person_id AS filtered_person_id,+ e.filtered_pageview_count AS filtered_pageview_count,+ s.breakdown_value AS breakdown_value,+ e.session_id AS session_id,+ s.start_timestamp AS start_timestamp+FROM (+ SELECT+ {filtered_person_id} AS filtered_person_id,+ {filtered_pageview_count} AS filtered_pageview_count,+ if(+ equals(bitAnd(bitShiftRight(events.$session_id_uuid, 76), 15), 7),+ events.$session_id,+ NULL+ ) AS session_id+ FROM events+ WHERE and({inside_periods}, {event_where}, {all_properties})+ GROUP BY session_id+) AS e+LEFT JOIN (+ SELECT+ sessions.session_id AS session_id,+ sessions.$channel_type AS breakdown_value,+ sessions.$start_timestamp AS start_timestamp+ FROM sessions+ WHERE {sessions_in_range}+) AS s ON e.session_id = s.session_id+"""+# No-join variants of the PAGE-breakdown queries: counts come straight from events# (bucketed by event timestamp) and bounce comes straight from the sessions table# (bucketed by session start), instead of routing both through the events↔sessionsdiff --git a/products/web_analytics/backend/hogql_queries/stats_table.py b/products/web_analytics/backend/hogql_queries/stats_table.pyindex 6b4dcf6c69f1..13f281f2ce70 100644--- a/products/web_analytics/backend/hogql_queries/stats_table.py+++ b/products/web_analytics/backend/hogql_queries/stats_table.py@@ -42,6 +42,7 @@ from products.web_analytics.backend.hogql_queries.first_pageview_attribution impfrom products.web_analytics.backend.hogql_queries.stats_table_pre_aggregated import StatsTablePreAggregatedQueryBuilderfrom products.web_analytics.backend.hogql_queries.stats_table_strategies import (ChannelTypeStrategy,+ ChannelTypeTwoPhaseStrategy,FirstPageviewAttributionStrategy,FrustrationMetricsStrategy,NoJoinFirstPageviewAttributionStrategy,@@ -74,6 +75,11 @@ from products.web_analytics.backend.hogql_queries.web_stats_paths_lazy_precomputBREAKDOWN_NULL_DISPLAY = "(none)"BREAKDOWN_REFERRER_PREFIX = "referrer:"+# The two-phase channel query joins at session grain on the coordinating node, so that+# node's memory grows with the sessions in the scanned span (current plus compare+# period). Above this span the per-shard join spreads the memory better.+CHANNEL_TYPE_TWO_PHASE_MAX_SPAN_DAYS = 62+def _none_if_nan(value):"""avgMergeIf returns NaN for empty windows; replace with None so the@@ -179,7 +185,9 @@ class WebStatsTableQueryRunner(WebAnalyticsQueryRunner[WebStatsTableQueryResponsreturn "stats_table_path_bounce_and_avg_time"if isinstance(strategy, PathBounceStrategy):return "stats_table_path_bounce"- # ChannelTypeStrategy must be checked before SimpleBreakdownStrategy since it's a subclass.+ # ChannelTypeTwoPhaseStrategy subclasses ChannelTypeStrategy, which subclasses SimpleBreakdownStrategy.+ if isinstance(strategy, ChannelTypeTwoPhaseStrategy):+ return "stats_table_channel_type_two_phase"if isinstance(strategy, ChannelTypeStrategy):return "stats_table_channel_type"if isinstance(strategy, NoJoinFirstPageviewAttributionStrategy):@@ -241,6 +249,8 @@ class WebStatsTableQueryRunner(WebAnalyticsQueryRunner[WebStatsTableQueryResponsreturn SimpleBreakdownStrategy(self, breakdown_override=self._bounce_entry_pathname_breakdown())if breakdown == WebStatsBreakdown.INITIAL_CHANNEL_TYPE:+ if self._can_use_channel_type_two_phase():+ return ChannelTypeTwoPhaseStrategy(self)return ChannelTypeStrategy(self)# Breakdowns whose displayed columns are all event-derived don't need@@ -386,12 +396,25 @@ class WebStatsTableQueryRunner(WebAnalyticsQueryRunner[WebStatsTableQueryResponsexpr=parse_expr(""" "context.columns.visitors".1 / sum("context.columns.visitors".1) OVER ()"""),)- def _uses_session_fields(self) -> bool:- """True when the breakdown value or any filter reads a `session.*`- field. Those queries need the events↔sessions join — Initial* entry- breakdowns for the value itself, session-property filters for the- predicate. Routing them to the no-join strategy would make HogQL- silently re-add the lazy join, mislabeling the query tag."""+ def _can_use_channel_type_two_phase(self) -> bool:+ # The two-phase shape carries only visitors and views, and it filters the events+ # scan alone, so a filter or goal that reads a session field needs the join back.+ if not self._team_in_no_join_rollout():+ return False+ if self.query.includeBounceRate or self.query.includeTrafficMetrics or self.query.conversionGoal:+ return False+ if self._scanned_span_days() > CHANNEL_TYPE_TWO_PHASE_MAX_SPAN_DAYS:+ return False+ return not self._expr_reads_session_fields(self.all_properties())++ def _scanned_span_days(self) -> int:+ date_from = self.query_date_range.date_from()+ compare = self.query_compare_to_date_range+ if compare is not None:+ date_from = min(date_from, compare.date_from())+ return (self.query_date_range.date_to() - date_from).days++ def _expr_reads_session_fields(self, expr: ast.Expr) -> bool:found = Falseclass Visitor(TraversingVisitor):@@ -400,13 +423,20 @@ class WebStatsTableQueryRunner(WebAnalyticsQueryRunner[WebStatsTableQueryResponsif node.chain and node.chain[0] == "session":found = True- visitor = Visitor()- visitor.visit(self._counts_breakdown_value())- visitor.visit(self.all_properties())- if self.conversion_goal_expr is not None:- visitor.visit(self.conversion_goal_expr)+ Visitor().visit(expr)return found+ def _uses_session_fields(self) -> bool:+ """True when the breakdown value or any filter reads a `session.*`+ field. Those queries need the events↔sessions join — Initial* entry+ breakdowns for the value itself, session-property filters for the+ predicate. Routing them to the no-join strategy would make HogQL+ silently re-add the lazy join, mislabeling the query tag."""+ exprs = [self._counts_breakdown_value(), self.all_properties()]+ if self.conversion_goal_expr is not None:+ exprs.append(self.conversion_goal_expr)+ return any(self._expr_reads_session_fields(expr) for expr in exprs)+def _period_comparison_tuple(self, column, alias, function_name):return ast.Alias(alias=alias,diff --git a/products/web_analytics/backend/hogql_queries/stats_table_strategies.py b/products/web_analytics/backend/hogql_queries/stats_table_strategies.pyindex 1c258cb56cd4..61f36d39e018 100644--- a/products/web_analytics/backend/hogql_queries/stats_table_strategies.py+++ b/products/web_analytics/backend/hogql_queries/stats_table_strategies.py@@ -7,6 +7,7 @@ from posthog.hogql import astfrom posthog.hogql.parser import parse_expr, parse_selectfrom products.web_analytics.backend.hogql_queries.query_constants.stats_table_queries import (+ CHANNEL_TYPE_TWO_PHASE_INNER_QUERY,FIRST_PAGEVIEW_INNER_QUERY,FRUSTRATION_METRICS_INNER_QUERY,MAIN_INNER_QUERY,@@ -265,6 +266,53 @@ class ChannelTypeStrategy(SimpleBreakdownStrategy):SQL is the same template."""+class ChannelTypeTwoPhaseStrategy(ChannelTypeStrategy):+ """INITIAL_CHANNEL_TYPE breakdown without the per-event sessions join.++ Eligible when the tile shows only visitors and views: every other+ session-derived column would need the join back. The outer query is+ inherited unchanged; only the inner scan differs.+ """++ INNER_QUERY = CHANNEL_TYPE_TWO_PHASE_INNER_QUERY++ def build_query(self) -> ast.SelectQuery:+ WEB_ANALYTICS_NO_JOIN_SERVED.labels(family="stats_table_channel_type").inc()+ return super().build_query()++ def _inner_query(self, breakdown: ast.Expr) -> ast.SelectQuery:+ query = parse_select(+ self.INNER_QUERY,+ timings=self.runner.timings,+ placeholders={+ **self._event_aggregation_placeholders(),+ "event_where": self.runner.event_type_expr,+ "all_properties": self.runner.all_properties(),+ "inside_periods": self.runner._periods_expression(),+ "sessions_in_range": self._sessions_start_range(),+ },+ )+ assert isinstance(query, ast.SelectQuery)+ return query++ def _sessions_start_range(self) -> ast.Expr:+ # Without a compare period the outer query counts every in-range event, so a+ # session that started before the range still needs its channel. The lazy+ # events↔sessions join pads its session pushdown by 3 days; pad the same way+ # so both paths see the same sessions.+ date_from = self.runner.query_date_range.date_from()+ compare = self.runner.query_compare_to_date_range+ if compare is not None:+ date_from = min(date_from, compare.date_from())+ return parse_expr(+ "and(sessions.$start_timestamp >= {date_from} - toIntervalDay(3), sessions.$start_timestamp <= {date_to})",+ placeholders={+ "date_from": ast.Constant(value=date_from),+ "date_to": ast.Constant(value=self.runner.query_date_range.date_to()),+ },+ )++class FirstPageviewAttributionStrategy(SimpleBreakdownStrategy):"""Breakdowns attributed to the session's first ``$pageview``/``$screen``.diff --git a/products/web_analytics/backend/hogql_queries/test/test_web_stats_table.py b/products/web_analytics/backend/hogql_queries/test/test_web_stats_table.pyindex aff0ea2c3abb..90ad2c8c63b5 100644--- a/products/web_analytics/backend/hogql_queries/test/test_web_stats_table.py+++ b/products/web_analytics/backend/hogql_queries/test/test_web_stats_table.py@@ -2964,6 +2964,146 @@ class TestWebStatsTableNoJoinFastPath(ClickhouseTestMixin, APIBaseTest):runner = self._make_runner()assert not runner.query_strategy().startswith("stats_table_no_join")+ def _create_channel_sessions(self):+ with time_machine.travel("2025-01-01T09:00:00Z", tick=False):+ _create_person(team_id=self.team.pk, distinct_ids=["organic"])+ _create_person(team_id=self.team.pk, distinct_ids=["paid"])+ _create_person(team_id=self.team.pk, distinct_ids=["direct"])+ for distinct_id, session_day, entry_props, timestamps in [+ ("organic", "2025-01-03", {"$referring_domain": "www.google.com"}, ["2025-01-03T10:00:00Z"]),+ (+ "organic",+ "2025-01-10",+ {"$referring_domain": "www.google.com"},+ ["2025-01-10T10:00:00Z", "2025-01-10T10:05:00Z"],+ ),+ ("paid", "2025-01-11", {"utm_source": "google", "utm_medium": "cpc"}, ["2025-01-11T09:00:00Z"]),+ (+ "direct",+ "2025-01-12",+ {"$referring_domain": "$direct"},+ ["2025-01-12T12:00:00Z", "2025-01-12T12:01:00Z"],+ ),+ # Starts before the queried range, with a pageview inside it.+ (+ "paid",+ "2025-01-07T23:50:00Z",+ {"utm_source": "google", "utm_medium": "cpc"},+ ["2025-01-07T23:50:00Z", "2025-01-08T00:10:00Z"],+ ),+ ]:+ session_id = str(uuid7(session_day))+ for index, ts in enumerate(timestamps):+ pathname = "/" if index == 0 else "/pricing"+ _create_event(+ team=self.team,+ event="$pageview",+ distinct_id=distinct_id,+ timestamp=ts,+ properties={+ "$session_id": session_id,+ "$pathname": pathname,+ "$current_url": f"https://example.com{pathname}",+ **(entry_props if index == 0 else {}),+ },+ )+ _create_event(+ team=self.team,+ event="$pageview",+ distinct_id="direct",+ timestamp="2025-01-12T13:00:00Z",+ properties={"$pathname": "/"},+ )+ _create_event(+ team=self.team,+ event="$pageview",+ distinct_id="paid",+ timestamp="2025-01-11T09:30:00Z",+ properties={"$session_id": "legacy-session", "$pathname": "/"},+ )+ flush_persons_and_events()++ @parameterized.expand(+ [+ ("no_filters", [], False),+ ("compare", [], True),+ ("event_filter", [EventPropertyFilter(key="$pathname", operator=PropertyOperator.EXACT, value="/")], True),+ ]+ )+ def test_channel_type_two_phase_matches_join_path(self, _name, properties, compare):+ self._create_channel_sessions()++ def results(allowlisted: bool, expected_strategy: str) -> list:+ with override_settings(WEB_ANALYTICS_NO_JOIN_TEAM_IDS=[self.team.pk] if allowlisted else []):+ runner = self._make_runner(+ breakdownBy=WebStatsBreakdown.INITIAL_CHANNEL_TYPE,+ includeBounceRate=False,+ properties=properties,+ compareFilter=CompareFilter(compare=compare),+ )+ assert runner.query_strategy() == expected_strategy+ return sorted(runner.calculate().results, key=str)++ with time_machine.travel(self.QUERY_TIMESTAMP, tick=False):+ two_phase = results(True, "stats_table_channel_type_two_phase")+ join = results(False, "stats_table_channel_type")++ assert two_phase == join+ assert len(join) >= 2++ @parameterized.expand(+ [+ ("visitors_and_views", {}, True, "stats_table_channel_type_two_phase"),+ ("bounce_rate", {"includeBounceRate": True}, True, "stats_table_channel_type"),+ ("traffic_metrics", {"includeTrafficMetrics": True}, True, "stats_table_channel_type"),+ (+ "conversion_goal",+ {"conversionGoal": CustomEventConversionGoal(customEventName="purchase")},+ True,+ "stats_table_channel_type",+ ),+ (+ "session_property_filter",+ {+ "properties": [+ SessionPropertyFilter(key="$channel_type", value="Direct", operator=PropertyOperator.EXACT)+ ]+ },+ True,+ "stats_table_channel_type",+ ),+ ("team_not_in_rollout", {}, False, "stats_table_channel_type"),+ (+ "thirty_days_with_compare",+ {+ "dateRange": DateRange(date_from="2024-12-30", date_to="2025-01-29"),+ "compareFilter": CompareFilter(compare=True),+ },+ True,+ "stats_table_channel_type_two_phase",+ ),+ (+ "ninety_days",+ {"dateRange": DateRange(date_from="2024-10-31", date_to="2025-01-29")},+ True,+ "stats_table_channel_type",+ ),+ ]+ )+ def test_channel_type_two_phase_selection(self, _name, query_kwargs, allowlisted, expected_strategy):+ with override_settings(WEB_ANALYTICS_NO_JOIN_TEAM_IDS=[self.team.pk] if allowlisted else []):+ runner = WebStatsTableQueryRunner(+ team=self.team,+ query=WebStatsTableQuery(+ dateRange=query_kwargs.pop("dateRange", DateRange(date_from="2025-01-08", date_to="2025-01-15")),+ breakdownBy=WebStatsBreakdown.INITIAL_CHANNEL_TYPE,+ includeBounceRate=query_kwargs.pop("includeBounceRate", False),+ properties=query_kwargs.pop("properties", []),+ **query_kwargs,+ ),+ )+ assert runner.query_strategy() == expected_strategy+@parameterized.expand([("device_type", WebStatsBreakdown.DEVICE_TYPE, [], True),