Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(web-analytics): serve the channel breakdown without the per-event sessions join (#109169)

Lucas Ricoyaef69ec161ec
diff --git a/products/web_analytics/backend/hogql_queries/query_constants/stats_table_queries.py b/products/web_analytics/backend/hogql_queries/query_constants/stats_table_queries.py
index 0d5bf14810a3..4431ce0a15fb 100644
--- a/products/web_analytics/backend/hogql_queries/query_constants/stats_table_queries.py
+++ b/products/web_analytics/backend/hogql_queries/query_constants/stats_table_queries.py
@@ -288,6 +288,43 @@ WHERE and({inside_periods}, {event_where}, {all_properties})
GROUP BY session_id, breakdown_value
"""
+# Channel-type breakdown in two phases. `$channel_type` is a session field, so the
+# sessions side cannot be dropped, but the lazy events↔sessions join re-runs the
+# sessions subquery on every events shard and probes it once per event. Here the
+# events are first reduced to one row per session with no join (same session key
+# as NO_JOIN_MAIN_INNER_QUERY), then joined once to the sessions that start in the
+# queried range. The LEFT JOIN keeps sessionless and non-UUIDv7 events under a
+# NULL session, as the lazy join does.
+CHANNEL_TYPE_TWO_PHASE_INNER_QUERY = """
+SELECT
+ e.filtered_person_id AS filtered_person_id,
+ e.filtered_pageview_count AS filtered_pageview_count,
+ s.breakdown_value AS breakdown_value,
+ e.session_id AS session_id,
+ s.start_timestamp AS start_timestamp
+FROM (
+ SELECT
+ {filtered_person_id} AS filtered_person_id,
+ {filtered_pageview_count} AS filtered_pageview_count,
+ if(
+ equals(bitAnd(bitShiftRight(events.$session_id_uuid, 76), 15), 7),
+ events.$session_id,
+ NULL
+ ) AS session_id
+ FROM events
+ WHERE and({inside_periods}, {event_where}, {all_properties})
+ GROUP BY session_id
+) AS e
+LEFT JOIN (
+ SELECT
+ sessions.session_id AS session_id,
+ sessions.$channel_type AS breakdown_value,
+ sessions.$start_timestamp AS start_timestamp
+ FROM sessions
+ WHERE {sessions_in_range}
+) AS s ON e.session_id = s.session_id
+"""
+
# No-join variants of the PAGE-breakdown queries: counts come straight from events
# (bucketed by event timestamp) and bounce comes straight from the sessions table
# (bucketed by session start), instead of routing both through the events↔sessions
diff --git a/products/web_analytics/backend/hogql_queries/stats_table.py b/products/web_analytics/backend/hogql_queries/stats_table.py
index 6b4dcf6c69f1..13f281f2ce70 100644
--- a/products/web_analytics/backend/hogql_queries/stats_table.py
+++ b/products/web_analytics/backend/hogql_queries/stats_table.py
@@ -42,6 +42,7 @@ from products.web_analytics.backend.hogql_queries.first_pageview_attribution imp
from products.web_analytics.backend.hogql_queries.stats_table_pre_aggregated import StatsTablePreAggregatedQueryBuilder
from products.web_analytics.backend.hogql_queries.stats_table_strategies import (
ChannelTypeStrategy,
+ ChannelTypeTwoPhaseStrategy,
FirstPageviewAttributionStrategy,
FrustrationMetricsStrategy,
NoJoinFirstPageviewAttributionStrategy,
@@ -74,6 +75,11 @@ from products.web_analytics.backend.hogql_queries.web_stats_paths_lazy_precomput
BREAKDOWN_NULL_DISPLAY = "(none)"
BREAKDOWN_REFERRER_PREFIX = "referrer:"
+# The two-phase channel query joins at session grain on the coordinating node, so that
+# node's memory grows with the sessions in the scanned span (current plus compare
+# period). Above this span the per-shard join spreads the memory better.
+CHANNEL_TYPE_TWO_PHASE_MAX_SPAN_DAYS = 62
+
def _none_if_nan(value):
"""avgMergeIf returns NaN for empty windows; replace with None so the
@@ -179,7 +185,9 @@ class WebStatsTableQueryRunner(WebAnalyticsQueryRunner[WebStatsTableQueryRespons
return "stats_table_path_bounce_and_avg_time"
if isinstance(strategy, PathBounceStrategy):
return "stats_table_path_bounce"
- # ChannelTypeStrategy must be checked before SimpleBreakdownStrategy since it's a subclass.
+ # ChannelTypeTwoPhaseStrategy subclasses ChannelTypeStrategy, which subclasses SimpleBreakdownStrategy.
+ if isinstance(strategy, ChannelTypeTwoPhaseStrategy):
+ return "stats_table_channel_type_two_phase"
if isinstance(strategy, ChannelTypeStrategy):
return "stats_table_channel_type"
if isinstance(strategy, NoJoinFirstPageviewAttributionStrategy):
@@ -241,6 +249,8 @@ class WebStatsTableQueryRunner(WebAnalyticsQueryRunner[WebStatsTableQueryRespons
return SimpleBreakdownStrategy(self, breakdown_override=self._bounce_entry_pathname_breakdown())
if breakdown == WebStatsBreakdown.INITIAL_CHANNEL_TYPE:
+ if self._can_use_channel_type_two_phase():
+ return ChannelTypeTwoPhaseStrategy(self)
return ChannelTypeStrategy(self)
# Breakdowns whose displayed columns are all event-derived don't need
@@ -386,12 +396,25 @@ class WebStatsTableQueryRunner(WebAnalyticsQueryRunner[WebStatsTableQueryRespons
expr=parse_expr(""" "context.columns.visitors".1 / sum("context.columns.visitors".1) OVER ()"""),
)
- def _uses_session_fields(self) -> bool:
- """True when the breakdown value or any filter reads a `session.*`
- field. Those queries need the events↔sessions join — Initial* entry
- breakdowns for the value itself, session-property filters for the
- predicate. Routing them to the no-join strategy would make HogQL
- silently re-add the lazy join, mislabeling the query tag."""
+ def _can_use_channel_type_two_phase(self) -> bool:
+ # The two-phase shape carries only visitors and views, and it filters the events
+ # scan alone, so a filter or goal that reads a session field needs the join back.
+ if not self._team_in_no_join_rollout():
+ return False
+ if self.query.includeBounceRate or self.query.includeTrafficMetrics or self.query.conversionGoal:
+ return False
+ if self._scanned_span_days() > CHANNEL_TYPE_TWO_PHASE_MAX_SPAN_DAYS:
+ return False
+ return not self._expr_reads_session_fields(self.all_properties())
+
+ def _scanned_span_days(self) -> int:
+ date_from = self.query_date_range.date_from()
+ compare = self.query_compare_to_date_range
+ if compare is not None:
+ date_from = min(date_from, compare.date_from())
+ return (self.query_date_range.date_to() - date_from).days
+
+ def _expr_reads_session_fields(self, expr: ast.Expr) -> bool:
found = False
class Visitor(TraversingVisitor):
@@ -400,13 +423,20 @@ class WebStatsTableQueryRunner(WebAnalyticsQueryRunner[WebStatsTableQueryRespons
if node.chain and node.chain[0] == "session":
found = True
- visitor = Visitor()
- visitor.visit(self._counts_breakdown_value())
- visitor.visit(self.all_properties())
- if self.conversion_goal_expr is not None:
- visitor.visit(self.conversion_goal_expr)
+ Visitor().visit(expr)
return found
+ def _uses_session_fields(self) -> bool:
+ """True when the breakdown value or any filter reads a `session.*`
+ field. Those queries need the events↔sessions join — Initial* entry
+ breakdowns for the value itself, session-property filters for the
+ predicate. Routing them to the no-join strategy would make HogQL
+ silently re-add the lazy join, mislabeling the query tag."""
+ exprs = [self._counts_breakdown_value(), self.all_properties()]
+ if self.conversion_goal_expr is not None:
+ exprs.append(self.conversion_goal_expr)
+ return any(self._expr_reads_session_fields(expr) for expr in exprs)
+
def _period_comparison_tuple(self, column, alias, function_name):
return ast.Alias(
alias=alias,
diff --git a/products/web_analytics/backend/hogql_queries/stats_table_strategies.py b/products/web_analytics/backend/hogql_queries/stats_table_strategies.py
index 1c258cb56cd4..61f36d39e018 100644
--- a/products/web_analytics/backend/hogql_queries/stats_table_strategies.py
+++ b/products/web_analytics/backend/hogql_queries/stats_table_strategies.py
@@ -7,6 +7,7 @@ from posthog.hogql import ast
from posthog.hogql.parser import parse_expr, parse_select
from products.web_analytics.backend.hogql_queries.query_constants.stats_table_queries import (
+ CHANNEL_TYPE_TWO_PHASE_INNER_QUERY,
FIRST_PAGEVIEW_INNER_QUERY,
FRUSTRATION_METRICS_INNER_QUERY,
MAIN_INNER_QUERY,
@@ -265,6 +266,53 @@ class ChannelTypeStrategy(SimpleBreakdownStrategy):
SQL is the same template."""
+class ChannelTypeTwoPhaseStrategy(ChannelTypeStrategy):
+ """INITIAL_CHANNEL_TYPE breakdown without the per-event sessions join.
+
+ Eligible when the tile shows only visitors and views: every other
+ session-derived column would need the join back. The outer query is
+ inherited unchanged; only the inner scan differs.
+ """
+
+ INNER_QUERY = CHANNEL_TYPE_TWO_PHASE_INNER_QUERY
+
+ def build_query(self) -> ast.SelectQuery:
+ WEB_ANALYTICS_NO_JOIN_SERVED.labels(family="stats_table_channel_type").inc()
+ return super().build_query()
+
+ def _inner_query(self, breakdown: ast.Expr) -> ast.SelectQuery:
+ query = parse_select(
+ self.INNER_QUERY,
+ timings=self.runner.timings,
+ placeholders={
+ **self._event_aggregation_placeholders(),
+ "event_where": self.runner.event_type_expr,
+ "all_properties": self.runner.all_properties(),
+ "inside_periods": self.runner._periods_expression(),
+ "sessions_in_range": self._sessions_start_range(),
+ },
+ )
+ assert isinstance(query, ast.SelectQuery)
+ return query
+
+ def _sessions_start_range(self) -> ast.Expr:
+ # Without a compare period the outer query counts every in-range event, so a
+ # session that started before the range still needs its channel. The lazy
+ # events↔sessions join pads its session pushdown by 3 days; pad the same way
+ # so both paths see the same sessions.
+ date_from = self.runner.query_date_range.date_from()
+ compare = self.runner.query_compare_to_date_range
+ if compare is not None:
+ date_from = min(date_from, compare.date_from())
+ return parse_expr(
+ "and(sessions.$start_timestamp >= {date_from} - toIntervalDay(3), sessions.$start_timestamp <= {date_to})",
+ placeholders={
+ "date_from": ast.Constant(value=date_from),
+ "date_to": ast.Constant(value=self.runner.query_date_range.date_to()),
+ },
+ )
+
+
class FirstPageviewAttributionStrategy(SimpleBreakdownStrategy):
"""Breakdowns attributed to the session's first ``$pageview``/``$screen``.
diff --git a/products/web_analytics/backend/hogql_queries/test/test_web_stats_table.py b/products/web_analytics/backend/hogql_queries/test/test_web_stats_table.py
index aff0ea2c3abb..90ad2c8c63b5 100644
--- a/products/web_analytics/backend/hogql_queries/test/test_web_stats_table.py
+++ b/products/web_analytics/backend/hogql_queries/test/test_web_stats_table.py
@@ -2964,6 +2964,146 @@ class TestWebStatsTableNoJoinFastPath(ClickhouseTestMixin, APIBaseTest):
runner = self._make_runner()
assert not runner.query_strategy().startswith("stats_table_no_join")
+ def _create_channel_sessions(self):
+ with time_machine.travel("2025-01-01T09:00:00Z", tick=False):
+ _create_person(team_id=self.team.pk, distinct_ids=["organic"])
+ _create_person(team_id=self.team.pk, distinct_ids=["paid"])
+ _create_person(team_id=self.team.pk, distinct_ids=["direct"])
+ for distinct_id, session_day, entry_props, timestamps in [
+ ("organic", "2025-01-03", {"$referring_domain": "www.google.com"}, ["2025-01-03T10:00:00Z"]),
+ (
+ "organic",
+ "2025-01-10",
+ {"$referring_domain": "www.google.com"},
+ ["2025-01-10T10:00:00Z", "2025-01-10T10:05:00Z"],
+ ),
+ ("paid", "2025-01-11", {"utm_source": "google", "utm_medium": "cpc"}, ["2025-01-11T09:00:00Z"]),
+ (
+ "direct",
+ "2025-01-12",
+ {"$referring_domain": "$direct"},
+ ["2025-01-12T12:00:00Z", "2025-01-12T12:01:00Z"],
+ ),
+ # Starts before the queried range, with a pageview inside it.
+ (
+ "paid",
+ "2025-01-07T23:50:00Z",
+ {"utm_source": "google", "utm_medium": "cpc"},
+ ["2025-01-07T23:50:00Z", "2025-01-08T00:10:00Z"],
+ ),
+ ]:
+ session_id = str(uuid7(session_day))
+ for index, ts in enumerate(timestamps):
+ pathname = "/" if index == 0 else "/pricing"
+ _create_event(
+ team=self.team,
+ event="$pageview",
+ distinct_id=distinct_id,
+ timestamp=ts,
+ properties={
+ "$session_id": session_id,
+ "$pathname": pathname,
+ "$current_url": f"https://example.com{pathname}",
+ **(entry_props if index == 0 else {}),
+ },
+ )
+ _create_event(
+ team=self.team,
+ event="$pageview",
+ distinct_id="direct",
+ timestamp="2025-01-12T13:00:00Z",
+ properties={"$pathname": "/"},
+ )
+ _create_event(
+ team=self.team,
+ event="$pageview",
+ distinct_id="paid",
+ timestamp="2025-01-11T09:30:00Z",
+ properties={"$session_id": "legacy-session", "$pathname": "/"},
+ )
+ flush_persons_and_events()
+
+ @parameterized.expand(
+ [
+ ("no_filters", [], False),
+ ("compare", [], True),
+ ("event_filter", [EventPropertyFilter(key="$pathname", operator=PropertyOperator.EXACT, value="/")], True),
+ ]
+ )
+ def test_channel_type_two_phase_matches_join_path(self, _name, properties, compare):
+ self._create_channel_sessions()
+
+ def results(allowlisted: bool, expected_strategy: str) -> list:
+ with override_settings(WEB_ANALYTICS_NO_JOIN_TEAM_IDS=[self.team.pk] if allowlisted else []):
+ runner = self._make_runner(
+ breakdownBy=WebStatsBreakdown.INITIAL_CHANNEL_TYPE,
+ includeBounceRate=False,
+ properties=properties,
+ compareFilter=CompareFilter(compare=compare),
+ )
+ assert runner.query_strategy() == expected_strategy
+ return sorted(runner.calculate().results, key=str)
+
+ with time_machine.travel(self.QUERY_TIMESTAMP, tick=False):
+ two_phase = results(True, "stats_table_channel_type_two_phase")
+ join = results(False, "stats_table_channel_type")
+
+ assert two_phase == join
+ assert len(join) >= 2
+
+ @parameterized.expand(
+ [
+ ("visitors_and_views", {}, True, "stats_table_channel_type_two_phase"),
+ ("bounce_rate", {"includeBounceRate": True}, True, "stats_table_channel_type"),
+ ("traffic_metrics", {"includeTrafficMetrics": True}, True, "stats_table_channel_type"),
+ (
+ "conversion_goal",
+ {"conversionGoal": CustomEventConversionGoal(customEventName="purchase")},
+ True,
+ "stats_table_channel_type",
+ ),
+ (
+ "session_property_filter",
+ {
+ "properties": [
+ SessionPropertyFilter(key="$channel_type", value="Direct", operator=PropertyOperator.EXACT)
+ ]
+ },
+ True,
+ "stats_table_channel_type",
+ ),
+ ("team_not_in_rollout", {}, False, "stats_table_channel_type"),
+ (
+ "thirty_days_with_compare",
+ {
+ "dateRange": DateRange(date_from="2024-12-30", date_to="2025-01-29"),
+ "compareFilter": CompareFilter(compare=True),
+ },
+ True,
+ "stats_table_channel_type_two_phase",
+ ),
+ (
+ "ninety_days",
+ {"dateRange": DateRange(date_from="2024-10-31", date_to="2025-01-29")},
+ True,
+ "stats_table_channel_type",
+ ),
+ ]
+ )
+ def test_channel_type_two_phase_selection(self, _name, query_kwargs, allowlisted, expected_strategy):
+ with override_settings(WEB_ANALYTICS_NO_JOIN_TEAM_IDS=[self.team.pk] if allowlisted else []):
+ runner = WebStatsTableQueryRunner(
+ team=self.team,
+ query=WebStatsTableQuery(
+ dateRange=query_kwargs.pop("dateRange", DateRange(date_from="2025-01-08", date_to="2025-01-15")),
+ breakdownBy=WebStatsBreakdown.INITIAL_CHANNEL_TYPE,
+ includeBounceRate=query_kwargs.pop("includeBounceRate", False),
+ properties=query_kwargs.pop("properties", []),
+ **query_kwargs,
+ ),
+ )
+ assert runner.query_strategy() == expected_strategy
+
@parameterized.expand(
[
("device_type", WebStatsBreakdown.DEVICE_TYPE, [], True),