Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

9.0 KB
Raw Blame
from functools import cached_property
from typing import NamedTuple

from posthog.schema import CachedLogsQueryResponse, LogsQuery

from posthog.hogql import ast
from posthog.hogql.constants import HogQLGlobalSettings
from posthog.hogql.parser import parse_select
from posthog.hogql.query import execute_hogql_query

from posthog.clickhouse.client.connection import Workload
from posthog.hogql_queries.query_runner import AnalyticsQueryRunner

from products.logs.backend.group_by_query_runner import GroupByDimension, LogsGroupBySource
from products.logs.backend.logs_query_runner import (
    LogsQueryResponse,
    LogsQueryRunnerMixin,
    fail_fast_aggregate_settings,
)
from products.logs.backend.models import resolved_distinct_id_attribute_keys, resolved_session_id_attribute_keys

# Enough top values for a drill-down popover; the full distribution belongs to Group mode.
TOP_IDENTITY_VALUES = 5


class _IdentityCandidate(NamedTuple):
    field: str
    map_key: str
    dimension: GroupByDimension


def _identity_candidates(attribute_keys: list[str]) -> list[_IdentityCandidate]:
    # One candidate per (key, map), checking the log attributes before the resource
    # attributes for each key — the same precedence getSessionIdWithKey applies in
    # products/logs/frontend/utils.tsx, so the counts cover the logs the viewer renders
    # as links. attributes_map_str keys carry the ingestion MV's `__str` type suffix
    # (posthog/clickhouse/logs/logs34.py); resource_attributes is a plain map with
    # unsuffixed keys. The query reports a match by its position in this list, so the
    # order is a contract between the SQL and the response.
    candidates: list[_IdentityCandidate] = []
    for attribute_key in attribute_keys:
        candidates.append(
            _IdentityCandidate(
                field="attributes_map_str",
                map_key=f"{attribute_key}__str",
                dimension=GroupByDimension(key=attribute_key, source=LogsGroupBySource.LOG.value),
            )
        )
        candidates.append(
            _IdentityCandidate(
                field="resource_attributes",
                map_key=attribute_key,
                dimension=GroupByDimension(key=attribute_key, source=LogsGroupBySource.RESOURCE.value),
            )
        )
    return candidates


def _identity_read(candidate: _IdentityCandidate) -> ast.Expr:
    # Both maps are read with a bare arrayElement: the property-resolver route wraps map
    # reads in a has() guard that defeats the bucketed serialization and reads every key
    # bucket (see group_by_query_runner._dimension_expr). A missing key reads as '', which
    # nullIf scrubs to NULL so the aggregates can skip identity-less rows.
    read = ast.Call(
        name="arrayElement", args=[ast.Field(chain=[candidate.field]), ast.Constant(value=candidate.map_key)]
    )
    return ast.Call(name="nullIf", args=[read, ast.Constant(value="")])


def _identity_value_expr(candidates: list[_IdentityCandidate]) -> ast.Expr:
    # First non-empty value across the candidates.
    return ast.Call(name="coalesce", args=[_identity_read(candidate) for candidate in candidates])


def _identity_index_expr(candidates: list[_IdentityCandidate]) -> ast.Expr:
    # Position of the candidate the value expr matched, NULL when none matched. An integer
    # rather than the key itself, because naming the key in the row builds and hashes one
    # string per scanned row. The reads are the same expressions the value expr uses, so
    # ClickHouse evaluates each one once and the extra column adds nothing to the scan.
    args: list[ast.Expr] = []
    for index, candidate in enumerate(candidates):
        args.extend(
            [ast.Call(name="isNotNull", args=[_identity_read(candidate)]), ast.Constant(value=index)],
        )
    args.append(ast.Constant(value=None))
    return ast.Call(name="multiIf", args=args)


def _top_values(entries: list[tuple] | None) -> list[dict]:
    # topK(..., 'counts') rows are (value, count, error) tuples; the error margin is
    # noise for a popover, so only value and count survive.
    return [{"value": value, "count": int(count)} for value, count, _error in entries or []]


def _group_key(candidates: list[_IdentityCandidate], indexes: list[int] | None) -> dict | None:
    # The most frequent candidate position names the dimension that carries the counts.
    if not indexes:
        return None
    return candidates[indexes[0]].dimension._asdict()


class ImpactQueryRunner(AnalyticsQueryRunner[LogsQueryResponse], LogsQueryRunnerMixin):
    """Counts the unique sessions and users behind the log entries matching the given filters.

    Kept as its own scan instead of folding into the sparkline query: the sparkline
    aggregates over toStartOfMinute(timestamp) so ClickHouse serves it from the
    minute-aggregate projection, and that projection does not carry the attribute maps
    the identity expressions read. A fold would push every sparkline onto a full scan,
    also for teams that have the impact strip flag off.
    """

    query: LogsQuery
    cached_response: CachedLogsQueryResponse

    @cached_property
    def settings(self) -> HogQLGlobalSettings:
        # Fail-fast caps like CountQueryRunner, plus the uncompressed block cache: unlike a
        # bare count, this query decompresses the two attribute-map columns over the whole
        # window and re-runs against a mostly identical window on every filter tweak.
        return fail_fast_aggregate_settings(use_uncompressed_cache=True)

    @cached_property
    def _session_candidates(self) -> list[_IdentityCandidate]:
        # The query and the response index into the same list, and resolved_* reads Postgres.
        return _identity_candidates(resolved_session_id_attribute_keys(self.team))

    @cached_property
    def _person_candidates(self) -> list[_IdentityCandidate]:
        return _identity_candidates(resolved_distinct_id_attribute_keys(self.team))

    def _calculate(self) -> LogsQueryResponse:
        response = execute_hogql_query(
            query_type="LogsQuery",
            query=self.to_query(),
            modifiers=self.modifiers,
            team=self.team,
            workload=Workload.LOGS,
            timings=self.timings,
            limit_context=self.limit_context,
            settings=self.settings,
        )
        (
            total,
            logs_with_session_id,
            sessions,
            logs_with_distinct_id,
            users,
            top_sessions,
            top_users,
            session_indexes,
            person_indexes,
        ) = response.results[0] if response.results else (0, 0, 0, 0, 0, [], [], [], [])
        return LogsQueryResponse(
            results={
                "total": total,
                "logsWithSessionId": logs_with_session_id,
                "sessions": sessions,
                "logsWithDistinctId": logs_with_distinct_id,
                "users": users,
                "topSessions": _top_values(top_sessions),
                "topUsers": _top_values(top_users),
                "sessionGroupKey": _group_key(self._session_candidates, session_indexes),
                "personGroupKey": _group_key(self._person_candidates, person_indexes),
            }
        )

    def to_query(self) -> ast.SelectQuery:
        # uniq() is HLL-based and ~1-2% off vs exact count(DISTINCT) on high-cardinality
        # ids, but much cheaper — the same tradeoff the error tracking aggregates accept.
        # topK is approximate in the same way. count(x)/uniq(x)/topK(x) skip NULLs, so rows
        # without an identity need no explicit predicate and stay out of the top lists.
        query = parse_select(
            """
            SELECT
                count() AS total,
                count(session_value) AS logs_with_session_id,
                uniq(session_value) AS sessions,
                count(person_value) AS logs_with_distinct_id,
                uniq(person_value) AS users,
                topK({top_n}, 3, 'counts')(session_value) AS top_sessions,
                topK({top_n}, 3, 'counts')(person_value) AS top_users,
                topK(1)(session_index) AS session_indexes,
                topK(1)(person_index) AS person_indexes
            FROM (
                SELECT
                    {session_value} AS session_value,
                    {person_value} AS person_value,
                    {session_index} AS session_index,
                    {person_index} AS person_index
                FROM logs
                WHERE {where}
            )
            """,
            placeholders={
                "session_value": _identity_value_expr(self._session_candidates),
                "person_value": _identity_value_expr(self._person_candidates),
                "session_index": _identity_index_expr(self._session_candidates),
                "person_index": _identity_index_expr(self._person_candidates),
                "where": self.where_with_timestamp_bounds(),
                "top_n": ast.Constant(value=TOP_IDENTITY_VALUES),
            },
        )
        assert isinstance(query, ast.SelectQuery)
        return query