Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(experiments): name the cause of a query-limit failure (#125934)

Marcel Poelker43333f81da94
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index 4b9e86ee5697..dae06d9a5550 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -10058,7 +10058,7 @@ export namespace Schemas {
*/
subcode: string | null;
/**
- * A stable qualifier below the subcode: where a sample ratio mismatch is located ('library', 'page'), what leaves nobody exposed ('test_account_filter'), or why a check could not read what it needs ('no_history_access'). Null when the finding names none.
+ * A stable qualifier below the subcode: where a sample ratio mismatch is located ('library', 'page'), what leaves nobody exposed ('test_account_filter'), what made a metric reach a query limit ('metric', 'exposure_criteria'), or why a check could not read what it needs ('no_history_access'). Null when the finding names none.
*/
reason: string | null;
/**
diff --git a/products/experiments/backend/facade/enums.py b/products/experiments/backend/facade/enums.py
index d307dfda7ac6..35083590d065 100644
--- a/products/experiments/backend/facade/enums.py
+++ b/products/experiments/backend/facade/enums.py
@@ -13,6 +13,14 @@ MIN_DEFAULT_CONFIDENCE_LEVEL = Decimal("0.01")
MAX_DEFAULT_CONFIDENCE_LEVEL = Decimal("0.99")
+class ExposuresPath(StrEnum):
+ """Where a metric query read the exposures. Tagged on the query as `experiment_exposures_path`, and stored with a
+ failed metric in `metric_errors`."""
+
+ PRECOMPUTED = "precomputed"
+ DIRECT_SCAN = "direct_scan"
+
+
class PrecomputeSkipReason(StrEnum):
"""Why a metric read never attempted precompute. Tagged on the query as
`experiment_precompute_skip_reason` and counted per reason by the
diff --git a/products/experiments/backend/health/checks/metric_calculation_failed.py b/products/experiments/backend/health/checks/metric_calculation_failed.py
index ce76ea4bc5ec..7fed8f8edbb5 100644
--- a/products/experiments/backend/health/checks/metric_calculation_failed.py
+++ b/products/experiments/backend/health/checks/metric_calculation_failed.py
@@ -17,12 +17,21 @@ from posthog.exceptions import ClickHouseClusterMemoryLimitExceeded
from products.experiments.backend.facade.contracts import (
ExperimentHealthFinding,
+ ExperimentHealthFindingActionKind,
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
ExperimentHealthMetricCompileErrorKind,
)
+from products.experiments.backend.facade.enums import ExposuresPath
+from products.experiments.backend.health.checks.events_received import joined
from products.experiments.backend.health.checks.metric_cannot_compile import MetricCompileError, word_compile_error
-from products.experiments.backend.health.context import HealthContext, MetricCalculationFailure
+from products.experiments.backend.health.context import (
+ ExposureFilterSource,
+ ExposureFilterWithoutColumn,
+ HealthContext,
+ LatestCalculation,
+ MetricCalculationFailure,
+)
from products.experiments.backend.health.skips import (
NOT_ON_ARCHIVED,
RUNS_AFTER_LAUNCH,
@@ -47,6 +56,12 @@ class MetricCalculationFailureKind(StrEnum):
UNEXPECTED_ERROR = "unexpected_error"
+class QueryLimitCause(StrEnum):
+ # pinned: the finding's `reason` when its subcode is `query_limit`.
+ EXPOSURE_CRITERIA = "exposure_criteria"
+ METRIC = "metric"
+
+
# The kinds that a change of the metric or the experiment fixes. The others go away with time or a retry.
_NEEDS_A_FIX = frozenset(
{
@@ -61,6 +76,11 @@ _LIMIT_NEEDS = {
"memory_limit": "needed more memory",
"time_limit": "took longer",
}
+_FILTER_OWNERS = {
+ ExposureFilterSource.EXPOSURE_CRITERIA: "the exposure criteria's",
+ ExposureFilterSource.EXPOSURE_ACTION: "the exposure action's",
+ ExposureFilterSource.TEST_ACCOUNTS: "the test account",
+}
_QUERY_SIZE_LIMIT = "query_size"
# A failure stored before calculations stored the class of the error has only its message, and some classes are too
@@ -105,6 +125,10 @@ class _Wording:
# Whether a setting of the experiment causes the failure, which then fails every metric, so the finding says the
# explanation once and names no metric in it.
every_metric: bool = False
+ # What caused the failure, when the words name it.
+ reason: str | None = None
+ # The filters that the words name.
+ exposure_filters: tuple[ExposureFilterWithoutColumn, ...] = ()
_UNEXPECTED = _Wording(
@@ -166,11 +190,14 @@ def metric_calculation_failed(ctx: HealthContext) -> CheckOutcome:
]
if not worded:
return not_applicable('Reported under "Invalid metric definitions"')
+ worded = _with_query_limit_cause(worded, calculation, ctx.exposure_filters_without_column)
worded.sort(key=lambda item: (item[0].role != "primary", item[1].kind not in _NEEDS_A_FIX))
count = len(worded)
+ exposure_filters = worded[0][1].exposure_filters
return ExperimentHealthFinding(
code=ExperimentHealthFindingCode.METRIC_CALCULATION_FAILED,
subcode=worded[0][1].kind.value,
+ reason=worded[0][1].reason,
# A retry or more data fixes the other kinds. A kind that needs a fix stays a warning after the end, because
# the calculation after the fix reads the events up to the end and brings back the results.
severity=ExperimentHealthFindingSeverity.WARNING
@@ -178,9 +205,13 @@ def metric_calculation_failed(ctx: HealthContext) -> CheckOutcome:
else ExperimentHealthFindingSeverity.INFO,
title=f"Latest calculation failed for {count} {'metric' if count == 1 else 'metrics'}",
detail=" ".join(dict.fromkeys(_sentence(failure, wording) for failure, wording in worded)),
- evidence={"calculated_at": calculation.finished_at.isoformat(), **_evidence(worded)},
+ evidence={
+ "calculated_at": calculation.finished_at.isoformat(),
+ **_evidence(worded),
+ **({"properties": ", ".join(dict.fromkeys(f.key for f in exposure_filters))} if exposure_filters else {}),
+ },
# No action opens a given metric.
- actions=(),
+ actions=(ExperimentHealthFindingActionKind.EDIT_EXPOSURE_CRITERIA,) if exposure_filters else (),
diagnostic_ref=None,
)
@@ -265,18 +296,118 @@ def _limit_explanation(limit: str, failure: MetricCalculationFailure) -> str:
"Its query is longer than PostHog allows. A filter with a long list of values can cause this. Shorten "
"the filters of the metric or of the exposure criteria, or contact support for help."
)
- example = (
+ return (
+ f"Its query {_LIMIT_NEEDS[limit]} than PostHog allows for one calculation. A query reads more data the longer "
+ f"an experiment runs and the more traffic it gets. Simplify the metric, for example with {_simpler(failure)}, "
+ "or contact support for help."
+ )
+
+
+def _simpler(failure: MetricCalculationFailure) -> str:
+ return (
"fewer funnel steps or narrower filters"
if failure.definition.get("metric_type") == "funnel"
else "narrower filters"
)
+
+
+def _with_query_limit_cause(
+ worded: list[tuple[MetricCalculationFailure, _Wording]],
+ calculation: LatestCalculation,
+ filters: tuple[ExposureFilterWithoutColumn, ...] | None,
+) -> list[tuple[MetricCalculationFailure, _Wording]]:
+ """The words of each failure that reached the read, memory or time limit of a query, with the cause that the rest
+ of the calculation shows.
+
+ Every metric's query reads the exposures. So when another metric of the calculation completed the same read, that
+ read fits the limits, and the failing metric's own part of its query reached them. When every metric reached a
+ limit and a query that read the events applies a filter that reads all the properties, that filter is the likely
+ cause, unless a query read precomputed exposures. A metric that ran out of the workflow's time stores no read, so
+ the other failures show the read.
+ """
+ limited = {
+ failure.metric_id: wording.error
+ for failure, wording in worded
+ if wording.kind == MetricCalculationFailureKind.QUERY_LIMIT
+ and wording.error is not None
+ and wording.error in _LIMIT_NEEDS
+ }
+ if calculation.completed_count:
+ return [
+ (failure, _metric_cause(failure, limited[failure.metric_id], calculation.completed_count))
+ if failure.metric_id in limited and failure.exposures_path in calculation.completed_exposures_paths
+ else (failure, wording)
+ for failure, wording in worded
+ ]
+ if (
+ limited
+ and len(limited) == calculation.metric_count
+ and filters
+ and {failure.exposures_path for failure, _ in worded} - {None} == {ExposuresPath.DIRECT_SCAN}
+ ):
+ # With one metric the words name it, as for a failure of the metric.
+ every_metric = calculation.metric_count > 1
+ explanation = _exposure_explanation(set(limited.values()), filters, every_metric=every_metric)
+ return [
+ (
+ failure,
+ _Wording(
+ kind=MetricCalculationFailureKind.QUERY_LIMIT,
+ error=limited[failure.metric_id],
+ explanation=explanation,
+ every_metric=every_metric,
+ reason=QueryLimitCause.EXPOSURE_CRITERIA.value,
+ exposure_filters=filters,
+ ),
+ )
+ for failure, _ in worded
+ ]
+ return worded
+
+
+def _metric_cause(failure: MetricCalculationFailure, limit: str, completed_count: int) -> _Wording:
+ others = "another metric" if completed_count == 1 else "other metrics"
+ return _Wording(
+ kind=MetricCalculationFailureKind.QUERY_LIMIT,
+ error=limit,
+ explanation=f"Its query {_LIMIT_NEEDS[limit]} than PostHog allows for one calculation, while {others} of the "
+ f"experiment returned results. Simplify this metric, for example with {_simpler(failure)}, or contact support "
+ "for help.",
+ reason=QueryLimitCause.METRIC.value,
+ )
+
+
+def _exposure_explanation(
+ limits: set[str], filters: tuple[ExposureFilterWithoutColumn, ...], *, every_metric: bool
+) -> str:
+ needs = _LIMIT_NEEDS[next(iter(limits))] if len(limits) == 1 else "needed more resources"
+ named = joined(
+ [
+ _named(owner, owned)
+ for source, owner in _FILTER_OWNERS.items()
+ if (owned := [f for f in filters if f.source == source])
+ ]
+ )
+ read = joined(list(dict.fromkeys(f.type for f in filters)))
+ one = len(filters) == 1
+ the_filters = "the filter" if one else "these filters"
+ query, each_query = ("Every metric's query", "each query") if every_metric else ("Its query", "the query")
+ fix = (
+ f"Change {the_filters}, or turn off the test account filter in the exposure criteria"
+ if any(f.source == ExposureFilterSource.TEST_ACCOUNTS for f in filters)
+ else f"Remove or change {the_filters}"
+ )
return (
- f"Its query {_LIMIT_NEEDS[limit]} than PostHog allows for one calculation. A query reads more data the longer "
- f"an experiment runs and the more traffic it gets. Simplify the metric, for example with {example}, or "
- "contact support for help."
+ f"{query} {needs} than PostHog allows for one calculation. The likely cause is {named}, which "
+ f"{'makes' if one else 'make'} {each_query} read all {read} properties. {fix}, or contact support for help."
)
+def _named(owner: str, filters: list[ExposureFilterWithoutColumn]) -> str:
+ properties = joined([f'the {f.type} property "{f.key}"' for f in filters])
+ return f"{owner} {'filter' if len(filters) == 1 else 'filters'} on {properties}"
+
+
def _settings(failure: MetricCalculationFailure) -> _Wording | None:
"""The words for a failure that a setting of the metric or the experiment caused, or None when the failure isn't
known to be one."""
diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.py
index a5ab3175d6a9..909c2e5b50a8 100644
--- a/products/experiments/backend/health/context.py
+++ b/products/experiments/backend/health/context.py
@@ -34,6 +34,7 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFlagChangeKind,
ExperimentHealthMetricCompileErrorKind,
)
+from products.experiments.backend.facade.enums import ExposuresPath
from products.experiments.backend.health.delays import ENDED_FLAG_QUIET_PERIOD, LAST_CALLED_AT_LAG_DURATION
from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimension
from products.experiments.backend.hogql_queries.experiment_query_builder import get_exposure_config_params_for_builder
@@ -326,6 +327,9 @@ class MetricCalculationFailure:
retriable: bool
# The effective definition now, in which the wording looks for the names that the message gives.
definition: Mapping[str, Any]
+ # Where the query read the exposures. None for a failure stored before calculations stored it, and for a query that
+ # failed before it was built.
+ exposures_path: ExposuresPath | None
@frozen
@@ -338,6 +342,10 @@ class LatestCalculation:
finished_at: datetime | None
# The metrics of the experiment now whose latest calculation failed, in calculation order.
failures: tuple[MetricCalculationFailure, ...]
+ # The metrics of the experiment now whose result of this calculation the page shows.
+ completed_count: int
+ # Where the queries of those results read the exposures, when there is a failure to compare them with.
+ completed_exposures_paths: frozenset[ExposuresPath]
@frozen
@@ -349,6 +357,23 @@ class FlagKeyFilter:
ExposurePropertyType = Literal["event", "person"]
+ExposureFilterPropertyType = ExposurePropertyType | Literal["group"]
+
+
+class ExposureFilterSource(StrEnum):
+ EXPOSURE_CRITERIA = "exposure_criteria"
+ EXPOSURE_ACTION = "exposure_action"
+ TEST_ACCOUNTS = "test_accounts"
+
+
+@frozen
+class ExposureFilterWithoutColumn:
+ """A property that the exposure query filters on and reads from the JSON of all the properties of each event, person
+ or group."""
+
+ type: ExposureFilterPropertyType
+ key: str
+ source: ExposureFilterSource
@frozen
@@ -506,6 +531,9 @@ class HealthContext:
# None when not loaded or unknown. Only the health endpoints load it, and only for a launched experiment that
# isn't archived.
latest_calculation: LatestCalculation | None
+ # None when not loaded or unknown. Only the health endpoints load it, and only when no metric of the latest
+ # calculation completed and a failed query read the events.
+ exposure_filters_without_column: tuple[ExposureFilterWithoutColumn, ...] | None
def parse_flag_state(
@@ -965,4 +993,5 @@ def load_health_context(
excluded_variant_keys=frozenset(experiment.excluded_variants or []),
metric_compiles=None,
latest_calculation=None,
+ exposure_filters_without_column=None,
)
diff --git a/products/experiments/backend/health/exposure_columns.py b/products/experiments/backend/health/exposure_columns.py
new file mode 100644
index 000000000000..308e5235151c
--- /dev/null
+++ b/products/experiments/backend/health/exposure_columns.py
@@ -0,0 +1,127 @@
+from typing import cast
+
+import pydantic
+
+from posthog.schema import ActionsNode, ExperimentExposureCriteria
+
+from posthog.hogql import ast
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.database import Database
+from posthog.hogql.modifiers import create_default_modifiers_for_team
+from posthog.hogql.property_planner import PropertyAccessPlan, PropertyScope, PropertySourceKind, plan_property_access
+from posthog.hogql.resolver import resolve_types
+from posthog.hogql.transforms.property_types import build_property_swapper
+from posthog.hogql.visitor import GetFieldsTraverser
+
+from posthog.exceptions_capture import capture_exception
+from posthog.models.event.new_events_schema import use_new_events_schema
+from posthog.schema_enums import MaterializationMode
+
+from products.experiments.backend.health.context import (
+ ExposureFilterPropertyType,
+ ExposureFilterSource,
+ ExposureFilterWithoutColumn,
+)
+from products.experiments.backend.hogql_queries.base_query_utils import event_or_action_to_filter
+from products.experiments.backend.hogql_queries.experiment_query_builder import get_exposure_config_params_for_builder
+from products.experiments.backend.hogql_queries.exposure_query_logic import (
+ get_test_accounts_filter,
+ normalize_to_exposure_criteria,
+)
+from products.experiments.backend.models.experiment import Experiment
+
+_FILTER_TYPES: dict[PropertyScope, ExposureFilterPropertyType] = {
+ PropertyScope.EVENT: "event",
+ PropertyScope.PERSON: "person",
+ PropertyScope.GROUP: "group",
+}
+# The exposure query always reads the flag key, so a filter on it costs nothing more and removing it changes nothing.
+_FLAG_KEY_PROPERTY = "$feature_flag"
+
+
+def _reads_all_properties(plan: PropertyAccessPlan, context: HogQLContext) -> bool:
+ if plan.source.kind != PropertySourceKind.JSON:
+ return False
+ # Before the lazy persons table expands, the planner can't match the columns of the `person` table.
+ if plan.scope == PropertyScope.PERSON and plan.source.table_name is None:
+ metadata = context.property_metadata
+ return (
+ metadata is None
+ or context.modifiers.materializationMode == MaterializationMode.DISABLED
+ or metadata.materialized_column("person", "properties", plan.property_name) is None
+ )
+ # The planner calls a read of the native events table JSON, but each property there is a subcolumn of its own.
+ if plan.source.table_name == "events" and context.uses_new_events_schema():
+ return False
+ return True
+
+
+def _named_properties(expr: ast.Expr) -> set[str | int]:
+ """The properties that a filter names. A `$virt` property expands to reads of other properties when its type
+ resolves, and the filter names none of those."""
+ return {field.chain[-1] for field in GetFieldsTraverser(expr).fields if len(field.chain) > 1}
+
+
+def _filters_without_column(
+ experiment: Experiment, criteria: ExperimentExposureCriteria | None
+) -> tuple[ExposureFilterWithoutColumn, ...]:
+ team = experiment.team
+ # The configs that the exposure query builds from, as the filters that it applies.
+ params = get_exposure_config_params_for_builder(criteria, team, experiment.start_date)
+ filters = [
+ (
+ event_or_action_to_filter(team, config),
+ ExposureFilterSource.EXPOSURE_ACTION
+ if isinstance(config, ActionsNode)
+ else ExposureFilterSource.EXPOSURE_CRITERIA,
+ )
+ for config in (params.exposure_config, params.activation_config)
+ if config
+ ]
+ filters += [(expr, ExposureFilterSource.TEST_ACCOUNTS) for expr in get_test_accounts_filter(team, criteria)]
+ named_by_filter = [_named_properties(expr) for expr, _ in filters]
+ if not any(named_by_filter):
+ return ()
+ modifiers = create_default_modifiers_for_team(team)
+ # The runner resolves this modifier, and the catalog cache key includes it, so a health read shares the compile's entry.
+ modifiers.useNewEventsSchema = use_new_events_schema(team.pk, modifiers)
+ context = HogQLContext(
+ team_id=team.pk,
+ team=team,
+ database=Database.create_for(team=team, modifiers=modifiers, use_cached_sources=True),
+ enable_select_queries=True,
+ modifiers=modifiers,
+ )
+ query = ast.SelectQuery(
+ select=[ast.Constant(value=1)],
+ select_from=ast.JoinExpr(table=ast.Field(chain=["events"])),
+ where=ast.And(exprs=[expr for expr, _ in filters]),
+ )
+ resolved = resolve_types(query, context, dialect="clickhouse")
+ build_property_swapper(resolved, context)
+ where = cast(ast.And, resolved.where)
+
+ found: dict[ExposureFilterWithoutColumn, None] = {}
+ for expr, (_, source), named_keys in zip(where.exprs, filters, named_by_filter, strict=True):
+ for field in GetFieldsTraverser(expr).fields:
+ plan = plan_property_access(field, context)
+ if plan is None or plan.property_name not in named_keys or plan.property_name == _FLAG_KEY_PROPERTY:
+ continue
+ filter_type = _FILTER_TYPES.get(plan.scope)
+ if filter_type is not None and _reads_all_properties(plan, context):
+ found[ExposureFilterWithoutColumn(type=filter_type, key=plan.property_name, source=source)] = None
+ return tuple(found)
+
+
+def load_filters_without_column(experiment: Experiment) -> tuple[ExposureFilterWithoutColumn, ...] | None:
+ """The filters of a query that reads the exposures from the events, for each property that the query reads from the
+ JSON of all the properties. None when the criteria don't parse or the lookup fails. Never raises."""
+ try:
+ criteria = normalize_to_exposure_criteria(experiment.exposure_criteria)
+ except pydantic.ValidationError:
+ return None
+ try:
+ return _filters_without_column(experiment, criteria)
+ except Exception as error:
+ capture_exception(error)
+ return None
diff --git a/products/experiments/backend/health/latest_calculation.py b/products/experiments/backend/health/latest_calculation.py
index ee7bf4ac1fd9..1445722070d5 100644
--- a/products/experiments/backend/health/latest_calculation.py
+++ b/products/experiments/backend/health/latest_calculation.py
@@ -3,6 +3,7 @@ from typing import Any
from posthog.exceptions_capture import capture_exception
+from products.experiments.backend.facade.enums import ExposuresPath
from products.experiments.backend.health.context import LatestCalculation, MetricCalculationFailure
from products.experiments.backend.health.metric_compiles import can_hold_evaluation_rules, without_available_fields
from products.experiments.backend.metric_calculation.config import (
@@ -43,19 +44,38 @@ def _load_latest_calculation(experiment: Experiment) -> LatestCalculation:
# failures, or else the run in progress, whose errors can still change, so the check waits for a finished run.
run = get_latest_recalculation(experiment)
if run is None:
- return LatestCalculation(metric_count=len(configs), finished_at=None, failures=())
+ return LatestCalculation(
+ metric_count=len(configs),
+ finished_at=None,
+ failures=(),
+ completed_count=0,
+ completed_exposures_paths=frozenset(),
+ )
metric_errors = run.metric_errors if isinstance(run.metric_errors, dict) else {}
- rows = {
- row.metric_uuid: row for row in MetricResultStore(experiment_id=experiment.id).for_run(run, with_result=False)
- }
- failures = (
- _shown_failure(experiment, calculation_config, metric_errors.get(metric_id), rows.get(metric_id))
+ results = MetricResultStore(experiment_id=experiment.id)
+ rows = {row.metric_uuid: row for row in results.for_run(run, with_result=False)}
+ failures = tuple(
+ failure
for metric_id, calculation_config in configs.items()
+ if (
+ failure := _shown_failure(experiment, calculation_config, metric_errors.get(metric_id), rows.get(metric_id))
+ )
+ is not None
)
+ failed = {failure.metric_id for failure in failures}
+ completed = [
+ row
+ for metric_id in configs
+ if metric_id not in failed
+ and (row := rows.get(metric_id)) is not None
+ and row.status == ExperimentMetricResult.Status.COMPLETED
+ ]
return LatestCalculation(
metric_count=len(configs),
finished_at=run.completed_at,
- failures=tuple(failure for failure in failures if failure is not None),
+ failures=failures,
+ completed_count=len(completed),
+ completed_exposures_paths=results.exposures_paths(completed) if failures else frozenset(),
)
@@ -81,6 +101,7 @@ def _shown_failure(
return None
error_type = error.get("error_type")
step = error.get("step")
+ exposures_path = error.get("exposures_path")
return MetricCalculationFailure(
metric_id=calculation_config.metric_id,
name=calculation_metric_name(experiment, calculation_config),
@@ -91,4 +112,7 @@ def _shown_failure(
can_quote_evaluation_rules=can_hold_evaluation_rules(calculation_config.definition),
retriable=error.get("retriable") is True,
definition=calculation_config.definition,
+ exposures_path=ExposuresPath(exposures_path)
+ if isinstance(exposures_path, str) and exposures_path in ExposuresPath
+ else None,
)
diff --git a/products/experiments/backend/health/loader.py b/products/experiments/backend/health/loader.py
index 959e5fafb5e2..f0b79e63b39b 100644
--- a/products/experiments/backend/health/loader.py
+++ b/products/experiments/backend/health/loader.py
@@ -5,8 +5,10 @@ from posthog.dataclasses import frozen
from posthog.models.user import User
from products.experiments.backend.facade.contracts import ExperimentHealthReaderAccess
+from products.experiments.backend.facade.enums import ExposuresPath
from products.experiments.backend.health.condition_set_traffic import load_condition_set_traffic
from products.experiments.backend.health.context import HealthContext, load_health_context
+from products.experiments.backend.health.exposure_columns import load_filters_without_column
from products.experiments.backend.health.exposure_gate import load_flag_history
from products.experiments.backend.health.exposure_inputs import ExposureInputs, load_exposure_inputs
from products.experiments.backend.health.latest_calculation import load_latest_calculation
@@ -73,11 +75,17 @@ def _health_context(
flag_history = load_flag_history(experiment, reader_access=reader_access)
context = load_health_context(experiment, flag_history=flag_history)
traffic = load_condition_set_traffic(experiment, context, reader_access=reader_access)
+ latest_calculation = load_latest_calculation(experiment)
return replace(
context,
condition_set_traffic=traffic,
pause_history=load_pause_history(experiment, reader_access=reader_access),
primary_metric=load_primary_metric(experiment),
metric_compiles=load_metric_compiles(experiment, user=user),
- latest_calculation=load_latest_calculation(experiment),
+ latest_calculation=latest_calculation,
+ exposure_filters_without_column=load_filters_without_column(experiment)
+ if latest_calculation is not None
+ and not latest_calculation.completed_count
+ and any(failure.exposures_path == ExposuresPath.DIRECT_SCAN for failure in latest_calculation.failures)
+ else None,
)
diff --git a/products/experiments/backend/health/test/factories.py b/products/experiments/backend/health/test/factories.py
index c55fdb4a5e1f..0a3ad42f0622 100644
--- a/products/experiments/backend/health/test/factories.py
+++ b/products/experiments/backend/health/test/factories.py
@@ -47,6 +47,7 @@ RUNNING_CONTEXT = HealthContext(
excluded_variant_keys=frozenset(),
metric_compiles=None,
latest_calculation=None,
+ exposure_filters_without_column=None,
)
diff --git a/products/experiments/backend/health/test/test_exposure_columns.py b/products/experiments/backend/health/test/test_exposure_columns.py
new file mode 100644
index 000000000000..600116c67c1c
--- /dev/null
+++ b/products/experiments/backend/health/test/test_exposure_columns.py
@@ -0,0 +1,282 @@
+from collections.abc import Callable
+from datetime import timedelta
+from typing import Any
+
+from posthog.test.base import BaseTest
+from unittest.mock import patch
+
+from django.utils import timezone
+
+from parameterized import parameterized
+
+from posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode, PropertyGroupsMode
+
+from posthog.models.event.new_events_schema import events_route_scope
+from posthog.models.property import TableColumn
+from posthog.schema_enums import MaterializationMode
+
+from products.actions.backend.models.action import Action
+from products.experiments.backend.health.context import ExposureFilterSource
+from products.experiments.backend.health.exposure_columns import load_filters_without_column
+from products.experiments.backend.models.experiment import Experiment
+from products.feature_flags.backend.models.feature_flag import FeatureFlag
+
+from ee.clickhouse.materialized_columns.columns import MaterializedColumn, MaterializedColumnDetails
+
+ON_EVENTS = PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS
+JOINED = PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED
+_SOURCE_LABELS = {
+ ExposureFilterSource.EXPOSURE_CRITERIA: "",
+ ExposureFilterSource.EXPOSURE_ACTION: " (action)",
+ ExposureFilterSource.TEST_ACCOUNTS: " (test account)",
+}
+
+
+def _column(table_column: TableColumn, key: str) -> MaterializedColumn:
+ return MaterializedColumn(
+ name=f"mat_{key}", details=MaterializedColumnDetails(table_column, key, is_disabled=False), is_nullable=False
+ )
+
+
+MATERIALIZED_COLUMNS = {
+ "events": {
+ ("plan", "properties"): _column("properties", "plan"),
+ ("tier", "person_properties"): _column("person_properties", "tier"),
+ },
+ "person": {("tier", "properties"): _column("properties", "tier")},
+}
+
+
+def _filter(key: str, property_type: str, **rest: Any) -> dict[str, Any]:
+ return {"key": key, "type": property_type, "operator": "is_set", **rest}
+
+
+class TestLoadFiltersWithoutColumn(BaseTest):
+ def _experiment(self, *, filter_test_accounts: bool = True) -> Experiment:
+ return Experiment.objects.create(
+ team=self.team,
+ name="Pricing page",
+ feature_flag=FeatureFlag.objects.create(
+ team=self.team, key="pricing-page", created_by=self.user, filters={}
+ ),
+ start_date=timezone.now() - timedelta(days=3),
+ exposure_criteria={
+ "filterTestAccounts": filter_test_accounts,
+ "exposure_config": {
+ "kind": "ExperimentEventExposureConfig",
+ "event": "$pageview",
+ "properties": [
+ _filter("plan", "event"),
+ _filter("$plan_source", "event"),
+ _filter("campaign", "event"),
+ _filter("tier", "person"),
+ _filter("$initial_os", "person"),
+ _filter("industry", "group", group_type_index=0),
+ # None of these reads the JSON: the query reads the flag key anyway, and the others are columns.
+ _filter("$feature_flag", "event"),
+ _filter("$elements_chain", "event"),
+ _filter("$virt_is_bot", "event"),
+ _filter("distinct_id", "person"),
+ _filter("$group_key", "group", group_type_index=0),
+ ],
+ },
+ },
+ )
+
+ @parameterized.expand(
+ [
+ (
+ "without_property_groups",
+ ON_EVENTS,
+ None,
+ False,
+ True,
+ [
+ "event:$plan_source",
+ "event:campaign",
+ "person:$initial_os",
+ "group:industry",
+ "person:email (test account)",
+ ],
+ ),
+ (
+ "property_groups_hold_the_custom_properties",
+ ON_EVENTS,
+ PropertyGroupsMode.OPTIMIZED,
+ False,
+ True,
+ ["event:$plan_source", "person:$initial_os", "group:industry"],
+ ),
+ (
+ "persons_joined",
+ JOINED,
+ None,
+ False,
+ True,
+ [
+ "event:$plan_source",
+ "event:campaign",
+ "person:$initial_os",
+ "group:industry",
+ "person:email (test account)",
+ ],
+ ),
+ (
+ "native_events_table",
+ ON_EVENTS,
+ None,
+ True,
+ True,
+ ["group:industry"],
+ ),
+ (
+ "test_accounts_not_filtered",
+ ON_EVENTS,
+ None,
+ False,
+ False,
+ ["event:$plan_source", "event:campaign", "person:$initial_os", "group:industry"],
+ ),
+ (
+ "persons_joined_materialization_disabled",
+ JOINED,
+ None,
+ False,
+ True,
+ [
+ "event:plan",
+ "event:$plan_source",
+ "event:campaign",
+ "person:tier",
+ "person:$initial_os",
+ "group:industry",
+ "person:email (test account)",
+ "person:tier (test account)",
+ ],
+ MaterializationMode.DISABLED,
+ ),
+ ]
+ )
+ def test_names_the_filters_that_read_all_the_properties(
+ self,
+ _name: str,
+ persons_on_events_mode: PersonsOnEventsMode,
+ property_groups_mode: PropertyGroupsMode | None,
+ new_events_schema: bool,
+ filter_test_accounts: bool,
+ expected: list[str],
+ materialization_mode: MaterializationMode | None = None,
+ ) -> None:
+ self.team.modifiers = {
+ "personsOnEventsMode": persons_on_events_mode,
+ "propertyGroupsMode": property_groups_mode,
+ "materializationMode": materialization_mode,
+ }
+ self.team.test_account_filters = [_filter("email", "person"), _filter("tier", "person")]
+ self.team.save()
+ experiment = self._experiment(filter_test_accounts=filter_test_accounts)
+
+ with (
+ events_route_scope(HogQLQueryModifiers(useNewEventsSchema=new_events_schema)),
+ patch(
+ "posthog.clickhouse.materialized_columns.get_enabled_materialized_columns_by_table",
+ return_value=MATERIALIZED_COLUMNS,
+ ),
+ ):
+ filters = load_filters_without_column(experiment)
+
+ assert filters is not None
+ self.assertEqual(
+ [f"{found.type}:{found.key}{_SOURCE_LABELS[found.source]}" for found in filters],
+ expected,
+ )
+
+ @parameterized.expand(
+ [
+ ("criteria_do_not_parse", {"exposure_config": {"kind": "Unknown"}}, None),
+ (
+ "criteria_have_no_exposure_query",
+ {
+ "exposure_config": {
+ "kind": "ExperimentWarehouseNativeExposureConfig",
+ "connection_id": "source",
+ "query": "SELECT 1",
+ }
+ },
+ None,
+ ),
+ (
+ "criteria_have_no_filters",
+ {"filterTestAccounts": False},
+ (),
+ ),
+ ]
+ )
+ def test_criteria_without_a_filter_to_name(
+ self, _name: str, exposure_criteria: dict[str, Any], expected: tuple[()] | None
+ ) -> None:
+ experiment = self._experiment()
+ experiment.exposure_criteria = exposure_criteria
+
+ self.assertEqual(load_filters_without_column(experiment), expected)
+
+ @parameterized.expand(
+ [
+ (
+ "exposure_action",
+ lambda action: {"exposure_config": action},
+ ["event:campaign (action)", "event:$current_url (action)", "event:source (action)"],
+ ),
+ (
+ "activation_action",
+ lambda action: {"exposure_config": None, "activation_config": action},
+ ["event:campaign (action)", "event:$current_url (action)", "event:source (action)"],
+ ),
+ ("deleted_action", lambda action: {"exposure_config": {**action, "id": 0}}, ["event:source (action)"]),
+ ]
+ )
+ def test_names_the_filters_of_an_action(
+ self, _name: str, criteria: Callable[[dict[str, Any]], dict[str, Any]], expected: list[str]
+ ) -> None:
+ action = Action.objects.create(
+ team=self.team,
+ name="Started checkout",
+ steps_json=[
+ {"event": "checkout", "properties": [_filter("campaign", "event")]},
+ {"event": "$pageview", "url": "/checkout", "properties": [_filter("$virt_is_bot", "event")]},
+ ],
+ )
+ experiment = self._experiment()
+ experiment.exposure_criteria = {
+ "filterTestAccounts": False,
+ **criteria(
+ {"kind": "ActionsNode", "id": action.pk, "properties": [_filter("source", "event")]},
+ ),
+ }
+
+ with events_route_scope(HogQLQueryModifiers(useNewEventsSchema=False)):
+ filters = load_filters_without_column(experiment)
+
+ assert filters is not None
+ self.assertEqual([f"{found.type}:{found.key}{_SOURCE_LABELS[found.source]}" for found in filters], expected)
+
+ def test_a_virtual_property_does_not_take_a_read_that_another_filter_names(self) -> None:
+ self.team.test_account_filters = [_filter("$virt_is_bot", "event")]
+ self.team.save()
+ experiment = self._experiment()
+ experiment.exposure_criteria = {
+ "filterTestAccounts": True,
+ "exposure_config": {
+ "kind": "ExperimentEventExposureConfig",
+ "event": "$pageview",
+ "properties": [_filter("$lib", "event")],
+ },
+ }
+
+ with events_route_scope(HogQLQueryModifiers(useNewEventsSchema=False)):
+ filters = load_filters_without_column(experiment)
+
+ assert filters is not None
+ self.assertEqual(
+ [f"{found.type}:{found.key}{_SOURCE_LABELS[found.source]}" for found in filters], ["event:$lib"]
+ )
diff --git a/products/experiments/backend/health/test/test_metric_calculation_failed.py b/products/experiments/backend/health/test/test_metric_calculation_failed.py
index 1bf40e74416f..27aafec73bb0 100644
--- a/products/experiments/backend/health/test/test_metric_calculation_failed.py
+++ b/products/experiments/backend/health/test/test_metric_calculation_failed.py
@@ -23,8 +23,11 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthMetricCompileErrorKind,
ExperimentHealthReaderAccess,
)
+from products.experiments.backend.facade.enums import ExposuresPath
from products.experiments.backend.health.checks.metric_calculation_failed import metric_calculation_failed
from products.experiments.backend.health.context import (
+ ExposureFilterSource,
+ ExposureFilterWithoutColumn,
LatestCalculation,
MetricCalculationFailure,
MetricCompileFailure,
@@ -45,6 +48,11 @@ NO_QUERIES = ExperimentHealthReaderAccess(
can_read_flag_history=False, can_query_events=False, denied_event_properties=frozenset()
)
FINISHED_AT = datetime(2026, 10, 26, 9, 30, tzinfo=UTC)
+OUT_OF_MEMORY = "This query ran out of memory before it could finish."
+PLAN_FILTER = ExposureFilterWithoutColumn(type="group", key="plan", source=ExposureFilterSource.EXPOSURE_CRITERIA)
+DIRECT = ExposuresPath.DIRECT_SCAN
+PRECOMPUTED = ExposuresPath.PRECOMPUTED
+NO_CAUSE = (None, 'primary metric "Revenue" failed. Its query needed more memory than PostHog allows for one')
CLICKHOUSE_MESSAGE = (
"Code: 215. DB::Exception: Column `properties` is not under aggregate function and not in GROUP BY keys. "
"In query SELECT properties FROM events"
@@ -59,6 +67,7 @@ def _failure(
role: str = "primary",
retriable: bool = True,
can_quote_evaluation_rules: bool = False,
+ exposures_path: ExposuresPath | None = None,
) -> MetricCalculationFailure:
return MetricCalculationFailure(
metric_id=f"{role}-metric",
@@ -70,11 +79,24 @@ def _failure(
can_quote_evaluation_rules=can_quote_evaluation_rules,
retriable=retriable,
definition={"kind": "ExperimentMetric", "metric_type": "mean"},
+ exposures_path=exposures_path,
)
-def _outcome(*failures: MetricCalculationFailure, **changes: Any) -> ExperimentHealthFinding | CheckSkip | None:
- calculation = LatestCalculation(metric_count=len(failures), finished_at=FINISHED_AT, failures=failures)
+def _limited(exposures_path: ExposuresPath | None, role: str = "primary") -> MetricCalculationFailure:
+ return _failure(OUT_OF_MEMORY, error_type="out_of_memory", role=role, exposures_path=exposures_path)
+
+
+def _outcome(
+ *failures: MetricCalculationFailure, completed: tuple[ExposuresPath | None, ...] = (), **changes: Any
+) -> ExperimentHealthFinding | CheckSkip | None:
+ calculation = LatestCalculation(
+ metric_count=len(failures) + len(completed),
+ finished_at=FINISHED_AT,
+ failures=failures,
+ completed_count=len(completed),
+ completed_exposures_paths=frozenset(path for path in completed if path is not None),
+ )
return metric_calculation_failed(replace(RUNNING_CONTEXT, latest_calculation=calculation, **changes))
@@ -319,8 +341,174 @@ class TestMetricCalculationFailed(TestCase):
self.assertEqual(finding.detail.count(words), 1)
self.assertNotIn("Revenue", finding.detail)
+ @parameterized.expand(
+ [
+ (
+ "every_metric_read_the_events_and_reached_a_limit",
+ (_limited(DIRECT), _limited(DIRECT, "secondary")),
+ (),
+ (PLAN_FILTER,),
+ ("exposure_criteria", 'filter on the group property "plan", which makes each query read all group'),
+ ),
+ (
+ "the_only_metric_read_the_events_and_reached_a_limit",
+ (_limited(DIRECT),),
+ (),
+ (PLAN_FILTER,),
+ (
+ "exposure_criteria",
+ 'primary metric "Revenue" failed. Its query needed more memory than PostHog allows for one '
+ 'calculation. The likely cause is the exposure criteria\'s filter on the group property "plan"',
+ ),
+ ),
+ (
+ "two_limits_and_two_filters_of_different_kinds",
+ (
+ _limited(DIRECT),
+ _failure(
+ "Limit for rows or bytes to read exceeded",
+ error_type="byte_limit",
+ role="secondary",
+ exposures_path=DIRECT,
+ ),
+ ),
+ (),
+ (
+ PLAN_FILTER,
+ ExposureFilterWithoutColumn(type="event", key="$device", source=ExposureFilterSource.TEST_ACCOUNTS),
+ ),
+ (
+ "exposure_criteria",
+ "Every metric's query needed more resources than PostHog allows for one calculation. The likely "
+ 'cause is the exposure criteria\'s filter on the group property "plan" and the test account filter '
+ 'on the event property "$device", which make each query read all group and event properties. '
+ "Change these filters, or turn off the test account filter in the exposure criteria, or contact",
+ ),
+ ),
+ (
+ "the_filter_is_in_the_exposure_action",
+ (_limited(DIRECT),),
+ (),
+ (
+ ExposureFilterWithoutColumn(
+ type="event", key="campaign", source=ExposureFilterSource.EXPOSURE_ACTION
+ ),
+ ),
+ (
+ "exposure_criteria",
+ 'The likely cause is the exposure action\'s filter on the event property "campaign", which makes '
+ "the query read all event properties. Remove or change the filter, or contact support for help.",
+ ),
+ ),
+ (
+ "one_property_in_the_exposure_criteria_and_the_test_account_filters",
+ (_limited(DIRECT),),
+ (),
+ (
+ ExposureFilterWithoutColumn(
+ type="event", key="plan", source=ExposureFilterSource.EXPOSURE_CRITERIA
+ ),
+ ExposureFilterWithoutColumn(type="event", key="plan", source=ExposureFilterSource.TEST_ACCOUNTS),
+ ),
+ (
+ "exposure_criteria",
+ 'the exposure criteria\'s filter on the event property "plan" and the test account filter on the '
+ 'event property "plan", which make the query read all event properties. Change these filters, or '
+ "turn off the test account filter in the exposure criteria",
+ ),
+ ),
+ (
+ "one_query_read_precomputed_exposures",
+ (_limited(DIRECT), _limited(PRECOMPUTED, "secondary")),
+ (),
+ (PLAN_FILTER,),
+ NO_CAUSE,
+ ),
+ (
+ "every_query_read_precomputed_exposures",
+ (_limited(PRECOMPUTED), _limited(PRECOMPUTED, "secondary")),
+ (),
+ (PLAN_FILTER,),
+ NO_CAUSE,
+ ),
+ (
+ "failures_do_not_say_where_they_read",
+ (_limited(None), _limited(None, "secondary")),
+ (),
+ (PLAN_FILTER,),
+ NO_CAUSE,
+ ),
+ ("no_filter_reads_all_the_properties", (_limited(DIRECT), _limited(DIRECT, "secondary")), (), (), NO_CAUSE),
+ ("filters_could_not_be_read", (_limited(DIRECT), _limited(DIRECT, "secondary")), (), None, NO_CAUSE),
+ (
+ "a_metric_ran_out_of_the_workflow_time_without_a_stored_read",
+ (_limited(DIRECT), _limited(None, "secondary")),
+ (),
+ (PLAN_FILTER,),
+ ("exposure_criteria", 'filter on the group property "plan", which makes each query read all group'),
+ ),
+ (
+ "another_metric_failed_for_another_reason",
+ (
+ _limited(DIRECT),
+ _failure(
+ "Queries are a little too busy right now.",
+ error_type="rate_limited",
+ role="secondary",
+ exposures_path=DIRECT,
+ ),
+ ),
+ (),
+ (PLAN_FILTER,),
+ NO_CAUSE,
+ ),
+ (
+ "another_metric_completed_the_same_read_of_the_events",
+ (_limited(DIRECT),),
+ (DIRECT,),
+ (PLAN_FILTER,),
+ ("metric", "while another metric of the experiment returned results. Simplify this metric"),
+ ),
+ (
+ "another_metric_completed_the_same_read_of_precomputed_exposures",
+ (_limited(PRECOMPUTED),),
+ (PRECOMPUTED,),
+ None,
+ ("metric", "while another metric of the experiment returned results. Simplify this metric"),
+ ),
+ (
+ "completed_metric_read_the_exposures_another_way",
+ (_limited(DIRECT),),
+ (PRECOMPUTED,),
+ (PLAN_FILTER,),
+ NO_CAUSE,
+ ),
+ ("completed_result_does_not_say_where_it_read", (_limited(DIRECT),), (None,), (PLAN_FILTER,), NO_CAUSE),
+ ("failure_does_not_say_where_it_read", (_limited(None),), (DIRECT,), (PLAN_FILTER,), NO_CAUSE),
+ ]
+ )
+ def test_names_the_cause_of_a_query_limit(
+ self,
+ _name: str,
+ failures: tuple[MetricCalculationFailure, ...],
+ completed: tuple[ExposuresPath | None, ...],
+ filters: tuple[ExposureFilterWithoutColumn, ...] | None,
+ expected: tuple[str | None, str],
+ ) -> None:
+ finding = _outcome(*failures, completed=completed, exposure_filters_without_column=filters)
+
+ assert isinstance(finding, ExperimentHealthFinding)
+ reason, words = expected
+ self.assertEqual((finding.subcode, finding.severity, finding.reason), ("query_limit", WARNING, reason))
+ self.assertEqual(finding.detail.count(words), 1)
+ names_the_filters = reason == "exposure_criteria"
+ self.assertEqual("properties" in finding.evidence, names_the_filters)
+ self.assertEqual(bool(finding.actions), names_the_filters)
+
def test_waits_for_a_metric_that_the_experiment_calculates(self) -> None:
- calculation = LatestCalculation(metric_count=0, finished_at=None, failures=())
+ calculation = LatestCalculation(
+ metric_count=0, finished_at=None, failures=(), completed_count=0, completed_exposures_paths=frozenset()
+ )
outcome = metric_calculation_failed(
replace(RUNNING_CONTEXT, latest_calculation=calculation, primary_metric_count=0, secondary_metric_count=0)
@@ -398,6 +586,7 @@ class TestMetricCalculationFailedFromStoredRuns(BaseTest):
metric_errors: dict[str, dict[str, Any]] | None = None,
failed_rows: tuple[str, ...] = (),
completed_rows: tuple[str, ...] = (),
+ completed_read_precomputed: bool = False,
) -> None:
with team_scope(self.team.id):
query_to = created_at + timedelta(minutes=1)
@@ -429,7 +618,7 @@ class TestMetricCalculationFailedFromStoredRuns(BaseTest):
calculation_config,
window=query_to,
query_from=query_from,
- result={"variant_results": []},
+ result={"variant_results": [], "is_precomputed": completed_read_precomputed},
query_id=None,
)
terminal = status in (
@@ -472,6 +661,78 @@ class TestMetricCalculationFailedFromStoredRuns(BaseTest):
self.assertNotIn("purchase_total", finding.detail + str(finding.evidence))
self.assertIsNone(finding.evidence["metric_2_message"])
+ @parameterized.expand(
+ [
+ (
+ "every_metric_read_the_events",
+ (),
+ "direct_scan",
+ False,
+ ("exposure_criteria", 'filter on the group property "plan"'),
+ ),
+ (
+ "other_metrics_completed_the_same_read",
+ ("rejected", "secondary"),
+ "direct_scan",
+ False,
+ ("metric", "while other metrics of the experiment returned results"),
+ ),
+ (
+ "other_metrics_read_precomputed_exposures",
+ ("rejected", "secondary"),
+ "direct_scan",
+ True,
+ (None, "read more data than PostHog allows for one calculation"),
+ ),
+ (
+ "failures_stored_without_their_read",
+ (),
+ None,
+ False,
+ (None, "read more data than PostHog allows for one calculation"),
+ ),
+ ]
+ )
+ def test_names_the_cause_of_a_query_limit_from_the_stored_run(
+ self,
+ _name: str,
+ completed_rows: tuple[str, ...],
+ exposures_path: str | None,
+ completed_read_precomputed: bool,
+ expected: tuple[str | None, str],
+ ) -> None:
+ self.experiment.exposure_criteria = {
+ "exposure_config": {
+ "kind": "ExperimentEventExposureConfig",
+ "event": "$feature_flag_called",
+ "properties": [
+ {"key": "plan", "type": "group", "group_type_index": 0, "operator": "exact", "value": ["pro"]}
+ ],
+ }
+ }
+ self.experiment.save()
+ read_limit = {
+ "step": "calculation",
+ "message": "Query read too much data",
+ "error_type": "byte_limit",
+ "exposures_path": exposures_path,
+ }
+ self._run(
+ status=ExperimentMetricsRecalculation.Status.FAILED,
+ created_at=timezone.now() - timedelta(hours=2),
+ metric_errors={
+ uuid: read_limit for uuid in ("primary", "rejected", "secondary") if uuid not in completed_rows
+ },
+ completed_rows=completed_rows,
+ completed_read_precomputed=completed_read_precomputed,
+ )
+
+ [finding] = [finding for finding in self._health().findings if finding.code == CODE]
+
+ reason, words = expected
+ self.assertEqual(finding.reason, reason)
+ self.assertIn(words, finding.detail)
+
@parameterized.expand(
[
("no_finished_calculation", None, ExperimentHealthCheckStatus.NOT_APPLICABLE),
diff --git a/products/experiments/backend/hogql_queries/experiment_query_builder.py b/products/experiments/backend/hogql_queries/experiment_query_builder.py
index 169e30156840..c0b27a68eb5a 100644
--- a/products/experiments/backend/hogql_queries/experiment_query_builder.py
+++ b/products/experiments/backend/hogql_queries/experiment_query_builder.py
@@ -256,6 +256,9 @@ class ExperimentQueryBuilder:
query.limit = ast.Constant(value=self.QUERY_RESULT_LIMIT)
return query
+ def reads_precomputed_exposures(self, job_ids: list[str]) -> bool:
+ return self._exposure_query_builder(job_ids=job_ids).reads_precomputed()
+
def _exposure_query_builder(
self, maturity: MaturityGate | None = None, job_ids: list[str] | None = None
) -> ExposureQueryBuilder:
diff --git a/products/experiments/backend/hogql_queries/experiment_query_runner.py b/products/experiments/backend/hogql_queries/experiment_query_runner.py
index bf2efe1cac5a..71473d214567 100644
--- a/products/experiments/backend/hogql_queries/experiment_query_runner.py
+++ b/products/experiments/backend/hogql_queries/experiment_query_runner.py
@@ -41,7 +41,7 @@ from posthog.models.team.extensions import get_or_create_team_extension
from products.ai_observability.backend.facade.api import mentions_evaluation_pass_markers
from products.analytics_platform.backend.lazy_computation.lazy_computation_executor import LazyComputationResult
-from products.experiments.backend.facade.enums import PrecomputeSkipReason
+from products.experiments.backend.facade.enums import ExposuresPath, PrecomputeSkipReason
from products.experiments.backend.hogql_queries import MULTIPLE_VARIANT_KEY, get_baseline_variant_key
from products.experiments.backend.hogql_queries.base_query_utils import experiment_window, experiment_window_end
from products.experiments.backend.hogql_queries.cuped_config import get_cuped_config
@@ -196,6 +196,7 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
self.clickhouse_sql: str | None = None
self.hogql: str | None = None
self._is_precomputed: bool = False # exposures precompute
+ self.exposures_path: ExposuresPath | None = None
self._metric_events_precomputed: bool = False
def _get_breakdowns_for_builder(self) -> list | None:
@@ -343,7 +344,7 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
result = self._ensure_exposures_precomputed(builder)
if result.ready:
exposure_job_ids = [str(job_id) for job_id in result.job_ids]
- self._is_precomputed = True
+ self._is_precomputed = builder.reads_precomputed_exposures(exposure_job_ids)
elif result.capacity_rejected:
# ClickHouse refused the build because the pool is full. The direct scan
# below would hold a slot for minutes and starve the next builds too, so
@@ -510,14 +511,14 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
experiment_query_ast = self._get_experiment_query()
# Tag after _get_experiment_query() which sets the precompute flags
- exposures_path = "precomputed" if self._is_precomputed else "direct_scan"
+ self.exposures_path = ExposuresPath.PRECOMPUTED if self._is_precomputed else ExposuresPath.DIRECT_SCAN
metric_events_path = self.metric_events_path
skip_reason = self._precompute_skip_reason()
tag_queries(
- experiment_exposures_path=exposures_path,
+ experiment_exposures_path=self.exposures_path.value,
experiment_metric_events_path=metric_events_path,
experiment_metric_events_skip_reason=self._metric_events_ineligibility_reason(),
- experiment_execution_path=exposures_path,
+ experiment_execution_path=self.exposures_path.value,
experiment_precompute_skip_reason=skip_reason.value if skip_reason is not None else None,
experiment_scan_date_from=self.date_range.date_from,
experiment_scan_date_to=self.date_range.date_to,
diff --git a/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.py b/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.py
index 94c559e65c57..8894ddfd7003 100644
--- a/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.py
+++ b/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.py
@@ -1,6 +1,7 @@
from collections.abc import Callable
from datetime import UTC, datetime
from typing import Any, cast
+from uuid import uuid4
from posthog.test.base import _create_event, _create_person
from unittest.mock import patch
@@ -11,6 +12,8 @@ from django.test import override_settings
from parameterized import parameterized
from posthog.schema import (
+ Breakdown,
+ BreakdownFilter,
EventsNode,
ExperimentExposureQuery,
ExperimentFunnelMetric,
@@ -34,6 +37,7 @@ from products.analytics_platform.backend.lazy_computation.lazy_computation_execu
ensure_precomputed,
)
from products.analytics_platform.backend.models import PreaggregationJob
+from products.experiments.backend.facade.enums import ExposuresPath
from products.experiments.backend.hogql_queries.base_query_utils import experiment_window
from products.experiments.backend.hogql_queries.experiment_exposures_query_runner import ExperimentExposuresQueryRunner
from products.experiments.backend.hogql_queries.experiment_query_builder import (
@@ -865,6 +869,58 @@ class TestExperimentExposurePreaggregation(ExperimentQueryRunnerBaseTest):
assert direct_result.variant_results is not None
assert direct_result.variant_results[0].number_of_samples == 1
+ def test_breakdown_scans_the_events_when_the_exposure_job_is_ready(self):
+ feature_flag = self.create_feature_flag()
+ experiment = self.create_experiment(
+ feature_flag=feature_flag,
+ start_date=datetime(2024, 1, 1),
+ end_date=datetime(2024, 1, 5),
+ )
+ self._enable_precomputation()
+ metric = ExperimentMeanMetric(
+ source=EventsNode(event="purchase"),
+ breakdownFilter=BreakdownFilter(breakdowns=[Breakdown(property="$browser", type="event")]),
+ )
+ experiment.metrics = [metric.model_dump(mode="json")]
+ experiment.save()
+
+ for variant in ("control", "test"):
+ for i in range(3):
+ distinct_id = f"user_{variant}_{i}"
+ _create_person(distinct_ids=[distinct_id], team_id=self.team.pk)
+ self._create_exposure_event(
+ distinct_id, feature_flag, variant, datetime(2024, 1, 2, 12, 0, 0, tzinfo=UTC)
+ )
+ _create_event(
+ team=self.team,
+ event="purchase",
+ distinct_id=distinct_id,
+ timestamp=datetime(2024, 1, 2, 13, 0, 0, tzinfo=UTC),
+ properties={f"$feature/{feature_flag.key}": variant, "$browser": "Chrome"},
+ )
+
+ with (
+ patch.object(
+ ExperimentQueryRunner,
+ "_ensure_exposures_precomputed",
+ return_value=LazyComputationResult(ready=True, job_ids=[uuid4()]),
+ ),
+ patch.object(
+ ExperimentQueryRunner,
+ "_ensure_metric_events_precomputed",
+ return_value=LazyComputationResult(ready=False, job_ids=[]),
+ ),
+ ):
+ runner = ExperimentQueryRunner(
+ query=ExperimentQuery(experiment_id=experiment.id, kind="ExperimentQuery", metric=metric),
+ team=self.team,
+ )
+ result = cast(ExperimentQueryResponse, runner.calculate())
+
+ assert result.variant_results is not None
+ assert result.is_precomputed is False
+ assert runner.exposures_path == ExposuresPath.DIRECT_SCAN
+
@parameterized.expand(
[
("exposure", "_ensure_exposures_precomputed", "_ensure_metric_events_precomputed"),
@@ -925,12 +981,17 @@ class TestExperimentExposurePreaggregation(ExperimentQueryRunnerBaseTest):
"products.experiments.backend.hogql_queries.experiment_query_runner.capture_exception"
) as mock_capture,
):
- result = self._run_experiment(experiment, metric)
+ runner = ExperimentQueryRunner(
+ query=ExperimentQuery(experiment_id=experiment.id, kind="ExperimentQuery", metric=metric),
+ team=self.team,
+ )
+ result = cast(ExperimentQueryResponse, runner.calculate())
assert result.baseline is not None
assert result.baseline.number_of_samples == 3
assert result.variant_results is not None
assert result.variant_results[0].number_of_samples == 3
+ assert runner.exposures_path == ExposuresPath.DIRECT_SCAN
# The fallback keeps the query working, but the precomputation failure must still
# be surfaced to error tracking rather than silently masked.
diff --git a/products/experiments/backend/metric_calculation/results.py b/products/experiments/backend/metric_calculation/results.py
index a932515684d3..9b904bb60f6e 100644
--- a/products/experiments/backend/metric_calculation/results.py
+++ b/products/experiments/backend/metric_calculation/results.py
@@ -40,6 +40,7 @@ from posthog.schema import ExperimentQueryResponse
from posthog.dataclasses import frozen
+from products.experiments.backend.facade.enums import ExposuresPath
from products.experiments.backend.logic.result_stats import MetricResultStats, parse_stored_response
from products.experiments.backend.metric_calculation.config import MetricCalculationConfig, build_calculation_configs
from products.experiments.backend.models.experiment import (
@@ -140,6 +141,18 @@ class MetricResultStore:
rows = rows.defer("result")
return list(_newest_write_first(rows, "metric_uuid").distinct("metric_uuid"))
+ def exposures_paths(self, rows: Iterable[ExperimentMetricResult]) -> frozenset[ExposuresPath]:
+ """Where the queries of these rows read the exposures. It reads one key of `result`, so the rows of `for_run` can
+ leave `result` unloaded. A result without the key adds nothing."""
+ stored = ExperimentMetricResult.objects.filter(
+ experiment_id=self.experiment_id, pk__in=[row.pk for row in rows]
+ ).values_list("result__is_precomputed", flat=True)
+ return frozenset(
+ ExposuresPath.PRECOMPUTED if is_precomputed else ExposuresPath.DIRECT_SCAN
+ for is_precomputed in stored
+ if is_precomputed is not None
+ )
+
def has_completed(self, calculation_config: MetricCalculationConfig, *, window: datetime) -> bool:
"""Whether a recalculation already stored a completed result for this calculation config at this window."""
return ExperimentMetricResult.objects.filter(
diff --git a/products/experiments/backend/presentation/serializers.py b/products/experiments/backend/presentation/serializers.py
index a8b04af886b4..ac4eb401a334 100644
--- a/products/experiments/backend/presentation/serializers.py
+++ b/products/experiments/backend/presentation/serializers.py
@@ -224,8 +224,9 @@ class ExperimentHealthFindingSerializer(serializers.Serializer):
allow_null=True,
help_text=(
"A stable qualifier below the subcode: where a sample ratio mismatch is located ('library', 'page'), "
- "what leaves nobody exposed ('test_account_filter'), or why a check could not read what it needs "
- "('no_history_access'). Null when the finding names none."
+ "what leaves nobody exposed ('test_account_filter'), what made a metric reach a query limit "
+ "('metric', 'exposure_criteria'), or why a check could not read what it needs ('no_history_access'). "
+ "Null when the finding names none."
),
)
severity = serializers.ChoiceField(
@@ -1649,9 +1650,11 @@ class _ExperimentMetricsRecalculationBaseSerializer(serializers.Serializer):
metric_errors = serializers.JSONField(
read_only=True,
help_text=(
- "Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp}. retriable is "
- "true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can "
- "succeed; false when the metric config, the data, or a resource limit must change first"
+ "Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp, exposures_path}. "
+ "retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry "
+ "can succeed; false when the metric config, the data, or a resource limit must change first. "
+ "exposures_path is where the query read the exposures, and is missing when the query failed before it "
+ "was built"
),
)
created_at = serializers.DateTimeField(read_only=True, help_text="When the job was created")
diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.py
index 051e41a1939b..b21cf20209f5 100644
--- a/products/experiments/backend/presentation/views.py
+++ b/products/experiments/backend/presentation/views.py
@@ -1024,7 +1024,13 @@ class EnterpriseExperimentsViewSet(
wrote for users), and `calculated_at` is when the calculation finished. It is a
warning when a primary metric fails for a kind that a change fixes (`query_limit`,
`settings`, `experiment_too_old`), also after the end, because the calculation after the fix
- reads the events up to the end. It is info otherwise.
+ reads the events up to the end. It is info otherwise. When the first failure reached the
+ read, memory or time limit, `reason` names the cause that the calculation shows: `metric`
+ when another metric of the same calculation completed the same read of the exposures, or
+ `exposure_criteria` when every metric reached a limit and an exposure or test account filter
+ makes each query read all the properties of an event, person or group. `evidence.properties`
+ then names the filters. A query that read precomputed exposures doesn't apply those filters,
+ so it can't get `exposure_criteria`. It is null otherwise.
The severity of the `some_conditions_pinned` finding follows the largest share that the
users of the pinned release conditions make up among the users who got one variant from
diff --git a/products/experiments/backend/temporal/ARCHITECTURE.md b/products/experiments/backend/temporal/ARCHITECTURE.md
index 33235cfb6b55..ecd934c99fab 100644
--- a/products/experiments/backend/temporal/ARCHITECTURE.md
+++ b/products/experiments/backend/temporal/ARCHITECTURE.md
@@ -204,7 +204,7 @@ Red nodes advance the window; blue nodes reuse it.
#### Retriable failures
-A metric's terminal failure lands in `metric_errors` as `{step, message, error_type, retriable, timestamp}`.
+A metric's terminal failure lands in `metric_errors` as `{step, message, error_type, retriable, timestamp}`, plus `exposures_path` (`precomputed` or `direct_scan`) once the runner has built the query.
`error_type` is the same taxonomy as the `experiment metric error` event (`classify_experiment_query_error`).
`retriable` is the backend's own decision, not a frontend mapping of `error_type`: it is true when a transient error (`timeout`, `rate_limited`, `server_error`) used its final attempt, and false when the calc activity failed permanently (`validation_error`, `out_of_memory`, `byte_limit`, `insufficient_data`, a `ValueError`, or a discovery failure) or the request was rejected (`recalculation_window_too_long`).
A retriable failure reads as a gap, so the page load heals it; a non-retriable one waits for the user, because a new run with the same inputs fails the same way.
diff --git a/products/experiments/backend/temporal/recalculation_logic.py b/products/experiments/backend/temporal/recalculation_logic.py
index f0b8523170d8..b7d0511f7a8c 100644
--- a/products/experiments/backend/temporal/recalculation_logic.py
+++ b/products/experiments/backend/temporal/recalculation_logic.py
@@ -33,6 +33,7 @@ from posthog.models.scoping import team_scope
from posthog.sync import database_sync_to_async_pool
from posthog.temporal.common.errors import NonReportableApplicationError
+from products.experiments.backend.facade.enums import ExposuresPath
from products.experiments.backend.hogql_queries.base_query_utils import experiment_window_end
from products.experiments.backend.hogql_queries.error_handling import (
classify_experiment_query_error,
@@ -397,7 +398,14 @@ def _capture_results_refresh_completed(update: RecalculationProgressUpdate) -> N
def _record_failure(
- recalculation_id: str, metric_uuid: str, step: str, message: str, *, error_type: str, retriable: bool
+ recalculation_id: str,
+ metric_uuid: str,
+ step: str,
+ message: str,
+ *,
+ error_type: str,
+ retriable: bool,
+ exposures_path: ExposuresPath | None = None,
) -> None:
"""Merge the error entry into metric_errors under a row lock (no lost updates between concurrent failures).
@@ -416,6 +424,8 @@ def _record_failure(
"retriable": retriable,
"timestamp": timezone.now().isoformat(),
}
+ if exposures_path is not None:
+ metric_errors[metric_uuid]["exposures_path"] = exposures_path.value
recalc.metric_errors = metric_errors
recalc.save(update_fields=["metric_errors"])
@@ -704,6 +714,8 @@ def _calculate_experiment_metric_for_recalculation_sync(
calc_started_at = time.perf_counter()
query_from = experiment.start_date
+ runner: ExperimentQueryRunner | None = None
+
def record_terminal_error(message: str, error_type: str, *, retriable: bool) -> None:
results.record_run_failure(
recalculation_id,
@@ -714,7 +726,13 @@ def _calculate_experiment_metric_for_recalculation_sync(
query_id=client_query_id,
)
_record_failure(
- recalculation_id, metric_uuid, "calculation", message, error_type=error_type, retriable=retriable
+ recalculation_id,
+ metric_uuid,
+ "calculation",
+ message,
+ error_type=error_type,
+ retriable=retriable,
+ exposures_path=runner.exposures_path if runner is not None else None,
)
_capture_experiment_metric_event(
experiment,
diff --git a/products/experiments/backend/temporal/test_recalculation_activities.py b/products/experiments/backend/temporal/test_recalculation_activities.py
index 756986649959..81708b382095 100644
--- a/products/experiments/backend/temporal/test_recalculation_activities.py
+++ b/products/experiments/backend/temporal/test_recalculation_activities.py
@@ -1,6 +1,7 @@
import asyncio
import threading
import dataclasses
+from collections.abc import Iterator
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from datetime import datetime, timedelta
@@ -28,6 +29,7 @@ from posthog.exceptions import ClickHouseAtCapacity, ClickHouseQueryMemoryLimitE
from posthog.temporal.common.errors import NonReportableError
from posthog.temporal.tests.utils.asyncify import restored_asyncify_executor
+from products.experiments.backend.facade.enums import ExposuresPath
from products.experiments.backend.metric_calculation.config import get_metric_calculation_config
from products.experiments.backend.metric_calculation.results import _recalc_fingerprint
from products.experiments.backend.models.experiment import (
@@ -99,6 +101,14 @@ def _calculate(
)
+@contextmanager
+def _patched_runner() -> Iterator[MagicMock]:
+ with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:
+ # A MagicMock in this attribute can't be stored in `metric_errors`.
+ mock_runner.return_value.exposures_path = None
+ yield mock_runner
+
+
@pytest.mark.django_db(transaction=True)
class TestRecalculationActivities(BaseTest):
def _flag(self, key: str) -> FeatureFlag:
@@ -734,7 +744,7 @@ class TestCalculateActivity(BaseTest):
ExperimentToSavedMetric.objects.create(experiment=exp, saved_metric=saved, metadata={"type": "primary"})
recalc = self._recalc(exp, metric_uuids=["s1"])
- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:
+ with _patched_runner() as mock_runner:
mock_runner.return_value.run.side_effect = RuntimeError("kaboom")
with pytest.raises(RuntimeError, match="kaboom"):
_calculate(exp.id, "s1", str(recalc.id), _QUERY_TO)
@@ -750,7 +760,7 @@ class TestCalculateActivity(BaseTest):
exp = self._experiment(flag_key="calc-transient", metrics=[_mean_metric("m1")])
recalc = self._recalc(exp, metric_uuids=["m1"])
- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:
+ with _patched_runner() as mock_runner:
mock_runner.return_value.run.side_effect = RuntimeError("transient blip")
# Non-final attempt: re-raises, but nothing is recorded.
@@ -819,7 +829,7 @@ class TestCalculateActivity(BaseTest):
recalc = self._recalc(exp, metric_uuids=["m1"])
with (
- patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner,
+ _patched_runner() as mock_runner,
patch("products.experiments.backend.temporal.recalculation_logic.random.uniform", return_value=47.5),
):
mock_runner.return_value.run.side_effect = exc
@@ -836,7 +846,7 @@ class TestCalculateActivity(BaseTest):
next_retry_at = datetime.fromisoformat(entry["next_retry_at"])
assert (next_retry_at - timezone.now()).total_seconds() == expected_delay_seconds
- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:
+ with _patched_runner() as mock_runner:
mock_runner.return_value.run.return_value.model_dump.return_value = {}
result = _calculate(exp.id, "m1", str(recalc.id), _QUERY_TO, attempt=attempt + 1)
@@ -859,8 +869,9 @@ class TestCalculateActivity(BaseTest):
exp = self._experiment(flag_key=f"calc-terminal-{name}", metrics=[_mean_metric("m1")])
recalc = self._recalc(exp, metric_uuids=["m1"])
- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:
+ with _patched_runner() as mock_runner:
mock_runner.return_value.run.side_effect = exc
+ mock_runner.return_value.exposures_path = ExposuresPath.DIRECT_SCAN
with pytest.raises(Exception):
_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO, is_final_attempt=True)
@@ -868,6 +879,7 @@ class TestCalculateActivity(BaseTest):
entry = recalc.metric_errors["m1"]
assert entry["error_type"] == expected_error_type
assert entry["retriable"] is expected_retriable
+ assert entry["exposures_path"] == "direct_scan"
def test_terminal_failure_clears_retry_state(self):
# A metric that retried and then failed for good must not keep a stale "retrying" entry alongside
@@ -875,7 +887,7 @@ class TestCalculateActivity(BaseTest):
exp = self._experiment(flag_key="calc-retry-terminal", metrics=[_mean_metric("m1")])
recalc = self._recalc(exp, metric_uuids=["m1"])
- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:
+ with _patched_runner() as mock_runner:
mock_runner.return_value.run.side_effect = RuntimeError("transient blip")
with pytest.raises(RuntimeError):
_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO, is_final_attempt=False, attempt=2)
@@ -902,7 +914,7 @@ class TestCalculateActivity(BaseTest):
):
exp = self._experiment(flag_key=f"calc-out-of-time-{name}", metrics=[_mean_metric("m1")])
recalc = self._recalc(exp, metric_uuids=["m1"])
- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:
+ with _patched_runner() as mock_runner:
if last_attempt_error is None:
mock_runner.return_value.run.return_value.model_dump.return_value = {}
_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO, is_final_attempt=False, attempt=2)
@@ -950,7 +962,7 @@ class TestCalculateActivity(BaseTest):
)
)
- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:
+ with _patched_runner() as mock_runner:
mock_runner.return_value.run.return_value.model_dump.return_value = {}
result = _calculate(exp.id, "m1", str(recalc.id), _QUERY_TO, is_final_attempt=False, attempt=3)
@@ -977,7 +989,7 @@ class TestCalculateActivity(BaseTest):
recalc = self._recalc(exp, metric_uuids=["m1"])
with (
- patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner,
+ _patched_runner() as mock_runner,
patch("products.experiments.backend.temporal.recalculation_logic.capture_exception") as mock_capture,
patch("products.experiments.backend.temporal.recalculation_logic.random.uniform", return_value=47.5),
):
@@ -1013,7 +1025,7 @@ class TestCalculateActivity(BaseTest):
exp = self._experiment(flag_key=f"calc-class-cap-{name}", metrics=[_mean_metric("m1")])
recalc = self._recalc(exp, metric_uuids=["m1"])
- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:
+ with _patched_runner() as mock_runner:
mock_runner.return_value.run.side_effect = exc
with pytest.raises((type(exc), ApplicationError)):
_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO, is_final_attempt=False, attempt=cap - 1)
@@ -1049,7 +1061,7 @@ class TestCalculateActivity(BaseTest):
exp = self._experiment(flag_key=f"calc-permanent-{name}", metrics=[_mean_metric("m1")])
recalc = self._recalc(exp, metric_uuids=["m1"])
- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:
+ with _patched_runner() as mock_runner:
mock_runner.return_value.run.side_effect = exc
with pytest.raises(ApplicationError) as exc_info:
@@ -1074,7 +1086,7 @@ class TestCalculateActivity(BaseTest):
exp = self._experiment(flag_key="calc-as-of", metrics=[_mean_metric("m1")])
recalc = self._recalc(exp, metric_uuids=["m1"])
- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:
+ with _patched_runner() as mock_runner:
mock_runner.return_value.run.return_value.model_dump.return_value = {}
_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO)
@@ -1093,7 +1105,7 @@ class TestCalculateActivity(BaseTest):
exp = self._experiment(flag_key="calc-query-from", metrics=[_mean_metric("m1")])
recalc = self._recalc(exp, metric_uuids=["m1"])
- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:
+ with _patched_runner() as mock_runner:
mock_runner.return_value.run.return_value.model_dump.return_value = {}
_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO)
@@ -1110,7 +1122,7 @@ class TestCalculateActivity(BaseTest):
exp = self._experiment(flag_key="calc-query-id", metrics=[_mean_metric("m1")])
recalc = self._recalc(exp, metric_uuids=["m1"])
- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:
+ with _patched_runner() as mock_runner:
mock_runner.return_value.run.return_value.model_dump.return_value = {}
_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO)
@@ -1138,7 +1150,7 @@ class TestCalculateActivity(BaseTest):
with (
patch("products.experiments.backend.recalculation.feature_enabled_or_false", **flag),
patch("posthog.clickhouse.client.execute.get_client_from_pool") as client_from_pool,
- patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner,
+ _patched_runner() as mock_runner,
):
client_from_pool.return_value.__enter__.return_value.execute.return_value = []
mock_runner.return_value.run.side_effect = run_metric_and_precompute_queries
@@ -1184,7 +1196,7 @@ class TestCalculateActivity(BaseTest):
recalc.metric_retries = {"m1": {"attempt": 3, "max_attempts": 8}}
recalc.save(update_fields=["metric_retries"])
- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:
+ with _patched_runner() as mock_runner:
result = _calculate(exp.id, "m1", str(recalc.id), _QUERY_TO)
mock_runner.assert_not_called()
@@ -1215,7 +1227,7 @@ class TestCalculateActivity(BaseTest):
)
recalc = self._recalc(exp, metric_uuids=["m1"])
- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:
+ with _patched_runner() as mock_runner:
mock_runner.return_value.run.return_value.model_dump.return_value = {}
_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO)
@@ -1241,7 +1253,7 @@ class TestCalculateActivity(BaseTest):
exp.save(update_fields=["excluded_variants"])
recalc = self._recalc(exp, metric_uuids=["m1"])
- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:
+ with _patched_runner() as mock_runner:
mock_runner.return_value.run.return_value.model_dump.return_value = {}
_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO)
@@ -1269,9 +1281,7 @@ class TestCalculateActivity(BaseTest):
def _run() -> None:
if mock_runner_failure:
- with patch(
- "products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner"
- ) as mock_runner:
+ with _patched_runner() as mock_runner:
mock_runner.return_value.run.side_effect = RuntimeError("kaboom")
# Calc-step failures now re-raise so Temporal can retry transient errors. The record is
# written before the re-raise, so the assertions below still hold across attempts.
@@ -1351,7 +1361,7 @@ class TestCalculateActivity(BaseTest):
recalc = self._recalc(exp, metric_uuids=["m1"])
with (
- patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner,
+ _patched_runner() as mock_runner,
patch("products.experiments.backend.temporal.recalculation_logic.capture_exception") as mock_capture,
):
mock_runner.return_value.run.side_effect = RuntimeError("x" * 5000)
@@ -1714,9 +1724,7 @@ class TestRecalculationAnalytics(BaseTest):
recalc = self._recalc(exp, metric_uuids=["m1"])
with _record_captures() as captured:
- with patch(
- "products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner"
- ) as mock_runner:
+ with _patched_runner() as mock_runner:
mock_runner.return_value.run.return_value.model_dump.return_value = {}
_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO)
@@ -1737,9 +1745,7 @@ class TestRecalculationAnalytics(BaseTest):
recalc = self._recalc(exp, metric_uuids=["m1"])
with _record_captures() as captured:
- with patch(
- "products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner"
- ) as mock_runner:
+ with _patched_runner() as mock_runner:
mock_runner.return_value.run.side_effect = StatisticError("not enough data")
_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO)
@@ -1757,9 +1763,7 @@ class TestRecalculationAnalytics(BaseTest):
recalc = self._recalc(exp, metric_uuids=["s1"])
with _record_captures() as captured:
- with patch(
- "products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner"
- ) as mock_runner:
+ with _patched_runner() as mock_runner:
mock_runner.return_value.run.return_value.model_dump.return_value = {}
_calculate(exp.id, "s1", str(recalc.id), _QUERY_TO, role="secondary")
@@ -1807,7 +1811,7 @@ class TestRecalculationAnalytics(BaseTest):
with _record_captures() as captured:
with (
- patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner,
+ _patched_runner() as mock_runner,
patch("products.experiments.backend.temporal.recalculation_logic.random.uniform", return_value=47.5),
):
mock_runner.return_value.run.side_effect = exc
@@ -1850,9 +1854,7 @@ class TestRecalculationAnalytics(BaseTest):
recalc = self._recalc(exp, metric_uuids=["m1"])
with _record_captures() as captured:
- with patch(
- "products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner"
- ) as mock_runner:
+ with _patched_runner() as mock_runner:
mock_runner.return_value.run.side_effect = exc
with pytest.raises(ApplicationError if raises_application_error else type(exc)):
_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO, is_final_attempt=True)
diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.ts
index bdd51818d7ea..0eedcc283447 100644
--- a/products/experiments/frontend/generated/api.schemas.ts
+++ b/products/experiments/frontend/generated/api.schemas.ts
@@ -2422,7 +2422,7 @@ export interface ExperimentHealthFindingApi {
*/
subcode: string | null
/**
- * A stable qualifier below the subcode: where a sample ratio mismatch is located ('library', 'page'), what leaves nobody exposed ('test_account_filter'), or why a check could not read what it needs ('no_history_access'). Null when the finding names none.
+ * A stable qualifier below the subcode: where a sample ratio mismatch is located ('library', 'page'), what leaves nobody exposed ('test_account_filter'), what made a metric reach a query limit ('metric', 'exposure_criteria'), or why a check could not read what it needs ('no_history_access'). Null when the finding names none.
* @nullable
*/
reason: string | null
@@ -3871,7 +3871,7 @@ export interface ExperimentMetricsRecalculationJobApi {
readonly completed_metrics: number
/** Number of failed metrics in this run (derived): FAILED result rows plus discovery-step failures that never made it to a result row */
readonly failed_metrics: number
- /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first */
+ /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp, exposures_path}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first. exposures_path is where the query read the exposures, and is missing when the query failed before it was built */
readonly metric_errors: unknown
/** When the job was created */
readonly created_at: string
@@ -3950,7 +3950,7 @@ export interface ExperimentMetricsRecalculationRunApi {
readonly completed_metrics: number
/** Number of failed metrics in this run (derived): FAILED result rows plus discovery-step failures that never made it to a result row */
readonly failed_metrics: number
- /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first */
+ /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp, exposures_path}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first. exposures_path is where the query read the exposures, and is missing when the query failed before it was built */
readonly metric_errors: unknown
/** When the job was created */
readonly created_at: string
@@ -4032,7 +4032,7 @@ export interface ExperimentMetricsRecalculationLatestApi {
readonly completed_metrics: number
/** Number of failed metrics in this run (derived): FAILED result rows plus discovery-step failures that never made it to a result row */
readonly failed_metrics: number
- /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first */
+ /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp, exposures_path}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first. exposures_path is where the query read the exposures, and is missing when the query failed before it was built */
readonly metric_errors: unknown
/** When the job was created */
readonly created_at: string
diff --git a/products/experiments/frontend/generated/api.ts b/products/experiments/frontend/generated/api.ts
index 994be9c77257..b2eac705f422 100644
--- a/products/experiments/frontend/generated/api.ts
+++ b/products/experiments/frontend/generated/api.ts
@@ -769,7 +769,13 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =
* wrote for users), and `calculated_at` is when the calculation finished. It is a
* warning when a primary metric fails for a kind that a change fixes (`query_limit`,
* `settings`, `experiment_too_old`), also after the end, because the calculation after the fix
- * reads the events up to the end. It is info otherwise.
+ * reads the events up to the end. It is info otherwise. When the first failure reached the
+ * read, memory or time limit, `reason` names the cause that the calculation shows: `metric`
+ * when another metric of the same calculation completed the same read of the exposures, or
+ * `exposure_criteria` when every metric reached a limit and an exposure or test account filter
+ * makes each query read all the properties of an event, person or group. `evidence.properties`
+ * then names the filters. A query that read precomputed exposures doesn't apply those filters,
+ * so it can't get `exposure_criteria`. It is null otherwise.
*
* The severity of the `some_conditions_pinned` finding follows the largest share that the
* users of the pinned release conditions make up among the users who got one variant from
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 2901ef310e6a..1e2bcc25c069 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -47860,7 +47860,7 @@ export namespace Schemas {
*/
subcode: string | null;
/**
- * A stable qualifier below the subcode: where a sample ratio mismatch is located ('library', 'page'), what leaves nobody exposed ('test_account_filter'), or why a check could not read what it needs ('no_history_access'). Null when the finding names none.
+ * A stable qualifier below the subcode: where a sample ratio mismatch is located ('library', 'page'), what leaves nobody exposed ('test_account_filter'), what made a metric reach a query limit ('metric', 'exposure_criteria'), or why a check could not read what it needs ('no_history_access'). Null when the finding names none.
* @nullable
*/
reason: string | null;
@@ -49202,7 +49202,7 @@ export namespace Schemas {
readonly completed_metrics: number;
/** Number of failed metrics in this run (derived): FAILED result rows plus discovery-step failures that never made it to a result row */
readonly failed_metrics: number;
- /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first */
+ /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp, exposures_path}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first. exposures_path is where the query read the exposures, and is missing when the query failed before it was built */
readonly metric_errors: unknown;
/** When the job was created */
readonly created_at: string;
@@ -49293,7 +49293,7 @@ export namespace Schemas {
readonly completed_metrics: number;
/** Number of failed metrics in this run (derived): FAILED result rows plus discovery-step failures that never made it to a result row */
readonly failed_metrics: number;
- /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first */
+ /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp, exposures_path}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first. exposures_path is where the query read the exposures, and is missing when the query failed before it was built */
readonly metric_errors: unknown;
/** When the job was created */
readonly created_at: string;
@@ -49378,7 +49378,7 @@ export namespace Schemas {
readonly completed_metrics: number;
/** Number of failed metrics in this run (derived): FAILED result rows plus discovery-step failures that never made it to a result row */
readonly failed_metrics: number;
- /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first */
+ /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp, exposures_path}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first. exposures_path is where the query read the exposures, and is missing when the query failed before it was built */
readonly metric_errors: unknown;
/** When the job was created */
readonly created_at: string;