:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(experiments): name the cause of a query-limit failure (#125934)
Marcel Poelker43333f81da94
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.tsindex 4b9e86ee5697..dae06d9a5550 100644--- a/products/desktop/packages/api-client/src/generated.ts+++ b/products/desktop/packages/api-client/src/generated.ts@@ -10058,7 +10058,7 @@ export namespace Schemas {*/subcode: string | null;/**- * A stable qualifier below the subcode: where a sample ratio mismatch is located ('library', 'page'), what leaves nobody exposed ('test_account_filter'), or why a check could not read what it needs ('no_history_access'). Null when the finding names none.+ * A stable qualifier below the subcode: where a sample ratio mismatch is located ('library', 'page'), what leaves nobody exposed ('test_account_filter'), what made a metric reach a query limit ('metric', 'exposure_criteria'), or why a check could not read what it needs ('no_history_access'). Null when the finding names none.*/reason: string | null;/**diff --git a/products/experiments/backend/facade/enums.py b/products/experiments/backend/facade/enums.pyindex d307dfda7ac6..35083590d065 100644--- a/products/experiments/backend/facade/enums.py+++ b/products/experiments/backend/facade/enums.py@@ -13,6 +13,14 @@ MIN_DEFAULT_CONFIDENCE_LEVEL = Decimal("0.01")MAX_DEFAULT_CONFIDENCE_LEVEL = Decimal("0.99")+class ExposuresPath(StrEnum):+ """Where a metric query read the exposures. Tagged on the query as `experiment_exposures_path`, and stored with a+ failed metric in `metric_errors`."""++ PRECOMPUTED = "precomputed"+ DIRECT_SCAN = "direct_scan"++class PrecomputeSkipReason(StrEnum):"""Why a metric read never attempted precompute. Tagged on the query as`experiment_precompute_skip_reason` and counted per reason by thediff --git a/products/experiments/backend/health/checks/metric_calculation_failed.py b/products/experiments/backend/health/checks/metric_calculation_failed.pyindex ce76ea4bc5ec..7fed8f8edbb5 100644--- a/products/experiments/backend/health/checks/metric_calculation_failed.py+++ b/products/experiments/backend/health/checks/metric_calculation_failed.py@@ -17,12 +17,21 @@ from posthog.exceptions import ClickHouseClusterMemoryLimitExceededfrom products.experiments.backend.facade.contracts import (ExperimentHealthFinding,+ ExperimentHealthFindingActionKind,ExperimentHealthFindingCode,ExperimentHealthFindingSeverity,ExperimentHealthMetricCompileErrorKind,)+from products.experiments.backend.facade.enums import ExposuresPath+from products.experiments.backend.health.checks.events_received import joinedfrom products.experiments.backend.health.checks.metric_cannot_compile import MetricCompileError, word_compile_error-from products.experiments.backend.health.context import HealthContext, MetricCalculationFailure+from products.experiments.backend.health.context import (+ ExposureFilterSource,+ ExposureFilterWithoutColumn,+ HealthContext,+ LatestCalculation,+ MetricCalculationFailure,+)from products.experiments.backend.health.skips import (NOT_ON_ARCHIVED,RUNS_AFTER_LAUNCH,@@ -47,6 +56,12 @@ class MetricCalculationFailureKind(StrEnum):UNEXPECTED_ERROR = "unexpected_error"+class QueryLimitCause(StrEnum):+ # pinned: the finding's `reason` when its subcode is `query_limit`.+ EXPOSURE_CRITERIA = "exposure_criteria"+ METRIC = "metric"++# The kinds that a change of the metric or the experiment fixes. The others go away with time or a retry._NEEDS_A_FIX = frozenset({@@ -61,6 +76,11 @@ _LIMIT_NEEDS = {"memory_limit": "needed more memory","time_limit": "took longer",}+_FILTER_OWNERS = {+ ExposureFilterSource.EXPOSURE_CRITERIA: "the exposure criteria's",+ ExposureFilterSource.EXPOSURE_ACTION: "the exposure action's",+ ExposureFilterSource.TEST_ACCOUNTS: "the test account",+}_QUERY_SIZE_LIMIT = "query_size"# A failure stored before calculations stored the class of the error has only its message, and some classes are too@@ -105,6 +125,10 @@ class _Wording:# Whether a setting of the experiment causes the failure, which then fails every metric, so the finding says the# explanation once and names no metric in it.every_metric: bool = False+ # What caused the failure, when the words name it.+ reason: str | None = None+ # The filters that the words name.+ exposure_filters: tuple[ExposureFilterWithoutColumn, ...] = ()_UNEXPECTED = _Wording(@@ -166,11 +190,14 @@ def metric_calculation_failed(ctx: HealthContext) -> CheckOutcome:]if not worded:return not_applicable('Reported under "Invalid metric definitions"')+ worded = _with_query_limit_cause(worded, calculation, ctx.exposure_filters_without_column)worded.sort(key=lambda item: (item[0].role != "primary", item[1].kind not in _NEEDS_A_FIX))count = len(worded)+ exposure_filters = worded[0][1].exposure_filtersreturn ExperimentHealthFinding(code=ExperimentHealthFindingCode.METRIC_CALCULATION_FAILED,subcode=worded[0][1].kind.value,+ reason=worded[0][1].reason,# A retry or more data fixes the other kinds. A kind that needs a fix stays a warning after the end, because# the calculation after the fix reads the events up to the end and brings back the results.severity=ExperimentHealthFindingSeverity.WARNING@@ -178,9 +205,13 @@ def metric_calculation_failed(ctx: HealthContext) -> CheckOutcome:else ExperimentHealthFindingSeverity.INFO,title=f"Latest calculation failed for {count} {'metric' if count == 1 else 'metrics'}",detail=" ".join(dict.fromkeys(_sentence(failure, wording) for failure, wording in worded)),- evidence={"calculated_at": calculation.finished_at.isoformat(), **_evidence(worded)},+ evidence={+ "calculated_at": calculation.finished_at.isoformat(),+ **_evidence(worded),+ **({"properties": ", ".join(dict.fromkeys(f.key for f in exposure_filters))} if exposure_filters else {}),+ },# No action opens a given metric.- actions=(),+ actions=(ExperimentHealthFindingActionKind.EDIT_EXPOSURE_CRITERIA,) if exposure_filters else (),diagnostic_ref=None,)@@ -265,18 +296,118 @@ def _limit_explanation(limit: str, failure: MetricCalculationFailure) -> str:"Its query is longer than PostHog allows. A filter with a long list of values can cause this. Shorten ""the filters of the metric or of the exposure criteria, or contact support for help.")- example = (+ return (+ f"Its query {_LIMIT_NEEDS[limit]} than PostHog allows for one calculation. A query reads more data the longer "+ f"an experiment runs and the more traffic it gets. Simplify the metric, for example with {_simpler(failure)}, "+ "or contact support for help."+ )+++def _simpler(failure: MetricCalculationFailure) -> str:+ return ("fewer funnel steps or narrower filters"if failure.definition.get("metric_type") == "funnel"else "narrower filters")+++def _with_query_limit_cause(+ worded: list[tuple[MetricCalculationFailure, _Wording]],+ calculation: LatestCalculation,+ filters: tuple[ExposureFilterWithoutColumn, ...] | None,+) -> list[tuple[MetricCalculationFailure, _Wording]]:+ """The words of each failure that reached the read, memory or time limit of a query, with the cause that the rest+ of the calculation shows.++ Every metric's query reads the exposures. So when another metric of the calculation completed the same read, that+ read fits the limits, and the failing metric's own part of its query reached them. When every metric reached a+ limit and a query that read the events applies a filter that reads all the properties, that filter is the likely+ cause, unless a query read precomputed exposures. A metric that ran out of the workflow's time stores no read, so+ the other failures show the read.+ """+ limited = {+ failure.metric_id: wording.error+ for failure, wording in worded+ if wording.kind == MetricCalculationFailureKind.QUERY_LIMIT+ and wording.error is not None+ and wording.error in _LIMIT_NEEDS+ }+ if calculation.completed_count:+ return [+ (failure, _metric_cause(failure, limited[failure.metric_id], calculation.completed_count))+ if failure.metric_id in limited and failure.exposures_path in calculation.completed_exposures_paths+ else (failure, wording)+ for failure, wording in worded+ ]+ if (+ limited+ and len(limited) == calculation.metric_count+ and filters+ and {failure.exposures_path for failure, _ in worded} - {None} == {ExposuresPath.DIRECT_SCAN}+ ):+ # With one metric the words name it, as for a failure of the metric.+ every_metric = calculation.metric_count > 1+ explanation = _exposure_explanation(set(limited.values()), filters, every_metric=every_metric)+ return [+ (+ failure,+ _Wording(+ kind=MetricCalculationFailureKind.QUERY_LIMIT,+ error=limited[failure.metric_id],+ explanation=explanation,+ every_metric=every_metric,+ reason=QueryLimitCause.EXPOSURE_CRITERIA.value,+ exposure_filters=filters,+ ),+ )+ for failure, _ in worded+ ]+ return worded+++def _metric_cause(failure: MetricCalculationFailure, limit: str, completed_count: int) -> _Wording:+ others = "another metric" if completed_count == 1 else "other metrics"+ return _Wording(+ kind=MetricCalculationFailureKind.QUERY_LIMIT,+ error=limit,+ explanation=f"Its query {_LIMIT_NEEDS[limit]} than PostHog allows for one calculation, while {others} of the "+ f"experiment returned results. Simplify this metric, for example with {_simpler(failure)}, or contact support "+ "for help.",+ reason=QueryLimitCause.METRIC.value,+ )+++def _exposure_explanation(+ limits: set[str], filters: tuple[ExposureFilterWithoutColumn, ...], *, every_metric: bool+) -> str:+ needs = _LIMIT_NEEDS[next(iter(limits))] if len(limits) == 1 else "needed more resources"+ named = joined(+ [+ _named(owner, owned)+ for source, owner in _FILTER_OWNERS.items()+ if (owned := [f for f in filters if f.source == source])+ ]+ )+ read = joined(list(dict.fromkeys(f.type for f in filters)))+ one = len(filters) == 1+ the_filters = "the filter" if one else "these filters"+ query, each_query = ("Every metric's query", "each query") if every_metric else ("Its query", "the query")+ fix = (+ f"Change {the_filters}, or turn off the test account filter in the exposure criteria"+ if any(f.source == ExposureFilterSource.TEST_ACCOUNTS for f in filters)+ else f"Remove or change {the_filters}"+ )return (- f"Its query {_LIMIT_NEEDS[limit]} than PostHog allows for one calculation. A query reads more data the longer "- f"an experiment runs and the more traffic it gets. Simplify the metric, for example with {example}, or "- "contact support for help."+ f"{query} {needs} than PostHog allows for one calculation. The likely cause is {named}, which "+ f"{'makes' if one else 'make'} {each_query} read all {read} properties. {fix}, or contact support for help.")+def _named(owner: str, filters: list[ExposureFilterWithoutColumn]) -> str:+ properties = joined([f'the {f.type} property "{f.key}"' for f in filters])+ return f"{owner} {'filter' if len(filters) == 1 else 'filters'} on {properties}"++def _settings(failure: MetricCalculationFailure) -> _Wording | None:"""The words for a failure that a setting of the metric or the experiment caused, or None when the failure isn'tknown to be one."""diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.pyindex a5ab3175d6a9..909c2e5b50a8 100644--- a/products/experiments/backend/health/context.py+++ b/products/experiments/backend/health/context.py@@ -34,6 +34,7 @@ from products.experiments.backend.facade.contracts import (ExperimentHealthFlagChangeKind,ExperimentHealthMetricCompileErrorKind,)+from products.experiments.backend.facade.enums import ExposuresPathfrom products.experiments.backend.health.delays import ENDED_FLAG_QUIET_PERIOD, LAST_CALLED_AT_LAG_DURATIONfrom products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimensionfrom products.experiments.backend.hogql_queries.experiment_query_builder import get_exposure_config_params_for_builder@@ -326,6 +327,9 @@ class MetricCalculationFailure:retriable: bool# The effective definition now, in which the wording looks for the names that the message gives.definition: Mapping[str, Any]+ # Where the query read the exposures. None for a failure stored before calculations stored it, and for a query that+ # failed before it was built.+ exposures_path: ExposuresPath | None@frozen@@ -338,6 +342,10 @@ class LatestCalculation:finished_at: datetime | None# The metrics of the experiment now whose latest calculation failed, in calculation order.failures: tuple[MetricCalculationFailure, ...]+ # The metrics of the experiment now whose result of this calculation the page shows.+ completed_count: int+ # Where the queries of those results read the exposures, when there is a failure to compare them with.+ completed_exposures_paths: frozenset[ExposuresPath]@frozen@@ -349,6 +357,23 @@ class FlagKeyFilter:ExposurePropertyType = Literal["event", "person"]+ExposureFilterPropertyType = ExposurePropertyType | Literal["group"]+++class ExposureFilterSource(StrEnum):+ EXPOSURE_CRITERIA = "exposure_criteria"+ EXPOSURE_ACTION = "exposure_action"+ TEST_ACCOUNTS = "test_accounts"+++@frozen+class ExposureFilterWithoutColumn:+ """A property that the exposure query filters on and reads from the JSON of all the properties of each event, person+ or group."""++ type: ExposureFilterPropertyType+ key: str+ source: ExposureFilterSource@frozen@@ -506,6 +531,9 @@ class HealthContext:# None when not loaded or unknown. Only the health endpoints load it, and only for a launched experiment that# isn't archived.latest_calculation: LatestCalculation | None+ # None when not loaded or unknown. Only the health endpoints load it, and only when no metric of the latest+ # calculation completed and a failed query read the events.+ exposure_filters_without_column: tuple[ExposureFilterWithoutColumn, ...] | Nonedef parse_flag_state(@@ -965,4 +993,5 @@ def load_health_context(excluded_variant_keys=frozenset(experiment.excluded_variants or []),metric_compiles=None,latest_calculation=None,+ exposure_filters_without_column=None,)diff --git a/products/experiments/backend/health/exposure_columns.py b/products/experiments/backend/health/exposure_columns.pynew file mode 100644index 000000000000..308e5235151c--- /dev/null+++ b/products/experiments/backend/health/exposure_columns.py@@ -0,0 +1,127 @@+from typing import cast++import pydantic++from posthog.schema import ActionsNode, ExperimentExposureCriteria++from posthog.hogql import ast+from posthog.hogql.context import HogQLContext+from posthog.hogql.database.database import Database+from posthog.hogql.modifiers import create_default_modifiers_for_team+from posthog.hogql.property_planner import PropertyAccessPlan, PropertyScope, PropertySourceKind, plan_property_access+from posthog.hogql.resolver import resolve_types+from posthog.hogql.transforms.property_types import build_property_swapper+from posthog.hogql.visitor import GetFieldsTraverser++from posthog.exceptions_capture import capture_exception+from posthog.models.event.new_events_schema import use_new_events_schema+from posthog.schema_enums import MaterializationMode++from products.experiments.backend.health.context import (+ ExposureFilterPropertyType,+ ExposureFilterSource,+ ExposureFilterWithoutColumn,+)+from products.experiments.backend.hogql_queries.base_query_utils import event_or_action_to_filter+from products.experiments.backend.hogql_queries.experiment_query_builder import get_exposure_config_params_for_builder+from products.experiments.backend.hogql_queries.exposure_query_logic import (+ get_test_accounts_filter,+ normalize_to_exposure_criteria,+)+from products.experiments.backend.models.experiment import Experiment++_FILTER_TYPES: dict[PropertyScope, ExposureFilterPropertyType] = {+ PropertyScope.EVENT: "event",+ PropertyScope.PERSON: "person",+ PropertyScope.GROUP: "group",+}+# The exposure query always reads the flag key, so a filter on it costs nothing more and removing it changes nothing.+_FLAG_KEY_PROPERTY = "$feature_flag"+++def _reads_all_properties(plan: PropertyAccessPlan, context: HogQLContext) -> bool:+ if plan.source.kind != PropertySourceKind.JSON:+ return False+ # Before the lazy persons table expands, the planner can't match the columns of the `person` table.+ if plan.scope == PropertyScope.PERSON and plan.source.table_name is None:+ metadata = context.property_metadata+ return (+ metadata is None+ or context.modifiers.materializationMode == MaterializationMode.DISABLED+ or metadata.materialized_column("person", "properties", plan.property_name) is None+ )+ # The planner calls a read of the native events table JSON, but each property there is a subcolumn of its own.+ if plan.source.table_name == "events" and context.uses_new_events_schema():+ return False+ return True+++def _named_properties(expr: ast.Expr) -> set[str | int]:+ """The properties that a filter names. A `$virt` property expands to reads of other properties when its type+ resolves, and the filter names none of those."""+ return {field.chain[-1] for field in GetFieldsTraverser(expr).fields if len(field.chain) > 1}+++def _filters_without_column(+ experiment: Experiment, criteria: ExperimentExposureCriteria | None+) -> tuple[ExposureFilterWithoutColumn, ...]:+ team = experiment.team+ # The configs that the exposure query builds from, as the filters that it applies.+ params = get_exposure_config_params_for_builder(criteria, team, experiment.start_date)+ filters = [+ (+ event_or_action_to_filter(team, config),+ ExposureFilterSource.EXPOSURE_ACTION+ if isinstance(config, ActionsNode)+ else ExposureFilterSource.EXPOSURE_CRITERIA,+ )+ for config in (params.exposure_config, params.activation_config)+ if config+ ]+ filters += [(expr, ExposureFilterSource.TEST_ACCOUNTS) for expr in get_test_accounts_filter(team, criteria)]+ named_by_filter = [_named_properties(expr) for expr, _ in filters]+ if not any(named_by_filter):+ return ()+ modifiers = create_default_modifiers_for_team(team)+ # The runner resolves this modifier, and the catalog cache key includes it, so a health read shares the compile's entry.+ modifiers.useNewEventsSchema = use_new_events_schema(team.pk, modifiers)+ context = HogQLContext(+ team_id=team.pk,+ team=team,+ database=Database.create_for(team=team, modifiers=modifiers, use_cached_sources=True),+ enable_select_queries=True,+ modifiers=modifiers,+ )+ query = ast.SelectQuery(+ select=[ast.Constant(value=1)],+ select_from=ast.JoinExpr(table=ast.Field(chain=["events"])),+ where=ast.And(exprs=[expr for expr, _ in filters]),+ )+ resolved = resolve_types(query, context, dialect="clickhouse")+ build_property_swapper(resolved, context)+ where = cast(ast.And, resolved.where)++ found: dict[ExposureFilterWithoutColumn, None] = {}+ for expr, (_, source), named_keys in zip(where.exprs, filters, named_by_filter, strict=True):+ for field in GetFieldsTraverser(expr).fields:+ plan = plan_property_access(field, context)+ if plan is None or plan.property_name not in named_keys or plan.property_name == _FLAG_KEY_PROPERTY:+ continue+ filter_type = _FILTER_TYPES.get(plan.scope)+ if filter_type is not None and _reads_all_properties(plan, context):+ found[ExposureFilterWithoutColumn(type=filter_type, key=plan.property_name, source=source)] = None+ return tuple(found)+++def load_filters_without_column(experiment: Experiment) -> tuple[ExposureFilterWithoutColumn, ...] | None:+ """The filters of a query that reads the exposures from the events, for each property that the query reads from the+ JSON of all the properties. None when the criteria don't parse or the lookup fails. Never raises."""+ try:+ criteria = normalize_to_exposure_criteria(experiment.exposure_criteria)+ except pydantic.ValidationError:+ return None+ try:+ return _filters_without_column(experiment, criteria)+ except Exception as error:+ capture_exception(error)+ return Nonediff --git a/products/experiments/backend/health/latest_calculation.py b/products/experiments/backend/health/latest_calculation.pyindex ee7bf4ac1fd9..1445722070d5 100644--- a/products/experiments/backend/health/latest_calculation.py+++ b/products/experiments/backend/health/latest_calculation.py@@ -3,6 +3,7 @@ from typing import Anyfrom posthog.exceptions_capture import capture_exception+from products.experiments.backend.facade.enums import ExposuresPathfrom products.experiments.backend.health.context import LatestCalculation, MetricCalculationFailurefrom products.experiments.backend.health.metric_compiles import can_hold_evaluation_rules, without_available_fieldsfrom products.experiments.backend.metric_calculation.config import (@@ -43,19 +44,38 @@ def _load_latest_calculation(experiment: Experiment) -> LatestCalculation:# failures, or else the run in progress, whose errors can still change, so the check waits for a finished run.run = get_latest_recalculation(experiment)if run is None:- return LatestCalculation(metric_count=len(configs), finished_at=None, failures=())+ return LatestCalculation(+ metric_count=len(configs),+ finished_at=None,+ failures=(),+ completed_count=0,+ completed_exposures_paths=frozenset(),+ )metric_errors = run.metric_errors if isinstance(run.metric_errors, dict) else {}- rows = {- row.metric_uuid: row for row in MetricResultStore(experiment_id=experiment.id).for_run(run, with_result=False)- }- failures = (- _shown_failure(experiment, calculation_config, metric_errors.get(metric_id), rows.get(metric_id))+ results = MetricResultStore(experiment_id=experiment.id)+ rows = {row.metric_uuid: row for row in results.for_run(run, with_result=False)}+ failures = tuple(+ failurefor metric_id, calculation_config in configs.items()+ if (+ failure := _shown_failure(experiment, calculation_config, metric_errors.get(metric_id), rows.get(metric_id))+ )+ is not None)+ failed = {failure.metric_id for failure in failures}+ completed = [+ row+ for metric_id in configs+ if metric_id not in failed+ and (row := rows.get(metric_id)) is not None+ and row.status == ExperimentMetricResult.Status.COMPLETED+ ]return LatestCalculation(metric_count=len(configs),finished_at=run.completed_at,- failures=tuple(failure for failure in failures if failure is not None),+ failures=failures,+ completed_count=len(completed),+ completed_exposures_paths=results.exposures_paths(completed) if failures else frozenset(),)@@ -81,6 +101,7 @@ def _shown_failure(return Noneerror_type = error.get("error_type")step = error.get("step")+ exposures_path = error.get("exposures_path")return MetricCalculationFailure(metric_id=calculation_config.metric_id,name=calculation_metric_name(experiment, calculation_config),@@ -91,4 +112,7 @@ def _shown_failure(can_quote_evaluation_rules=can_hold_evaluation_rules(calculation_config.definition),retriable=error.get("retriable") is True,definition=calculation_config.definition,+ exposures_path=ExposuresPath(exposures_path)+ if isinstance(exposures_path, str) and exposures_path in ExposuresPath+ else None,)diff --git a/products/experiments/backend/health/loader.py b/products/experiments/backend/health/loader.pyindex 959e5fafb5e2..f0b79e63b39b 100644--- a/products/experiments/backend/health/loader.py+++ b/products/experiments/backend/health/loader.py@@ -5,8 +5,10 @@ from posthog.dataclasses import frozenfrom posthog.models.user import Userfrom products.experiments.backend.facade.contracts import ExperimentHealthReaderAccess+from products.experiments.backend.facade.enums import ExposuresPathfrom products.experiments.backend.health.condition_set_traffic import load_condition_set_trafficfrom products.experiments.backend.health.context import HealthContext, load_health_context+from products.experiments.backend.health.exposure_columns import load_filters_without_columnfrom products.experiments.backend.health.exposure_gate import load_flag_historyfrom products.experiments.backend.health.exposure_inputs import ExposureInputs, load_exposure_inputsfrom products.experiments.backend.health.latest_calculation import load_latest_calculation@@ -73,11 +75,17 @@ def _health_context(flag_history = load_flag_history(experiment, reader_access=reader_access)context = load_health_context(experiment, flag_history=flag_history)traffic = load_condition_set_traffic(experiment, context, reader_access=reader_access)+ latest_calculation = load_latest_calculation(experiment)return replace(context,condition_set_traffic=traffic,pause_history=load_pause_history(experiment, reader_access=reader_access),primary_metric=load_primary_metric(experiment),metric_compiles=load_metric_compiles(experiment, user=user),- latest_calculation=load_latest_calculation(experiment),+ latest_calculation=latest_calculation,+ exposure_filters_without_column=load_filters_without_column(experiment)+ if latest_calculation is not None+ and not latest_calculation.completed_count+ and any(failure.exposures_path == ExposuresPath.DIRECT_SCAN for failure in latest_calculation.failures)+ else None,)diff --git a/products/experiments/backend/health/test/factories.py b/products/experiments/backend/health/test/factories.pyindex c55fdb4a5e1f..0a3ad42f0622 100644--- a/products/experiments/backend/health/test/factories.py+++ b/products/experiments/backend/health/test/factories.py@@ -47,6 +47,7 @@ RUNNING_CONTEXT = HealthContext(excluded_variant_keys=frozenset(),metric_compiles=None,latest_calculation=None,+ exposure_filters_without_column=None,)diff --git a/products/experiments/backend/health/test/test_exposure_columns.py b/products/experiments/backend/health/test/test_exposure_columns.pynew file mode 100644index 000000000000..600116c67c1c--- /dev/null+++ b/products/experiments/backend/health/test/test_exposure_columns.py@@ -0,0 +1,282 @@+from collections.abc import Callable+from datetime import timedelta+from typing import Any++from posthog.test.base import BaseTest+from unittest.mock import patch++from django.utils import timezone++from parameterized import parameterized++from posthog.schema import HogQLQueryModifiers, PersonsOnEventsMode, PropertyGroupsMode++from posthog.models.event.new_events_schema import events_route_scope+from posthog.models.property import TableColumn+from posthog.schema_enums import MaterializationMode++from products.actions.backend.models.action import Action+from products.experiments.backend.health.context import ExposureFilterSource+from products.experiments.backend.health.exposure_columns import load_filters_without_column+from products.experiments.backend.models.experiment import Experiment+from products.feature_flags.backend.models.feature_flag import FeatureFlag++from ee.clickhouse.materialized_columns.columns import MaterializedColumn, MaterializedColumnDetails++ON_EVENTS = PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS+JOINED = PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED+_SOURCE_LABELS = {+ ExposureFilterSource.EXPOSURE_CRITERIA: "",+ ExposureFilterSource.EXPOSURE_ACTION: " (action)",+ ExposureFilterSource.TEST_ACCOUNTS: " (test account)",+}+++def _column(table_column: TableColumn, key: str) -> MaterializedColumn:+ return MaterializedColumn(+ name=f"mat_{key}", details=MaterializedColumnDetails(table_column, key, is_disabled=False), is_nullable=False+ )+++MATERIALIZED_COLUMNS = {+ "events": {+ ("plan", "properties"): _column("properties", "plan"),+ ("tier", "person_properties"): _column("person_properties", "tier"),+ },+ "person": {("tier", "properties"): _column("properties", "tier")},+}+++def _filter(key: str, property_type: str, **rest: Any) -> dict[str, Any]:+ return {"key": key, "type": property_type, "operator": "is_set", **rest}+++class TestLoadFiltersWithoutColumn(BaseTest):+ def _experiment(self, *, filter_test_accounts: bool = True) -> Experiment:+ return Experiment.objects.create(+ team=self.team,+ name="Pricing page",+ feature_flag=FeatureFlag.objects.create(+ team=self.team, key="pricing-page", created_by=self.user, filters={}+ ),+ start_date=timezone.now() - timedelta(days=3),+ exposure_criteria={+ "filterTestAccounts": filter_test_accounts,+ "exposure_config": {+ "kind": "ExperimentEventExposureConfig",+ "event": "$pageview",+ "properties": [+ _filter("plan", "event"),+ _filter("$plan_source", "event"),+ _filter("campaign", "event"),+ _filter("tier", "person"),+ _filter("$initial_os", "person"),+ _filter("industry", "group", group_type_index=0),+ # None of these reads the JSON: the query reads the flag key anyway, and the others are columns.+ _filter("$feature_flag", "event"),+ _filter("$elements_chain", "event"),+ _filter("$virt_is_bot", "event"),+ _filter("distinct_id", "person"),+ _filter("$group_key", "group", group_type_index=0),+ ],+ },+ },+ )++ @parameterized.expand(+ [+ (+ "without_property_groups",+ ON_EVENTS,+ None,+ False,+ True,+ [+ "event:$plan_source",+ "event:campaign",+ "person:$initial_os",+ "group:industry",+ "person:email (test account)",+ ],+ ),+ (+ "property_groups_hold_the_custom_properties",+ ON_EVENTS,+ PropertyGroupsMode.OPTIMIZED,+ False,+ True,+ ["event:$plan_source", "person:$initial_os", "group:industry"],+ ),+ (+ "persons_joined",+ JOINED,+ None,+ False,+ True,+ [+ "event:$plan_source",+ "event:campaign",+ "person:$initial_os",+ "group:industry",+ "person:email (test account)",+ ],+ ),+ (+ "native_events_table",+ ON_EVENTS,+ None,+ True,+ True,+ ["group:industry"],+ ),+ (+ "test_accounts_not_filtered",+ ON_EVENTS,+ None,+ False,+ False,+ ["event:$plan_source", "event:campaign", "person:$initial_os", "group:industry"],+ ),+ (+ "persons_joined_materialization_disabled",+ JOINED,+ None,+ False,+ True,+ [+ "event:plan",+ "event:$plan_source",+ "event:campaign",+ "person:tier",+ "person:$initial_os",+ "group:industry",+ "person:email (test account)",+ "person:tier (test account)",+ ],+ MaterializationMode.DISABLED,+ ),+ ]+ )+ def test_names_the_filters_that_read_all_the_properties(+ self,+ _name: str,+ persons_on_events_mode: PersonsOnEventsMode,+ property_groups_mode: PropertyGroupsMode | None,+ new_events_schema: bool,+ filter_test_accounts: bool,+ expected: list[str],+ materialization_mode: MaterializationMode | None = None,+ ) -> None:+ self.team.modifiers = {+ "personsOnEventsMode": persons_on_events_mode,+ "propertyGroupsMode": property_groups_mode,+ "materializationMode": materialization_mode,+ }+ self.team.test_account_filters = [_filter("email", "person"), _filter("tier", "person")]+ self.team.save()+ experiment = self._experiment(filter_test_accounts=filter_test_accounts)++ with (+ events_route_scope(HogQLQueryModifiers(useNewEventsSchema=new_events_schema)),+ patch(+ "posthog.clickhouse.materialized_columns.get_enabled_materialized_columns_by_table",+ return_value=MATERIALIZED_COLUMNS,+ ),+ ):+ filters = load_filters_without_column(experiment)++ assert filters is not None+ self.assertEqual(+ [f"{found.type}:{found.key}{_SOURCE_LABELS[found.source]}" for found in filters],+ expected,+ )++ @parameterized.expand(+ [+ ("criteria_do_not_parse", {"exposure_config": {"kind": "Unknown"}}, None),+ (+ "criteria_have_no_exposure_query",+ {+ "exposure_config": {+ "kind": "ExperimentWarehouseNativeExposureConfig",+ "connection_id": "source",+ "query": "SELECT 1",+ }+ },+ None,+ ),+ (+ "criteria_have_no_filters",+ {"filterTestAccounts": False},+ (),+ ),+ ]+ )+ def test_criteria_without_a_filter_to_name(+ self, _name: str, exposure_criteria: dict[str, Any], expected: tuple[()] | None+ ) -> None:+ experiment = self._experiment()+ experiment.exposure_criteria = exposure_criteria++ self.assertEqual(load_filters_without_column(experiment), expected)++ @parameterized.expand(+ [+ (+ "exposure_action",+ lambda action: {"exposure_config": action},+ ["event:campaign (action)", "event:$current_url (action)", "event:source (action)"],+ ),+ (+ "activation_action",+ lambda action: {"exposure_config": None, "activation_config": action},+ ["event:campaign (action)", "event:$current_url (action)", "event:source (action)"],+ ),+ ("deleted_action", lambda action: {"exposure_config": {**action, "id": 0}}, ["event:source (action)"]),+ ]+ )+ def test_names_the_filters_of_an_action(+ self, _name: str, criteria: Callable[[dict[str, Any]], dict[str, Any]], expected: list[str]+ ) -> None:+ action = Action.objects.create(+ team=self.team,+ name="Started checkout",+ steps_json=[+ {"event": "checkout", "properties": [_filter("campaign", "event")]},+ {"event": "$pageview", "url": "/checkout", "properties": [_filter("$virt_is_bot", "event")]},+ ],+ )+ experiment = self._experiment()+ experiment.exposure_criteria = {+ "filterTestAccounts": False,+ **criteria(+ {"kind": "ActionsNode", "id": action.pk, "properties": [_filter("source", "event")]},+ ),+ }++ with events_route_scope(HogQLQueryModifiers(useNewEventsSchema=False)):+ filters = load_filters_without_column(experiment)++ assert filters is not None+ self.assertEqual([f"{found.type}:{found.key}{_SOURCE_LABELS[found.source]}" for found in filters], expected)++ def test_a_virtual_property_does_not_take_a_read_that_another_filter_names(self) -> None:+ self.team.test_account_filters = [_filter("$virt_is_bot", "event")]+ self.team.save()+ experiment = self._experiment()+ experiment.exposure_criteria = {+ "filterTestAccounts": True,+ "exposure_config": {+ "kind": "ExperimentEventExposureConfig",+ "event": "$pageview",+ "properties": [_filter("$lib", "event")],+ },+ }++ with events_route_scope(HogQLQueryModifiers(useNewEventsSchema=False)):+ filters = load_filters_without_column(experiment)++ assert filters is not None+ self.assertEqual(+ [f"{found.type}:{found.key}{_SOURCE_LABELS[found.source]}" for found in filters], ["event:$lib"]+ )diff --git a/products/experiments/backend/health/test/test_metric_calculation_failed.py b/products/experiments/backend/health/test/test_metric_calculation_failed.pyindex 1bf40e74416f..27aafec73bb0 100644--- a/products/experiments/backend/health/test/test_metric_calculation_failed.py+++ b/products/experiments/backend/health/test/test_metric_calculation_failed.py@@ -23,8 +23,11 @@ from products.experiments.backend.facade.contracts import (ExperimentHealthMetricCompileErrorKind,ExperimentHealthReaderAccess,)+from products.experiments.backend.facade.enums import ExposuresPathfrom products.experiments.backend.health.checks.metric_calculation_failed import metric_calculation_failedfrom products.experiments.backend.health.context import (+ ExposureFilterSource,+ ExposureFilterWithoutColumn,LatestCalculation,MetricCalculationFailure,MetricCompileFailure,@@ -45,6 +48,11 @@ NO_QUERIES = ExperimentHealthReaderAccess(can_read_flag_history=False, can_query_events=False, denied_event_properties=frozenset())FINISHED_AT = datetime(2026, 10, 26, 9, 30, tzinfo=UTC)+OUT_OF_MEMORY = "This query ran out of memory before it could finish."+PLAN_FILTER = ExposureFilterWithoutColumn(type="group", key="plan", source=ExposureFilterSource.EXPOSURE_CRITERIA)+DIRECT = ExposuresPath.DIRECT_SCAN+PRECOMPUTED = ExposuresPath.PRECOMPUTED+NO_CAUSE = (None, 'primary metric "Revenue" failed. Its query needed more memory than PostHog allows for one')CLICKHOUSE_MESSAGE = ("Code: 215. DB::Exception: Column `properties` is not under aggregate function and not in GROUP BY keys. ""In query SELECT properties FROM events"@@ -59,6 +67,7 @@ def _failure(role: str = "primary",retriable: bool = True,can_quote_evaluation_rules: bool = False,+ exposures_path: ExposuresPath | None = None,) -> MetricCalculationFailure:return MetricCalculationFailure(metric_id=f"{role}-metric",@@ -70,11 +79,24 @@ def _failure(can_quote_evaluation_rules=can_quote_evaluation_rules,retriable=retriable,definition={"kind": "ExperimentMetric", "metric_type": "mean"},+ exposures_path=exposures_path,)-def _outcome(*failures: MetricCalculationFailure, **changes: Any) -> ExperimentHealthFinding | CheckSkip | None:- calculation = LatestCalculation(metric_count=len(failures), finished_at=FINISHED_AT, failures=failures)+def _limited(exposures_path: ExposuresPath | None, role: str = "primary") -> MetricCalculationFailure:+ return _failure(OUT_OF_MEMORY, error_type="out_of_memory", role=role, exposures_path=exposures_path)+++def _outcome(+ *failures: MetricCalculationFailure, completed: tuple[ExposuresPath | None, ...] = (), **changes: Any+) -> ExperimentHealthFinding | CheckSkip | None:+ calculation = LatestCalculation(+ metric_count=len(failures) + len(completed),+ finished_at=FINISHED_AT,+ failures=failures,+ completed_count=len(completed),+ completed_exposures_paths=frozenset(path for path in completed if path is not None),+ )return metric_calculation_failed(replace(RUNNING_CONTEXT, latest_calculation=calculation, **changes))@@ -319,8 +341,174 @@ class TestMetricCalculationFailed(TestCase):self.assertEqual(finding.detail.count(words), 1)self.assertNotIn("Revenue", finding.detail)+ @parameterized.expand(+ [+ (+ "every_metric_read_the_events_and_reached_a_limit",+ (_limited(DIRECT), _limited(DIRECT, "secondary")),+ (),+ (PLAN_FILTER,),+ ("exposure_criteria", 'filter on the group property "plan", which makes each query read all group'),+ ),+ (+ "the_only_metric_read_the_events_and_reached_a_limit",+ (_limited(DIRECT),),+ (),+ (PLAN_FILTER,),+ (+ "exposure_criteria",+ 'primary metric "Revenue" failed. Its query needed more memory than PostHog allows for one '+ 'calculation. The likely cause is the exposure criteria\'s filter on the group property "plan"',+ ),+ ),+ (+ "two_limits_and_two_filters_of_different_kinds",+ (+ _limited(DIRECT),+ _failure(+ "Limit for rows or bytes to read exceeded",+ error_type="byte_limit",+ role="secondary",+ exposures_path=DIRECT,+ ),+ ),+ (),+ (+ PLAN_FILTER,+ ExposureFilterWithoutColumn(type="event", key="$device", source=ExposureFilterSource.TEST_ACCOUNTS),+ ),+ (+ "exposure_criteria",+ "Every metric's query needed more resources than PostHog allows for one calculation. The likely "+ 'cause is the exposure criteria\'s filter on the group property "plan" and the test account filter '+ 'on the event property "$device", which make each query read all group and event properties. '+ "Change these filters, or turn off the test account filter in the exposure criteria, or contact",+ ),+ ),+ (+ "the_filter_is_in_the_exposure_action",+ (_limited(DIRECT),),+ (),+ (+ ExposureFilterWithoutColumn(+ type="event", key="campaign", source=ExposureFilterSource.EXPOSURE_ACTION+ ),+ ),+ (+ "exposure_criteria",+ 'The likely cause is the exposure action\'s filter on the event property "campaign", which makes '+ "the query read all event properties. Remove or change the filter, or contact support for help.",+ ),+ ),+ (+ "one_property_in_the_exposure_criteria_and_the_test_account_filters",+ (_limited(DIRECT),),+ (),+ (+ ExposureFilterWithoutColumn(+ type="event", key="plan", source=ExposureFilterSource.EXPOSURE_CRITERIA+ ),+ ExposureFilterWithoutColumn(type="event", key="plan", source=ExposureFilterSource.TEST_ACCOUNTS),+ ),+ (+ "exposure_criteria",+ 'the exposure criteria\'s filter on the event property "plan" and the test account filter on the '+ 'event property "plan", which make the query read all event properties. Change these filters, or '+ "turn off the test account filter in the exposure criteria",+ ),+ ),+ (+ "one_query_read_precomputed_exposures",+ (_limited(DIRECT), _limited(PRECOMPUTED, "secondary")),+ (),+ (PLAN_FILTER,),+ NO_CAUSE,+ ),+ (+ "every_query_read_precomputed_exposures",+ (_limited(PRECOMPUTED), _limited(PRECOMPUTED, "secondary")),+ (),+ (PLAN_FILTER,),+ NO_CAUSE,+ ),+ (+ "failures_do_not_say_where_they_read",+ (_limited(None), _limited(None, "secondary")),+ (),+ (PLAN_FILTER,),+ NO_CAUSE,+ ),+ ("no_filter_reads_all_the_properties", (_limited(DIRECT), _limited(DIRECT, "secondary")), (), (), NO_CAUSE),+ ("filters_could_not_be_read", (_limited(DIRECT), _limited(DIRECT, "secondary")), (), None, NO_CAUSE),+ (+ "a_metric_ran_out_of_the_workflow_time_without_a_stored_read",+ (_limited(DIRECT), _limited(None, "secondary")),+ (),+ (PLAN_FILTER,),+ ("exposure_criteria", 'filter on the group property "plan", which makes each query read all group'),+ ),+ (+ "another_metric_failed_for_another_reason",+ (+ _limited(DIRECT),+ _failure(+ "Queries are a little too busy right now.",+ error_type="rate_limited",+ role="secondary",+ exposures_path=DIRECT,+ ),+ ),+ (),+ (PLAN_FILTER,),+ NO_CAUSE,+ ),+ (+ "another_metric_completed_the_same_read_of_the_events",+ (_limited(DIRECT),),+ (DIRECT,),+ (PLAN_FILTER,),+ ("metric", "while another metric of the experiment returned results. Simplify this metric"),+ ),+ (+ "another_metric_completed_the_same_read_of_precomputed_exposures",+ (_limited(PRECOMPUTED),),+ (PRECOMPUTED,),+ None,+ ("metric", "while another metric of the experiment returned results. Simplify this metric"),+ ),+ (+ "completed_metric_read_the_exposures_another_way",+ (_limited(DIRECT),),+ (PRECOMPUTED,),+ (PLAN_FILTER,),+ NO_CAUSE,+ ),+ ("completed_result_does_not_say_where_it_read", (_limited(DIRECT),), (None,), (PLAN_FILTER,), NO_CAUSE),+ ("failure_does_not_say_where_it_read", (_limited(None),), (DIRECT,), (PLAN_FILTER,), NO_CAUSE),+ ]+ )+ def test_names_the_cause_of_a_query_limit(+ self,+ _name: str,+ failures: tuple[MetricCalculationFailure, ...],+ completed: tuple[ExposuresPath | None, ...],+ filters: tuple[ExposureFilterWithoutColumn, ...] | None,+ expected: tuple[str | None, str],+ ) -> None:+ finding = _outcome(*failures, completed=completed, exposure_filters_without_column=filters)++ assert isinstance(finding, ExperimentHealthFinding)+ reason, words = expected+ self.assertEqual((finding.subcode, finding.severity, finding.reason), ("query_limit", WARNING, reason))+ self.assertEqual(finding.detail.count(words), 1)+ names_the_filters = reason == "exposure_criteria"+ self.assertEqual("properties" in finding.evidence, names_the_filters)+ self.assertEqual(bool(finding.actions), names_the_filters)+def test_waits_for_a_metric_that_the_experiment_calculates(self) -> None:- calculation = LatestCalculation(metric_count=0, finished_at=None, failures=())+ calculation = LatestCalculation(+ metric_count=0, finished_at=None, failures=(), completed_count=0, completed_exposures_paths=frozenset()+ )outcome = metric_calculation_failed(replace(RUNNING_CONTEXT, latest_calculation=calculation, primary_metric_count=0, secondary_metric_count=0)@@ -398,6 +586,7 @@ class TestMetricCalculationFailedFromStoredRuns(BaseTest):metric_errors: dict[str, dict[str, Any]] | None = None,failed_rows: tuple[str, ...] = (),completed_rows: tuple[str, ...] = (),+ completed_read_precomputed: bool = False,) -> None:with team_scope(self.team.id):query_to = created_at + timedelta(minutes=1)@@ -429,7 +618,7 @@ class TestMetricCalculationFailedFromStoredRuns(BaseTest):calculation_config,window=query_to,query_from=query_from,- result={"variant_results": []},+ result={"variant_results": [], "is_precomputed": completed_read_precomputed},query_id=None,)terminal = status in (@@ -472,6 +661,78 @@ class TestMetricCalculationFailedFromStoredRuns(BaseTest):self.assertNotIn("purchase_total", finding.detail + str(finding.evidence))self.assertIsNone(finding.evidence["metric_2_message"])+ @parameterized.expand(+ [+ (+ "every_metric_read_the_events",+ (),+ "direct_scan",+ False,+ ("exposure_criteria", 'filter on the group property "plan"'),+ ),+ (+ "other_metrics_completed_the_same_read",+ ("rejected", "secondary"),+ "direct_scan",+ False,+ ("metric", "while other metrics of the experiment returned results"),+ ),+ (+ "other_metrics_read_precomputed_exposures",+ ("rejected", "secondary"),+ "direct_scan",+ True,+ (None, "read more data than PostHog allows for one calculation"),+ ),+ (+ "failures_stored_without_their_read",+ (),+ None,+ False,+ (None, "read more data than PostHog allows for one calculation"),+ ),+ ]+ )+ def test_names_the_cause_of_a_query_limit_from_the_stored_run(+ self,+ _name: str,+ completed_rows: tuple[str, ...],+ exposures_path: str | None,+ completed_read_precomputed: bool,+ expected: tuple[str | None, str],+ ) -> None:+ self.experiment.exposure_criteria = {+ "exposure_config": {+ "kind": "ExperimentEventExposureConfig",+ "event": "$feature_flag_called",+ "properties": [+ {"key": "plan", "type": "group", "group_type_index": 0, "operator": "exact", "value": ["pro"]}+ ],+ }+ }+ self.experiment.save()+ read_limit = {+ "step": "calculation",+ "message": "Query read too much data",+ "error_type": "byte_limit",+ "exposures_path": exposures_path,+ }+ self._run(+ status=ExperimentMetricsRecalculation.Status.FAILED,+ created_at=timezone.now() - timedelta(hours=2),+ metric_errors={+ uuid: read_limit for uuid in ("primary", "rejected", "secondary") if uuid not in completed_rows+ },+ completed_rows=completed_rows,+ completed_read_precomputed=completed_read_precomputed,+ )++ [finding] = [finding for finding in self._health().findings if finding.code == CODE]++ reason, words = expected+ self.assertEqual(finding.reason, reason)+ self.assertIn(words, finding.detail)+@parameterized.expand([("no_finished_calculation", None, ExperimentHealthCheckStatus.NOT_APPLICABLE),diff --git a/products/experiments/backend/hogql_queries/experiment_query_builder.py b/products/experiments/backend/hogql_queries/experiment_query_builder.pyindex 169e30156840..c0b27a68eb5a 100644--- a/products/experiments/backend/hogql_queries/experiment_query_builder.py+++ b/products/experiments/backend/hogql_queries/experiment_query_builder.py@@ -256,6 +256,9 @@ class ExperimentQueryBuilder:query.limit = ast.Constant(value=self.QUERY_RESULT_LIMIT)return query+ def reads_precomputed_exposures(self, job_ids: list[str]) -> bool:+ return self._exposure_query_builder(job_ids=job_ids).reads_precomputed()+def _exposure_query_builder(self, maturity: MaturityGate | None = None, job_ids: list[str] | None = None) -> ExposureQueryBuilder:diff --git a/products/experiments/backend/hogql_queries/experiment_query_runner.py b/products/experiments/backend/hogql_queries/experiment_query_runner.pyindex bf2efe1cac5a..71473d214567 100644--- a/products/experiments/backend/hogql_queries/experiment_query_runner.py+++ b/products/experiments/backend/hogql_queries/experiment_query_runner.py@@ -41,7 +41,7 @@ from posthog.models.team.extensions import get_or_create_team_extensionfrom products.ai_observability.backend.facade.api import mentions_evaluation_pass_markersfrom products.analytics_platform.backend.lazy_computation.lazy_computation_executor import LazyComputationResult-from products.experiments.backend.facade.enums import PrecomputeSkipReason+from products.experiments.backend.facade.enums import ExposuresPath, PrecomputeSkipReasonfrom products.experiments.backend.hogql_queries import MULTIPLE_VARIANT_KEY, get_baseline_variant_keyfrom products.experiments.backend.hogql_queries.base_query_utils import experiment_window, experiment_window_endfrom products.experiments.backend.hogql_queries.cuped_config import get_cuped_config@@ -196,6 +196,7 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):self.clickhouse_sql: str | None = Noneself.hogql: str | None = Noneself._is_precomputed: bool = False # exposures precompute+ self.exposures_path: ExposuresPath | None = Noneself._metric_events_precomputed: bool = Falsedef _get_breakdowns_for_builder(self) -> list | None:@@ -343,7 +344,7 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):result = self._ensure_exposures_precomputed(builder)if result.ready:exposure_job_ids = [str(job_id) for job_id in result.job_ids]- self._is_precomputed = True+ self._is_precomputed = builder.reads_precomputed_exposures(exposure_job_ids)elif result.capacity_rejected:# ClickHouse refused the build because the pool is full. The direct scan# below would hold a slot for minutes and starve the next builds too, so@@ -510,14 +511,14 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):experiment_query_ast = self._get_experiment_query()# Tag after _get_experiment_query() which sets the precompute flags- exposures_path = "precomputed" if self._is_precomputed else "direct_scan"+ self.exposures_path = ExposuresPath.PRECOMPUTED if self._is_precomputed else ExposuresPath.DIRECT_SCANmetric_events_path = self.metric_events_pathskip_reason = self._precompute_skip_reason()tag_queries(- experiment_exposures_path=exposures_path,+ experiment_exposures_path=self.exposures_path.value,experiment_metric_events_path=metric_events_path,experiment_metric_events_skip_reason=self._metric_events_ineligibility_reason(),- experiment_execution_path=exposures_path,+ experiment_execution_path=self.exposures_path.value,experiment_precompute_skip_reason=skip_reason.value if skip_reason is not None else None,experiment_scan_date_from=self.date_range.date_from,experiment_scan_date_to=self.date_range.date_to,diff --git a/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.py b/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.pyindex 94c559e65c57..8894ddfd7003 100644--- a/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.py+++ b/products/experiments/backend/hogql_queries/test/test_experiment_exposure_preaggregation.py@@ -1,6 +1,7 @@from collections.abc import Callablefrom datetime import UTC, datetimefrom typing import Any, cast+from uuid import uuid4from posthog.test.base import _create_event, _create_personfrom unittest.mock import patch@@ -11,6 +12,8 @@ from django.test import override_settingsfrom parameterized import parameterizedfrom posthog.schema import (+ Breakdown,+ BreakdownFilter,EventsNode,ExperimentExposureQuery,ExperimentFunnelMetric,@@ -34,6 +37,7 @@ from products.analytics_platform.backend.lazy_computation.lazy_computation_execuensure_precomputed,)from products.analytics_platform.backend.models import PreaggregationJob+from products.experiments.backend.facade.enums import ExposuresPathfrom products.experiments.backend.hogql_queries.base_query_utils import experiment_windowfrom products.experiments.backend.hogql_queries.experiment_exposures_query_runner import ExperimentExposuresQueryRunnerfrom products.experiments.backend.hogql_queries.experiment_query_builder import (@@ -865,6 +869,58 @@ class TestExperimentExposurePreaggregation(ExperimentQueryRunnerBaseTest):assert direct_result.variant_results is not Noneassert direct_result.variant_results[0].number_of_samples == 1+ def test_breakdown_scans_the_events_when_the_exposure_job_is_ready(self):+ feature_flag = self.create_feature_flag()+ experiment = self.create_experiment(+ feature_flag=feature_flag,+ start_date=datetime(2024, 1, 1),+ end_date=datetime(2024, 1, 5),+ )+ self._enable_precomputation()+ metric = ExperimentMeanMetric(+ source=EventsNode(event="purchase"),+ breakdownFilter=BreakdownFilter(breakdowns=[Breakdown(property="$browser", type="event")]),+ )+ experiment.metrics = [metric.model_dump(mode="json")]+ experiment.save()++ for variant in ("control", "test"):+ for i in range(3):+ distinct_id = f"user_{variant}_{i}"+ _create_person(distinct_ids=[distinct_id], team_id=self.team.pk)+ self._create_exposure_event(+ distinct_id, feature_flag, variant, datetime(2024, 1, 2, 12, 0, 0, tzinfo=UTC)+ )+ _create_event(+ team=self.team,+ event="purchase",+ distinct_id=distinct_id,+ timestamp=datetime(2024, 1, 2, 13, 0, 0, tzinfo=UTC),+ properties={f"$feature/{feature_flag.key}": variant, "$browser": "Chrome"},+ )++ with (+ patch.object(+ ExperimentQueryRunner,+ "_ensure_exposures_precomputed",+ return_value=LazyComputationResult(ready=True, job_ids=[uuid4()]),+ ),+ patch.object(+ ExperimentQueryRunner,+ "_ensure_metric_events_precomputed",+ return_value=LazyComputationResult(ready=False, job_ids=[]),+ ),+ ):+ runner = ExperimentQueryRunner(+ query=ExperimentQuery(experiment_id=experiment.id, kind="ExperimentQuery", metric=metric),+ team=self.team,+ )+ result = cast(ExperimentQueryResponse, runner.calculate())++ assert result.variant_results is not None+ assert result.is_precomputed is False+ assert runner.exposures_path == ExposuresPath.DIRECT_SCAN+@parameterized.expand([("exposure", "_ensure_exposures_precomputed", "_ensure_metric_events_precomputed"),@@ -925,12 +981,17 @@ class TestExperimentExposurePreaggregation(ExperimentQueryRunnerBaseTest):"products.experiments.backend.hogql_queries.experiment_query_runner.capture_exception") as mock_capture,):- result = self._run_experiment(experiment, metric)+ runner = ExperimentQueryRunner(+ query=ExperimentQuery(experiment_id=experiment.id, kind="ExperimentQuery", metric=metric),+ team=self.team,+ )+ result = cast(ExperimentQueryResponse, runner.calculate())assert result.baseline is not Noneassert result.baseline.number_of_samples == 3assert result.variant_results is not Noneassert result.variant_results[0].number_of_samples == 3+ assert runner.exposures_path == ExposuresPath.DIRECT_SCAN# The fallback keeps the query working, but the precomputation failure must still# be surfaced to error tracking rather than silently masked.diff --git a/products/experiments/backend/metric_calculation/results.py b/products/experiments/backend/metric_calculation/results.pyindex a932515684d3..9b904bb60f6e 100644--- a/products/experiments/backend/metric_calculation/results.py+++ b/products/experiments/backend/metric_calculation/results.py@@ -40,6 +40,7 @@ from posthog.schema import ExperimentQueryResponsefrom posthog.dataclasses import frozen+from products.experiments.backend.facade.enums import ExposuresPathfrom products.experiments.backend.logic.result_stats import MetricResultStats, parse_stored_responsefrom products.experiments.backend.metric_calculation.config import MetricCalculationConfig, build_calculation_configsfrom products.experiments.backend.models.experiment import (@@ -140,6 +141,18 @@ class MetricResultStore:rows = rows.defer("result")return list(_newest_write_first(rows, "metric_uuid").distinct("metric_uuid"))+ def exposures_paths(self, rows: Iterable[ExperimentMetricResult]) -> frozenset[ExposuresPath]:+ """Where the queries of these rows read the exposures. It reads one key of `result`, so the rows of `for_run` can+ leave `result` unloaded. A result without the key adds nothing."""+ stored = ExperimentMetricResult.objects.filter(+ experiment_id=self.experiment_id, pk__in=[row.pk for row in rows]+ ).values_list("result__is_precomputed", flat=True)+ return frozenset(+ ExposuresPath.PRECOMPUTED if is_precomputed else ExposuresPath.DIRECT_SCAN+ for is_precomputed in stored+ if is_precomputed is not None+ )+def has_completed(self, calculation_config: MetricCalculationConfig, *, window: datetime) -> bool:"""Whether a recalculation already stored a completed result for this calculation config at this window."""return ExperimentMetricResult.objects.filter(diff --git a/products/experiments/backend/presentation/serializers.py b/products/experiments/backend/presentation/serializers.pyindex a8b04af886b4..ac4eb401a334 100644--- a/products/experiments/backend/presentation/serializers.py+++ b/products/experiments/backend/presentation/serializers.py@@ -224,8 +224,9 @@ class ExperimentHealthFindingSerializer(serializers.Serializer):allow_null=True,help_text=("A stable qualifier below the subcode: where a sample ratio mismatch is located ('library', 'page'), "- "what leaves nobody exposed ('test_account_filter'), or why a check could not read what it needs "- "('no_history_access'). Null when the finding names none."+ "what leaves nobody exposed ('test_account_filter'), what made a metric reach a query limit "+ "('metric', 'exposure_criteria'), or why a check could not read what it needs ('no_history_access'). "+ "Null when the finding names none."),)severity = serializers.ChoiceField(@@ -1649,9 +1650,11 @@ class _ExperimentMetricsRecalculationBaseSerializer(serializers.Serializer):metric_errors = serializers.JSONField(read_only=True,help_text=(- "Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp}. retriable is "- "true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can "- "succeed; false when the metric config, the data, or a resource limit must change first"+ "Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp, exposures_path}. "+ "retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry "+ "can succeed; false when the metric config, the data, or a resource limit must change first. "+ "exposures_path is where the query read the exposures, and is missing when the query failed before it "+ "was built"),)created_at = serializers.DateTimeField(read_only=True, help_text="When the job was created")diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.pyindex 051e41a1939b..b21cf20209f5 100644--- a/products/experiments/backend/presentation/views.py+++ b/products/experiments/backend/presentation/views.py@@ -1024,7 +1024,13 @@ class EnterpriseExperimentsViewSet(wrote for users), and `calculated_at` is when the calculation finished. It is awarning when a primary metric fails for a kind that a change fixes (`query_limit`,`settings`, `experiment_too_old`), also after the end, because the calculation after the fix- reads the events up to the end. It is info otherwise.+ reads the events up to the end. It is info otherwise. When the first failure reached the+ read, memory or time limit, `reason` names the cause that the calculation shows: `metric`+ when another metric of the same calculation completed the same read of the exposures, or+ `exposure_criteria` when every metric reached a limit and an exposure or test account filter+ makes each query read all the properties of an event, person or group. `evidence.properties`+ then names the filters. A query that read precomputed exposures doesn't apply those filters,+ so it can't get `exposure_criteria`. It is null otherwise.The severity of the `some_conditions_pinned` finding follows the largest share that theusers of the pinned release conditions make up among the users who got one variant fromdiff --git a/products/experiments/backend/temporal/ARCHITECTURE.md b/products/experiments/backend/temporal/ARCHITECTURE.mdindex 33235cfb6b55..ecd934c99fab 100644--- a/products/experiments/backend/temporal/ARCHITECTURE.md+++ b/products/experiments/backend/temporal/ARCHITECTURE.md@@ -204,7 +204,7 @@ Red nodes advance the window; blue nodes reuse it.#### Retriable failures-A metric's terminal failure lands in `metric_errors` as `{step, message, error_type, retriable, timestamp}`.+A metric's terminal failure lands in `metric_errors` as `{step, message, error_type, retriable, timestamp}`, plus `exposures_path` (`precomputed` or `direct_scan`) once the runner has built the query.`error_type` is the same taxonomy as the `experiment metric error` event (`classify_experiment_query_error`).`retriable` is the backend's own decision, not a frontend mapping of `error_type`: it is true when a transient error (`timeout`, `rate_limited`, `server_error`) used its final attempt, and false when the calc activity failed permanently (`validation_error`, `out_of_memory`, `byte_limit`, `insufficient_data`, a `ValueError`, or a discovery failure) or the request was rejected (`recalculation_window_too_long`).A retriable failure reads as a gap, so the page load heals it; a non-retriable one waits for the user, because a new run with the same inputs fails the same way.diff --git a/products/experiments/backend/temporal/recalculation_logic.py b/products/experiments/backend/temporal/recalculation_logic.pyindex f0b8523170d8..b7d0511f7a8c 100644--- a/products/experiments/backend/temporal/recalculation_logic.py+++ b/products/experiments/backend/temporal/recalculation_logic.py@@ -33,6 +33,7 @@ from posthog.models.scoping import team_scopefrom posthog.sync import database_sync_to_async_poolfrom posthog.temporal.common.errors import NonReportableApplicationError+from products.experiments.backend.facade.enums import ExposuresPathfrom products.experiments.backend.hogql_queries.base_query_utils import experiment_window_endfrom products.experiments.backend.hogql_queries.error_handling import (classify_experiment_query_error,@@ -397,7 +398,14 @@ def _capture_results_refresh_completed(update: RecalculationProgressUpdate) -> Ndef _record_failure(- recalculation_id: str, metric_uuid: str, step: str, message: str, *, error_type: str, retriable: bool+ recalculation_id: str,+ metric_uuid: str,+ step: str,+ message: str,+ *,+ error_type: str,+ retriable: bool,+ exposures_path: ExposuresPath | None = None,) -> None:"""Merge the error entry into metric_errors under a row lock (no lost updates between concurrent failures).@@ -416,6 +424,8 @@ def _record_failure("retriable": retriable,"timestamp": timezone.now().isoformat(),}+ if exposures_path is not None:+ metric_errors[metric_uuid]["exposures_path"] = exposures_path.valuerecalc.metric_errors = metric_errorsrecalc.save(update_fields=["metric_errors"])@@ -704,6 +714,8 @@ def _calculate_experiment_metric_for_recalculation_sync(calc_started_at = time.perf_counter()query_from = experiment.start_date+ runner: ExperimentQueryRunner | None = None+def record_terminal_error(message: str, error_type: str, *, retriable: bool) -> None:results.record_run_failure(recalculation_id,@@ -714,7 +726,13 @@ def _calculate_experiment_metric_for_recalculation_sync(query_id=client_query_id,)_record_failure(- recalculation_id, metric_uuid, "calculation", message, error_type=error_type, retriable=retriable+ recalculation_id,+ metric_uuid,+ "calculation",+ message,+ error_type=error_type,+ retriable=retriable,+ exposures_path=runner.exposures_path if runner is not None else None,)_capture_experiment_metric_event(experiment,diff --git a/products/experiments/backend/temporal/test_recalculation_activities.py b/products/experiments/backend/temporal/test_recalculation_activities.pyindex 756986649959..81708b382095 100644--- a/products/experiments/backend/temporal/test_recalculation_activities.py+++ b/products/experiments/backend/temporal/test_recalculation_activities.py@@ -1,6 +1,7 @@import asyncioimport threadingimport dataclasses+from collections.abc import Iteratorfrom concurrent.futures import ThreadPoolExecutorfrom contextlib import contextmanagerfrom datetime import datetime, timedelta@@ -28,6 +29,7 @@ from posthog.exceptions import ClickHouseAtCapacity, ClickHouseQueryMemoryLimitEfrom posthog.temporal.common.errors import NonReportableErrorfrom posthog.temporal.tests.utils.asyncify import restored_asyncify_executor+from products.experiments.backend.facade.enums import ExposuresPathfrom products.experiments.backend.metric_calculation.config import get_metric_calculation_configfrom products.experiments.backend.metric_calculation.results import _recalc_fingerprintfrom products.experiments.backend.models.experiment import (@@ -99,6 +101,14 @@ def _calculate()+@contextmanager+def _patched_runner() -> Iterator[MagicMock]:+ with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:+ # A MagicMock in this attribute can't be stored in `metric_errors`.+ mock_runner.return_value.exposures_path = None+ yield mock_runner++@pytest.mark.django_db(transaction=True)class TestRecalculationActivities(BaseTest):def _flag(self, key: str) -> FeatureFlag:@@ -734,7 +744,7 @@ class TestCalculateActivity(BaseTest):ExperimentToSavedMetric.objects.create(experiment=exp, saved_metric=saved, metadata={"type": "primary"})recalc = self._recalc(exp, metric_uuids=["s1"])- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:+ with _patched_runner() as mock_runner:mock_runner.return_value.run.side_effect = RuntimeError("kaboom")with pytest.raises(RuntimeError, match="kaboom"):_calculate(exp.id, "s1", str(recalc.id), _QUERY_TO)@@ -750,7 +760,7 @@ class TestCalculateActivity(BaseTest):exp = self._experiment(flag_key="calc-transient", metrics=[_mean_metric("m1")])recalc = self._recalc(exp, metric_uuids=["m1"])- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:+ with _patched_runner() as mock_runner:mock_runner.return_value.run.side_effect = RuntimeError("transient blip")# Non-final attempt: re-raises, but nothing is recorded.@@ -819,7 +829,7 @@ class TestCalculateActivity(BaseTest):recalc = self._recalc(exp, metric_uuids=["m1"])with (- patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner,+ _patched_runner() as mock_runner,patch("products.experiments.backend.temporal.recalculation_logic.random.uniform", return_value=47.5),):mock_runner.return_value.run.side_effect = exc@@ -836,7 +846,7 @@ class TestCalculateActivity(BaseTest):next_retry_at = datetime.fromisoformat(entry["next_retry_at"])assert (next_retry_at - timezone.now()).total_seconds() == expected_delay_seconds- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:+ with _patched_runner() as mock_runner:mock_runner.return_value.run.return_value.model_dump.return_value = {}result = _calculate(exp.id, "m1", str(recalc.id), _QUERY_TO, attempt=attempt + 1)@@ -859,8 +869,9 @@ class TestCalculateActivity(BaseTest):exp = self._experiment(flag_key=f"calc-terminal-{name}", metrics=[_mean_metric("m1")])recalc = self._recalc(exp, metric_uuids=["m1"])- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:+ with _patched_runner() as mock_runner:mock_runner.return_value.run.side_effect = exc+ mock_runner.return_value.exposures_path = ExposuresPath.DIRECT_SCANwith pytest.raises(Exception):_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO, is_final_attempt=True)@@ -868,6 +879,7 @@ class TestCalculateActivity(BaseTest):entry = recalc.metric_errors["m1"]assert entry["error_type"] == expected_error_typeassert entry["retriable"] is expected_retriable+ assert entry["exposures_path"] == "direct_scan"def test_terminal_failure_clears_retry_state(self):# A metric that retried and then failed for good must not keep a stale "retrying" entry alongside@@ -875,7 +887,7 @@ class TestCalculateActivity(BaseTest):exp = self._experiment(flag_key="calc-retry-terminal", metrics=[_mean_metric("m1")])recalc = self._recalc(exp, metric_uuids=["m1"])- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:+ with _patched_runner() as mock_runner:mock_runner.return_value.run.side_effect = RuntimeError("transient blip")with pytest.raises(RuntimeError):_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO, is_final_attempt=False, attempt=2)@@ -902,7 +914,7 @@ class TestCalculateActivity(BaseTest):):exp = self._experiment(flag_key=f"calc-out-of-time-{name}", metrics=[_mean_metric("m1")])recalc = self._recalc(exp, metric_uuids=["m1"])- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:+ with _patched_runner() as mock_runner:if last_attempt_error is None:mock_runner.return_value.run.return_value.model_dump.return_value = {}_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO, is_final_attempt=False, attempt=2)@@ -950,7 +962,7 @@ class TestCalculateActivity(BaseTest):))- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:+ with _patched_runner() as mock_runner:mock_runner.return_value.run.return_value.model_dump.return_value = {}result = _calculate(exp.id, "m1", str(recalc.id), _QUERY_TO, is_final_attempt=False, attempt=3)@@ -977,7 +989,7 @@ class TestCalculateActivity(BaseTest):recalc = self._recalc(exp, metric_uuids=["m1"])with (- patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner,+ _patched_runner() as mock_runner,patch("products.experiments.backend.temporal.recalculation_logic.capture_exception") as mock_capture,patch("products.experiments.backend.temporal.recalculation_logic.random.uniform", return_value=47.5),):@@ -1013,7 +1025,7 @@ class TestCalculateActivity(BaseTest):exp = self._experiment(flag_key=f"calc-class-cap-{name}", metrics=[_mean_metric("m1")])recalc = self._recalc(exp, metric_uuids=["m1"])- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:+ with _patched_runner() as mock_runner:mock_runner.return_value.run.side_effect = excwith pytest.raises((type(exc), ApplicationError)):_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO, is_final_attempt=False, attempt=cap - 1)@@ -1049,7 +1061,7 @@ class TestCalculateActivity(BaseTest):exp = self._experiment(flag_key=f"calc-permanent-{name}", metrics=[_mean_metric("m1")])recalc = self._recalc(exp, metric_uuids=["m1"])- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:+ with _patched_runner() as mock_runner:mock_runner.return_value.run.side_effect = excwith pytest.raises(ApplicationError) as exc_info:@@ -1074,7 +1086,7 @@ class TestCalculateActivity(BaseTest):exp = self._experiment(flag_key="calc-as-of", metrics=[_mean_metric("m1")])recalc = self._recalc(exp, metric_uuids=["m1"])- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:+ with _patched_runner() as mock_runner:mock_runner.return_value.run.return_value.model_dump.return_value = {}_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO)@@ -1093,7 +1105,7 @@ class TestCalculateActivity(BaseTest):exp = self._experiment(flag_key="calc-query-from", metrics=[_mean_metric("m1")])recalc = self._recalc(exp, metric_uuids=["m1"])- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:+ with _patched_runner() as mock_runner:mock_runner.return_value.run.return_value.model_dump.return_value = {}_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO)@@ -1110,7 +1122,7 @@ class TestCalculateActivity(BaseTest):exp = self._experiment(flag_key="calc-query-id", metrics=[_mean_metric("m1")])recalc = self._recalc(exp, metric_uuids=["m1"])- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:+ with _patched_runner() as mock_runner:mock_runner.return_value.run.return_value.model_dump.return_value = {}_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO)@@ -1138,7 +1150,7 @@ class TestCalculateActivity(BaseTest):with (patch("products.experiments.backend.recalculation.feature_enabled_or_false", **flag),patch("posthog.clickhouse.client.execute.get_client_from_pool") as client_from_pool,- patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner,+ _patched_runner() as mock_runner,):client_from_pool.return_value.__enter__.return_value.execute.return_value = []mock_runner.return_value.run.side_effect = run_metric_and_precompute_queries@@ -1184,7 +1196,7 @@ class TestCalculateActivity(BaseTest):recalc.metric_retries = {"m1": {"attempt": 3, "max_attempts": 8}}recalc.save(update_fields=["metric_retries"])- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:+ with _patched_runner() as mock_runner:result = _calculate(exp.id, "m1", str(recalc.id), _QUERY_TO)mock_runner.assert_not_called()@@ -1215,7 +1227,7 @@ class TestCalculateActivity(BaseTest):)recalc = self._recalc(exp, metric_uuids=["m1"])- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:+ with _patched_runner() as mock_runner:mock_runner.return_value.run.return_value.model_dump.return_value = {}_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO)@@ -1241,7 +1253,7 @@ class TestCalculateActivity(BaseTest):exp.save(update_fields=["excluded_variants"])recalc = self._recalc(exp, metric_uuids=["m1"])- with patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner:+ with _patched_runner() as mock_runner:mock_runner.return_value.run.return_value.model_dump.return_value = {}_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO)@@ -1269,9 +1281,7 @@ class TestCalculateActivity(BaseTest):def _run() -> None:if mock_runner_failure:- with patch(- "products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner"- ) as mock_runner:+ with _patched_runner() as mock_runner:mock_runner.return_value.run.side_effect = RuntimeError("kaboom")# Calc-step failures now re-raise so Temporal can retry transient errors. The record is# written before the re-raise, so the assertions below still hold across attempts.@@ -1351,7 +1361,7 @@ class TestCalculateActivity(BaseTest):recalc = self._recalc(exp, metric_uuids=["m1"])with (- patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner,+ _patched_runner() as mock_runner,patch("products.experiments.backend.temporal.recalculation_logic.capture_exception") as mock_capture,):mock_runner.return_value.run.side_effect = RuntimeError("x" * 5000)@@ -1714,9 +1724,7 @@ class TestRecalculationAnalytics(BaseTest):recalc = self._recalc(exp, metric_uuids=["m1"])with _record_captures() as captured:- with patch(- "products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner"- ) as mock_runner:+ with _patched_runner() as mock_runner:mock_runner.return_value.run.return_value.model_dump.return_value = {}_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO)@@ -1737,9 +1745,7 @@ class TestRecalculationAnalytics(BaseTest):recalc = self._recalc(exp, metric_uuids=["m1"])with _record_captures() as captured:- with patch(- "products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner"- ) as mock_runner:+ with _patched_runner() as mock_runner:mock_runner.return_value.run.side_effect = StatisticError("not enough data")_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO)@@ -1757,9 +1763,7 @@ class TestRecalculationAnalytics(BaseTest):recalc = self._recalc(exp, metric_uuids=["s1"])with _record_captures() as captured:- with patch(- "products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner"- ) as mock_runner:+ with _patched_runner() as mock_runner:mock_runner.return_value.run.return_value.model_dump.return_value = {}_calculate(exp.id, "s1", str(recalc.id), _QUERY_TO, role="secondary")@@ -1807,7 +1811,7 @@ class TestRecalculationAnalytics(BaseTest):with _record_captures() as captured:with (- patch("products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner") as mock_runner,+ _patched_runner() as mock_runner,patch("products.experiments.backend.temporal.recalculation_logic.random.uniform", return_value=47.5),):mock_runner.return_value.run.side_effect = exc@@ -1850,9 +1854,7 @@ class TestRecalculationAnalytics(BaseTest):recalc = self._recalc(exp, metric_uuids=["m1"])with _record_captures() as captured:- with patch(- "products.experiments.backend.temporal.recalculation_logic.ExperimentQueryRunner"- ) as mock_runner:+ with _patched_runner() as mock_runner:mock_runner.return_value.run.side_effect = excwith pytest.raises(ApplicationError if raises_application_error else type(exc)):_calculate(exp.id, "m1", str(recalc.id), _QUERY_TO, is_final_attempt=True)diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.tsindex bdd51818d7ea..0eedcc283447 100644--- a/products/experiments/frontend/generated/api.schemas.ts+++ b/products/experiments/frontend/generated/api.schemas.ts@@ -2422,7 +2422,7 @@ export interface ExperimentHealthFindingApi {*/subcode: string | null/**- * A stable qualifier below the subcode: where a sample ratio mismatch is located ('library', 'page'), what leaves nobody exposed ('test_account_filter'), or why a check could not read what it needs ('no_history_access'). Null when the finding names none.+ * A stable qualifier below the subcode: where a sample ratio mismatch is located ('library', 'page'), what leaves nobody exposed ('test_account_filter'), what made a metric reach a query limit ('metric', 'exposure_criteria'), or why a check could not read what it needs ('no_history_access'). Null when the finding names none.* @nullable*/reason: string | null@@ -3871,7 +3871,7 @@ export interface ExperimentMetricsRecalculationJobApi {readonly completed_metrics: number/** Number of failed metrics in this run (derived): FAILED result rows plus discovery-step failures that never made it to a result row */readonly failed_metrics: number- /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first */+ /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp, exposures_path}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first. exposures_path is where the query read the exposures, and is missing when the query failed before it was built */readonly metric_errors: unknown/** When the job was created */readonly created_at: string@@ -3950,7 +3950,7 @@ export interface ExperimentMetricsRecalculationRunApi {readonly completed_metrics: number/** Number of failed metrics in this run (derived): FAILED result rows plus discovery-step failures that never made it to a result row */readonly failed_metrics: number- /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first */+ /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp, exposures_path}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first. exposures_path is where the query read the exposures, and is missing when the query failed before it was built */readonly metric_errors: unknown/** When the job was created */readonly created_at: string@@ -4032,7 +4032,7 @@ export interface ExperimentMetricsRecalculationLatestApi {readonly completed_metrics: number/** Number of failed metrics in this run (derived): FAILED result rows plus discovery-step failures that never made it to a result row */readonly failed_metrics: number- /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first */+ /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp, exposures_path}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first. exposures_path is where the query read the exposures, and is missing when the query failed before it was built */readonly metric_errors: unknown/** When the job was created */readonly created_at: stringdiff --git a/products/experiments/frontend/generated/api.ts b/products/experiments/frontend/generated/api.tsindex 994be9c77257..b2eac705f422 100644--- a/products/experiments/frontend/generated/api.ts+++ b/products/experiments/frontend/generated/api.ts@@ -769,7 +769,13 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =* wrote for users), and `calculated_at` is when the calculation finished. It is a* warning when a primary metric fails for a kind that a change fixes (`query_limit`,* `settings`, `experiment_too_old`), also after the end, because the calculation after the fix- * reads the events up to the end. It is info otherwise.+ * reads the events up to the end. It is info otherwise. When the first failure reached the+ * read, memory or time limit, `reason` names the cause that the calculation shows: `metric`+ * when another metric of the same calculation completed the same read of the exposures, or+ * `exposure_criteria` when every metric reached a limit and an exposure or test account filter+ * makes each query read all the properties of an event, person or group. `evidence.properties`+ * then names the filters. A query that read precomputed exposures doesn't apply those filters,+ * so it can't get `exposure_criteria`. It is null otherwise.** The severity of the `some_conditions_pinned` finding follows the largest share that the* users of the pinned release conditions make up among the users who got one variant fromdiff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 2901ef310e6a..1e2bcc25c069 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -47860,7 +47860,7 @@ export namespace Schemas {*/subcode: string | null;/**- * A stable qualifier below the subcode: where a sample ratio mismatch is located ('library', 'page'), what leaves nobody exposed ('test_account_filter'), or why a check could not read what it needs ('no_history_access'). Null when the finding names none.+ * A stable qualifier below the subcode: where a sample ratio mismatch is located ('library', 'page'), what leaves nobody exposed ('test_account_filter'), what made a metric reach a query limit ('metric', 'exposure_criteria'), or why a check could not read what it needs ('no_history_access'). Null when the finding names none.* @nullable*/reason: string | null;@@ -49202,7 +49202,7 @@ export namespace Schemas {readonly completed_metrics: number;/** Number of failed metrics in this run (derived): FAILED result rows plus discovery-step failures that never made it to a result row */readonly failed_metrics: number;- /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first */+ /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp, exposures_path}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first. exposures_path is where the query read the exposures, and is missing when the query failed before it was built */readonly metric_errors: unknown;/** When the job was created */readonly created_at: string;@@ -49293,7 +49293,7 @@ export namespace Schemas {readonly completed_metrics: number;/** Number of failed metrics in this run (derived): FAILED result rows plus discovery-step failures that never made it to a result row */readonly failed_metrics: number;- /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first */+ /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp, exposures_path}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first. exposures_path is where the query read the exposures, and is missing when the query failed before it was built */readonly metric_errors: unknown;/** When the job was created */readonly created_at: string;@@ -49378,7 +49378,7 @@ export namespace Schemas {readonly completed_metrics: number;/** Number of failed metrics in this run (derived): FAILED result rows plus discovery-step failures that never made it to a result row */readonly failed_metrics: number;- /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first */+ /** Terminal failure per metric_uuid: {step, message, error_type, retriable, timestamp, exposures_path}. retriable is true when a transient error exhausted its attempts, so a heal_latest_run or manual_retry can succeed; false when the metric config, the data, or a resource limit must change first. exposures_path is where the query read the exposures, and is missing when the query failed before it was built */readonly metric_errors: unknown;/** When the job was created */readonly created_at: string;