Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(experiments): flag metrics whose query doesn't compile (#123282)

Marcel Poelkere56f965e6f16
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index c440cedfd9b9..ec18ef9120f4 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -11173,9 +11173,9 @@ snapshots:
scenes-app-experiments--experiment-frequentist-two-variants--light:
hash: v1.k794b7964.4ddea1c7537af9d50c57b4b9a76d4bcba39f743a23b4715683c0ce59d4c24f57.0ueR8XMLqRby_u6I0EyNIiUwXkbjmWMH3vBdoFNYmII
scenes-app-experiments--experiment-health-debug-tab--dark:
- hash: v1.k794b7964.c42640b2d757ecf6208b3658848ee0ac7f212b167ce614aaa41f36e0f935736b.opgqM_7IOcqH8n65gLUXLw9kkYbvtRJbVjmi7qukMG8
+ hash: v1.k794b7964.5edf9e51dc2fcaf83180f5ebad252d4f2e4db0759ac475691c7a06e7bcd47b77.aOSdT75x7w_nAKP96aVhXKbpOYrzivbbdmfc_PUUpgo
scenes-app-experiments--experiment-health-debug-tab--light:
- hash: v1.k794b7964.1249fadfb8bf915c349e9adb2500e6ee98550a973f6d044fab1b0ad1610e7091.GTAsiqSMl9rSqNoD3BBeuNvdhNMlzCSfXnT3FbHwd28
+ hash: v1.k794b7964.031fb28af0f7086c3c0e869b3e4368d01ba2edc8d2c3184ec7061fb2b0223ae6.OY6eoJZtfSlG9w6CbbP9HlvA16I7_J8VY31UkdS6JcY
scenes-app-experiments--experiment-recordings-dropped-metric--dark:
hash: v1.k794b7964.e0e7181f2c9e61a8324eff9aa7a9230fef25b3cf37e0b24ef875780186ed25dc.9VEvZ4iVieVJwmDol0gwtAsr3wgZYW5gj2KXAGZJfiw
scenes-app-experiments--experiment-recordings-dropped-metric--light:
diff --git a/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx b/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx
index 7473e7ba87f5..15ca3bed0a40 100644
--- a/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx
+++ b/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx
@@ -77,6 +77,13 @@ const HEALTH_INPUTS: ExperimentHealthInputsApi = {
activation: null,
},
events_received: {},
+ metric_compiles: {
+ metric_count: 1,
+ unknown_count: 0,
+ no_access_count: 0,
+ exposure_query_fails: false,
+ failures: [],
+ },
flag_history: {
checked: true,
not_checked_reason: null,
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index db2aecae7159..db602de4703f 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -9854,6 +9854,7 @@ export namespace Schemas {
* * `multiple_variants_share` - Multiple Variants Share
* * `no_metric` - No Metric
* * `metric_event_not_ingested` - Metric Event Not Ingested
+ * * `metric_cannot_compile` - Metric Cannot Compile
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `srm` - Sample Ratio Mismatch
* * `zero_exposures` - Zero Exposures
@@ -9874,6 +9875,7 @@ export namespace Schemas {
| "multiple_variants_share"
| "no_metric"
| "metric_event_not_ingested"
+ | "metric_cannot_compile"
| "exposure_criteria_cannot_match"
| "srm"
| "zero_exposures"
@@ -9924,6 +9926,7 @@ export namespace Schemas {
* * `multiple_variants_share` - Multiple Variants Share
* * `no_metric` - No Metric
* * `metric_event_not_ingested` - Metric Event Not Ingested
+ * * `metric_cannot_compile` - Metric Cannot Compile
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `srm` - Sample Ratio Mismatch
* * `zero_exposures` - Zero Exposures
@@ -9980,6 +9983,7 @@ export namespace Schemas {
* * `new_flag_no_traffic` - New Flag No Traffic
* * `no_metric` - No Metric
* * `metric_event_not_ingested` - Metric Event Not Ingested
+ * * `metric_cannot_compile` - Metric Cannot Compile
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `early_significance` - Early Significance
* * `zero_exposures` - Zero Exposures
@@ -9998,6 +10002,7 @@ export namespace Schemas {
| "new_flag_no_traffic"
| "no_metric"
| "metric_event_not_ingested"
+ | "metric_cannot_compile"
| "exposure_criteria_cannot_match"
| "early_significance"
| "zero_exposures"
@@ -10029,6 +10034,7 @@ export namespace Schemas {
* * `new_flag_no_traffic` - New Flag No Traffic
* * `no_metric` - No Metric
* * `metric_event_not_ingested` - Metric Event Not Ingested
+ * * `metric_cannot_compile` - Metric Cannot Compile
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `early_significance` - Early Significance
* * `zero_exposures` - Zero Exposures
diff --git a/products/experiments/backend/facade/api.py b/products/experiments/backend/facade/api.py
index 9ea98e6414ac..26a10ef21e2d 100644
--- a/products/experiments/backend/facade/api.py
+++ b/products/experiments/backend/facade/api.py
@@ -143,11 +143,14 @@ def count_running_experiments_on_feature_flag_called(organization_id: UUID) -> i
def get_experiment_health(
- *, team_id: int, experiment_id: int, reader_access: ExperimentHealthReaderAccess
+ *, team_id: int, experiment_id: int, reader_access: ExperimentHealthReaderAccess, user: User | None
) -> ExperimentHealth:
"""The findings and the result of each health check that reads the experiment, its flag, its linked metrics,
- and the changes of the flag since launch and the flag's recent calls, as far as `reader_access` allows."""
- context = load_experiment_health_context(team_id=team_id, experiment_id=experiment_id, reader_access=reader_access)
+ and the changes of the flag since launch and the flag's recent calls, as far as `reader_access` allows. The
+ metric queries compile with `user`'s warehouse access."""
+ context = load_experiment_health_context(
+ team_id=team_id, experiment_id=experiment_id, reader_access=reader_access, user=user
+ )
return evaluate_health(context, EXPERIMENT_HEALTH_CHECKS)
diff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.py
index ea3a06b2b63c..65af16684b56 100644
--- a/products/experiments/backend/facade/contracts.py
+++ b/products/experiments/backend/facade/contracts.py
@@ -106,6 +106,7 @@ class ExperimentHealthFindingCode(LabeledStrEnum):
MULTIPLE_VARIANTS_SHARE = "multiple_variants_share"
NO_METRIC = "no_metric"
METRIC_EVENT_NOT_INGESTED = "metric_event_not_ingested"
+ METRIC_CANNOT_COMPILE = "metric_cannot_compile"
EXPOSURE_CRITERIA_CANNOT_MATCH = "exposure_criteria_cannot_match"
SRM = "srm", "Sample Ratio Mismatch"
ZERO_EXPOSURES = "zero_exposures"
@@ -118,6 +119,17 @@ class ExperimentHealthFindingCode(LabeledStrEnum):
EARLY_SIGNIFICANCE = "early_significance"
+class ExperimentHealthMetricCompileErrorKind(LabeledStrEnum):
+ # A validation error of the query builder, written for the user.
+ REJECTED = "rejected"
+ # HogQL can't parse an expression of the metric.
+ SYNTAX = "syntax"
+ # HogQL can't compile the query.
+ QUERY = "query"
+ # The stored definition doesn't parse into a metric.
+ DEFINITION = "definition"
+
+
class HealthExposureInputsStatus(LabeledStrEnum):
READY = "ready"
# The cached answer is past its age. A recalculation runs in the background, or the last one failed.
@@ -197,6 +209,7 @@ class ExperimentHealthCheckKey(LabeledStrEnum):
NEW_FLAG_NO_TRAFFIC = "new_flag_no_traffic"
NO_METRIC = "no_metric"
METRIC_EVENT_NOT_INGESTED = "metric_event_not_ingested"
+ METRIC_CANNOT_COMPILE = "metric_cannot_compile"
EXPOSURE_CRITERIA_CANNOT_MATCH = "exposure_criteria_cannot_match"
EARLY_SIGNIFICANCE = "early_significance"
ZERO_EXPOSURES = "zero_exposures"
diff --git a/products/experiments/backend/health/checks/metric_cannot_compile.py b/products/experiments/backend/health/checks/metric_cannot_compile.py
new file mode 100644
index 000000000000..a24f6cdb97f6
--- /dev/null
+++ b/products/experiments/backend/health/checks/metric_cannot_compile.py
@@ -0,0 +1,281 @@
+"""
+Metrics whose query doesn't compile, so that every calculation of the metric fails. Pure functions, no I/O.
+
+Nothing checks this at save: a metric fails only when its query runs, and before launch no query runs. The compiler's
+messages are written for HogQL, so the finding puts the known ones in words. A message that names a field, a function,
+a cohort, a regular expression or a table gets those words only when the metric's definition holds that name, because
+a query can also fail on a name that PostHog generates.
+"""
+
+import re
+from collections.abc import Callable, Mapping
+from enum import StrEnum
+from typing import Any
+
+from posthog.dataclasses import frozen
+
+from products.experiments.backend.facade.contracts import (
+ ExperimentHealthFinding,
+ ExperimentHealthFindingCode,
+ ExperimentHealthFindingSeverity,
+ ExperimentHealthMetricCompileErrorKind,
+)
+from products.experiments.backend.health.context import HealthContext, MetricCompileFailure
+from products.experiments.backend.health.skips import NOT_ON_ARCHIVED, CheckOutcome, not_applicable, not_checked
+
+
+class MetricCompileError(StrEnum):
+ # pinned: the finding's subcode and `evidence.metric_N_error`.
+ COMPOUND_AGGREGATION = "compound_aggregation"
+ AGGREGATION_ARGUMENTS = "aggregation_arguments"
+ UNKNOWN_FUNCTION = "unknown_function"
+ UNKNOWN_FIELD = "unknown_field"
+ SYNTAX_ERROR = "syntax_error"
+ DELETED_COHORT = "deleted_cohort"
+ INVALID_REGEX = "invalid_regex"
+ INCOMPLETE_BEHAVIORAL_FILTER = "incomplete_behavioral_filter"
+ UNSUPPORTED_ELEMENT_OPERATOR = "unsupported_element_operator"
+ MISSING_WAREHOUSE_TABLE = "missing_warehouse_table"
+ INVALID_DEFINITION = "invalid_definition"
+ # A message that the query builder writes for the user, shown as it is.
+ REJECTED = "rejected"
+ OTHER = "other"
+
+
+@frozen
+class _Wording:
+ error: MetricCompileError
+ # What is wrong and what fixes it, after the sentence that names the metric.
+ explanation: str
+
+
+_COMPOUND_AGGREGATION = "must be a single aggregation"
+_MAX_EVIDENCE_MESSAGE_LENGTH = 300
+_ELEMENT_LABELS = {"selector": "CSS selector", "tag_name": "tag name", "href": "link target", "text": "text"}
+
+
+def metric_cannot_compile(ctx: HealthContext) -> CheckOutcome:
+ if ctx.archived:
+ return not_applicable(NOT_ON_ARCHIVED)
+ compiles = ctx.metric_compiles
+ if compiles is None:
+ return not_checked("Couldn't check because the metric queries couldn't be built")
+ if compiles.metric_count == 0:
+ # The calculation leaves out a metric in the legacy format, which the no-metric check counts.
+ if ctx.primary_metric_count + ctx.secondary_metric_count:
+ return not_applicable("Runs once the experiment has a metric in the current format")
+ return not_applicable("Runs once the experiment has a metric")
+ if compiles.exposure_query_fails:
+ return not_checked("Couldn't check because the exposure criteria can't be built into a query")
+ if not compiles.failures:
+ if compiles.no_access_count:
+ reads = "reads" if compiles.no_access_count == 1 else "read"
+ return not_checked(
+ f"Couldn't check {_metrics(compiles.no_access_count)} that {reads} data you don't have access to"
+ )
+ if compiles.unknown_count:
+ return not_checked(f"Couldn't check {_metrics(compiles.unknown_count)}")
+ return None
+ failures = sorted(compiles.failures, key=lambda failure: failure.role != "primary")
+ worded = [(failure, _word(failure)) for failure in failures]
+ return ExperimentHealthFinding(
+ code=ExperimentHealthFindingCode.METRIC_CANNOT_COMPILE,
+ subcode=worded[0][1].error.value,
+ # The metric fails every calculation, so the warning holds before launch too. After the end the experiment
+ # collects nothing more, so the finding is info.
+ severity=ExperimentHealthFindingSeverity.WARNING
+ if not ctx.has_ended and any(failure.role == "primary" for failure in failures)
+ else ExperimentHealthFindingSeverity.INFO,
+ title="PostHog can't calculate a metric"
+ if len(failures) == 1
+ else f"PostHog can't calculate {_metrics(len(failures))}",
+ detail=" ".join(
+ f'PostHog can\'t calculate the {failure.role} metric "{failure.name}". {wording.explanation}'
+ for failure, wording in worded
+ ),
+ evidence=_evidence(worded),
+ # No action opens a given metric.
+ actions=(),
+ diagnostic_ref=None,
+ )
+
+
+def _metrics(count: int) -> str:
+ return "1 metric" if count == 1 else f"{count} metrics"
+
+
+def _evidence(worded: list[tuple[MetricCompileFailure, _Wording]]) -> dict[str, str | None]:
+ evidence: dict[str, str | None] = {}
+ for number, (failure, wording) in enumerate(worded, start=1):
+ message = failure.message
+ evidence[f"metric_{number}"] = failure.name
+ evidence[f"metric_{number}_role"] = failure.role
+ evidence[f"metric_{number}_error"] = wording.error.value
+ evidence[f"metric_{number}_message"] = message[:_MAX_EVIDENCE_MESSAGE_LENGTH] if message else None
+ return evidence
+
+
+def _word(failure: MetricCompileFailure) -> _Wording:
+ message = failure.message
+ match failure.kind:
+ case ExperimentHealthMetricCompileErrorKind.DEFINITION if message is not None:
+ return _Wording(
+ error=MetricCompileError.INVALID_DEFINITION,
+ explanation=f"Its saved settings aren't valid: {_sentence(message)} Create the metric again, or edit it "
+ "to fix the setting.",
+ )
+ case ExperimentHealthMetricCompileErrorKind.REJECTED if (
+ message is not None and _COMPOUND_AGGREGATION in message
+ ):
+ return _Wording(
+ error=MetricCompileError.COMPOUND_AGGREGATION,
+ explanation="Its SQL expression combines several aggregations, such as sum(a) / count(). The expression "
+ "can hold one aggregation, such as sum(properties.revenue). To divide one sum by another, use a ratio "
+ "metric.",
+ )
+ case ExperimentHealthMetricCompileErrorKind.REJECTED if message is not None:
+ return _Wording(error=MetricCompileError.REJECTED, explanation=_sentence(message))
+ case ExperimentHealthMetricCompileErrorKind.SYNTAX if message is not None:
+ return _Wording(
+ error=MetricCompileError.SYNTAX_ERROR,
+ explanation=f"An SQL expression of the metric has a syntax error: {_sentence(message)} Correct the "
+ "expression.",
+ )
+ case ExperimentHealthMetricCompileErrorKind.QUERY if message is not None:
+ for pattern, subject, word in _QUERY_RULES:
+ found = pattern.search(message)
+ if found and (subject is None or _mentions(failure.definition, found.group(subject))):
+ return word(found, message)
+ return _Wording(
+ error=MetricCompileError.OTHER,
+ explanation=(f"Building its query fails: {_sentence(message)}" if message else "Building its query fails.")
+ + " Check the metric's settings, or contact support if they look right.",
+ )
+
+
+def _sentence(text: str) -> str:
+ text = text.strip()
+ return text if text.endswith((".", "?", "!")) else f"{text}."
+
+
+def _mentions(definition: Mapping[str, Any], subject: str) -> bool:
+ """Whether a value of the definition holds `subject`, such as a field name inside an SQL expression."""
+
+ def values(node: object) -> list[str]:
+ if isinstance(node, Mapping):
+ return [text for value in node.values() for text in values(value)]
+ if isinstance(node, list):
+ return [text for value in node for text in values(value)]
+ return [] if node is None else [str(node)]
+
+ return any(subject in value for value in values(definition))
+
+
+def _aggregation_arguments(match: re.Match[str], _message: str) -> _Wording:
+ name = match.group("name")
+ # An aggregation with a condition, such as uniqIf, comes from the plain one with the suffix If.
+ plain = name[:-2] if len(name) > 2 and name.endswith("If") else None
+ fix = (
+ f" Use {plain}, and set the condition as a filter of the metric."
+ if plain
+ else " Use an aggregation that takes one argument, such as sum or max."
+ )
+ return _Wording(
+ error=MetricCompileError.AGGREGATION_ARGUMENTS,
+ explanation=f"Its SQL expression uses {name}, and PostHog passes only the first argument of the aggregation "
+ f"to it, so {name} gets too few arguments.{fix}",
+ )
+
+
+def _unknown_function(match: re.Match[str], _message: str) -> _Wording:
+ suggestion = match.group("suggestion")
+ return _Wording(
+ error=MetricCompileError.UNKNOWN_FUNCTION,
+ explanation=f"Its SQL expression calls {match.group('name')}, which PostHog doesn't support."
+ + (f" Did you mean {suggestion}?" if suggestion else " Check the name of the function."),
+ )
+
+
+_SUGGESTIONS = re.compile(r"Did you mean: (?P<first>[^,?]+)")
+_PROPERTY_HINT = re.compile(r"is a custom property, read it as (?P<path>.+?)\.(?:\s|$)")
+
+
+def _unknown_field(match: re.Match[str], message: str) -> _Wording:
+ name = match.group("name")
+ suggestion = _SUGGESTIONS.search(message)
+ hint = _PROPERTY_HINT.search(message)
+ if suggestion:
+ fix = f" Did you mean {suggestion.group('first')}?"
+ elif hint:
+ fix = f" If {name} is an event property, write {hint.group('path')}."
+ else:
+ fix = " Check the name of the field."
+ return _Wording(
+ error=MetricCompileError.UNKNOWN_FIELD,
+ explanation=f'It reads "{name}", which isn\'t a field that PostHog knows.{fix}',
+ )
+
+
+def _deleted_cohort(match: re.Match[str], _message: str) -> _Wording:
+ return _Wording(
+ error=MetricCompileError.DELETED_COHORT,
+ explanation=f"A filter of the metric uses a cohort that doesn't exist anymore (ID {match.group('id')}). "
+ "Remove the filter, or choose another cohort.",
+ )
+
+
+def _invalid_regex(match: re.Match[str], _message: str) -> _Wording:
+ return _Wording(
+ error=MetricCompileError.INVALID_REGEX,
+ explanation=f'A filter of the metric uses the regular expression "{match.group("regex")}", which PostHog '
+ "can't run. PostHog supports the RE2 syntax, which has no lookaheads, such as (?!...), and no backreferences. "
+ "Rewrite the expression.",
+ )
+
+
+def _incomplete_behavioral_filter(_match: re.Match[str], _message: str) -> _Wording:
+ return _Wording(
+ error=MetricCompileError.INCOMPLETE_BEHAVIORAL_FILTER,
+ explanation="A behavioral filter of the metric is incomplete. Set all of its fields, such as the time "
+ "range, or remove the filter.",
+ )
+
+
+def _unsupported_element_operator(match: re.Match[str], _message: str) -> _Wording:
+ key = match.group("key")
+ return _Wording(
+ error=MetricCompileError.UNSUPPORTED_ELEMENT_OPERATOR,
+ explanation=f"A filter of the metric on the element's {_ELEMENT_LABELS.get(key, key)} uses an operator "
+ 'that this filter doesn\'t support. Use "equals" or "doesn\'t equal".',
+ )
+
+
+def _missing_warehouse_table(match: re.Match[str], message: str) -> _Wording:
+ suggestion = _SUGGESTIONS.search(message)
+ return _Wording(
+ error=MetricCompileError.MISSING_WAREHOUSE_TABLE,
+ explanation=f'It reads the data warehouse table "{match.group("table")}", which this project doesn\'t '
+ "have." + (f" Did you mean {suggestion.group('first')}?" if suggestion else " Choose another table."),
+ )
+
+
+# Each rule is a pattern of the compiler's message, the group that the definition must hold (None when the message
+# names nothing), and the words. The first rule that matches applies.
+_QUERY_RULES: tuple[tuple[re.Pattern[str], str | None, Callable[[re.Match[str], str], _Wording]], ...] = (
+ (re.compile(r"Aggregation '(?P<name>\w+)' expects"), "name", _aggregation_arguments),
+ (
+ re.compile(
+ r"Unsupported function call '(?P<name>[^'(]+)\(\.\.\.\)'"
+ r"(?:\. Perhaps you meant '(?P<suggestion>[^'(]+)\(\.\.\.\)'\?)?"
+ ),
+ "name",
+ _unknown_function,
+ ),
+ (re.compile(r"(?:Unable to resolve field|Field not found): `?(?P<name>[^\s`.]+)"), "name", _unknown_field),
+ (re.compile(r"Cohort (?P<id>\d+) does not exist"), "id", _deleted_cohort),
+ (re.compile(r"Could not find cohort with ID (?P<id>\d+)"), "id", _deleted_cohort),
+ (re.compile(r"Invalid regular expression: '(?P<regex>.*)'"), "regex", _invalid_regex),
+ (re.compile(r"Invalid behavioral property filter"), None, _incomplete_behavioral_filter),
+ (re.compile(r"property_to_expr for element (?P<key>\w+) only supports"), None, _unsupported_element_operator),
+ (re.compile(r"Unknown table `(?P<table>[^`]+)`"), "table", _missing_warehouse_table),
+)
diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.py
index a9d058734785..de3db4bfc7ad 100644
--- a/products/experiments/backend/health/context.py
+++ b/products/experiments/backend/health/context.py
@@ -20,7 +20,10 @@ from posthog.dataclasses import frozen
from posthog.models.activity_logging.activity_log import ActivityLog
from posthog.models.team.team import Team
-from products.experiments.backend.facade.contracts import ExperimentHealthFlagChangeKind
+from products.experiments.backend.facade.contracts import (
+ ExperimentHealthFlagChangeKind,
+ ExperimentHealthMetricCompileErrorKind,
+)
from products.experiments.backend.hogql_queries.exposure_query_logic import (
DEFAULT_EXPOSURE_EVENT,
EXPERIMENT_EXPOSURE_EVENT,
@@ -239,6 +242,34 @@ class MetricEvents:
events: tuple[str, ...]
+@frozen
+class MetricCompileFailure:
+ """A metric whose query doesn't compile, so every calculation of the metric fails."""
+
+ name: str
+ role: MetricRole
+ kind: ExperimentHealthMetricCompileErrorKind
+ # The compiler's message. None when it can hold the evaluation rules that the markers keep out of the browser.
+ message: str | None
+ # The effective definition, in which the wording looks for the names that the message gives.
+ definition: Mapping[str, Any]
+
+
+@frozen
+class MetricCompiles:
+ """The compile of the query of each metric that the experiment calculates, without running it."""
+
+ metric_count: int
+ failures: tuple[MetricCompileFailure, ...]
+ # The metrics whose compile raised an error that names no problem of the metric, such as a defect of PostHog.
+ unknown_count: int
+ # The metrics that read a warehouse table or view that the reader may not query.
+ no_access_count: int
+ # Whether the exposure query alone doesn't compile. A failure then says nothing about the metric, so `failures`
+ # is empty.
+ exposure_query_fails: bool
+
+
@frozen
class FlagKeyFilter:
"""An event property filter of the exposure criteria on `$feature_flag`, the key of the called flag."""
@@ -311,6 +342,8 @@ class HealthContext:
primary_metric: PrimaryMetric | None
# The variants that the analysis leaves out of the results.
excluded_variant_keys: frozenset[str]
+ # None when not loaded or unknown. Only the health endpoint loads it, as the reader.
+ metric_compiles: MetricCompiles | None
def parse_flag_state(
@@ -595,4 +628,5 @@ def load_health_context(
pause_history=None,
primary_metric=None,
excluded_variant_keys=frozenset(experiment.excluded_variants or []),
+ metric_compiles=None,
)
diff --git a/products/experiments/backend/health/inputs_payload.py b/products/experiments/backend/health/inputs_payload.py
index 7b31a6bd6b45..f362375ccbd8 100644
--- a/products/experiments/backend/health/inputs_payload.py
+++ b/products/experiments/backend/health/inputs_payload.py
@@ -17,6 +17,7 @@ from products.experiments.backend.health.context import (
FlagHistory,
FlagHistoryNotChecked,
FlagState,
+ MetricCompiles,
)
from products.experiments.backend.health.exposure_inputs import ExposureInputs
from products.experiments.backend.health.loader import HealthInputs
@@ -205,6 +206,19 @@ def _health_inputs_exposures(exposures: ExposureInputs) -> dict[str, Any]:
}
+def _health_inputs_metric_compiles(compiles: MetricCompiles) -> dict[str, Any]:
+ return {
+ "metric_count": compiles.metric_count,
+ "unknown_count": compiles.unknown_count,
+ "no_access_count": compiles.no_access_count,
+ "exposure_query_fails": compiles.exposure_query_fails,
+ "failures": [
+ {"name": failure.name, "role": failure.role, "kind": failure.kind.value, "message": failure.message}
+ for failure in compiles.failures
+ ],
+ }
+
+
def health_inputs_payload(inputs: HealthInputs) -> dict[str, Any]:
"""The inputs as plain data in the shape of `ExperimentHealthInputsSerializer`, so the presentation layer renders
them without reading the health package's types."""
@@ -232,6 +246,7 @@ def health_inputs_payload(inputs: HealthInputs) -> dict[str, Any]:
if ctx.exposure_events
else None,
"events_received": dict(ctx.events_received),
+ "metric_compiles": _health_inputs_metric_compiles(ctx.metric_compiles) if ctx.metric_compiles else None,
"flag_history": _health_inputs_flag_history(ctx.flag_history) if ctx.flag_history else None,
"condition_set_traffic_status": _health_inputs_traffic_status(inputs),
"condition_set_traffic": _health_inputs_traffic(ctx.condition_set_traffic)
diff --git a/products/experiments/backend/health/loader.py b/products/experiments/backend/health/loader.py
index c82f70bc7f83..a6fe1dd4c5c6 100644
--- a/products/experiments/backend/health/loader.py
+++ b/products/experiments/backend/health/loader.py
@@ -9,6 +9,7 @@ from products.experiments.backend.health.condition_set_traffic import load_condi
from products.experiments.backend.health.context import HealthContext, load_health_context
from products.experiments.backend.health.exposure_gate import load_flag_history
from products.experiments.backend.health.exposure_inputs import ExposureInputs, load_exposure_inputs
+from products.experiments.backend.health.metric_compiles import load_metric_compiles
from products.experiments.backend.health.pause_history import load_pause_history
from products.experiments.backend.health.primary_metric import load_primary_metric
from products.experiments.backend.models.experiment import Experiment
@@ -31,11 +32,12 @@ class HealthInputs:
def load_experiment_health_context(
- *, team_id: int, experiment_id: int, reader_access: ExperimentHealthReaderAccess
+ *, team_id: int, experiment_id: int, reader_access: ExperimentHealthReaderAccess, user: User | None
) -> HealthContext:
- """The context that the checks of the health endpoint read: the experiment, its flag, its linked metrics, the
- changes of the flag since launch and the flag's recent calls, as far as `reader_access` allows."""
- return _health_context(_load_experiment(team_id, experiment_id), reader_access)
+ """The context that the checks of the health endpoint read: the experiment, its flag, its linked metrics and the
+ compile of their queries, the changes of the flag since launch and the flag's recent calls, as far as
+ `reader_access` and `user`'s warehouse access allow."""
+ return _health_context(_load_experiment(team_id, experiment_id), reader_access, user)
def load_experiment_health_inputs(
@@ -43,7 +45,7 @@ def load_experiment_health_inputs(
) -> HealthInputs:
experiment = _load_experiment(team_id, experiment_id)
return HealthInputs(
- context=_health_context(experiment, reader_access),
+ context=_health_context(experiment, reader_access, user),
exposures=load_exposure_inputs(experiment, user=user, can_query_events=reader_access.can_query_events),
reader_access=reader_access,
flag_version=experiment.feature_flag.version if experiment.feature_flag_id else None,
@@ -64,7 +66,9 @@ def _load_experiment(team_id: int, experiment_id: int) -> Experiment:
)
-def _health_context(experiment: Experiment, reader_access: ExperimentHealthReaderAccess) -> HealthContext:
+def _health_context(
+ experiment: Experiment, reader_access: ExperimentHealthReaderAccess, user: User | None
+) -> HealthContext:
flag_history = load_flag_history(experiment, reader_access=reader_access)
context = load_health_context(experiment, flag_history=flag_history)
traffic = load_condition_set_traffic(experiment, context, reader_access=reader_access)
@@ -73,4 +77,5 @@ def _health_context(experiment: Experiment, reader_access: ExperimentHealthReade
condition_set_traffic=traffic,
pause_history=load_pause_history(experiment, reader_access=reader_access),
primary_metric=load_primary_metric(experiment),
+ metric_compiles=load_metric_compiles(experiment, user=user),
)
diff --git a/products/experiments/backend/health/metric_compiles.py b/products/experiments/backend/health/metric_compiles.py
new file mode 100644
index 000000000000..bb3d5766f547
--- /dev/null
+++ b/products/experiments/backend/health/metric_compiles.py
@@ -0,0 +1,276 @@
+"""
+Compiles the query of each metric that the experiment calculates, without running it, so that a metric that every
+calculation fails on shows before launch. The compile is the query runner's own, so it fails where a calculation
+fails before ClickHouse receives the query. A failure inside ClickHouse, such as a type that the data holds, doesn't
+show here.
+
+The compile runs as the reader, with the reader's warehouse access. The compiler's messages, such as its "Did you mean"
+suggestions, can name any table or column that the compile can see, so a reader only sees names they may query. A
+metric that reads a table or a view that the reader may not query is not checked for them.
+
+Only an answer without a compiler message is cached, per reader and per input of the compile. A failure's message can
+name the tables that the reader may query, and a denial is a decision on the reader's access, so both compile again on
+every read and follow the reader's current access. The cache key doesn't hold the cohorts, actions and warehouse tables
+that a metric names, or the reader's access, so a metric that compiled can show as passing until its answer expires,
+also after one of them is deleted or the reader loses access.
+"""
+
+import re
+import json
+import hashlib
+from datetime import timedelta
+from enum import StrEnum
+from typing import Any
+
+from django.core.cache import cache
+
+import pydantic
+from rest_framework.exceptions import ValidationError
+
+from posthog.schema import ExperimentQuery
+
+from posthog.hogql.database.database import Database
+from posthog.hogql.errors import (
+ ExposedHogQLError,
+ NotImplementedError as HogQLNotImplementedError,
+ SyntaxError as HogQLSyntaxError,
+ TableAccessDeniedError,
+)
+
+from posthog.dataclasses import frozen
+from posthog.exceptions_capture import capture_exception
+from posthog.models.user import User
+
+from products.ai_observability.backend.facade.api import mentions_evaluation_pass_markers
+from products.experiments.backend.facade.contracts import ExperimentHealthMetricCompileErrorKind
+from products.experiments.backend.health.context import MetricCompileFailure, MetricCompiles
+from products.experiments.backend.hogql_queries.error_handling import get_user_friendly_message
+from products.experiments.backend.hogql_queries.experiment_query_runner import ExperimentQueryRunner
+from products.experiments.backend.metric_calculation.config import (
+ MetricCalculationConfig,
+ build_calculation_configs,
+ calculation_metric_name,
+ team_experiments_configs,
+)
+from products.experiments.backend.metric_resolution import build_metric
+from products.experiments.backend.models.experiment import Experiment
+from products.experiments.backend.models.team_experiments_config import TeamExperimentsConfig
+
+# Part of the cache key. Raise it when the cached answer changes shape.
+_ANSWER_SHAPE = 1
+_MAX_MESSAGE_LENGTH = 1000
+
+
+class _Outcome(StrEnum):
+ COMPILES = "compiles"
+ # The compile raised an error that names no problem of the metric, such as a defect of PostHog.
+ UNKNOWN = "unknown"
+ # The metric reads a warehouse table or view that the reader may not query.
+ NO_ACCESS = "no_access"
+ EXPOSURE_QUERY_FAILS = "exposure_query_fails"
+
+
+@frozen
+class _Failure:
+ kind: ExperimentHealthMetricCompileErrorKind
+ message: str
+
+
+# A failure and NO_ACCESS depend on the reader's current access, as the module docstring explains, so they aren't cached.
+_CACHE_SECONDS = {
+ _Outcome.COMPILES: int(timedelta(days=1).total_seconds()),
+ # The cause can go away without an edit of the metric, for example when a deleted cohort is restored.
+ _Outcome.UNKNOWN: int(timedelta(hours=1).total_seconds()),
+ _Outcome.EXPOSURE_QUERY_FAILS: int(timedelta(hours=1).total_seconds()),
+}
+# The errors that the configuration of the experiment causes. HogQL raises its NotImplementedError for a filter that
+# combines settings it doesn't support, such as an element selector with "contains".
+_CONFIGURATION_ERRORS = (ValidationError, ExposedHogQLError, HogQLNotImplementedError, pydantic.ValidationError)
+
+
+def load_metric_compiles(experiment: Experiment, *, user: User | None) -> MetricCompiles | None:
+ """The compile of each metric that the experiment calculates, as `user`, or None when it fails. Never raises: the
+ check then reports that it couldn't check. Without a user, a metric that reads the warehouse is not checked."""
+ try:
+ return _load_metric_compiles(experiment, user)
+ except Exception as error:
+ capture_exception(error)
+ return None
+
+
+def _load_metric_compiles(experiment: Experiment, user: User | None) -> MetricCompiles:
+ team_config = team_experiments_configs([experiment.team_id])[experiment.team_id]
+ # Metrics that share a uuid share its first definition, so one compile covers them.
+ configs: dict[str, MetricCalculationConfig] = {}
+ for calculation_config in build_calculation_configs(experiment, team_config=team_config):
+ configs.setdefault(calculation_config.metric_id, calculation_config)
+ compiler = _MetricCompiler(experiment, team_config=team_config, user=user)
+ failures: list[MetricCompileFailure] = []
+ unknown_count = 0
+ no_access_count = 0
+ exposure_query_fails = False
+ for calculation_config in configs.values():
+ match _answer(compiler, calculation_config):
+ case _Outcome.UNKNOWN:
+ unknown_count += 1
+ case _Outcome.NO_ACCESS:
+ no_access_count += 1
+ case _Outcome.EXPOSURE_QUERY_FAILS:
+ exposure_query_fails = True
+ case _Failure(kind=kind, message=message):
+ failures.append(
+ MetricCompileFailure(
+ name=calculation_metric_name(experiment, calculation_config),
+ role=calculation_config.role,
+ kind=kind,
+ message=None if _can_hold_evaluation_rules(kind, calculation_config) else message,
+ definition=calculation_config.definition,
+ )
+ )
+ return MetricCompiles(
+ metric_count=len(configs),
+ failures=() if exposure_query_fails else tuple(failures),
+ unknown_count=unknown_count,
+ no_access_count=no_access_count,
+ exposure_query_fails=exposure_query_fails,
+ )
+
+
+def _can_hold_evaluation_rules(
+ kind: ExperimentHealthMetricCompileErrorKind, calculation_config: MetricCalculationConfig
+) -> bool:
+ # The compile replaces each marker with the team's evaluation rules, so a HogQL message can quote them. The query
+ # builder's own messages don't.
+ return kind != ExperimentHealthMetricCompileErrorKind.REJECTED and mentions_evaluation_pass_markers(
+ json.dumps(calculation_config.definition, default=str)
+ )
+
+
+def _answer(compiler: "_MetricCompiler", calculation_config: MetricCalculationConfig) -> _Outcome | _Failure:
+ cache_key = _cache_key(compiler.experiment.team_id, compiler.user, calculation_config)
+ cached = cache.get(cache_key)
+ if isinstance(cached, str) and cached in _CACHE_SECONDS:
+ return _Outcome(cached)
+ answer: _Outcome | _Failure
+ try:
+ answer = compiler.compile(calculation_config.definition)
+ except Exception as error:
+ capture_exception(error)
+ answer = _Outcome.UNKNOWN
+ if isinstance(answer, _Outcome) and answer in _CACHE_SECONDS:
+ cache.set(cache_key, answer.value, _CACHE_SECONDS[answer])
+ return answer
+
+
+class _MetricCompiler:
+ """Compiles the metrics of one experiment as one reader, against one HogQL database, which is the costly part of
+ a compile."""
+
+ def __init__(self, experiment: Experiment, *, team_config: TeamExperimentsConfig, user: User | None) -> None:
+ self.experiment = experiment
+ self.user = user
+ self._team_config = team_config
+ self._database: Database | None = None
+
+ def compile(self, definition: dict[str, Any]) -> _Outcome | _Failure:
+ try:
+ metric = build_metric(definition)
+ except pydantic.ValidationError as error:
+ return _failure(ExperimentHealthMetricCompileErrorKind.DEFINITION, _definition_message(error))
+ runner = ExperimentQueryRunner(
+ query=ExperimentQuery(experiment_id=self.experiment.id, metric=metric),
+ team=self.experiment.team,
+ user=self.user,
+ user_facing=False,
+ error_event_context=None,
+ experiment=self.experiment,
+ team_experiments_config=self._team_config,
+ )
+ if self._database is None:
+ # A compile only validates the query, as editor assist does, so it can read the team's catalog from HogQL's
+ # short-lived cache. HogQL applies the reader's current access on every read.
+ self._database = runner.create_database(runner.query_modifiers(), use_cached_sources=True)
+ try:
+ runner.compile_query(self._database)
+ except TableAccessDeniedError:
+ return _Outcome.NO_ACCESS
+ except _CONFIGURATION_ERRORS as error:
+ # The exposure criteria and the project's test account filters are part of every metric's query, so a
+ # failure in them fails every metric and isn't a problem of the metric.
+ if not self._exposure_query_compiles(runner):
+ return _Outcome.EXPOSURE_QUERY_FAILS
+ return _failure(_error_kind(error), _error_message(error))
+ return _Outcome.COMPILES
+
+ def _exposure_query_compiles(self, runner: ExperimentQueryRunner) -> bool:
+ assert self._database is not None
+ try:
+ runner.compile_exposure_query(self._database)
+ except _CONFIGURATION_ERRORS:
+ return False
+ return True
+
+
+def _failure(kind: ExperimentHealthMetricCompileErrorKind, message: str) -> _Failure:
+ return _Failure(kind=kind, message=message[:_MAX_MESSAGE_LENGTH])
+
+
+def _error_kind(error: Exception) -> ExperimentHealthMetricCompileErrorKind:
+ if isinstance(error, ValidationError):
+ return ExperimentHealthMetricCompileErrorKind.REJECTED
+ if isinstance(error, HogQLSyntaxError):
+ return ExperimentHealthMetricCompileErrorKind.SYNTAX
+ if isinstance(error, pydantic.ValidationError):
+ return ExperimentHealthMetricCompileErrorKind.DEFINITION
+ return ExperimentHealthMetricCompileErrorKind.QUERY
+
+
+# The fields of the scope that a field doesn't resolve in, which name the columns that the query builder generates.
+# The list ends at the hints that follow it, or at the end of the message, because a column name can hold a dot.
+_AVAILABLE_FIELDS = re.compile(r"\. Available fields: .*?\.(?= If |$)", re.DOTALL)
+
+
+def _error_message(error: Exception) -> str:
+ if isinstance(error, ValidationError):
+ return get_user_friendly_message(error) or str(error)
+ if isinstance(error, pydantic.ValidationError):
+ return _definition_message(error)
+ return _AVAILABLE_FIELDS.sub(".", str(error))
+
+
+def _definition_message(error: pydantic.ValidationError) -> str:
+ # The location and the reason only. The full text echoes the stored values and runs to many lines.
+ details = error.errors()
+ # A part with brackets names the member of a union that pydantic tried, such as `str-enum[BaseMathType]`.
+ locations = list(
+ dict.fromkeys(".".join(str(part) for part in detail["loc"] if "[" not in str(part)) for detail in details)
+ )
+ if len(details) == 1:
+ return f"{locations[0]}: {details[0]['msg']}"
+ return f"PostHog doesn't accept the value of {', '.join(locations[:3])}"
+
+
+def _cache_key(team_id: int, user: User | None, calculation_config: MetricCalculationConfig) -> str:
+ settings = calculation_config.settings
+ # Every input of the compile except the dates, which change only constants of the query, so a draft's answer
+ # still holds after launch.
+ inputs = json.dumps(
+ [
+ _ANSWER_SHAPE,
+ calculation_config.metric,
+ settings.feature_flag_key,
+ settings.aggregation_group_type_index,
+ settings.variants,
+ settings.stored_exposure_criteria,
+ settings.test_account_filters,
+ settings.only_count_matured_users,
+ settings.cuped.enabled,
+ settings.cuped.lookback_days,
+ settings.timezone,
+ ],
+ sort_keys=True,
+ default=str,
+ )
+ digest = hashlib.sha256(inputs.encode()).hexdigest()
+ # The reader decides which warehouse tables the compile sees, so each reader has an answer of their own.
+ return f"experiment_metric_compile:{team_id}:{user.pk if user else 'none'}:{digest}"
diff --git a/products/experiments/backend/health/primary_metric.py b/products/experiments/backend/health/primary_metric.py
index 4b49191d9bda..7608b1ba48ea 100644
--- a/products/experiments/backend/health/primary_metric.py
+++ b/products/experiments/backend/health/primary_metric.py
@@ -2,13 +2,11 @@ from posthog.exceptions_capture import capture_exception
from products.experiments.backend.health.context import PrimaryMetric
from products.experiments.backend.metric_calculation.config import (
- MetricCalculationConfig,
build_primary_calculation_configs,
+ calculation_metric_name,
team_experiments_configs,
)
from products.experiments.backend.metric_calculation.results import MetricResultStore
-from products.experiments.backend.metric_resolution import saved_metric_links
-from products.experiments.backend.metric_utils import get_default_metric_title
from products.experiments.backend.models.experiment import Experiment
from products.experiments.backend.models.team_experiments_config import TeamExperimentsConfig
from products.experiments.backend.setup_context import PRODUCT_DEFAULT_MINIMUM_DETECTABLE_EFFECT
@@ -35,7 +33,7 @@ def _load_primary_metric(experiment: Experiment) -> PrimaryMetric | None:
return None
is_funnel = calculation_config.definition.get("metric_type") == "funnel"
return PrimaryMetric(
- name=_metric_name(experiment, calculation_config),
+ name=calculation_metric_name(experiment, calculation_config),
is_funnel=is_funnel,
minimum_detectable_effect=_minimum_detectable_effect(experiment, team_config),
manual_running_time=_manual_running_time(experiment),
@@ -43,18 +41,6 @@ def _load_primary_metric(experiment: Experiment) -> PrimaryMetric | None:
)
-def _metric_name(experiment: Experiment, calculation_config: MetricCalculationConfig) -> str:
- definition = calculation_config.definition
- # An inline metric wins over a saved metric with the same uuid, as in `get_metrics_for_calculation`.
- inline_uuids = {metric.get("uuid") for metric in experiment.metrics or [] if isinstance(metric, dict)}
- if calculation_config.metric_id not in inline_uuids:
- for link in saved_metric_links(experiment):
- query = link.saved_metric.query
- if isinstance(query, dict) and query.get("uuid") == calculation_config.metric_id:
- return link.saved_metric.name
- return definition.get("name") or get_default_metric_title(definition)
-
-
def _minimum_detectable_effect(experiment: Experiment, team_config: TeamExperimentsConfig) -> float:
running_time_calculation = experiment.running_time_calculation
planned = (
diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.py
index 61b6cdfd5797..28cc6e6c86fd 100644
--- a/products/experiments/backend/health/registry.py
+++ b/products/experiments/backend/health/registry.py
@@ -20,6 +20,7 @@ from products.experiments.backend.health.checks.flag_change_findings import flag
from products.experiments.backend.health.checks.flag_state import flag_state
from products.experiments.backend.health.checks.flag_variants import flag_variants_mismatch
from products.experiments.backend.health.checks.forced_variant import forced_variant_release_condition
+from products.experiments.backend.health.checks.metric_cannot_compile import metric_cannot_compile
from products.experiments.backend.health.checks.metric_event_not_ingested import metric_event_not_ingested
from products.experiments.backend.health.checks.multiple_variants import multiple_variants_share
from products.experiments.backend.health.checks.new_flag_no_traffic import new_flag_no_traffic
@@ -89,6 +90,11 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
name="Metric events",
run=metric_event_not_ingested,
),
+ HealthCheck(
+ key=ExperimentHealthCheckKey.METRIC_CANNOT_COMPILE,
+ name="Metric queries",
+ run=metric_cannot_compile,
+ ),
HealthCheck(
key=ExperimentHealthCheckKey.EXPOSURE_CRITERIA_CANNOT_MATCH,
name="Exposure criteria",
diff --git a/products/experiments/backend/health/test/factories.py b/products/experiments/backend/health/test/factories.py
index 3be50d362e20..99687fa7d098 100644
--- a/products/experiments/backend/health/test/factories.py
+++ b/products/experiments/backend/health/test/factories.py
@@ -40,6 +40,7 @@ RUNNING_CONTEXT = HealthContext(
pause_history=None,
primary_metric=None,
excluded_variant_keys=frozenset(),
+ metric_compiles=None,
)
diff --git a/products/experiments/backend/health/test/test_early_significance.py b/products/experiments/backend/health/test/test_early_significance.py
index b37759a019e1..65a829898529 100644
--- a/products/experiments/backend/health/test/test_early_significance.py
+++ b/products/experiments/backend/health/test/test_early_significance.py
@@ -265,7 +265,9 @@ class TestEarlySignificanceFromStoredResults(BaseTest):
},
)
- health = get_experiment_health(team_id=self.team.id, experiment_id=experiment.id, reader_access=NO_QUERIES)
+ health = get_experiment_health(
+ team_id=self.team.id, experiment_id=experiment.id, reader_access=NO_QUERIES, user=self.user
+ )
check = next(check for check in health.checks if check.key == ExperimentHealthCheckKey.EARLY_SIGNIFICANCE)
finding = next(
diff --git a/products/experiments/backend/health/test/test_events_received.py b/products/experiments/backend/health/test/test_events_received.py
index e8acd06b38f0..74db6bc4da1e 100644
--- a/products/experiments/backend/health/test/test_events_received.py
+++ b/products/experiments/backend/health/test/test_events_received.py
@@ -97,7 +97,9 @@ class TestEventsReceived(BaseTest):
)
ExperimentToSavedMetric.objects.create(experiment=experiment, saved_metric=shared, metadata={"type": "primary"})
- health = get_experiment_health(team_id=self.team.id, experiment_id=experiment.id, reader_access=NO_QUERIES)
+ health = get_experiment_health(
+ team_id=self.team.id, experiment_id=experiment.id, reader_access=NO_QUERIES, user=self.user
+ )
findings = {finding.code: finding for finding in health.findings}
metric = findings[Code.METRIC_EVENT_NOT_INGESTED]
@@ -140,7 +142,9 @@ class TestEventsReceived(BaseTest):
},
)
- health = get_experiment_health(team_id=self.team.id, experiment_id=experiment.id, reader_access=NO_QUERIES)
+ health = get_experiment_health(
+ team_id=self.team.id, experiment_id=experiment.id, reader_access=NO_QUERIES, user=self.user
+ )
self.assertNotIn(Code.EXPOSURE_CRITERIA_CANNOT_MATCH, {finding.code for finding in health.findings})
@@ -149,7 +153,7 @@ class TestEventsReceived(BaseTest):
experiment = Experiment.objects.create(team=self.team, name="Checkout", feature_flag=flag)
query_counts = []
- for number in range(3):
+ for number in range(4):
shared = ExperimentSavedMetric.objects.create(
team=self.team,
name=f"Shared {number}",
@@ -160,7 +164,11 @@ class TestEventsReceived(BaseTest):
experiment=experiment, saved_metric=shared, metadata={"type": "primary"}
)
with CaptureQueriesContext(connection) as queries:
- get_experiment_health(team_id=self.team.id, experiment_id=experiment.id, reader_access=NO_QUERIES)
+ get_experiment_health(
+ team_id=self.team.id, experiment_id=experiment.id, reader_access=NO_QUERIES, user=self.user
+ )
query_counts.append(len(queries))
- self.assertEqual(len(set(query_counts)), 1, query_counts)
+ # The first compile builds HogQL's database, which reads instance settings that the process caches and
+ # writes the team's revenue analytics settings row once.
+ self.assertEqual(len(set(query_counts[1:])), 1, query_counts)
diff --git a/products/experiments/backend/health/test/test_metric_cannot_compile.py b/products/experiments/backend/health/test/test_metric_cannot_compile.py
new file mode 100644
index 000000000000..bc40a1d887a9
--- /dev/null
+++ b/products/experiments/backend/health/test/test_metric_cannot_compile.py
@@ -0,0 +1,406 @@
+from contextlib import nullcontext
+from datetime import timedelta
+from typing import Any, Literal
+
+from posthog.test.base import BaseTest
+from unittest.mock import Mock, patch
+
+from django.core.cache import cache
+from django.utils import timezone
+
+from parameterized import parameterized
+
+from posthog.hogql.errors import InternalHogQLError
+
+from posthog.constants import AvailableFeature
+from posthog.models import OrganizationMembership
+
+from products.access_control.backend.models.access_control import AccessControl
+from products.analytics_platform.backend.lazy_computation.lazy_computation_executor import LazyComputationExecutor
+from products.experiments.backend.facade.api import get_experiment_health
+from products.experiments.backend.facade.contracts import (
+ ExperimentHealth,
+ ExperimentHealthCheckKey,
+ ExperimentHealthCheckStatus,
+ ExperimentHealthFindingCode,
+ ExperimentHealthFindingSeverity,
+ ExperimentHealthReaderAccess,
+)
+from products.experiments.backend.hogql_queries.experiment_query_runner import ExperimentQueryRunner
+from products.experiments.backend.models.experiment import Experiment
+from products.feature_flags.backend.models.feature_flag import FeatureFlag
+from products.warehouse_sources.backend.facade.models import DataWarehouseCredential, DataWarehouseTable
+
+NO_QUERIES = ExperimentHealthReaderAccess(
+ can_read_flag_history=False, can_query_events=False, denied_event_properties=frozenset()
+)
+CODE = ExperimentHealthFindingCode.METRIC_CANNOT_COMPILE
+KEY = ExperimentHealthCheckKey.METRIC_CANNOT_COMPILE
+WARNING = ExperimentHealthFindingSeverity.WARNING
+INFO = ExperimentHealthFindingSeverity.INFO
+NOT_CHECKED = ExperimentHealthCheckStatus.NOT_CHECKED
+NOT_APPLICABLE = ExperimentHealthCheckStatus.NOT_APPLICABLE
+COMPOUND = "sum(properties.revenue) / count()"
+
+
+def _events(**fields: Any) -> dict[str, Any]:
+ return {"kind": "EventsNode", "event": "purchase", **fields}
+
+
+def _hogql(expression: str) -> dict[str, Any]:
+ return _events(math="hogql", math_hogql=expression)
+
+
+def _mean(source: dict[str, Any], **fields: Any) -> dict[str, Any]:
+ return {
+ "kind": "ExperimentMetric",
+ "metric_type": "mean",
+ "uuid": "m",
+ "name": "Revenue",
+ "source": source,
+ **fields,
+ }
+
+
+def _filtered(event_filter: dict[str, Any]) -> dict[str, Any]:
+ return _mean(_events(properties=[event_filter]))
+
+
+def _cohort(cohort_id: int) -> dict[str, Any]:
+ return {"key": "id", "type": "cohort", "value": cohort_id}
+
+
+def _warehouse_sum(table_name: str) -> dict[str, Any]:
+ return _mean(
+ {
+ "kind": "ExperimentDataWarehouseNode",
+ "table_name": table_name,
+ "timestamp_field": "created",
+ "events_join_key": "person.properties.email",
+ "data_warehouse_join_key": "email",
+ "math": "sum",
+ "math_property": "amount",
+ }
+ )
+
+
+class TestMetricCannotCompile(BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ cache.clear()
+ self.flag = FeatureFlag.objects.create(
+ team=self.team,
+ key="checkout-test",
+ created_by=self.user,
+ filters={
+ "groups": [{"properties": [], "rollout_percentage": 100}],
+ "multivariate": {
+ "variants": [
+ {"key": "control", "rollout_percentage": 50},
+ {"key": "test", "rollout_percentage": 50},
+ ]
+ },
+ },
+ )
+
+ def _experiment(self, **fields: Any) -> Experiment:
+ return Experiment.objects.create(team=self.team, name="Checkout", feature_flag=self.flag, **fields)
+
+ def _health(self, experiment: Experiment) -> ExperimentHealth:
+ return get_experiment_health(
+ team_id=self.team.id, experiment_id=experiment.id, reader_access=NO_QUERIES, user=self.user
+ )
+
+ @parameterized.expand(
+ [
+ ("compiles", "metrics", _mean(_hogql("sum(toFloat(properties.revenue))")), None),
+ (
+ "compound_aggregation",
+ "metrics",
+ _mean(_hogql(COMPOUND)),
+ ("compound_aggregation", WARNING, "use a ratio metric"),
+ ),
+ (
+ "aggregation_with_a_condition",
+ "metrics",
+ _mean(_hogql("uniqIf(person_id, properties.plan = 'pro')")),
+ ("aggregation_arguments", WARNING, "Use uniq, and set the condition as a filter"),
+ ),
+ (
+ "unknown_function",
+ "metrics",
+ _mean(_hogql("sum(toFlot(properties.revenue))")),
+ ("unknown_function", WARNING, "Did you mean toFloat?"),
+ ),
+ (
+ "field_outside_properties",
+ "metrics",
+ _mean(_hogql("sum(order_total)")),
+ ("unknown_field", WARNING, "If order_total is an event property, write properties.order_total."),
+ ),
+ (
+ "syntax_error",
+ "metrics",
+ _mean(_hogql("sum(properties.revenue NOT)")),
+ ("syntax_error", WARNING, "has a syntax error"),
+ ),
+ (
+ "deleted_cohort_in_a_funnel_step",
+ "metrics",
+ {
+ "kind": "ExperimentMetric",
+ "metric_type": "funnel",
+ "uuid": "f",
+ "name": "Checkout funnel",
+ "series": [_events(properties=[_cohort(999999)])],
+ },
+ ("deleted_cohort", WARNING, "(ID 999999)"),
+ ),
+ (
+ "lookahead_regex",
+ "metrics",
+ _filtered({"key": "$current_url", "type": "event", "operator": "regex", "value": "^/a(?!b)"}),
+ ("invalid_regex", WARNING, '"^/a(?!b)"'),
+ ),
+ (
+ "element_selector_operator",
+ "metrics",
+ _filtered({"key": "selector", "type": "element", "operator": "icontains", "value": "button"}),
+ ("unsupported_element_operator", WARNING, "on the element's CSS selector"),
+ ),
+ (
+ "behavioral_filter_without_a_time_range",
+ "metrics",
+ _filtered(
+ {"key": "$pageview", "type": "behavioral", "value": "performed_event", "event_type": "events"}
+ ),
+ ("incomplete_behavioral_filter", WARNING, "behavioral filter of the metric is incomplete"),
+ ),
+ (
+ "missing_warehouse_table",
+ "metrics",
+ _warehouse_sum("stripe_charges"),
+ ("missing_warehouse_table", WARNING, '"stripe_charges"'),
+ ),
+ (
+ "rejected_by_the_builder",
+ "metrics",
+ _mean(_events(), breakdownFilter={"breakdowns": [{"type": "element", "property": "tag_name"}]}),
+ ("rejected", WARNING, "Element breakdowns are not supported"),
+ ),
+ (
+ "stored_definition_the_schema_rejects",
+ "metrics",
+ _mean(_events(math="median_of_sums")),
+ ("invalid_definition", WARNING, "source"),
+ ),
+ (
+ "unnamed_stored_definition_without_a_source",
+ "metrics",
+ {"kind": "ExperimentMetric", "metric_type": "mean", "uuid": "m", "source": None},
+ ("invalid_definition", WARNING, 'the primary metric "Mean Event"'),
+ ),
+ (
+ "secondary_metric_only",
+ "metrics_secondary",
+ _mean(_hogql(COMPOUND)),
+ ("compound_aggregation", INFO, 'the secondary metric "Revenue"'),
+ ),
+ (
+ "primary_metric_after_the_end",
+ "metrics",
+ _mean(_hogql(COMPOUND)),
+ ("compound_aggregation", INFO, 'the primary metric "Revenue"'),
+ {"start_date": timezone.now() - timedelta(days=10), "end_date": timezone.now() - timedelta(days=1)},
+ ),
+ ]
+ )
+ def test_words_why_a_metric_cannot_be_calculated(
+ self,
+ _name: str,
+ field: str,
+ metric: dict[str, Any],
+ expected: tuple[str, ExperimentHealthFindingSeverity, str] | None,
+ fields: dict[str, Any] | None = None,
+ ) -> None:
+ health = self._health(self._experiment(**{field: [metric]}, **(fields or {})))
+
+ findings = [finding for finding in health.findings if finding.code == CODE]
+ if expected is None:
+ self.assertEqual(findings, [])
+ self.assertEqual(
+ {check.key: check.status for check in health.checks}[KEY], ExperimentHealthCheckStatus.PASSED
+ )
+ return
+ subcode, severity, words = expected
+ [finding] = findings
+ self.assertEqual(
+ (finding.subcode, finding.severity, finding.evidence["metric_1_error"]), (subcode, severity, subcode)
+ )
+ self.assertIn(words, finding.detail)
+
+ def test_a_metric_with_evaluation_markers_shows_no_compiler_message(self) -> None:
+ health = self._health(self._experiment(metrics=[_mean(_hogql("sum(aiEvaluationPassed())"))]))
+
+ [finding] = [finding for finding in health.findings if finding.code == CODE]
+ self.assertEqual(
+ (finding.subcode, finding.detail, finding.evidence["metric_1_message"]),
+ (
+ "other",
+ "PostHog can't calculate the primary metric \"Revenue\". Building its query fails. Check the metric's "
+ "settings, or contact support if they look right.",
+ None,
+ ),
+ )
+
+ @parameterized.expand(
+ [
+ (
+ "exposure_criteria_that_do_not_compile",
+ {
+ "metrics": [_mean(_events())],
+ "exposure_criteria": {
+ "exposure_config": {
+ "kind": "ExperimentEventExposureConfig",
+ "event": "checkout_viewed",
+ "properties": [_cohort(999999)],
+ }
+ },
+ },
+ None,
+ (NOT_CHECKED, "Couldn't check because the exposure criteria can't be built into a query"),
+ ),
+ (
+ "error_of_the_compiler_itself",
+ {"metrics": [_mean(_hogql(COMPOUND))]},
+ InternalHogQLError("Unexpected node"),
+ (NOT_CHECKED, "Couldn't check 1 metric"),
+ ),
+ (
+ "legacy_metrics_only",
+ {"metrics": [{"kind": "ExperimentTrendsQuery"}]},
+ None,
+ (NOT_APPLICABLE, "Runs once the experiment has a metric in the current format"),
+ ),
+ ]
+ )
+ def test_does_not_blame_the_metric_for_what_it_did_not_cause(
+ self,
+ _name: str,
+ fields: dict[str, Any],
+ compile_error: Exception | None,
+ expected: tuple[ExperimentHealthCheckStatus, str],
+ ) -> None:
+ with (
+ patch.object(ExperimentQueryRunner, "compile_query", side_effect=compile_error)
+ if compile_error
+ else nullcontext()
+ ):
+ health = self._health(self._experiment(**fields))
+
+ self.assertNotIn(CODE, [finding.code for finding in health.findings])
+ [check] = [check for check in health.checks if check.key == KEY]
+ self.assertEqual((check.status, check.reason), expected)
+
+ @parameterized.expand(
+ [
+ (
+ "table_the_reader_may_not_query",
+ "stripe_charges",
+ "before_the_read",
+ (NOT_CHECKED, "Couldn't check 1 metric that reads data you don't have access to"),
+ ),
+ (
+ "suggestion_for_a_reader_who_may_query_the_table",
+ "stripe_charge",
+ "never",
+ (ExperimentHealthCheckStatus.FINDING, "Unknown table `stripe_charge`. Did you mean: stripe_charges?"),
+ ),
+ (
+ "no_suggestion_of_a_table_the_reader_may_not_query",
+ "stripe_charge",
+ "before_the_read",
+ (ExperimentHealthCheckStatus.FINDING, "Unknown table `stripe_charge`."),
+ ),
+ (
+ "no_suggestion_after_the_reader_loses_access",
+ "stripe_charge",
+ "after_a_first_read",
+ (ExperimentHealthCheckStatus.FINDING, "Unknown table `stripe_charge`."),
+ ),
+ ]
+ )
+ # The flag `hogql-warehouse-access-control` turns on warehouse access control.
+ @patch("posthoganalytics.feature_enabled", new=Mock(return_value=True))
+ def test_compiles_with_the_readers_warehouse_access(
+ self,
+ _name: str,
+ table_name: str,
+ denied: Literal["never", "before_the_read", "after_a_first_read"],
+ expected: tuple[ExperimentHealthCheckStatus, str],
+ ) -> None:
+ self.organization.available_product_features = [
+ {"key": AvailableFeature.ACCESS_CONTROL, "name": AvailableFeature.ACCESS_CONTROL},
+ {"key": AvailableFeature.ROLE_BASED_ACCESS, "name": AvailableFeature.ROLE_BASED_ACCESS},
+ ]
+ self.organization.save()
+ OrganizationMembership.objects.filter(user=self.user, organization=self.organization).update(
+ level=OrganizationMembership.Level.MEMBER
+ )
+ DataWarehouseTable.objects.create(
+ team=self.team,
+ name="stripe_charges",
+ columns={
+ "email": {"hogql": "StringDatabaseField", "clickhouse": "Nullable(String)"},
+ "amount": {"hogql": "FloatDatabaseField", "clickhouse": "Nullable(Float64)"},
+ "created": {"hogql": "DateTimeDatabaseField", "clickhouse": "Nullable(DateTime64(3, 'UTC'))"},
+ },
+ credential=DataWarehouseCredential.objects.create(team=self.team, access_key="key", access_secret="secret"),
+ url_pattern="",
+ )
+ experiment = self._experiment(metrics=[_warehouse_sum(table_name)])
+ if denied == "after_a_first_read":
+ self._health(experiment)
+ if denied != "never":
+ AccessControl.objects.create(team=self.team, resource="warehouse_objects", access_level="none")
+
+ health = self._health(experiment)
+
+ [check] = [check for check in health.checks if check.key == KEY]
+ findings = [finding for finding in health.findings if finding.code == CODE]
+ self.assertEqual(
+ (check.status, findings[0].evidence["metric_1_message"] if findings else check.reason), expected
+ )
+
+ def test_a_metric_that_compiles_is_cached_until_the_metric_changes(self) -> None:
+ experiment = self._experiment(metrics=[_mean(_hogql("sum(toFloat(properties.revenue))"))])
+
+ with patch.object(
+ ExperimentQueryRunner, "compile_query", autospec=True, side_effect=ExperimentQueryRunner.compile_query
+ ) as compile_query:
+ first = self._health(experiment)
+ second = self._health(experiment)
+ experiment.metrics = [_mean(_hogql(COMPOUND))]
+ experiment.save()
+ broken = self._health(experiment)
+
+ self.assertEqual(compile_query.call_count, 2)
+ self.assertEqual(first.findings, second.findings)
+ self.assertNotIn(CODE, [finding.code for finding in first.findings])
+ self.assertIn(CODE, [finding.code for finding in broken.findings])
+
+ def test_the_compile_runs_no_precompute(self) -> None:
+ # With this modifier, HogQL precomputes a subquery of daily unique pageview persons before it prints the query.
+ self.team.modifiers = {"usePreaggregatedIntermediateResults": True}
+ self.team.save()
+ daily_uniques = (
+ "1 IN (SELECT uniqExact(person_id) FROM events WHERE event = '$pageview' "
+ "AND timestamp >= '2026-01-01' AND timestamp < '2026-01-08' GROUP BY toStartOfDay(timestamp))"
+ )
+ experiment = self._experiment(metrics=[_filtered({"type": "hogql", "key": daily_uniques})])
+
+ with patch.object(LazyComputationExecutor, "execute", return_value=Mock(ready=False)) as precompute:
+ health = self._health(experiment)
+
+ precompute.assert_not_called()
+ self.assertEqual({check.key: check.status for check in health.checks}[KEY], ExperimentHealthCheckStatus.PASSED)
diff --git a/products/experiments/backend/health/test/test_new_flag_no_traffic.py b/products/experiments/backend/health/test/test_new_flag_no_traffic.py
index 2aa5e1baf623..da33d4369cbc 100644
--- a/products/experiments/backend/health/test/test_new_flag_no_traffic.py
+++ b/products/experiments/backend/health/test/test_new_flag_no_traffic.py
@@ -101,7 +101,9 @@ class TestNewFlagNoTraffic(BaseTest):
experiment = Experiment.objects.create(team=self.team, name="draft", feature_flag=flag)
with override_settings(FEATURE_FLAG_LAST_CALLED_AT_SYNC_SOURCE=sync_source):
- health = get_experiment_health(team_id=self.team.id, experiment_id=experiment.id, reader_access=FULL_ACCESS)
+ health = get_experiment_health(
+ team_id=self.team.id, experiment_id=experiment.id, reader_access=FULL_ACCESS, user=self.user
+ )
check = next(check for check in health.checks if check.key == ExperimentHealthCheckKey.NEW_FLAG_NO_TRAFFIC)
self.assertEqual(check.status, expected)
diff --git a/products/experiments/backend/hogql_queries/experiment_query_runner.py b/products/experiments/backend/hogql_queries/experiment_query_runner.py
index 8e75e95e2287..298d097c98bf 100644
--- a/products/experiments/backend/hogql_queries/experiment_query_runner.py
+++ b/products/experiments/backend/hogql_queries/experiment_query_runner.py
@@ -19,6 +19,7 @@ from posthog.schema import (
ExperimentRatioMetric,
ExperimentRetentionMetric,
ExperimentStatsBase,
+ HogQLQueryModifiers,
IntervalType,
MultipleVariantHandling,
PrecomputationMode,
@@ -28,7 +29,8 @@ from posthog.hogql import ast
from posthog.hogql.constants import HogQLGlobalSettings
from posthog.hogql.context import HogQLContext
from posthog.hogql.database.database import Database
-from posthog.hogql.query import execute_hogql_query
+from posthog.hogql.errors import ExposedHogQLError, QueryError, TableAccessDeniedError
+from posthog.hogql.query import HogQLQueryExecutor, execute_hogql_query
from posthog.clickhouse.query_tagging import Product, tag_queries, tags_context
from posthog.exceptions import ClickHouseAtCapacity
@@ -111,6 +113,8 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
max_execution_time: Optional[int] = None,
bypass_warehouse_access_control: bool = False,
error_event_context: Optional[str] = "ui",
+ experiment: Optional[Experiment] = None,
+ team_experiments_config: Optional[TeamExperimentsConfig] = None,
**kwargs,
):
super().__init__(*args, **kwargs)
@@ -127,10 +131,19 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
if not self.query.experiment_id:
raise ValidationError("experiment_id is required")
- try:
- self.experiment = Experiment.objects.get(id=self.query.experiment_id, team=self.team)
- except Experiment.DoesNotExist:
- raise ValidationError(f"Experiment with id {self.query.experiment_id} not found")
+ # A caller that runs several metrics of one experiment passes the experiment, with its flag loaded, and the
+ # team's settings, so each runner doesn't read them again.
+ if team_experiments_config is not None:
+ self._team_experiments_config = team_experiments_config
+ if experiment is not None:
+ if experiment.id != self.query.experiment_id or experiment.team_id != self.team.pk:
+ raise ValueError("The experiment doesn't match the query's experiment and team")
+ self.experiment = experiment
+ else:
+ try:
+ self.experiment = Experiment.objects.get(id=self.query.experiment_id, team=self.team)
+ except Experiment.DoesNotExist:
+ raise ValidationError(f"Experiment with id {self.query.experiment_id} not found")
# Evaluation point for the analysis window; experiment_window_end caps it at end_date. An
# explicit as_of is a recalc's frozen run snapshot or a timeseries backfill's per-day point.
@@ -280,7 +293,7 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
return "not_applicable"
return "precomputed" if self._metric_events_precomputed else "direct_scan"
- def _get_experiment_query(self) -> ast.SelectQuery:
+ def _query_builder(self, *, with_metric: bool = True) -> ExperimentQueryBuilder:
assert isinstance(
self.metric,
ExperimentFunnelMetric | ExperimentMeanMetric | ExperimentRatioMetric | ExperimentRetentionMetric,
@@ -290,7 +303,7 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
self.experiment.exposure_criteria, self.team, self.experiment.start_date
)
- builder = ExperimentQueryBuilder(
+ return ExperimentQueryBuilder(
team=self.team,
feature_flag_key=self.feature_flag_key,
exposure_config=exposure_params.exposure_config,
@@ -299,13 +312,16 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
variants=self.variants,
date_range_query=self.date_range_query,
entity_key=self.entity_key,
- metric=self.metric,
- breakdowns=self._get_breakdowns_for_builder(),
+ metric=self.metric if with_metric else None,
+ breakdowns=self._get_breakdowns_for_builder() if with_metric else None,
only_count_matured_users=self.experiment.only_count_matured_users,
cuped_config=self.cuped_config,
activation_config=exposure_params.activation_config,
)
+ def _get_experiment_query(self) -> ast.SelectQuery:
+ builder = self._query_builder()
+
should_precompute = self._should_precompute()
exposure_job_ids: list[str] | None = None
@@ -393,6 +409,80 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
)
)
+ def query_modifiers(self) -> HogQLQueryModifiers:
+ modifiers = self.modifiers.model_copy()
+ if posthoganalytics.feature_enabled(
+ "hogql-session-id-pushdown",
+ str(self.team.id),
+ groups={"project": str(self.team.id)},
+ group_properties={"project": {"id": str(self.team.id)}},
+ only_evaluate_locally=True,
+ send_feature_flag_events=False,
+ ):
+ modifiers.sessionIdPushdown = True
+ return modifiers
+
+ def create_database(self, modifiers: HogQLQueryModifiers, *, use_cached_sources: bool = False) -> Database:
+ return Database.create_for(
+ team=self.team,
+ user=self.user,
+ modifiers=modifiers,
+ timings=self.timings,
+ bypass_warehouse_access_control=self.bypass_warehouse_access_control,
+ use_cached_sources=use_cached_sources,
+ )
+
+ def compile_query(self, database: Database) -> None:
+ """
+ Build the query and compile it to ClickHouse SQL without running it or a precompute. Raises what a calculation
+ raises before ClickHouse receives the query. A calculation can still fail in ClickHouse, for example on a type
+ that the data holds.
+
+ `database` comes from `create_database(query_modifiers())` of a runner with the same team, user and warehouse
+ access, so a caller that compiles several metrics builds it once.
+ """
+ self._compile(self._query_builder().build_query(), database)
+ enforce_warehouse_metric_timestamp_types([self.metric.model_dump(mode="json")], database)
+
+ def compile_exposure_query(self, database: Database) -> None:
+ """Compile the exposure query without the metric and its breakdowns, as `compile_query` compiles the whole
+ query. When the whole query fails and this one compiles, the cause is in the metric."""
+ self._compile(self._query_builder(with_metric=False).get_exposure_timeseries_query(), database)
+
+ def _compile(self, query: ast.SelectQuery, database: Database) -> None:
+ context = HogQLContext(
+ team_id=self.team.pk,
+ user=self.user,
+ bypass_warehouse_access_control=self.bypass_warehouse_access_control,
+ database=database,
+ )
+ modifiers = self.query_modifiers()
+ # With this modifier, the executor writes preaggregated results to ClickHouse and waits for them before it
+ # prints the query, and a compile must run nothing. When no results are ready, the executor keeps the query
+ # unchanged, so the compile of the unchanged query checks what a calculation can fall back to.
+ modifiers.usePreaggregatedIntermediateResults = False
+ try:
+ HogQLQueryExecutor(
+ query_type="ExperimentQuery",
+ query=query,
+ team=self.team,
+ user=self.user,
+ bypass_warehouse_access_control=self.bypass_warehouse_access_control,
+ context=context,
+ timings=self.timings,
+ modifiers=modifiers,
+ workload=self.workload,
+ ).generate_clickhouse_sql()
+ except TableAccessDeniedError:
+ # A denied table must reach the caller as a denial, not as a field error that the HogQL pass recorded.
+ raise
+ except ExposedHogQLError as error:
+ # The HogQL pass records a field that it can't resolve and goes on. A later reference to the column that
+ # holds the field then raises an error that names the column, such as "Field not found: entity_id".
+ if context.errors:
+ raise QueryError(context.errors[0].message) from error
+ raise
+
def _evaluate_experiment_query(
self,
) -> tuple[list[tuple], list[str]]:
@@ -429,16 +519,7 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
experiment_scan_date_from=self.date_range.date_from,
experiment_scan_date_to=self.date_range.date_to,
)
- modifiers = self.modifiers.model_copy()
- if posthoganalytics.feature_enabled(
- "hogql-session-id-pushdown",
- str(self.team.id),
- groups={"project": str(self.team.id)},
- group_properties={"project": {"id": str(self.team.id)}},
- only_evaluate_locally=True,
- send_feature_flag_events=False,
- ):
- modifiers.sessionIdPushdown = True
+ modifiers = self.query_modifiers()
# The debug SQL would expose the evaluation rules that the markers keep out of the browser.
if not self._uses_evaluation_pass_markers():
@@ -464,13 +545,7 @@ class ExperimentQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
# Built once here and handed to the executor, so the timestamp check reads the same catalog
# the query runs against. The column can change type after the metric was saved.
- database = Database.create_for(
- team=self.team,
- user=self.user,
- modifiers=modifiers,
- timings=self.timings,
- bypass_warehouse_access_control=self.bypass_warehouse_access_control,
- )
+ database = self.create_database(modifiers)
enforce_warehouse_metric_timestamp_types([self.metric.model_dump(mode="json")], database)
response = execute_hogql_query(
diff --git a/products/experiments/backend/metric_calculation/config.py b/products/experiments/backend/metric_calculation/config.py
index 1b9ee46eae3f..cf0e061124e0 100644
--- a/products/experiments/backend/metric_calculation/config.py
+++ b/products/experiments/backend/metric_calculation/config.py
@@ -48,6 +48,7 @@ from products.experiments.backend.metric_resolution import (
saved_metric_link_role,
saved_metric_links,
)
+from products.experiments.backend.metric_utils import get_default_metric_title
from products.experiments.backend.models.experiment import Experiment, metric_display_rank
from products.experiments.backend.models.team_experiments_config import TeamExperimentsConfig
@@ -325,6 +326,18 @@ def get_metric_calculation_config(experiment: Experiment, metric_uuid: str) -> M
return settings.build_metric_config(metric_id=metric.uuid, role=metric.role, definition=metric.definition)
+def calculation_metric_name(experiment: Experiment, calculation_config: MetricCalculationConfig) -> str:
+ definition = calculation_config.definition
+ # An inline metric wins over a saved metric with the same uuid, as in `get_metrics_for_calculation`.
+ inline_uuids = {metric.get("uuid") for metric in experiment.metrics or [] if isinstance(metric, dict)}
+ if calculation_config.metric_id not in inline_uuids:
+ for link in saved_metric_links(experiment):
+ query = link.saved_metric.query
+ if isinstance(query, dict) and query.get("uuid") == calculation_config.metric_id:
+ return link.saved_metric.name
+ return definition.get("name") or get_default_metric_title(definition)
+
+
def stamp_calculation_keys(
metrics: list[dict[str, Any]], role: MetricRole, settings: ExperimentCalculationSettings
) -> list[dict[str, Any]]:
diff --git a/products/experiments/backend/metric_utils.py b/products/experiments/backend/metric_utils.py
index 2a2919306563..671401234878 100644
--- a/products/experiments/backend/metric_utils.py
+++ b/products/experiments/backend/metric_utils.py
@@ -9,8 +9,12 @@ from posthog.models.team.team import Team
from products.actions.backend.models.action import Action
-def _get_source_name(source: dict) -> str:
+def _get_source_name(source: object) -> str:
"""Extract a display name from an event/action/data warehouse source dict."""
+ # A stored definition can hold null where a source belongs. The health checks name such a metric to report it as
+ # invalid, so it gets a generic name instead of an error.
+ if not isinstance(source, dict):
+ return "Event"
kind = source.get("kind", "")
if kind == "ExperimentDataWarehouseNode":
return source.get("table_name") or "Table"
@@ -24,8 +28,8 @@ def get_default_metric_title(metric_dict: dict) -> str:
"""Generate a default title for a metric based on its configuration."""
metric_type = metric_dict.get("metric_type", "")
if metric_type == "funnel":
- series = metric_dict.get("series", [])
- if series:
+ series = metric_dict.get("series")
+ if isinstance(series, list) and series:
first_event = _get_source_name(series[0])
last_event = _get_source_name(series[-1])
if len(series) == 1:
diff --git a/products/experiments/backend/presentation/serializers.py b/products/experiments/backend/presentation/serializers.py
index b9a434f1624f..c6b04291e80c 100644
--- a/products/experiments/backend/presentation/serializers.py
+++ b/products/experiments/backend/presentation/serializers.py
@@ -52,6 +52,7 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingSeverity,
ExperimentHealthFlagChangeKind,
ExperimentHealthInputsTrafficStatus,
+ ExperimentHealthMetricCompileErrorKind,
HealthExposureInputsStatus,
)
from products.experiments.backend.facade.timeseries import METRIC_BUILDERS, apply_saved_metric_overrides
@@ -3302,6 +3303,44 @@ class ExperimentHealthInputsExposureEventsSerializer(serializers.Serializer):
)
+class ExperimentHealthInputsMetricCompileFailureSerializer(serializers.Serializer):
+ name = serializers.CharField(help_text="The metric's name as the experiment shows it.")
+ role = serializers.CharField(help_text="'primary' or 'secondary'.")
+ kind = serializers.ChoiceField(
+ choices=ExperimentHealthMetricCompileErrorKind.choices,
+ help_text=(
+ "Who rejected the query: 'rejected' is a validation of the query builder, 'syntax' and 'query' come "
+ "from HogQL, 'definition' means the stored definition doesn't parse into a metric."
+ ),
+ )
+ message = serializers.CharField(
+ allow_null=True,
+ help_text=(
+ "The compiler's message, without the list of a scope's fields. Null when it can hold the evaluation rules "
+ "that the AI evaluation markers keep out of the browser."
+ ),
+ )
+
+
+class ExperimentHealthInputsMetricCompilesSerializer(serializers.Serializer):
+ metric_count = serializers.IntegerField(help_text="The metrics that the experiment calculates, one per uuid.")
+ unknown_count = serializers.IntegerField(
+ help_text="The metrics whose compile raised an error that names no problem of the metric, such as a defect."
+ )
+ no_access_count = serializers.IntegerField(
+ help_text="The metrics that read a warehouse table or view that the reader may not query, so they aren't checked."
+ )
+ exposure_query_fails = serializers.BooleanField(
+ help_text=(
+ "Whether the exposure query alone doesn't compile. Every metric then fails for that reason, so no "
+ "failure is listed."
+ )
+ )
+ failures = ExperimentHealthInputsMetricCompileFailureSerializer(
+ many=True, help_text="The metrics whose query doesn't compile, with the compiler's message."
+ )
+
+
class ExperimentHealthInputsReaderAccessSerializer(serializers.Serializer):
can_read_flag_history = serializers.BooleanField(
help_text="Whether the reader may read the flag's history, which the flag-change checks need."
@@ -3427,6 +3466,14 @@ class ExperimentHealthInputsSerializer(serializers.Serializer):
"name. A name that ingestion keeps no record of is absent."
),
)
+ metric_compiles = ExperimentHealthInputsMetricCompilesSerializer(
+ allow_null=True,
+ help_text=(
+ "The compile of each metric's query without running it, as the reader, which the metric queries check "
+ "reads. A metric that compiles is cached per reader for a day, and a failing metric compiles again on "
+ "every request. Null when the compile itself failed."
+ ),
+ )
flag_history = ExperimentHealthInputsFlagHistorySerializer(
allow_null=True,
help_text=(
diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.py
index f98bac419cef..5e9aeed937df 100644
--- a/products/experiments/backend/presentation/views.py
+++ b/products/experiments/backend/presentation/views.py
@@ -976,6 +976,16 @@ class EnterpriseExperimentsViewSet(
hours after launch, and when only secondary metrics read it, and a warning otherwise.
`filter_on_another_flag` is a warning in every state, and the other findings stop at the end.
+ `metric_cannot_compile` builds and compiles the query of each metric without running it, as
+ the caller, so a metric that every calculation fails on shows before launch. A metric that
+ reads a warehouse table or view the caller may not query is not checked for them. The finding
+ names each failing metric in the `evidence` keys `metric_N`, `metric_N_role`, `metric_N_error`
+ (the error class, which is also the subcode of the first metric) and `metric_N_message` (the
+ compiler's message). It is a warning when a primary metric fails before the end, and info
+ otherwise. A failure inside ClickHouse, such as a type that the data holds, isn't found. A
+ failing metric compiles again on every request. A metric that compiles is cached per caller
+ for a day, so a cohort or table that is deleted after a compile can take up to a day to show.
+
The severity of the `some_conditions_pinned` finding follows the largest share that the
users of the pinned release conditions make up among the users who got one variant from
the flag, in the last 7 days, or since launch or the flag's last save if later. It reads
@@ -1019,6 +1029,7 @@ class EnterpriseExperimentsViewSet(
team_id=self.team_id,
experiment_id=experiment.id,
reader_access=self._health_reader_access(request, experiment),
+ user=request.user if isinstance(request.user, User) else None,
)
return Response(ExperimentHealthSerializer(health).data)
diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.ts
index e1dce09b73d7..7d951512f6f4 100644
--- a/products/experiments/frontend/generated/api.schemas.ts
+++ b/products/experiments/frontend/generated/api.schemas.ts
@@ -2278,6 +2278,7 @@ export type _ExperimentApiMetricsListApi = ExperimentApiMetricApi[]
* * `multiple_variants_share` - Multiple Variants Share
* * `no_metric` - No Metric
* * `metric_event_not_ingested` - Metric Event Not Ingested
+ * * `metric_cannot_compile` - Metric Cannot Compile
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `srm` - Sample Ratio Mismatch
* * `zero_exposures` - Zero Exposures
@@ -2301,6 +2302,7 @@ export const ExperimentHealthFindingCodeEnumApi = {
MultipleVariantsShare: 'multiple_variants_share',
NoMetric: 'no_metric',
MetricEventNotIngested: 'metric_event_not_ingested',
+ MetricCannotCompile: 'metric_cannot_compile',
ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',
Srm: 'srm',
ZeroExposures: 'zero_exposures',
@@ -2367,6 +2369,7 @@ export interface ExperimentHealthFindingApi {
* * `multiple_variants_share` - Multiple Variants Share
* * `no_metric` - No Metric
* * `metric_event_not_ingested` - Metric Event Not Ingested
+ * * `metric_cannot_compile` - Metric Cannot Compile
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `srm` - Sample Ratio Mismatch
* * `zero_exposures` - Zero Exposures
@@ -2415,6 +2418,7 @@ export interface ExperimentHealthFindingApi {
* * `new_flag_no_traffic` - New Flag No Traffic
* * `no_metric` - No Metric
* * `metric_event_not_ingested` - Metric Event Not Ingested
+ * * `metric_cannot_compile` - Metric Cannot Compile
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `early_significance` - Early Significance
* * `zero_exposures` - Zero Exposures
@@ -2436,6 +2440,7 @@ export const ExperimentHealthCheckKeyEnumApi = {
NewFlagNoTraffic: 'new_flag_no_traffic',
NoMetric: 'no_metric',
MetricEventNotIngested: 'metric_event_not_ingested',
+ MetricCannotCompile: 'metric_cannot_compile',
ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',
EarlySignificance: 'early_significance',
ZeroExposures: 'zero_exposures',
@@ -2473,6 +2478,7 @@ export interface ExperimentHealthCheckResultApi {
* * `new_flag_no_traffic` - New Flag No Traffic
* * `no_metric` - No Metric
* * `metric_event_not_ingested` - Metric Event Not Ingested
+ * * `metric_cannot_compile` - Metric Cannot Compile
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `early_significance` - Early Significance
* * `zero_exposures` - Zero Exposures
@@ -3207,6 +3213,54 @@ export interface ExperimentHealthInputsExposureEventsApi {
activation: string | null
}
+/**
+ * * `rejected` - Rejected
+ * * `syntax` - Syntax
+ * * `query` - Query
+ * * `definition` - Definition
+ */
+export type ExperimentHealthMetricCompileErrorKindEnumApi =
+ (typeof ExperimentHealthMetricCompileErrorKindEnumApi)[keyof typeof ExperimentHealthMetricCompileErrorKindEnumApi]
+
+export const ExperimentHealthMetricCompileErrorKindEnumApi = {
+ Rejected: 'rejected',
+ Syntax: 'syntax',
+ Query: 'query',
+ Definition: 'definition',
+} as const
+
+export interface ExperimentHealthInputsMetricCompileFailureApi {
+ /** The metric's name as the experiment shows it. */
+ name: string
+ /** 'primary' or 'secondary'. */
+ role: string
+ /** Who rejected the query: 'rejected' is a validation of the query builder, 'syntax' and 'query' come from HogQL, 'definition' means the stored definition doesn't parse into a metric.
+ *
+ * * `rejected` - Rejected
+ * * `syntax` - Syntax
+ * * `query` - Query
+ * * `definition` - Definition */
+ kind: ExperimentHealthMetricCompileErrorKindEnumApi
+ /**
+ * The compiler's message, without the list of a scope's fields. Null when it can hold the evaluation rules that the AI evaluation markers keep out of the browser.
+ * @nullable
+ */
+ message: string | null
+}
+
+export interface ExperimentHealthInputsMetricCompilesApi {
+ /** The metrics that the experiment calculates, one per uuid. */
+ metric_count: number
+ /** The metrics whose compile raised an error that names no problem of the metric, such as a defect. */
+ unknown_count: number
+ /** The metrics that read a warehouse table or view that the reader may not query, so they aren't checked. */
+ no_access_count: number
+ /** Whether the exposure query alone doesn't compile. Every metric then fails for that reason, so no failure is listed. */
+ exposure_query_fails: boolean
+ /** The metrics whose query doesn't compile, with the compiler's message. */
+ failures: ExperimentHealthInputsMetricCompileFailureApi[]
+}
+
/**
* * `split` - Split
* * `bucketing` - Bucketing
@@ -3617,6 +3671,8 @@ export interface ExperimentHealthInputsApi {
exposure_events: ExperimentHealthInputsExposureEventsApi | null
/** Whether the project has received each event that the metrics and the exposure criteria name, by event name. A name that ingestion keeps no record of is absent. */
events_received: ExperimentHealthInputsApiEventsReceived
+ /** The compile of each metric's query without running it, as the reader, which the metric queries check reads. A metric that compiles is cached per reader for a day, and a failing metric compiles again on every request. Null when the compile itself failed. */
+ metric_compiles: ExperimentHealthInputsMetricCompilesApi | null
/** The changes of the flag since launch that can move exposed people. Null when the experiment is not running, is archived, its flag is missing or deleted, or the read failed. */
flag_history: ExperimentHealthInputsFlagHistoryApi | null
/** Whether the condition-set traffic was read, or why not. The forced-variant check needs it when the experiment is launched and some but not all release conditions pin a variant that has a share of the split and is not excluded from the analysis. The release-condition check needs it when such an override was removed after launch.
diff --git a/products/experiments/frontend/generated/api.ts b/products/experiments/frontend/generated/api.ts
index 2d2e04a74204..5874deaeb68c 100644
--- a/products/experiments/frontend/generated/api.ts
+++ b/products/experiments/frontend/generated/api.ts
@@ -726,6 +726,16 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =
* hours after launch, and when only secondary metrics read it, and a warning otherwise.
* `filter_on_another_flag` is a warning in every state, and the other findings stop at the end.
*
+ * `metric_cannot_compile` builds and compiles the query of each metric without running it, as
+ * the caller, so a metric that every calculation fails on shows before launch. A metric that
+ * reads a warehouse table or view the caller may not query is not checked for them. The finding
+ * names each failing metric in the `evidence` keys `metric_N`, `metric_N_role`, `metric_N_error`
+ * (the error class, which is also the subcode of the first metric) and `metric_N_message` (the
+ * compiler's message). It is a warning when a primary metric fails before the end, and info
+ * otherwise. A failure inside ClickHouse, such as a type that the data holds, isn't found. A
+ * failing metric compiles again on every request. A metric that compiles is cached per caller
+ * for a day, so a cohort or table that is deleted after a compile can take up to a day to show.
+ *
* The severity of the `some_conditions_pinned` finding follows the largest share that the
* users of the pinned release conditions make up among the users who got one variant from
* the flag, in the last 7 days, or since launch or the flag's last save if later. It reads
diff --git a/products/experiments/frontend/health/debug/healthDebugInputs.ts b/products/experiments/frontend/health/debug/healthDebugInputs.ts
index 3c6ebe59b07e..fb5099fa099c 100644
--- a/products/experiments/frontend/health/debug/healthDebugInputs.ts
+++ b/products/experiments/frontend/health/debug/healthDebugInputs.ts
@@ -308,9 +308,40 @@ export function buildMetricFacts(
label: 'Shared metric links',
value: `${countType('primary')} primary, ${countType('secondary')} secondary, ${countType('none')} without a type`,
},
+ {
+ label: 'Metric queries',
+ value: serverValue(state, (inputs) => describeMetricCompiles(inputs.metric_compiles)),
+ help: "The metric queries check compiles each metric's query as you, without running it. A metric that compiles is cached for a day. A failing metric compiles again on every load.",
+ },
+ ...(state.inputs?.metric_compiles?.failures ?? []).map((failure) => ({
+ label: `Doesn't compile: ${failure.name} (${failure.role})`,
+ value: `${failure.kind}: ${failure.message ?? 'message hidden, it can hold evaluation rules'}`,
+ })),
]
}
+function describeMetricCompiles(compiles: ExperimentHealthInputsApi['metric_compiles']): string {
+ if (!compiles) {
+ return 'Not compiled: the compile failed'
+ }
+ if (compiles.exposure_query_fails) {
+ return "Not compiled: the exposure query doesn't compile"
+ }
+ const failed = compiles.failures.length
+ const checked = compiles.metric_count - failed - compiles.unknown_count - compiles.no_access_count
+ const parts = [`${checked} of ${compiles.metric_count} compile`]
+ if (failed) {
+ parts.push(`${failed} ${failed === 1 ? "doesn't" : "don't"}`)
+ }
+ if (compiles.no_access_count) {
+ parts.push(`${compiles.no_access_count} not checked, they read data you can't query`)
+ }
+ if (compiles.unknown_count) {
+ parts.push(`${compiles.unknown_count} not checked`)
+ }
+ return parts.join(', ')
+}
+
const EXPOSURE_STATUS_LABELS: Record<HealthExposureInputsStatusEnumApi, string> = {
ready: 'Read from the cache',
stale: 'Read from the cache, which is past its age. A recalculation runs in the background.',
diff --git a/products/experiments/frontend/health/debug/healthDebugReport.ts b/products/experiments/frontend/health/debug/healthDebugReport.ts
index a7ae0e72c2f7..910f94795f18 100644
--- a/products/experiments/frontend/health/debug/healthDebugReport.ts
+++ b/products/experiments/frontend/health/debug/healthDebugReport.ts
@@ -233,6 +233,13 @@ function metricEventNote({ evidence }: ExperimentHealthFindingApi): string {
return `Not received: ${events.join(', ')}.`
}
+function metricCompileNote({ evidence }: ExperimentHealthFindingApi): string {
+ const errors = Object.entries(evidence)
+ .filter(([key]) => /^metric_\d+_error$/.test(key))
+ .map(([, value]) => value)
+ return `Doesn't compile: ${errors.join(', ')}.`
+}
+
function exposureCriteriaNote({ subcode, evidence }: ExperimentHealthFindingApi): string {
return subcode === 'filter_on_another_flag'
? `The query requires ${evidence.flag_key}, a filter requires ${evidence.filter_flag_keys}.`
@@ -336,6 +343,7 @@ export function buildHealthDebugChecks(input: HealthDebugInput): HealthDebugChec
serverOnlyCheck(input, 'new_flag_no_traffic', null),
noMetricCheck(input),
serverOnlyCheck(input, 'metric_event_not_ingested', metricEventNote, ['metrics', 'project']),
+ serverOnlyCheck(input, 'metric_cannot_compile', metricCompileNote, ['metrics', 'experiment']),
serverOnlyCheck(input, 'exposure_criteria_cannot_match', exposureCriteriaNote, ['experiment', 'project']),
serverOnlyCheck(input, 'early_significance', null, ['experiment', 'metrics']),
...exposureChecks(input),
diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
index 3380a836e12d..56cc5427649a 100644
--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts
+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
@@ -24,6 +24,7 @@ export type ExperimentHealthFindingCode =
| 'zero_exposures'
| 'no_metric'
| 'metric_event_not_ingested'
+ | 'metric_cannot_compile'
| 'exposure_criteria_cannot_match'
| 'forced_variant_release_condition'
| 'flag_variants_mismatch'
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 86a105d8e393..c56ac24e72b3 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -47030,6 +47030,7 @@ export namespace Schemas {
* * `multiple_variants_share` - Multiple Variants Share
* * `no_metric` - No Metric
* * `metric_event_not_ingested` - Metric Event Not Ingested
+ * * `metric_cannot_compile` - Metric Cannot Compile
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `srm` - Sample Ratio Mismatch
* * `zero_exposures` - Zero Exposures
@@ -47053,6 +47054,7 @@ export namespace Schemas {
MultipleVariantsShare: 'multiple_variants_share',
NoMetric: 'no_metric',
MetricEventNotIngested: 'metric_event_not_ingested',
+ MetricCannotCompile: 'metric_cannot_compile',
ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',
Srm: 'srm',
ZeroExposures: 'zero_exposures',
@@ -47119,6 +47121,7 @@ export namespace Schemas {
* * `multiple_variants_share` - Multiple Variants Share
* * `no_metric` - No Metric
* * `metric_event_not_ingested` - Metric Event Not Ingested
+ * * `metric_cannot_compile` - Metric Cannot Compile
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `srm` - Sample Ratio Mismatch
* * `zero_exposures` - Zero Exposures
@@ -47167,6 +47170,7 @@ export namespace Schemas {
* * `new_flag_no_traffic` - New Flag No Traffic
* * `no_metric` - No Metric
* * `metric_event_not_ingested` - Metric Event Not Ingested
+ * * `metric_cannot_compile` - Metric Cannot Compile
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `early_significance` - Early Significance
* * `zero_exposures` - Zero Exposures
@@ -47188,6 +47192,7 @@ export namespace Schemas {
NewFlagNoTraffic: 'new_flag_no_traffic',
NoMetric: 'no_metric',
MetricEventNotIngested: 'metric_event_not_ingested',
+ MetricCannotCompile: 'metric_cannot_compile',
ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',
EarlySignificance: 'early_significance',
ZeroExposures: 'zero_exposures',
@@ -47225,6 +47230,7 @@ export namespace Schemas {
* * `new_flag_no_traffic` - New Flag No Traffic
* * `no_metric` - No Metric
* * `metric_event_not_ingested` - Metric Event Not Ingested
+ * * `metric_cannot_compile` - Metric Cannot Compile
* * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
* * `early_significance` - Early Significance
* * `zero_exposures` - Zero Exposures
@@ -47910,6 +47916,54 @@ export namespace Schemas {
activation: string | null;
}
+ /**
+ * * `rejected` - Rejected
+ * * `syntax` - Syntax
+ * * `query` - Query
+ * * `definition` - Definition
+ */
+ export type ExperimentHealthMetricCompileErrorKindEnum = typeof ExperimentHealthMetricCompileErrorKindEnum[keyof typeof ExperimentHealthMetricCompileErrorKindEnum];
+
+
+ export const ExperimentHealthMetricCompileErrorKindEnum = {
+ Rejected: 'rejected',
+ Syntax: 'syntax',
+ Query: 'query',
+ Definition: 'definition',
+ } as const;
+
+ export interface ExperimentHealthInputsMetricCompileFailure {
+ /** The metric's name as the experiment shows it. */
+ name: string;
+ /** 'primary' or 'secondary'. */
+ role: string;
+ /** Who rejected the query: 'rejected' is a validation of the query builder, 'syntax' and 'query' come from HogQL, 'definition' means the stored definition doesn't parse into a metric.
+ *
+ * * `rejected` - Rejected
+ * * `syntax` - Syntax
+ * * `query` - Query
+ * * `definition` - Definition */
+ kind: ExperimentHealthMetricCompileErrorKindEnum;
+ /**
+ * The compiler's message, without the list of a scope's fields. Null when it can hold the evaluation rules that the AI evaluation markers keep out of the browser.
+ * @nullable
+ */
+ message: string | null;
+ }
+
+ export interface ExperimentHealthInputsMetricCompiles {
+ /** The metrics that the experiment calculates, one per uuid. */
+ metric_count: number;
+ /** The metrics whose compile raised an error that names no problem of the metric, such as a defect. */
+ unknown_count: number;
+ /** The metrics that read a warehouse table or view that the reader may not query, so they aren't checked. */
+ no_access_count: number;
+ /** Whether the exposure query alone doesn't compile. Every metric then fails for that reason, so no failure is listed. */
+ exposure_query_fails: boolean;
+ /** The metrics whose query doesn't compile, with the compiler's message. */
+ failures: ExperimentHealthInputsMetricCompileFailure[];
+ }
+
export interface ExperimentHealthInputsFlagChange {
/** The kind of change. A bucketing change is a change of a setting that decides the variant draw.
*
@@ -48307,6 +48361,8 @@ export namespace Schemas {
exposure_events: ExperimentHealthInputsExposureEvents | null;
/** Whether the project has received each event that the metrics and the exposure criteria name, by event name. A name that ingestion keeps no record of is absent. */
events_received: ExperimentHealthInputsEventsReceived;
+ /** The compile of each metric's query without running it, as the reader, which the metric queries check reads. A metric that compiles is cached per reader for a day, and a failing metric compiles again on every request. Null when the compile itself failed. */
+ metric_compiles: ExperimentHealthInputsMetricCompiles | null;
/** The changes of the flag since launch that can move exposed people. Null when the experiment is not running, is archived, its flag is missing or deleted, or the read failed. */
flag_history: ExperimentHealthInputsFlagHistory | null;
/** Whether the condition-set traffic was read, or why not. The forced-variant check needs it when the experiment is launched and some but not all release conditions pin a variant that has a share of the split and is not excluded from the analysis. The release-condition check needs it when such an override was removed after launch.