Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(experiments): describe each health check and its measured value (#128084)

Marcel Poelker92d2ff0caedf
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index 7bc747ac5130..722541cd417b 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -3400,30 +3400,38 @@ snapshots:
hash: v1.k794b7964.7cb2348662f942bc00e904aca1cf74990ac3b8e25549be308b29e907c156d0eb.WQ5R8L0lbbFdH9oG3RShU2S1nHfj_SHpnz9PrjLAx6A
errortracking-volumesparkline--events-before-data-range--light:
hash: v1.k794b7964.c88c4e3b028915efba3fdbbac7107a063662d8210dfb6ba8667e96d719fa3d4f.tzwEQFa4F1f1O9PGqBkcaqSavbFVgAwPAjyh9Uf9NL8
+ experiments-healthchecksoverview--check-details-of-a-check-that-does-not-apply--dark:
+ hash: v1.k794b7964.ebb5b817b0a9b4bb6bae2c3cd45a0b38818584e8866134b61c08ffd93f3362d5.MUjVFifBOedHd2GA6TtvfKp-Nx8Mj5I-UTOq1JPnVzY
+ experiments-healthchecksoverview--check-details-of-a-check-that-does-not-apply--light:
+ hash: v1.k794b7964.f5e4913a62e1e01741dc4e880598bb8c201005b0f704d7fe720d1797c9718fc1.VQCG4WEsoHK-jWyrAJAH1N5TdIS0d2F3ZiPiQO-k_cs
+ experiments-healthchecksoverview--check-details-with-a-measurement--dark:
+ hash: v1.k794b7964.beea1b208d24c851f82094bed9f3a39aae5ae3caad0481d24de610ca9a8253b1.Ok3EDpxjH9-hXggrp7MlVAZDCy_lmYhSr-cM_GVPdqw
+ experiments-healthchecksoverview--check-details-with-a-measurement--light:
+ hash: v1.k794b7964.38a1738c641bdffa21c9644f5b673732b5aedbd8d09c36f4a1cce484393614cd.bW-xQzxh7qk1_h1dsXbx_H9IaA0S21haMS3KfULxCts
experiments-healthchecksoverview--checks-that-could-not-run--dark:
- hash: v1.k794b7964.8146d36bf02449ae4b2cbc068d3a8c9b9a526fe98943acf0b57bd5dcaad5e6d5.fc6r4oDTtjgfC-OHKRDsugjSlC6k5cWSM54vvzQM9mo
+ hash: v1.k794b7964.f7edab6e955759c878ca746667f1eca6ec1db0c1fae2347081e5856d0bc6ed91.xWcWzMN0s2HjBOE6c2Js6i_vQZnPNevNCI05GBRw6KQ
experiments-healthchecksoverview--checks-that-could-not-run--light:
- hash: v1.k794b7964.06e0cff56ebe51ceff259f34b68d3264930fb0c11905b56b7fe9ceaf8f418bf6.7fZpFHvrBEgKqkNydDxXtJ_LKQKV6Fryti6eNfk57PM
+ hash: v1.k794b7964.e7f8cc191ade33ceec892f3a06bd905baec01a0e29ffef5e7ebddad7b9204723.eU-HWniM8QwPua44W2kUqH3juub-oTKnfi-kyhwKTmk
experiments-healthchecksoverview--draft--dark:
- hash: v1.k794b7964.8ce0d1b68bbf088f3f69dc2f88f2f3bb8ac9059ea99a2c3f4f0e6409f5546ee5.zIywLX5gjLEKZ2xB2qzK3WPrKvQZpFjwcKoWuoKZELU
+ hash: v1.k794b7964.d16ac34a2d5ace6fbd43f4e3326b70de8f2ef8136650781aa81b569a4a0ac379.upf1fasUVWnNtl__hRaoDQEjhdA5__LtqImlkqR7JYo
experiments-healthchecksoverview--draft--light:
- hash: v1.k794b7964.59103b098225cdf194d6e196b7ef86c6039e4f507f96ed2d8f98d336d56fbbbe.ftqfIR3wi-bslWbDUy9rG2Pc1Lo9j_xfNagx88xm5Ps
+ hash: v1.k794b7964.fbeeca6537a201435f6d6b87a88c52b26fdec333d84b731c4523c20290b34c0d.YlZcgD0vIoCVuaNCHq-GXYgUcplhODFha-FyOkfjMKY
experiments-healthchecksoverview--draft-with-events-not-received--dark:
- hash: v1.k794b7964.9c70afa920669d0b9bcced0e8cfc3323352add548e6754b3be6e26135ebe28ee.5-qXSWJuFs5bNVjJTb1kNSz7EpmxGj0cfVkKSShMKTU
+ hash: v1.k794b7964.593d931ef10e1dfa13c3b9e84fcac505808e5dcf9e251def8fd74f8e477a09b8.5TO3tRdAiqZRSkQ5z_fSXmh-tcU448WwZrpoQt0AhPA
experiments-healthchecksoverview--draft-with-events-not-received--light:
- hash: v1.k794b7964.a2dcf1155d438600b9b8b1b43916c33496e7fd32b65527fbd5c4ba1ac50b9b10.URRKvYDQ_6mxCNckg_4Vm94P6FR9aSHm5jYTyM9WIXI
+ hash: v1.k794b7964.cdbd81729b929387c92c3445151ffa0b44db0dbcb669edf5530a9218d42d88a5.zEyeK3-4p_p-O_-RziKypXwXiOr7_c_GsqMjGzjFMCk
experiments-healthchecksoverview--running-with-an-issue--dark:
- hash: v1.k794b7964.67ae51acb1a5a680cdf7e008911ce8c4c0828711a76727d21ed3889b07b26917.pK9afrH7eguEJ5gSocUl68zIoqovXYKRWfbxPvEgtSI
+ hash: v1.k794b7964.4eaa36867b90c26f36b146f519ddaa00e9668301064716efe0992beeb572f516.5uSKe33yLu7tuwvNIsF4GWofGFvge6niuE4tcyCDz4c
experiments-healthchecksoverview--running-with-an-issue--light:
- hash: v1.k794b7964.ac37db2eb5df0a36a0aeb9603c56b5a6d31f5d45afee36dca97cd9144235aecc.HA2SaQlI_HoSfw_kbDTh9RtAg2MeqQbuhoDgUWREwOs
+ hash: v1.k794b7964.36d898bc716c594dacd4998dae84414ff0b7e70171426e6b07e7c0b732bd98d3.bmoF1F9NGvYbtS6R_uKhPLBnKlwMCfROEPMcGaYDDMw
experiments-healthchecksoverview--running-with-metric-windows-not-complete--dark:
- hash: v1.k794b7964.c028d2802d8c6040c3b96acb60be867dda5dfa7a20c061f11c0806f8d13a3ffc.Xcx8Ntc7rjwkn8nCIk5F6mH0yLEykqFsa569tUnM8cA
+ hash: v1.k794b7964.3678a64cb528ceaca7d91459a0eddd12fe61c50c9190bd04c0b773db27771002.G4JwFDn0J8aXv3Qo3Jaf6M0tmhqwp_D4GP_QfqVmGAo
experiments-healthchecksoverview--running-with-metric-windows-not-complete--light:
- hash: v1.k794b7964.3d6323d20aafa349eaab62f5e0fd4408129f694ad65dcb51fe04373ee087cdf5.x5ZDA6dnw2p2pMO0DI7qeY80hY4-IuoROj9tlrGVMQ4
+ hash: v1.k794b7964.6f0f2095c2fac0466734599b45d96b043bb2a16b147df6346a27198e9d001aeb.Nf_8pb45Uk1zPZhHVpuMRLc7QSPmp2i5Wxei_PqSVRU
experiments-healthchecksoverview--running-with-significance-on-a-small-sample--dark:
- hash: v1.k794b7964.2a56cfad0fe2b8742792e7cc20dcd687883839ab474620f847b7c235ea440bfb.PfXKVn-zxcl2L9-HEffR5Ft5euEA6J8S5l9O-MzIyko
+ hash: v1.k794b7964.5e53a67703b82bcc386061e3cfb32654c7abe3deb61711f5047260de67ba950c.uakl2Ik2xiH0o4dIAjpjHsbnPipnwoYx6uNZX4uIcto
experiments-healthchecksoverview--running-with-significance-on-a-small-sample--light:
- hash: v1.k794b7964.748f4506e88aed65c9c075130598283823aa4e4e878d3282c6a5e5b9237fb40f.Bff__yx5A0zeNw3QFeNx-tqHiWL--GIIlEMYr0te510
+ hash: v1.k794b7964.1fc60ac88944e51bc7ac686cd4ea4bab33e265c0b6234562eb9ac2f7871d99c7.-B5a36kS4xMd0A0ZkCfd6Q3zwVLieVcNJjLbuUhfCv8
experiments-howtoreadtooltip--intervals-bayesian--dark:
hash: v1.k794b7964.a06d267cee9cb54bafc732335b35d461d3c1a59250c6daa8becace0fcb8492d2.vR5tecOozn6B51xE8LrByAFVsi03J4w-iUu-j2hQUTU
experiments-howtoreadtooltip--intervals-bayesian--light:
diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json
index 80406205b130..341ebe61d1b0 100644
--- a/frontend/src/queries/schema.json
+++ b/frontend/src/queries/schema.json
@@ -26453,10 +26453,25 @@
"additionalProperties": false,
"description": "What one health check concluded on the exposure answer. Same shape as the experiment's `health.checks`.",
"properties": {
+ "description": {
+ "description": "What the check looks for and why it matters. Missing on an answer cached before the field existed.",
+ "type": "string"
+ },
"key": {
"$ref": "#/definitions/ExperimentExposureHealthCheckKey",
"description": "Stable identifier of the check."
},
+ "measurement": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/ExperimentExposureHealthMeasurement"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "The value that decided a passed check. Null for the other statuses and for a pass without a value."
+ },
"name": {
"description": "The problem the check looks for, as the health panel lists it.",
"type": "string"
@@ -26565,6 +26580,38 @@
"enum": ["critical", "warning", "info"],
"type": "string"
},
+ "ExperimentExposureHealthMeasurement": {
+ "additionalProperties": false,
+ "description": "The value that decided a passed exposure check, against its threshold.",
+ "properties": {
+ "threshold": {
+ "description": "The value at which the check reports a finding.",
+ "type": "number"
+ },
+ "unit": {
+ "$ref": "#/definitions/ExperimentExposureHealthMeasurementUnit",
+ "description": "percentage: a share of the exposed users or groups, on the 0-100 scale. p_value: the p-value of a statistical test."
+ },
+ "value": {
+ "description": "The value that the check measured on the experiment.",
+ "type": "number"
+ },
+ "warns": {
+ "$ref": "#/definitions/ExperimentExposureHealthMeasurementSide",
+ "description": "The side of the threshold that reports a finding."
+ }
+ },
+ "required": ["value", "threshold", "unit", "warns"],
+ "type": "object"
+ },
+ "ExperimentExposureHealthMeasurementSide": {
+ "enum": ["above", "below"],
+ "type": "string"
+ },
+ "ExperimentExposureHealthMeasurementUnit": {
+ "enum": ["percentage", "p_value"],
+ "type": "string"
+ },
"ExperimentExposureNode": {
"additionalProperties": false,
"description": "Sentinel start source for retention metrics. It carries no event of its own: at query time it resolves to the experiment's exposure, so one shared metric anchors correctly on any experiment regardless of that experiment's exposure event.",
diff --git a/frontend/src/queries/schema/schema-general.ts b/frontend/src/queries/schema/schema-general.ts
index e47709b5b025..46d0ebd89720 100644
--- a/frontend/src/queries/schema/schema-general.ts
+++ b/frontend/src/queries/schema/schema-general.ts
@@ -6615,16 +6615,36 @@ export type ExperimentExposureHealthCheckKey =
export type ExperimentExposureHealthCheckStatus = 'passed' | 'finding' | 'not_applicable' | 'not_checked'
+export type ExperimentExposureHealthMeasurementUnit = 'percentage' | 'p_value'
+
+export type ExperimentExposureHealthMeasurementSide = 'above' | 'below'
+
+/** The value that decided a passed exposure check, against its threshold. */
+export interface ExperimentExposureHealthMeasurement {
+ /** The value that the check measured on the experiment. */
+ value: number
+ /** The value at which the check reports a finding. */
+ threshold: number
+ /** percentage: a share of the exposed users or groups, on the 0-100 scale. p_value: the p-value of a statistical test. */
+ unit: ExperimentExposureHealthMeasurementUnit
+ /** The side of the threshold that reports a finding. */
+ warns: ExperimentExposureHealthMeasurementSide
+}
+
/** What one health check concluded on the exposure answer. Same shape as the experiment's `health.checks`. */
export interface ExperimentExposureHealthCheck {
/** Stable identifier of the check. */
key: ExperimentExposureHealthCheckKey
/** The problem the check looks for, as the health panel lists it. */
name: string
+ /** What the check looks for and why it matters. Missing on an answer cached before the field existed. */
+ description?: string
/** passed: no problem found. finding: a finding in `health_findings`. not_applicable: the check doesn't cover the experiment's state. not_checked: an input was missing or the check failed. */
status: ExperimentExposureHealthCheckStatus
/** Why the check did not look, for the statuses not_applicable and not_checked, for example 'Runs after launch'. Null for passed and finding. */
reason: string | null
+ /** The value that decided a passed check. Null for the other statuses and for a pass without a value. */
+ measurement?: ExperimentExposureHealthMeasurement | null
}
export interface ExperimentExposureQueryResponse {
diff --git a/frontend/src/scenes/experiments/experimentLogic.tsx b/frontend/src/scenes/experiments/experimentLogic.tsx
index 250e14f98486..c16d378b9c00 100644
--- a/frontend/src/scenes/experiments/experimentLogic.tsx
+++ b/frontend/src/scenes/experiments/experimentLogic.tsx
@@ -83,6 +83,7 @@ import {
type ExperimentHealthFindingOpenKind,
type ExperimentHealthUi,
captureExperimentHealthChecked,
+ captureExperimentHealthCheckExpanded,
captureExperimentHealthChecksOpened,
captureExperimentHealthFindingActedOn,
captureExperimentHealthFindingOpened,
@@ -759,6 +760,9 @@ export interface experimentLogicActions {
experiment: Experiment
forceRefresh: boolean
}
+ reportHealthCheckExpanded: (check: HealthPanelCheck) => {
+ check: HealthPanelCheck
+ }
reportHealthChecksOpened: (openedFrom: ExperimentHealthChecksOpenedFrom) => {
openedFrom: ExperimentHealthChecksOpenedFrom
}
@@ -1089,6 +1093,7 @@ export const experimentLogic = kea<experimentLogicType>([
actionKind: ExperimentHealthFindingActionKind
) => ({ finding, actionKind }),
reportHealthChecksOpened: (openedFrom: ExperimentHealthChecksOpenedFrom) => ({ openedFrom }),
+ reportHealthCheckExpanded: (check: HealthPanelCheck) => ({ check }),
setExperimentMissing: true,
setExperiment: (experiment: Partial<Experiment>) => ({ experiment }),
setLaunchExperimentLoading: (loading: boolean) => ({ loading }),
@@ -1574,6 +1579,9 @@ export const experimentLogic = kea<experimentLogicType>([
reportHealthFindingActedOn: ({ finding, actionKind }) => {
captureExperimentHealthFindingActedOn(values.experiment, finding, values.healthUi, actionKind)
},
+ reportHealthCheckExpanded: ({ check }) => {
+ captureExperimentHealthCheckExpanded(values.experiment, check, values.healthUi)
+ },
reportHealthChecksOpened: ({ openedFrom }) => {
captureExperimentHealthChecksOpened(
values.experiment,
diff --git a/frontend/src/scenes/experiments/stories/ExperimentHealthTab.stories.tsx b/frontend/src/scenes/experiments/stories/ExperimentHealthTab.stories.tsx
index 94eee433b3f2..797082285855 100644
--- a/frontend/src/scenes/experiments/stories/ExperimentHealthTab.stories.tsx
+++ b/frontend/src/scenes/experiments/stories/ExperimentHealthTab.stories.tsx
@@ -13,9 +13,25 @@ import { NodeKind } from '~/queries/schema/schema-general'
import { recalculationMocks, resultsByMetricType } from './recalculationMocks'
-type Check = { key: string; name: string; status: string; reason: string | null }
+type Check = {
+ key: string
+ name: string
+ description: string
+ status: string
+ reason: string | null
+ measurement: { value: number; threshold: number; unit: string; warns: string } | null
+}
+
+const DESCRIPTION = 'Looks for a problem in the setup of the experiment that can make its results wrong.'
-const passed = (key: string, name: string): Check => ({ key, name, status: 'passed', reason: null })
+const passed = (key: string, name: string): Check => ({
+ key,
+ name,
+ description: DESCRIPTION,
+ status: 'passed',
+ reason: null,
+ measurement: null,
+})
const EXPERIMENT_CHECKS: Check[] = [
passed('flag_state', "Flag state that doesn't match the experiment"),
@@ -26,8 +42,7 @@ const EXPERIMENT_CHECKS: Check[] = [
passed('flag_variants_mismatch', 'Flag variants not set up'),
passed('forced_variant_release_condition', 'Release conditions that force a variant'),
{
- key: 'new_flag_no_traffic',
- name: 'Flag calls not received yet',
+ ...passed('new_flag_no_traffic', 'Flag calls not received yet'),
status: 'not_applicable',
reason: 'Runs before launch',
},
@@ -45,10 +60,12 @@ const EXPERIMENT_CHECKS: Check[] = [
const EXPOSURE_CHECKS: Check[] = [
passed('zero_exposures', 'Exposures not received since launch'),
passed('test_account_filter_removes_exposures', 'Test account filters that leave out most users'),
- passed('srm', 'Split of users across variants'),
{
- key: 'bias_risk_multiple_excluded',
- name: 'Bias from an uneven split',
+ ...passed('srm', 'Split of users across variants'),
+ measurement: { value: 0.71, threshold: 0.001, unit: 'p_value', warns: 'below' },
+ },
+ {
+ ...passed('bias_risk_multiple_excluded', 'Bias from an uneven split'),
status: 'not_applicable',
reason: 'The split is even',
},
diff --git a/posthog/schema.py b/posthog/schema.py
index 311ce12c81b8..41328e4904b1 100644
--- a/posthog/schema.py
+++ b/posthog/schema.py
@@ -133,6 +133,8 @@ from posthog.schema_enums import (
ExperimentExposureHealthFindingActionKind as ExperimentExposureHealthFindingActionKind,
ExperimentExposureHealthFindingCode as ExperimentExposureHealthFindingCode,
ExperimentExposureHealthFindingSeverity as ExperimentExposureHealthFindingSeverity,
+ ExperimentExposureHealthMeasurementSide as ExperimentExposureHealthMeasurementSide,
+ ExperimentExposureHealthMeasurementUnit as ExperimentExposureHealthMeasurementUnit,
ExperimentMetricGoal as ExperimentMetricGoal,
ExperimentMetricMathType as ExperimentMetricMathType,
ExperimentMetricType as ExperimentMetricType,
@@ -5845,33 +5847,6 @@ class ExperimentApiRetentionStart(BaseModel):
)
-class ExperimentExposureHealthCheck(BaseModel):
- model_config = ConfigDict(
- extra="forbid",
- )
- key: ExperimentExposureHealthCheckKey = Field(..., description="Stable identifier of the check.")
- name: str = Field(
- ...,
- description="The problem the check looks for, as the health panel lists it.",
- )
- reason: str | None = Field(
- ...,
- description=(
- "Why the check did not look, for the statuses not_applicable and"
- " not_checked, for example 'Runs after launch'. Null for passed and"
- " finding."
- ),
- )
- status: ExperimentExposureHealthCheckStatus = Field(
- ...,
- description=(
- "passed: no problem found. finding: a finding in `health_findings`."
- " not_applicable: the check doesn't cover the experiment's state."
- " not_checked: an input was missing or the check failed."
- ),
- )
-
-
class ExperimentExposureHealthFinding(BaseModel):
model_config = ConfigDict(
extra="forbid",
@@ -5920,32 +5895,21 @@ class ExperimentExposureHealthFinding(BaseModel):
title: str = Field(..., description="One-line summary of the problem.")
-class ExperimentExposureQueryResponse(BaseModel):
+class ExperimentExposureHealthMeasurement(BaseModel):
model_config = ConfigDict(
extra="forbid",
)
- bias_risk: BiasRisk | None = None
- date_range: DateRange
- health_checks: list[ExperimentExposureHealthCheck] | None = Field(
- default=None,
- description=(
- "The result of each check behind `health_findings`, so that a check that did not run is not read as passed."
- ),
- )
- health_findings: list[ExperimentExposureHealthFinding] | None = Field(
- default=None,
+ threshold: float = Field(..., description="The value at which the check reports a finding.")
+ unit: ExperimentExposureHealthMeasurementUnit = Field(
+ ...,
description=(
- "Health check diagnostics that read the exposures: zero exposures, a sample"
- " ratio mismatch, and bias. Empty when every check passed."
+ "percentage: a share of the exposed users or groups, on the 0-100 scale."
+ " p_value: the p-value of a statistical test."
),
)
- kind: Literal["ExperimentExposureQuery"] = "ExperimentExposureQuery"
- sample_ratio_mismatch: SampleRatioMismatch | None = None
- timeseries: list[ExperimentExposureTimeSeries]
- total_exposures: dict[str, float]
- warnings: list[DataWarehouseSyncWarning] | None = Field(
- default=None,
- description=("Data warehouse sync warnings — see AnalyticsQueryResponseBase.warnings for semantics."),
+ value: float = Field(..., description="The value that the check measured on the experiment.")
+ warns: ExperimentExposureHealthMeasurementSide = Field(
+ ..., description="The side of the threshold that reports a finding."
)
@@ -7698,35 +7662,6 @@ class QueryResponseAlternative10(BaseModel):
)
-class QueryResponseAlternative20(BaseModel):
- model_config = ConfigDict(
- extra="forbid",
- )
- bias_risk: BiasRisk | None = None
- date_range: DateRange
- health_checks: list[ExperimentExposureHealthCheck] | None = Field(
- default=None,
- description=(
- "The result of each check behind `health_findings`, so that a check that did not run is not read as passed."
- ),
- )
- health_findings: list[ExperimentExposureHealthFinding] | None = Field(
- default=None,
- description=(
- "Health check diagnostics that read the exposures: zero exposures, a sample"
- " ratio mismatch, and bias. Empty when every check passed."
- ),
- )
- kind: Literal["ExperimentExposureQuery"] = "ExperimentExposureQuery"
- sample_ratio_mismatch: SampleRatioMismatch | None = None
- timeseries: list[ExperimentExposureTimeSeries]
- total_exposures: dict[str, float]
- warnings: list[DataWarehouseSyncWarning] | None = Field(
- default=None,
- description=("Data warehouse sync warnings — see AnalyticsQueryResponseBase.warnings for semantics."),
- )
-
-
class QueryResponseAlternative31(BaseModel):
model_config = ConfigDict(
extra="forbid",
@@ -13050,54 +12985,6 @@ class CachedEventsQueryResponse(BaseModel):
)
-class CachedExperimentExposureQueryResponse(BaseModel):
- model_config = ConfigDict(
- extra="forbid",
- )
- bias_risk: BiasRisk | None = None
- cache_key: str
- cache_target_age: AwareDatetime | None = None
- calculation_trigger: str | None = Field(
- default=None,
- description=("What triggered the calculation of the query, leave empty if user/immediate"),
- )
- date_range: DateRange
- health_checks: list[ExperimentExposureHealthCheck] | None = Field(
- default=None,
- description=(
- "The result of each check behind `health_findings`, so that a check that did not run is not read as passed."
- ),
- )
- health_findings: list[ExperimentExposureHealthFinding] | None = Field(
- default=None,
- description=(
- "Health check diagnostics that read the exposures: zero exposures, a sample"
- " ratio mismatch, and bias. Empty when every check passed."
- ),
- )
- is_cached: bool
- kind: Literal["ExperimentExposureQuery"] = "ExperimentExposureQuery"
- last_refresh: AwareDatetime
- next_allowed_client_refresh: AwareDatetime
- query_metadata: dict[str, Any] | None = None
- query_scan: QueryScanSummary | None = Field(
- default=None,
- description=("The rows and time of the run that produced these results, with its analysis once it is stored."),
- )
- query_status: QueryStatus | None = Field(
- default=None,
- description=("Query status indicates whether next to the provided data, a query is still running."),
- )
- sample_ratio_mismatch: SampleRatioMismatch | None = None
- timeseries: list[ExperimentExposureTimeSeries]
- timezone: str
- total_exposures: dict[str, float]
- warnings: list[DataWarehouseSyncWarning] | None = Field(
- default=None,
- description=("Data warehouse sync warnings — see AnalyticsQueryResponseBase.warnings for semantics."),
- )
-
-
class CachedFunnelsQueryResponse(BaseModel):
model_config = ConfigDict(
extra="forbid",
@@ -19514,6 +19401,74 @@ class ExperimentBreakdownResult(BaseModel):
)
+class ExperimentExposureHealthCheck(BaseModel):
+ model_config = ConfigDict(
+ extra="forbid",
+ )
+ description: str | None = Field(
+ default=None,
+ description=(
+ "What the check looks for and why it matters. Missing on an answer cached before the field existed."
+ ),
+ )
+ key: ExperimentExposureHealthCheckKey = Field(..., description="Stable identifier of the check.")
+ measurement: ExperimentExposureHealthMeasurement | None = Field(
+ default=None,
+ description=(
+ "The value that decided a passed check. Null for the other statuses and for a pass without a value."
+ ),
+ )
+ name: str = Field(
+ ...,
+ description="The problem the check looks for, as the health panel lists it.",
+ )
+ reason: str | None = Field(
+ ...,
+ description=(
+ "Why the check did not look, for the statuses not_applicable and"
+ " not_checked, for example 'Runs after launch'. Null for passed and"
+ " finding."
+ ),
+ )
+ status: ExperimentExposureHealthCheckStatus = Field(
+ ...,
+ description=(
+ "passed: no problem found. finding: a finding in `health_findings`."
+ " not_applicable: the check doesn't cover the experiment's state."
+ " not_checked: an input was missing or the check failed."
+ ),
+ )
+
+
+class ExperimentExposureQueryResponse(BaseModel):
+ model_config = ConfigDict(
+ extra="forbid",
+ )
+ bias_risk: BiasRisk | None = None
+ date_range: DateRange
+ health_checks: list[ExperimentExposureHealthCheck] | None = Field(
+ default=None,
+ description=(
+ "The result of each check behind `health_findings`, so that a check that did not run is not read as passed."
+ ),
+ )
+ health_findings: list[ExperimentExposureHealthFinding] | None = Field(
+ default=None,
+ description=(
+ "Health check diagnostics that read the exposures: zero exposures, a sample"
+ " ratio mismatch, and bias. Empty when every check passed."
+ ),
+ )
+ kind: Literal["ExperimentExposureQuery"] = "ExperimentExposureQuery"
+ sample_ratio_mismatch: SampleRatioMismatch | None = None
+ timeseries: list[ExperimentExposureTimeSeries]
+ total_exposures: dict[str, float]
+ warnings: list[DataWarehouseSyncWarning] | None = Field(
+ default=None,
+ description=("Data warehouse sync warnings — see AnalyticsQueryResponseBase.warnings for semantics."),
+ )
+
+
class ExperimentQueryResponse(BaseModel):
model_config = ConfigDict(
extra="forbid",
@@ -22425,6 +22380,35 @@ class QueryResponseAlternative19(BaseModel):
)
+class QueryResponseAlternative20(BaseModel):
+ model_config = ConfigDict(
+ extra="forbid",
+ )
+ bias_risk: BiasRisk | None = None
+ date_range: DateRange
+ health_checks: list[ExperimentExposureHealthCheck] | None = Field(
+ default=None,
+ description=(
+ "The result of each check behind `health_findings`, so that a check that did not run is not read as passed."
+ ),
+ )
+ health_findings: list[ExperimentExposureHealthFinding] | None = Field(
+ default=None,
+ description=(
+ "Health check diagnostics that read the exposures: zero exposures, a sample"
+ " ratio mismatch, and bias. Empty when every check passed."
+ ),
+ )
+ kind: Literal["ExperimentExposureQuery"] = "ExperimentExposureQuery"
+ sample_ratio_mismatch: SampleRatioMismatch | None = None
+ timeseries: list[ExperimentExposureTimeSeries]
+ total_exposures: dict[str, float]
+ warnings: list[DataWarehouseSyncWarning] | None = Field(
+ default=None,
+ description=("Data warehouse sync warnings — see AnalyticsQueryResponseBase.warnings for semantics."),
+ )
+
+
class QueryResponseAlternative21(BaseModel):
model_config = ConfigDict(
extra="forbid",
@@ -28385,6 +28369,54 @@ class CachedErrorTrackingQueryResponse(BaseModel):
)
+class CachedExperimentExposureQueryResponse(BaseModel):
+ model_config = ConfigDict(
+ extra="forbid",
+ )
+ bias_risk: BiasRisk | None = None
+ cache_key: str
+ cache_target_age: AwareDatetime | None = None
+ calculation_trigger: str | None = Field(
+ default=None,
+ description=("What triggered the calculation of the query, leave empty if user/immediate"),
+ )
+ date_range: DateRange
+ health_checks: list[ExperimentExposureHealthCheck] | None = Field(
+ default=None,
+ description=(
+ "The result of each check behind `health_findings`, so that a check that did not run is not read as passed."
+ ),
+ )
+ health_findings: list[ExperimentExposureHealthFinding] | None = Field(
+ default=None,
+ description=(
+ "Health check diagnostics that read the exposures: zero exposures, a sample"
+ " ratio mismatch, and bias. Empty when every check passed."
+ ),
+ )
+ is_cached: bool
+ kind: Literal["ExperimentExposureQuery"] = "ExperimentExposureQuery"
+ last_refresh: AwareDatetime
+ next_allowed_client_refresh: AwareDatetime
+ query_metadata: dict[str, Any] | None = None
+ query_scan: QueryScanSummary | None = Field(
+ default=None,
+ description=("The rows and time of the run that produced these results, with its analysis once it is stored."),
+ )
+ query_status: QueryStatus | None = Field(
+ default=None,
+ description=("Query status indicates whether next to the provided data, a query is still running."),
+ )
+ sample_ratio_mismatch: SampleRatioMismatch | None = None
+ timeseries: list[ExperimentExposureTimeSeries]
+ timezone: str
+ total_exposures: dict[str, float]
+ warnings: list[DataWarehouseSyncWarning] | None = Field(
+ default=None,
+ description=("Data warehouse sync warnings — see AnalyticsQueryResponseBase.warnings for semantics."),
+ )
+
+
class CachedExperimentQueryResponse(BaseModel):
model_config = ConfigDict(
extra="forbid",
diff --git a/posthog/schema_enums.py b/posthog/schema_enums.py
index 9c79547ad1f7..2224f3340240 100644
--- a/posthog/schema_enums.py
+++ b/posthog/schema_enums.py
@@ -1504,6 +1504,16 @@ class ExperimentExposureHealthFindingSeverity(StrEnum):
INFO = "info"
+class ExperimentExposureHealthMeasurementSide(StrEnum):
+ ABOVE = "above"
+ BELOW = "below"
+
+
+class ExperimentExposureHealthMeasurementUnit(StrEnum):
+ PERCENTAGE = "percentage"
+ P_VALUE = "p_value"
+
+
class ExperimentMetricGoal(StrEnum):
INCREASE = "increase"
DECREASE = "decrease"
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index f235a73e211a..8458e4692d51 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -10205,6 +10205,40 @@ export namespace Schemas {
| "finding"
| "not_applicable"
| "not_checked";
+ /**
+ * * `percentage` - Percentage of exposed units
+ * * `p_value` - P-value of a statistical test
+ */
+ export type ExperimentHealthMeasurementUnitEnum = "percentage" | "p_value";
+ /**
+ * * `above` - Finding above the threshold
+ * * `below` - Finding below the threshold
+ */
+ export type ExperimentHealthMeasurementSideEnum = "above" | "below";
+ export type ExperimentHealthMeasurement = {
+ /**
+ * The value that the check measured on the experiment.
+ */
+ value: number;
+ /**
+ * The value at which the check reports a finding.
+ */
+ threshold: number;
+ /**
+ * percentage: a share of the exposed users or groups, on the 0-100 scale. p_value: the p-value of a statistical test.
+ *
+ * * `percentage` - Percentage of exposed units
+ * * `p_value` - P-value of a statistical test
+ */
+ unit: ExperimentHealthMeasurementUnitEnum;
+ /**
+ * The side of the threshold that reports a finding: above or below.
+ *
+ * * `above` - Finding above the threshold
+ * * `below` - Finding below the threshold
+ */
+ warns: ExperimentHealthMeasurementSideEnum;
+ };
export type ExperimentHealthCheckResult = {
/**
* Stable identifier of the check. A check can report several finding codes, so the key is not a finding code.
@@ -10238,6 +10272,10 @@ export namespace Schemas {
* The problem the check looks for, for example 'Metrics not set up'.
*/
name: string;
+ /**
+ * What the check looks for and why it matters. The same for every experiment and every status.
+ */
+ description: string;
/**
* passed: the check found no problem. finding: the check reported a finding in `findings`. not_applicable: the check does not cover the experiment's current state, for example a check of the changes since launch on a draft. not_checked: the check covers the experiment, but it could not read an input or it failed, so a problem it looks for can be present without a finding.
*
@@ -10251,6 +10289,10 @@ export namespace Schemas {
* Why the check did not look, for the statuses not_applicable and not_checked, for example 'Runs after launch'. Null for passed and finding.
*/
reason: string | null;
+ /**
+ * The value that decided a passed check, against its threshold. Null for the other statuses, and for a pass without a value. A finding carries its values in `evidence`.
+ */
+ measurement: ExperimentHealthMeasurement | null;
};
export type ExperimentHealth = {
/**
@@ -10594,16 +10636,40 @@ export namespace Schemas {
| "bias_risk_multiple_excluded"
| "multiple_variants_share"
| "test_account_filter_removes_exposures";
+ export type ExperimentExposureHealthMeasurementUnit =
+ | "percentage"
+ | "p_value";
+ export type ExperimentExposureHealthMeasurementSide = "above" | "below";
+ export type ExperimentExposureHealthMeasurement = {
+ /**
+ * The value at which the check reports a finding.
+ */
+ threshold: number;
+ /**
+ * percentage: a share of the exposed users or groups, on the 0-100 scale. p_value: the p-value of a statistical test.
+ */
+ unit: ExperimentExposureHealthMeasurementUnit;
+ /**
+ * The value that the check measured on the experiment.
+ */
+ value: number;
+ /**
+ * The side of the threshold that reports a finding.
+ */
+ warns: ExperimentExposureHealthMeasurementSide;
+ };
export type ExperimentExposureHealthCheckStatus =
| "passed"
| "finding"
| "not_applicable"
| "not_checked";
export type ExperimentExposureHealthCheck = {
+ description?: (string | null) | undefined;
/**
* Stable identifier of the check.
*/
key: ExperimentExposureHealthCheckKey;
+ measurement?: (ExperimentExposureHealthMeasurement | null) | undefined;
/**
* The problem the check looks for, as the health panel lists it.
*/
diff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.py
index de9d689d1d96..3b279db905d0 100644
--- a/products/experiments/backend/facade/contracts.py
+++ b/products/experiments/backend/facade/contracts.py
@@ -369,13 +369,38 @@ class ExperimentHealthCheckStatus(LabeledStrEnum):
NOT_CHECKED = "not_checked"
+class ExperimentHealthMeasurementUnit(LabeledStrEnum):
+ # On the 0-100 scale.
+ PERCENTAGE = "percentage", "Percentage of exposed units"
+ P_VALUE = "p_value", "P-value of a statistical test"
+
+
+class ExperimentHealthMeasurementSide(LabeledStrEnum):
+ ABOVE = "above", "Finding above the threshold"
+ BELOW = "below", "Finding below the threshold"
+
+
+@frozen
+class ExperimentHealthMeasurement:
+ """The value that decided a passed check, so a reader sees how far it is from a finding."""
+
+ value: float
+ threshold: float
+ unit: ExperimentHealthMeasurementUnit
+ # The side of the threshold that reports a finding.
+ warns: ExperimentHealthMeasurementSide
+
+
@frozen
class ExperimentHealthCheckResult:
key: ExperimentHealthCheckKey
name: str
+ description: str
status: ExperimentHealthCheckStatus
# Why the check did not look, for the statuses not_applicable and not_checked. None otherwise.
reason: str | None
+ # Only for the status passed, when the check shows the value that decided the pass.
+ measurement: ExperimentHealthMeasurement | None
@frozen
diff --git a/products/experiments/backend/health/checks/filtered_exposures.py b/products/experiments/backend/health/checks/filtered_exposures.py
index 3ddc1489cec0..568744401a15 100644
--- a/products/experiments/backend/health/checks/filtered_exposures.py
+++ b/products/experiments/backend/health/checks/filtered_exposures.py
@@ -3,12 +3,16 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingActionKind,
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
+ ExperimentHealthMeasurement,
+ ExperimentHealthMeasurementSide,
+ ExperimentHealthMeasurementUnit,
)
from products.experiments.backend.health.context import ExposureTotals, HealthContext
from products.experiments.backend.health.delays import ZERO_EXPOSURES_GRACE_HOURS
from products.experiments.backend.health.skips import (
RUNS_AFTER_LAUNCH,
CheckOutcome,
+ CheckPassed,
CheckSkip,
not_applicable,
not_checked,
@@ -43,8 +47,17 @@ def exposures_removed_by_test_account_filter(ctx: HealthContext) -> CheckOutcome
f"Couldn't check because the {unit}s exposed without the test account filter weren't counted"
)
counted = exposures.total
- if exposed < MINIMUM_EXPOSED_WITHOUT_FILTER or counted * 100 > exposed * KEPT_SHARE_THRESHOLD:
+ if exposed < MINIMUM_EXPOSED_WITHOUT_FILTER:
return None
+ if counted * 100 > exposed * KEPT_SHARE_THRESHOLD:
+ return CheckPassed(
+ measurement=ExperimentHealthMeasurement(
+ value=counted / exposed * 100,
+ threshold=KEPT_SHARE_THRESHOLD,
+ unit=ExperimentHealthMeasurementUnit.PERCENTAGE,
+ warns=ExperimentHealthMeasurementSide.BELOW,
+ )
+ )
return ExperimentHealthFinding(
code=ExperimentHealthFindingCode.TEST_ACCOUNT_FILTER_REMOVES_EXPOSURES,
subcode=None,
diff --git a/products/experiments/backend/health/checks/multiple_variants.py b/products/experiments/backend/health/checks/multiple_variants.py
index b7a24ed57ff1..669180f1947c 100644
--- a/products/experiments/backend/health/checks/multiple_variants.py
+++ b/products/experiments/backend/health/checks/multiple_variants.py
@@ -4,10 +4,19 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFinding,
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
+ ExperimentHealthMeasurement,
+ ExperimentHealthMeasurementSide,
+ ExperimentHealthMeasurementUnit,
)
from products.experiments.backend.health.checks.bias_risk import MULTIPLE_VARIANT_KEY
from products.experiments.backend.health.context import HealthContext
-from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckOutcome, not_applicable, not_checked
+from products.experiments.backend.health.skips import (
+ RUNS_AFTER_LAUNCH,
+ CheckOutcome,
+ CheckPassed,
+ not_applicable,
+ not_checked,
+)
# Some users get more than one ID in any experiment, for example one per device, so a low share is expected.
# The share grows as users return and log in, so a lower line fires on a large part of all experiments. The cases
@@ -31,7 +40,14 @@ def multiple_variants_share(ctx: HealthContext) -> CheckOutcome:
return None
percentage = multiple / total * 100
if percentage <= MULTIPLE_VARIANTS_SHARE_THRESHOLD:
- return None
+ return CheckPassed(
+ measurement=ExperimentHealthMeasurement(
+ value=percentage,
+ threshold=MULTIPLE_VARIANTS_SHARE_THRESHOLD,
+ unit=ExperimentHealthMeasurementUnit.PERCENTAGE,
+ warns=ExperimentHealthMeasurementSide.ABOVE,
+ )
+ )
# The flag gives every member of a group the same variant, so the causes of a person's several IDs do not apply.
if exposures.counts_groups:
title = "Many groups were exposed to multiple variants"
diff --git a/products/experiments/backend/health/checks/srm.py b/products/experiments/backend/health/checks/srm.py
index dfc1d995b324..df64e9b69dc6 100644
--- a/products/experiments/backend/health/checks/srm.py
+++ b/products/experiments/backend/health/checks/srm.py
@@ -17,6 +17,9 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFinding,
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
+ ExperimentHealthMeasurement,
+ ExperimentHealthMeasurementSide,
+ ExperimentHealthMeasurementUnit,
)
from products.experiments.backend.health.checks.forced_variant import (
SOME_CONDITIONS_PINNED,
@@ -33,7 +36,7 @@ from products.experiments.backend.health.context import (
ReachableCondition,
)
from products.experiments.backend.health.pinned_share import pinned_variant_keys
-from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckOutcome, not_applicable
+from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckOutcome, CheckPassed, not_applicable
from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimension
from products.feature_flags.backend.facade.edits import SplitChange
@@ -146,7 +149,14 @@ def srm(ctx: HealthContext) -> CheckOutcome:
return not_applicable("Needs two or more variants in the split")
return not_applicable("Runs once enough users are exposed")
if p_value >= SAMPLE_RATIO_MISMATCH_P_VALUE:
- return None
+ return CheckPassed(
+ measurement=ExperimentHealthMeasurement(
+ value=p_value,
+ threshold=SAMPLE_RATIO_MISMATCH_P_VALUE,
+ unit=ExperimentHealthMeasurementUnit.P_VALUE,
+ warns=ExperimentHealthMeasurementSide.BELOW,
+ )
+ )
share_off_split = _share_off_split(exposures.total_exposures, exposures.sample_ratio_expected)
split_change = _split_change_warning(ctx)
history = ctx.flag_history
diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.py
index c324bffbe738..cb4b293ab82b 100644
--- a/products/experiments/backend/health/registry.py
+++ b/products/experiments/backend/health/registry.py
@@ -10,6 +10,7 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthCheckResult,
ExperimentHealthCheckStatus,
ExperimentHealthFinding,
+ ExperimentHealthMeasurement,
)
from products.experiments.backend.health.checks.bias_risk import bias_risk_multiple_excluded
from products.experiments.backend.health.checks.bucketing_changed import bucketing_changed_mid_run
@@ -35,13 +36,15 @@ from products.experiments.backend.health.checks.split_changed import split_chang
from products.experiments.backend.health.checks.srm import srm
from products.experiments.backend.health.checks.zero_exposures import zero_exposures
from products.experiments.backend.health.context import HealthContext
-from products.experiments.backend.health.skips import CheckOutcome, CheckSkip
+from products.experiments.backend.health.skips import CheckOutcome, CheckPassed, CheckSkip
@frozen
class HealthCheck:
key: ExperimentHealthCheckKey
name: str
+ # What the check looks for and why it matters. The tab shows it for every status, so it holds for every experiment.
+ description: str
run: Callable[[HealthContext], CheckOutcome]
@@ -49,91 +52,169 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
HealthCheck(
key=ExperimentHealthCheckKey.FLAG_STATE,
name="Flag state that doesn't match the experiment",
+ description=(
+ "Checks that the feature flag matches the experiment's status. A running experiment needs a flag that is "
+ "on and serves every variant, or new users stop entering it. Before launch and after the end, a flag that "
+ "serves variants shows them to users that the results don't count."
+ ),
run=flag_state,
),
HealthCheck(
key=ExperimentHealthCheckKey.FLAG_SHARED,
name="Other experiments on the same flag",
+ description=(
+ "Looks for other running or draft experiments on the same feature flag. Pausing one, shipping a variant or "
+ "changing the split changes the flag for all of them, and experiments that run at the same time can count "
+ "the same users."
+ ),
run=flag_shared,
),
HealthCheck(
key=ExperimentHealthCheckKey.FLAG_HISTORY,
name="Unreadable flag history",
+ description=(
+ "Reads the feature flag's history since launch. The checks of changes since launch need it, so without it "
+ "they can't tell whether a change affected the results."
+ ),
run=flag_history_not_readable,
),
HealthCheck(
key=ExperimentHealthCheckKey.SPLIT_CHANGED_MID_RUN,
name="Variant split changes since launch",
+ description=(
+ "Looks for changes to the variant split after users were first exposed. A change can move users who "
+ "already saw one variant to another, which mixes their behavior across variants and biases the results."
+ ),
run=split_changed_mid_run,
),
HealthCheck(
key=ExperimentHealthCheckKey.BUCKETING_CHANGED_MID_RUN,
name="Variant assignment changes since launch",
+ description=(
+ 'Looks for changes after launch to how the flag assigns variants, such as its key or "Match by". Users who '
+ "were already exposed can then get a new variant, which mixes their behavior across variants and biases "
+ "the results."
+ ),
run=bucketing_changed_mid_run,
),
HealthCheck(
key=ExperimentHealthCheckKey.RELEASE_CONDITIONS_CHANGED_MID_RUN,
name="Release condition changes since launch",
+ description=(
+ "Looks for release condition changes after users were first exposed. A change can let in other users or "
+ "move users to another variant, so the results mix two different setups."
+ ),
run=release_conditions_changed_mid_run,
),
HealthCheck(
key=ExperimentHealthCheckKey.FLAG_VARIANTS_MISMATCH,
name="Flag variants not set up",
+ description=(
+ "Checks that the feature flag has the experiment's variants, including the baseline. Without them, PostHog "
+ "can't assign users to the variants or compare the variants with the baseline."
+ ),
run=flag_variants_mismatch,
),
HealthCheck(
key=ExperimentHealthCheckKey.FORCED_VARIANT_RELEASE_CONDITION,
name="Release conditions that force a variant",
+ description=(
+ "Looks for release conditions that give users a fixed variant instead of a random one. The comparison is "
+ "fair only for randomly assigned users, so a large override can bias the results."
+ ),
run=forced_variant_release_condition,
),
HealthCheck(
key=ExperimentHealthCheckKey.NEW_FLAG_NO_TRAFFIC,
name="Flag calls not received yet",
+ description=(
+ "Checks before launch that PostHog receives calls to the feature flag. The experiment counts users only "
+ "after your code evaluates the flag, so a launch before then starts the running time without users."
+ ),
run=new_flag_no_traffic,
),
HealthCheck(
key=ExperimentHealthCheckKey.NO_METRIC,
name="Metrics not set up",
+ description=(
+ "Checks that a launched experiment has at least one metric. Without a metric, the experiment counts "
+ "exposures but has nothing to compare the variants on."
+ ),
run=no_metric,
),
HealthCheck(
key=ExperimentHealthCheckKey.METRIC_EVENT_NOT_INGESTED,
name="Metric events not received yet",
+ description=(
+ "Checks that the project receives the events that each metric uses. A metric whose event never arrives "
+ "stays at zero, so it can't show a difference between the variants."
+ ),
run=metric_event_not_ingested,
),
HealthCheck(
key=ExperimentHealthCheckKey.METRIC_CANNOT_COMPILE,
name="Invalid metric definitions",
+ description=(
+ "Builds the query of each metric without running it, to find definitions that can't be calculated. Such a "
+ "metric shows no results until you fix it."
+ ),
run=metric_cannot_compile,
),
HealthCheck(
key=ExperimentHealthCheckKey.METRIC_CALCULATION_FAILED,
name="Failed metric calculations",
+ description=(
+ "Reads the latest calculation of each metric and reports the ones that failed. A failed metric shows old "
+ "results or none, and some failures repeat until you change the metric or a setting."
+ ),
run=metric_calculation_failed,
),
HealthCheck(
key=ExperimentHealthCheckKey.METRIC_ALL_ZERO,
name="Metrics at zero in every variant",
+ description=(
+ "Looks for metrics that stay at zero in every variant after enough users and time. This usually means that "
+ "the metric events have a different ID than the exposures, so the metric can't show a difference."
+ ),
run=metric_all_zero,
),
HealthCheck(
key=ExperimentHealthCheckKey.EXPOSURE_CRITERIA_CANNOT_MATCH,
name="Exposure criteria that can't match",
+ description=(
+ "Checks that events can match the exposure criteria, for example that the project receives the custom "
+ "exposure event. Criteria that no event matches leave the experiment without exposed users."
+ ),
run=exposure_criteria_cannot_match,
),
HealthCheck(
key=ExperimentHealthCheckKey.EXPOSURE_FILTERS,
name="Dynamic cohorts in the exposure criteria",
+ description=(
+ "Looks for dynamic cohorts in the exposure criteria. The results read a dynamic cohort's members at the "
+ "time of the calculation, not at each user's exposure, so users who join or leave during the run can bias "
+ "the results."
+ ),
run=exposure_filters,
),
HealthCheck(
key=ExperimentHealthCheckKey.EARLY_SIGNIFICANCE,
name="Significance on a small sample",
+ description=(
+ "Compares a significant result with the smallest difference that the sample can reliably detect. A "
+ "significant result on a small sample often shrinks or goes away as more users arrive, so a decision on it "
+ "can ship the wrong variant."
+ ),
run=early_significance,
),
HealthCheck(
key=ExperimentHealthCheckKey.METRICS_NOT_MATURED,
name="Metric windows not complete yet",
+ description=(
+ "Checks whether the primary metrics have results yet when the experiment waits for complete conversion or "
+ "retention windows. A user counts only after their window passes, so a new experiment can show no results "
+ "while nothing is wrong."
+ ),
run=metrics_not_matured,
),
)
@@ -146,26 +227,48 @@ EXPOSURE_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
HealthCheck(
key=ExperimentHealthCheckKey.ZERO_EXPOSURES,
name="Exposures not received since launch",
+ description=(
+ "Checks that users are exposed after launch. An experiment without exposures has no results, which usually "
+ "means that your code doesn't evaluate the flag or that no event matches the exposure criteria."
+ ),
run=zero_exposures,
),
HealthCheck(
key=ExperimentHealthCheckKey.TEST_ACCOUNT_FILTER_REMOVES_EXPOSURES,
name="Test account filters that leave out most users",
+ description=(
+ "Compares the exposed users with and without the test account filters. Filters that leave out most users "
+ "often match real users by mistake, so the results measure only a small part of your users."
+ ),
run=exposures_removed_by_test_account_filter,
),
HealthCheck(
key=ExperimentHealthCheckKey.SRM,
name="Split of users across variants",
+ description=(
+ "Tests whether exposed users are divided across variants in the percentages of the variant split. A "
+ "mismatch usually means that a variant override, the assignment or the tracking adds or loses users in "
+ "one variant, which can bias the results."
+ ),
run=srm,
),
HealthCheck(
key=ExperimentHealthCheckKey.BIAS_RISK_MULTIPLE_EXCLUDED,
name="Bias from an uneven split",
+ description=(
+ "Looks for users exposed to multiple variants when the variant split is uneven. The results leave these "
+ "users out, and the smaller variant loses a larger share of its users, which biases the comparison."
+ ),
run=bias_risk_multiple_excluded,
),
HealthCheck(
key=ExperimentHealthCheckKey.MULTIPLE_VARIANTS_SHARE,
name="Users exposed to multiple variants",
+ description=(
+ "Counts the users exposed to more than one variant, which the results leave out. A high share usually "
+ "means that one person has several IDs, and the variants can then lose different users, which biases the "
+ "results."
+ ),
run=multiple_variants_share,
),
)
@@ -192,6 +295,8 @@ def evaluate(ctx: HealthContext, checks: Sequence[HealthCheck]) -> ExperimentHea
case ExperimentHealthFinding():
findings.append(outcome)
results.append(_result(check, ExperimentHealthCheckStatus.FINDING, None))
+ case CheckPassed(measurement=measurement):
+ results.append(_result(check, ExperimentHealthCheckStatus.PASSED, None, measurement))
case None:
results.append(_result(check, ExperimentHealthCheckStatus.PASSED, None))
case _:
@@ -199,5 +304,17 @@ def evaluate(ctx: HealthContext, checks: Sequence[HealthCheck]) -> ExperimentHea
return ExperimentHealth(findings=tuple(findings), checks=tuple(results))
-def _result(check: HealthCheck, status: ExperimentHealthCheckStatus, reason: str | None) -> ExperimentHealthCheckResult:
- return ExperimentHealthCheckResult(key=check.key, name=check.name, status=status, reason=reason)
+def _result(
+ check: HealthCheck,
+ status: ExperimentHealthCheckStatus,
+ reason: str | None,
+ measurement: ExperimentHealthMeasurement | None = None,
+) -> ExperimentHealthCheckResult:
+ return ExperimentHealthCheckResult(
+ key=check.key,
+ name=check.name,
+ description=check.description,
+ status=status,
+ reason=reason,
+ measurement=measurement,
+ )
diff --git a/products/experiments/backend/health/skips.py b/products/experiments/backend/health/skips.py
index 89b742ca65eb..b8fb69bc699a 100644
--- a/products/experiments/backend/health/skips.py
+++ b/products/experiments/backend/health/skips.py
@@ -2,7 +2,11 @@ from typing import Literal
from posthog.dataclasses import frozen
-from products.experiments.backend.facade.contracts import ExperimentHealthCheckStatus, ExperimentHealthFinding
+from products.experiments.backend.facade.contracts import (
+ ExperimentHealthCheckStatus,
+ ExperimentHealthFinding,
+ ExperimentHealthMeasurement,
+)
from products.experiments.backend.health.context import FlagState, HealthContext
@@ -12,8 +16,14 @@ class CheckSkip:
reason: str
-# What a check returns: a finding, why the check doesn't look, or None when it looks and finds no problem.
-CheckOutcome = ExperimentHealthFinding | CheckSkip | None
+@frozen
+class CheckPassed:
+ measurement: ExperimentHealthMeasurement
+
+
+# What a check returns: a finding, why the check doesn't look, or a pass. CheckPassed is a pass that shows the value
+# that decided it against its threshold, and None is a pass without a value.
+CheckOutcome = ExperimentHealthFinding | CheckSkip | CheckPassed | None
RUNS_AFTER_LAUNCH = "Runs after launch"
RUNS_BEFORE_LAUNCH = "Runs only before launch"
diff --git a/products/experiments/backend/health/test/test_metric_all_zero.py b/products/experiments/backend/health/test/test_metric_all_zero.py
index 1b022a93603b..e813ae7647e9 100644
--- a/products/experiments/backend/health/test/test_metric_all_zero.py
+++ b/products/experiments/backend/health/test/test_metric_all_zero.py
@@ -21,7 +21,7 @@ from products.experiments.backend.facade.contracts import (
)
from products.experiments.backend.health.checks.metric_all_zero import metric_all_zero
from products.experiments.backend.health.context import LatestCalculation, ZeroMetric
-from products.experiments.backend.health.skips import CheckSkip
+from products.experiments.backend.health.skips import CheckOutcome, CheckSkip
from products.experiments.backend.health.test.factories import ENDED, RUNNING_CONTEXT
from products.experiments.backend.metric_calculation.config import get_metric_calculation_config
from products.experiments.backend.metric_calculation.results import MetricResultStore
@@ -57,9 +57,7 @@ def _metric(**changes: Any) -> ZeroMetric:
return ZeroMetric(**{**fields, **changes})
-def _outcome(
- *metrics: ZeroMetric, hours_of_data: float = 96, **changes: Any
-) -> ExperimentHealthFinding | CheckSkip | None:
+def _outcome(*metrics: ZeroMetric, hours_of_data: float = 96, **changes: Any) -> CheckOutcome:
calculation = LatestCalculation(
metric_count=len(metrics) or 1,
finished_at=LAST_ARRIVED,
diff --git a/products/experiments/backend/health/test/test_metric_calculation_failed.py b/products/experiments/backend/health/test/test_metric_calculation_failed.py
index 27aafec73bb0..23e6707cfecc 100644
--- a/products/experiments/backend/health/test/test_metric_calculation_failed.py
+++ b/products/experiments/backend/health/test/test_metric_calculation_failed.py
@@ -33,7 +33,7 @@ from products.experiments.backend.health.context import (
MetricCompileFailure,
MetricCompiles,
)
-from products.experiments.backend.health.skips import CheckSkip
+from products.experiments.backend.health.skips import CheckOutcome, CheckSkip
from products.experiments.backend.health.test.factories import ENDED, RUNNING_CONTEXT
from products.experiments.backend.metric_calculation.config import get_metric_calculation_config
from products.experiments.backend.metric_calculation.results import MetricResultStore
@@ -89,7 +89,7 @@ def _limited(exposures_path: ExposuresPath | None, role: str = "primary") -> Met
def _outcome(
*failures: MetricCalculationFailure, completed: tuple[ExposuresPath | None, ...] = (), **changes: Any
-) -> ExperimentHealthFinding | CheckSkip | None:
+) -> CheckOutcome:
calculation = LatestCalculation(
metric_count=len(failures) + len(completed),
finished_at=FINISHED_AT,
diff --git a/products/experiments/backend/health/test/test_registry.py b/products/experiments/backend/health/test/test_registry.py
index 573c915faf49..af1a3560ea78 100644
--- a/products/experiments/backend/health/test/test_registry.py
+++ b/products/experiments/backend/health/test/test_registry.py
@@ -13,6 +13,9 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthCheckKey,
ExperimentHealthCheckStatus,
ExperimentHealthFindingCode,
+ ExperimentHealthMeasurement,
+ ExperimentHealthMeasurementSide,
+ ExperimentHealthMeasurementUnit,
)
from products.experiments.backend.health import registry
from products.experiments.backend.health.checks.filtered_exposures import reads_share_kept_by_test_account_filter
@@ -112,7 +115,7 @@ def _raise(_ctx: HealthContext) -> Any:
MULTIPLE_VARIANTS_CHECK = next(check for check in EXPOSURE_HEALTH_CHECKS if check.key == Key.MULTIPLE_VARIANTS_SHARE)
-FAILING_CHECK = HealthCheck(key=Key.FLAG_STATE, name="Failing", run=_raise)
+FAILING_CHECK = HealthCheck(key=Key.FLAG_STATE, name="Failing", description="Fails.", run=_raise)
class TestEvaluate(TestCase):
@@ -341,6 +344,63 @@ class TestEvaluate(TestCase):
)
self.assertIn(share, finding.detail)
+ @parameterized.expand(
+ [
+ (
+ "srm_p_value",
+ Key.SRM,
+ replace(UNEVEN_SPLIT_TOTALS, sample_ratio_mismatch_p_value=0.71),
+ ExperimentHealthMeasurement(
+ value=0.71,
+ threshold=0.001,
+ unit=ExperimentHealthMeasurementUnit.P_VALUE,
+ warns=ExperimentHealthMeasurementSide.BELOW,
+ ),
+ ),
+ (
+ "multiple_variants_share",
+ Key.MULTIPLE_VARIANTS_SHARE,
+ _totals_with_multiple(30, 1000),
+ ExperimentHealthMeasurement(
+ value=3.0,
+ threshold=5.0,
+ unit=ExperimentHealthMeasurementUnit.PERCENTAGE,
+ warns=ExperimentHealthMeasurementSide.ABOVE,
+ ),
+ ),
+ (
+ "multiple_variants_below_the_minimum_count",
+ Key.MULTIPLE_VARIANTS_SHARE,
+ _totals_with_multiple(4, 20),
+ None,
+ ),
+ (
+ "kept_share",
+ Key.TEST_ACCOUNT_FILTER_REMOVES_EXPOSURES,
+ _filtered_totals(400, 1000),
+ ExperimentHealthMeasurement(
+ value=40.0,
+ threshold=10.0,
+ unit=ExperimentHealthMeasurementUnit.PERCENTAGE,
+ warns=ExperimentHealthMeasurementSide.BELOW,
+ ),
+ ),
+ (
+ "kept_share_below_the_minimum_exposed",
+ Key.TEST_ACCOUNT_FILTER_REMOVES_EXPOSURES,
+ _filtered_totals(5, 60),
+ None,
+ ),
+ ]
+ )
+ def test_a_pass_reports_its_value_only_when_the_threshold_decided_it(
+ self, _name: str, key: ExperimentHealthCheckKey, exposures: ExposureTotals, measurement: Any
+ ) -> None:
+ health = evaluate(replace(RUNNING_UNEVEN_SPLIT, exposures=exposures), EXPOSURE_HEALTH_CHECKS)
+
+ check = next(check for check in health.checks if check.key == key)
+ self.assertEqual((check.status, check.measurement), (Status.PASSED, measurement))
+
@parameterized.expand(
[
("persons", False, "Many users were exposed to multiple variants"),
diff --git a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
index 9c545900e64a..145f3b17601b 100644
--- a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
+++ b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
@@ -18,6 +18,9 @@ from posthog.schema import (
ExperimentExposureHealthFindingActionKind,
ExperimentExposureHealthFindingCode,
ExperimentExposureHealthFindingSeverity,
+ ExperimentExposureHealthMeasurement,
+ ExperimentExposureHealthMeasurementSide,
+ ExperimentExposureHealthMeasurementUnit,
ExperimentExposureQuery,
ExperimentExposureQueryResponse,
ExperimentExposureTimeSeries,
@@ -470,8 +473,17 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
ExperimentExposureHealthCheck(
key=ExperimentExposureHealthCheckKey(check.key.value),
name=check.name,
+ description=check.description,
status=ExperimentExposureHealthCheckStatus(check.status.value),
reason=check.reason,
+ measurement=ExperimentExposureHealthMeasurement(
+ value=check.measurement.value,
+ threshold=check.measurement.threshold,
+ unit=ExperimentExposureHealthMeasurementUnit(check.measurement.unit.value),
+ warns=ExperimentExposureHealthMeasurementSide(check.measurement.warns.value),
+ )
+ if check.measurement
+ else None,
)
for check in health.checks
]
diff --git a/products/experiments/backend/presentation/serializers.py b/products/experiments/backend/presentation/serializers.py
index c79c1360c9ff..80802d1ac9dc 100644
--- a/products/experiments/backend/presentation/serializers.py
+++ b/products/experiments/backend/presentation/serializers.py
@@ -53,6 +53,8 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingSeverity,
ExperimentHealthFlagChangeKind,
ExperimentHealthInputsTrafficStatus,
+ ExperimentHealthMeasurementSide,
+ ExperimentHealthMeasurementUnit,
ExperimentHealthMetricCompileErrorKind,
ExperimentValidationError,
HealthExposureInputsStatus,
@@ -257,6 +259,22 @@ class ExperimentHealthFindingSerializer(serializers.Serializer):
)
+class ExperimentHealthMeasurementSerializer(serializers.Serializer):
+ value = serializers.FloatField(help_text="The value that the check measured on the experiment.")
+ threshold = serializers.FloatField(help_text="The value at which the check reports a finding.")
+ unit = serializers.ChoiceField(
+ choices=ExperimentHealthMeasurementUnit.choices,
+ help_text=(
+ "percentage: a share of the exposed users or groups, on the 0-100 scale. p_value: the p-value of a "
+ "statistical test."
+ ),
+ )
+ warns = serializers.ChoiceField(
+ choices=ExperimentHealthMeasurementSide.choices,
+ help_text="The side of the threshold that reports a finding: above or below.",
+ )
+
+
class ExperimentHealthCheckResultSerializer(serializers.Serializer):
key = serializers.ChoiceField(
choices=ExperimentHealthCheckKey.choices,
@@ -266,6 +284,9 @@ class ExperimentHealthCheckResultSerializer(serializers.Serializer):
),
)
name = serializers.CharField(help_text="The problem the check looks for, for example 'Metrics not set up'.")
+ description = serializers.CharField(
+ help_text=("What the check looks for and why it matters. The same for every experiment and every status.")
+ )
status = serializers.ChoiceField(
choices=ExperimentHealthCheckStatus.choices,
help_text=(
@@ -282,6 +303,13 @@ class ExperimentHealthCheckResultSerializer(serializers.Serializer):
"'Runs after launch'. Null for passed and finding."
),
)
+ measurement = ExperimentHealthMeasurementSerializer(
+ allow_null=True,
+ help_text=(
+ "The value that decided a passed check, against its threshold. Null for the other statuses, and for a "
+ "pass without a value. A finding carries its values in `evidence`."
+ ),
+ )
class ExperimentHealthSerializer(serializers.Serializer):
diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.ts
index 922bffc9dc79..26e400380c5e 100644
--- a/products/experiments/frontend/generated/api.schemas.ts
+++ b/products/experiments/frontend/generated/api.schemas.ts
@@ -2534,6 +2534,47 @@ export const ExperimentHealthCheckStatusEnumApi = {
NotChecked: 'not_checked',
} as const
+/**
+ * * `percentage` - Percentage of exposed units
+ * * `p_value` - P-value of a statistical test
+ */
+export type ExperimentHealthMeasurementUnitEnumApi =
+ (typeof ExperimentHealthMeasurementUnitEnumApi)[keyof typeof ExperimentHealthMeasurementUnitEnumApi]
+
+export const ExperimentHealthMeasurementUnitEnumApi = {
+ Percentage: 'percentage',
+ PValue: 'p_value',
+} as const
+
+/**
+ * * `above` - Finding above the threshold
+ * * `below` - Finding below the threshold
+ */
+export type ExperimentHealthMeasurementSideEnumApi =
+ (typeof ExperimentHealthMeasurementSideEnumApi)[keyof typeof ExperimentHealthMeasurementSideEnumApi]
+
+export const ExperimentHealthMeasurementSideEnumApi = {
+ Above: 'above',
+ Below: 'below',
+} as const
+
+export interface ExperimentHealthMeasurementApi {
+ /** The value that the check measured on the experiment. */
+ value: number
+ /** The value at which the check reports a finding. */
+ threshold: number
+ /** percentage: a share of the exposed users or groups, on the 0-100 scale. p_value: the p-value of a statistical test.
+ *
+ * * `percentage` - Percentage of exposed units
+ * * `p_value` - P-value of a statistical test */
+ unit: ExperimentHealthMeasurementUnitEnumApi
+ /** The side of the threshold that reports a finding: above or below.
+ *
+ * * `above` - Finding above the threshold
+ * * `below` - Finding below the threshold */
+ warns: ExperimentHealthMeasurementSideEnumApi
+}
+
export interface ExperimentHealthCheckResultApi {
/** Stable identifier of the check. A check can report several finding codes, so the key is not a finding code.
*
@@ -2563,6 +2604,8 @@ export interface ExperimentHealthCheckResultApi {
key: ExperimentHealthCheckKeyEnumApi
/** The problem the check looks for, for example 'Metrics not set up'. */
name: string
+ /** What the check looks for and why it matters. The same for every experiment and every status. */
+ description: string
/** passed: the check found no problem. finding: the check reported a finding in `findings`. not_applicable: the check does not cover the experiment's current state, for example a check of the changes since launch on a draft. not_checked: the check covers the experiment, but it could not read an input or it failed, so a problem it looks for can be present without a finding.
*
* * `passed` - Passed
@@ -2575,6 +2618,8 @@ export interface ExperimentHealthCheckResultApi {
* @nullable
*/
reason: string | null
+ /** The value that decided a passed check, against its threshold. Null for the other statuses, and for a pass without a value. A finding carries its values in `evidence`. */
+ measurement: ExperimentHealthMeasurementApi | null
}
export interface ExperimentHealthApi {
diff --git a/products/experiments/frontend/health/HealthCheckList.tsx b/products/experiments/frontend/health/HealthCheckList.tsx
index 7a39045df677..addf0e339be0 100644
--- a/products/experiments/frontend/health/HealthCheckList.tsx
+++ b/products/experiments/frontend/health/HealthCheckList.tsx
@@ -1,7 +1,7 @@
import { IconCheck, IconMinus, IconQuestion } from '@posthog/icons'
-import { cn } from 'lib/utils/css-classes'
-
+import { HealthCheckPlainRow } from './HealthCheckPlainRow'
+import { HealthCheckRow } from './HealthCheckRow'
import type { ListedHealthCheck } from './healthPanelChecks'
type ListedStatus = ListedHealthCheck['status']
@@ -15,25 +15,17 @@ const STATUS_DISPLAY: Record<ListedStatus, { icon: JSX.Element; label: string }>
interface CheckGroup {
label: string
status: ListedStatus
- rows: { key: string; name: string }[]
+ checks: ListedHealthCheck[]
}
// Checks that share an outcome share a heading, so a reason such as "Runs after launch" shows once.
-function groupByOutcome(checks: ListedHealthCheck[], exposureChecksMissing: boolean): CheckGroup[] {
+function groupByOutcome(checks: ListedHealthCheck[]): CheckGroup[] {
const groups = new Map<string, CheckGroup>()
- if (exposureChecksMissing) {
- // The exposure checks come with the exposure answer, so without one the page has no names for them.
- groups.set('exposure_checks_missing', {
- label: "Couldn't check. Refresh the results to run these checks.",
- status: 'not_checked',
- rows: [{ key: 'exposure_checks', name: 'Exposure checks' }],
- })
- }
for (const check of checks) {
const label = check.reason ?? STATUS_DISPLAY[check.status].label
const key = `${check.status}:${label}`
- const group = groups.get(key) ?? { label, status: check.status, rows: [] }
- group.rows.push({ key: check.key, name: check.name })
+ const group = groups.get(key) ?? { label, status: check.status, checks: [] }
+ group.checks.push(check)
groups.set(key, group)
}
return [...groups.values()]
@@ -48,17 +40,25 @@ export function HealthCheckList({
}): JSX.Element {
return (
<div className="flex flex-col gap-4 px-3 py-2" data-attr="experiment-health-check-list">
- {groupByOutcome(checks, exposureChecksMissing).map((group) => (
+ {exposureChecksMissing && (
+ // The exposure checks come with the exposure answer, so without one the page has no names for them.
+ <div className="flex flex-col gap-1">
+ <span className="text-xs font-semibold text-secondary">
+ Couldn't check. Refresh the results to run these checks.
+ </span>
+ <ul className="m-0 p-0 list-none flex flex-col">
+ <HealthCheckPlainRow icon={STATUS_DISPLAY.not_checked.icon}>
+ <span>Exposure checks</span>
+ </HealthCheckPlainRow>
+ </ul>
+ </div>
+ )}
+ {groupByOutcome(checks).map((group) => (
<div key={`${group.status}:${group.label}`} className="flex flex-col gap-1">
<span className="text-xs font-semibold text-secondary">{group.label}</span>
- <ul className="m-0 p-0 list-none flex flex-col gap-1">
- {group.rows.map((row) => (
- <li key={row.key} className="flex items-center gap-2">
- <span className="flex shrink-0 text-base">{STATUS_DISPLAY[group.status].icon}</span>
- <span className={cn(group.status === 'not_applicable' && 'text-secondary')}>
- {row.name}
- </span>
- </li>
+ <ul className="m-0 p-0 list-none flex flex-col">
+ {group.checks.map((check) => (
+ <HealthCheckRow key={check.key} check={check} icon={STATUS_DISPLAY[group.status].icon} />
))}
</ul>
</div>
diff --git a/products/experiments/frontend/health/HealthCheckPlainRow.tsx b/products/experiments/frontend/health/HealthCheckPlainRow.tsx
new file mode 100644
index 000000000000..7dc49ccb4c3e
--- /dev/null
+++ b/products/experiments/frontend/health/HealthCheckPlainRow.tsx
@@ -0,0 +1,11 @@
+import type { ReactNode } from 'react'
+
+/** A row that opens nothing. It copies the height, gap and sizes of the xsmall button rows, so both kinds line up. */
+export function HealthCheckPlainRow({ icon, children }: { icon: JSX.Element; children: ReactNode }): JSX.Element {
+ return (
+ <li className="flex items-center gap-1 min-h-[1.625rem] pl-0.5 text-xs">
+ <span className="flex shrink-0 text-sm">{icon}</span>
+ {children}
+ </li>
+ )
+}
diff --git a/products/experiments/frontend/health/HealthCheckRow.tsx b/products/experiments/frontend/health/HealthCheckRow.tsx
new file mode 100644
index 000000000000..8d0d8e2ca90c
--- /dev/null
+++ b/products/experiments/frontend/health/HealthCheckRow.tsx
@@ -0,0 +1,49 @@
+import { useActions } from 'kea'
+import { useId, useState } from 'react'
+
+import { LemonButton } from '@posthog/lemon-ui'
+
+import { cn } from 'lib/utils/css-classes'
+import { experimentLogic } from 'scenes/experiments/experimentLogic'
+
+import { HealthCheckPlainRow } from './HealthCheckPlainRow'
+import { type ListedHealthCheck, healthMeasurementText } from './healthPanelChecks'
+
+export function HealthCheckRow({ check, icon }: { check: ListedHealthCheck; icon: JSX.Element }): JSX.Element {
+ const [isDetailOpen, setIsDetailOpen] = useState(false)
+ const detailId = useId()
+ const { reportHealthCheckExpanded } = useActions(experimentLogic)
+ const name = <span className={cn(check.status === 'not_applicable' && 'text-secondary')}>{check.name}</span>
+
+ if (!check.description) {
+ // An exposure answer cached before the descriptions existed has nothing to show.
+ return <HealthCheckPlainRow icon={icon}>{name}</HealthCheckPlainRow>
+ }
+
+ return (
+ <li className="flex flex-col">
+ <LemonButton
+ size="xsmall"
+ icon={icon}
+ className="-mx-1"
+ onClick={() => {
+ if (!isDetailOpen) {
+ reportHealthCheckExpanded(check)
+ }
+ setIsDetailOpen(!isDetailOpen)
+ }}
+ aria-expanded={isDetailOpen}
+ aria-controls={detailId}
+ data-attr="experiment-health-check"
+ >
+ {name}
+ </LemonButton>
+ {isDetailOpen && (
+ <div id={detailId} className="flex flex-col gap-1 pl-5 pb-1 text-xs">
+ <p className="m-0 text-secondary">{check.description}</p>
+ {check.measurement && <span>{healthMeasurementText(check.measurement)}</span>}
+ </div>
+ )}
+ </li>
+ )
+}
diff --git a/products/experiments/frontend/health/HealthChecksOverview.stories.tsx b/products/experiments/frontend/health/HealthChecksOverview.stories.tsx
index 59c7cded8156..561975e4d8f9 100644
--- a/products/experiments/frontend/health/HealthChecksOverview.stories.tsx
+++ b/products/experiments/frontend/health/HealthChecksOverview.stories.tsx
@@ -1,13 +1,33 @@
import type { Meta, StoryObj } from '@storybook/react'
+import { within } from '@testing-library/dom'
+
+import { userEvent } from 'storybook/test'
import { HealthChecksOverview } from './HealthChecksOverview'
import type { HealthPanelCheck } from './healthPanelChecks'
+const DESCRIPTIONS: Partial<Record<HealthPanelCheck['key'], string>> = {
+ flag_state:
+ "Checks that the feature flag matches the experiment's status. A running experiment needs a flag that is on and serves every variant, or new users stop entering it. Before launch and after the end, a flag that serves variants shows them to users that the results don't count.",
+ split_changed_mid_run:
+ 'Looks for changes to the variant split after users were first exposed. A change can move users who already saw one variant to another, which mixes their behavior across variants and biases the results.',
+ no_metric:
+ 'Checks that a launched experiment has at least one metric. Without a metric, the experiment counts exposures but has nothing to compare the variants on.',
+ srm: 'Tests whether exposed users are divided across variants in the percentages of the variant split. A mismatch usually means that a variant override, the assignment or the tracking adds or loses users in one variant, which can bias the results.',
+ multiple_variants_share:
+ 'Counts the users exposed to more than one variant, which the results leave out. A high share usually means that one person has several IDs, and the variants can then lose different users, which biases the results.',
+}
+
+const description = (key: HealthPanelCheck['key']): string =>
+ DESCRIPTIONS[key] ?? 'Looks for a problem in the setup of the experiment that can make its results wrong.'
+
const passed = (key: HealthPanelCheck['key'], name: string): HealthPanelCheck => ({
key,
name,
+ description: description(key),
status: 'passed',
reason: null,
+ measurement: null,
})
const skipped = (
@@ -15,7 +35,7 @@ const skipped = (
name: string,
status: 'not_applicable' | 'not_checked',
reason: string
-): HealthPanelCheck => ({ key, name, status, reason })
+): HealthPanelCheck => ({ key, name, description: description(key), status, reason, measurement: null })
const FLAG_SETUP_CHECKS = [
passed('flag_state', "Flag state that doesn't match the experiment"),
@@ -75,8 +95,10 @@ export const Draft: Story = {
const found = (key: HealthPanelCheck['key'], name: string): HealthPanelCheck => ({
key,
name,
+ description: description(key),
status: 'finding',
reason: null,
+ measurement: null,
})
export const DraftWithEventsNotReceived: Story = {
@@ -214,3 +236,50 @@ export const RunningWithAnIssue: Story = {
exposureChecksMissing: false,
},
}
+
+export const CheckDetailsWithAMeasurement: Story = {
+ args: {
+ checks: [
+ ...FLAG_SETUP_CHECKS,
+ passed('no_metric', 'Metrics not set up'),
+ {
+ ...passed('srm', 'Split of users across variants'),
+ measurement: { value: 0.71, threshold: 0.001, unit: 'p_value', warns: 'below' },
+ },
+ {
+ ...passed('multiple_variants_share', 'Users exposed to multiple variants'),
+ measurement: { value: 4.2, threshold: 5, unit: 'percentage', warns: 'above' },
+ },
+ ],
+ findings: [],
+ isDraft: false,
+ exposureChecksMissing: false,
+ },
+ play: async ({ canvasElement }) => {
+ await userEvent.click(
+ await within(canvasElement).findByRole('button', { name: /Users exposed to multiple variants/ })
+ )
+ },
+}
+
+export const CheckDetailsOfACheckThatDoesNotApply: Story = {
+ args: {
+ checks: [
+ ...FLAG_SETUP_CHECKS,
+ skipped(
+ 'split_changed_mid_run',
+ 'Variant split changes since launch',
+ 'not_applicable',
+ 'Runs after launch'
+ ),
+ ],
+ findings: [],
+ isDraft: true,
+ exposureChecksMissing: false,
+ },
+ play: async ({ canvasElement }) => {
+ await userEvent.click(
+ await within(canvasElement).findByRole('button', { name: /Variant split changes since launch/ })
+ )
+ },
+}
diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
index d588d83e7dce..e0540f56550d 100644
--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts
+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
@@ -210,6 +210,23 @@ export function captureExperimentHealthChecksOpened(
})
}
+export function captureExperimentHealthCheckExpanded(
+ experiment: Experiment,
+ check: HealthPanelCheck,
+ healthUi: ExperimentHealthUi
+): void {
+ // pinned: analytics event name, so renaming it breaks insights
+ posthog.capture('experiment health check expanded', {
+ experiment_id: experiment.id,
+ experiment_status: getExperimentStatus(experiment),
+ health_ui: healthUi,
+ check_key: check.key,
+ check_status: check.status,
+ check_has_measurement: Boolean(check.measurement),
+ surface: 'experiment_page',
+ })
+}
+
/**
* Compares the page's own rules with the server's checks for one experiment load. A `_browser` value is the
* result of the page's rule, and a `_server` value is the result of the server's check.
diff --git a/products/experiments/frontend/health/healthPanelChecks.test.ts b/products/experiments/frontend/health/healthPanelChecks.test.ts
index 2a9b2b46df5d..4e150d0b2c96 100644
--- a/products/experiments/frontend/health/healthPanelChecks.test.ts
+++ b/products/experiments/frontend/health/healthPanelChecks.test.ts
@@ -7,6 +7,7 @@ import {
type HealthPanelCheck,
exposureChecksState,
healthChipState,
+ healthMeasurementText,
healthPanelChecks,
} from './healthPanelChecks'
import type { HealthPanelFinding } from './healthPanelFindings'
@@ -14,7 +15,7 @@ import type { HealthPanelFinding } from './healthPanelFindings'
const check = (
key: ExperimentHealthCheckResultApi['key'],
status: ExperimentHealthCheckResultApi['status']
-): HealthPanelCheck => ({ key, name: key, status, reason: null })
+): ExperimentHealthCheckResultApi => ({ key, name: key, description: '', status, reason: null, measurement: null })
const finding = (severity: HealthPanelFinding['severity']): HealthPanelFinding => ({
code: 'no_metric',
@@ -169,4 +170,23 @@ describe('healthPanelChecks', () => {
])('$name', ({ findings, checks, exposureChecks, expected }) => {
expect(healthChipState(findings, checks, exposureChecks)).toMatchObject(expected)
})
+ it.each([
+ {
+ name: 'a share against its threshold',
+ measurement: { value: 4.23, threshold: 5, unit: 'percentage', warns: 'above' },
+ expected: 'Measured 4.2%. Warns above 5%.',
+ },
+ {
+ name: 'a share too small for one decimal',
+ measurement: { value: 0.04, threshold: 5, unit: 'percentage', warns: 'above' },
+ expected: 'Measured less than 0.1%. Warns above 5%.',
+ },
+ {
+ name: 'a small p-value',
+ measurement: { value: 0.003456, threshold: 0.001, unit: 'p_value', warns: 'below' },
+ expected: 'Measured p = 0.0035. Warns below 0.001.',
+ },
+ ] as const)('healthMeasurementText states $name', ({ measurement, expected }) => {
+ expect(healthMeasurementText(measurement)).toEqual(expected)
+ })
})
diff --git a/products/experiments/frontend/health/healthPanelChecks.ts b/products/experiments/frontend/health/healthPanelChecks.ts
index 0a1506f7a3b0..5b08df56d7a5 100644
--- a/products/experiments/frontend/health/healthPanelChecks.ts
+++ b/products/experiments/frontend/health/healthPanelChecks.ts
@@ -1,11 +1,16 @@
import { pluralize } from 'lib/utils/strings'
-import type { ExperimentExposureQueryResponse } from '~/queries/schema/schema-general'
+import type {
+ ExperimentExposureHealthCheck,
+ ExperimentExposureHealthMeasurement,
+ ExperimentExposureQueryResponse,
+} from '~/queries/schema/schema-general'
import type { ExperimentHealthApi, ExperimentHealthCheckResultApi } from '../generated/api.schemas'
import { type HealthPanelFinding, healthIssues } from './healthPanelFindings'
-export type HealthPanelCheck = ExperimentHealthCheckResultApi
+/** An exposure check from an answer cached before the server added a field has no description or measurement. */
+export type HealthPanelCheck = ExperimentHealthCheckResultApi | ExperimentExposureHealthCheck
/** A check in the list of checks. A check that reported a finding shows as that finding instead. */
export type ListedHealthCheck = HealthPanelCheck & { status: Exclude<HealthPanelCheck['status'], 'finding'> }
@@ -106,3 +111,20 @@ export function healthChipState(
.join(' · ')
return { kind: 'checks', label: label || 'No checks apply yet', checks, findings }
}
+
+function formatMeasuredValue(value: number, unit: ExperimentExposureHealthMeasurement['unit']): string {
+ if (unit === 'p_value') {
+ return `p = ${Number(value.toPrecision(2))}`
+ }
+ // One decimal would print a small share as 0%, which reads as none.
+ return value > 0 && value < 0.1 ? 'less than 0.1%' : `${Number(value.toFixed(1))}%`
+}
+
+/** The value that decided a passed check, against its threshold, for example "Measured 4.2%. Warns above 5%." */
+export function healthMeasurementText(measurement: ExperimentExposureHealthMeasurement): string {
+ const threshold =
+ measurement.unit === 'p_value'
+ ? `${measurement.threshold}`
+ : formatMeasuredValue(measurement.threshold, 'percentage')
+ return `Measured ${formatMeasuredValue(measurement.value, measurement.unit)}. Warns ${measurement.warns} ${threshold}.`
+}
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 0c6f70ae39b7..ae39e1f2feba 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -43145,6 +43145,47 @@ export namespace Schemas {
NotChecked: 'not_checked',
} as const;
+ /**
+ * * `percentage` - Percentage of exposed units
+ * * `p_value` - P-value of a statistical test
+ */
+ export type ExperimentHealthMeasurementUnitEnum = typeof ExperimentHealthMeasurementUnitEnum[keyof typeof ExperimentHealthMeasurementUnitEnum];
+
+
+ export const ExperimentHealthMeasurementUnitEnum = {
+ Percentage: 'percentage',
+ PValue: 'p_value',
+ } as const;
+
+ /**
+ * * `above` - Finding above the threshold
+ * * `below` - Finding below the threshold
+ */
+ export type ExperimentHealthMeasurementSideEnum = typeof ExperimentHealthMeasurementSideEnum[keyof typeof ExperimentHealthMeasurementSideEnum];
+
+
+ export const ExperimentHealthMeasurementSideEnum = {
+ Above: 'above',
+ Below: 'below',
+ } as const;
+
+ export interface ExperimentHealthMeasurement {
+ /** The value that the check measured on the experiment. */
+ value: number;
+ /** The value at which the check reports a finding. */
+ threshold: number;
+ /** percentage: a share of the exposed users or groups, on the 0-100 scale. p_value: the p-value of a statistical test.
+ *
+ * * `percentage` - Percentage of exposed units
+ * * `p_value` - P-value of a statistical test */
+ unit: ExperimentHealthMeasurementUnitEnum;
+ /** The side of the threshold that reports a finding: above or below.
+ *
+ * * `above` - Finding above the threshold
+ * * `below` - Finding below the threshold */
+ warns: ExperimentHealthMeasurementSideEnum;
+ }
+
export interface ExperimentHealthCheckResult {
/** Stable identifier of the check. A check can report several finding codes, so the key is not a finding code.
*
@@ -43174,6 +43215,8 @@ export namespace Schemas {
key: ExperimentHealthCheckKeyEnum;
/** The problem the check looks for, for example 'Metrics not set up'. */
name: string;
+ /** What the check looks for and why it matters. The same for every experiment and every status. */
+ description: string;
/** passed: the check found no problem. finding: the check reported a finding in `findings`. not_applicable: the check does not cover the experiment's current state, for example a check of the changes since launch on a draft. not_checked: the check covers the experiment, but it could not read an input or it failed, so a problem it looks for can be present without a finding.
*
* * `passed` - Passed
@@ -43186,6 +43229,8 @@ export namespace Schemas {
* @nullable
*/
reason: string | null;
+ /** The value that decided a passed check, against its threshold. Null for the other statuses, and for a pass without a value. A finding carries its values in `evidence`. */
+ measurement: ExperimentHealthMeasurement | null;
}
export interface ExperimentHealth {
@@ -43452,6 +43497,33 @@ export namespace Schemas {
TestAccountFilterRemovesExposures: 'test_account_filter_removes_exposures',
} as const;
+ export type ExperimentExposureHealthMeasurementUnit = typeof ExperimentExposureHealthMeasurementUnit[keyof typeof ExperimentExposureHealthMeasurementUnit];
+
+
+ export const ExperimentExposureHealthMeasurementUnit = {
+ Percentage: 'percentage',
+ PValue: 'p_value',
+ } as const;
+
+ export type ExperimentExposureHealthMeasurementSide = typeof ExperimentExposureHealthMeasurementSide[keyof typeof ExperimentExposureHealthMeasurementSide];
+
+
+ export const ExperimentExposureHealthMeasurementSide = {
+ Above: 'above',
+ Below: 'below',
+ } as const;
+
+ export interface ExperimentExposureHealthMeasurement {
+ /** The value at which the check reports a finding. */
+ threshold: number;
+ /** percentage: a share of the exposed users or groups, on the 0-100 scale. p_value: the p-value of a statistical test. */
+ unit: ExperimentExposureHealthMeasurementUnit;
+ /** The value that the check measured on the experiment. */
+ value: number;
+ /** The side of the threshold that reports a finding. */
+ warns: ExperimentExposureHealthMeasurementSide;
+ }
+
export type ExperimentExposureHealthCheckStatus = typeof ExperimentExposureHealthCheckStatus[keyof typeof ExperimentExposureHealthCheckStatus];
@@ -43463,8 +43535,12 @@ export namespace Schemas {
} as const;
export interface ExperimentExposureHealthCheck {
+ /** What the check looks for and why it matters. Missing on an answer cached before the field existed. */
+ description?: string | null;
/** Stable identifier of the check. */
key: ExperimentExposureHealthCheckKey;
+ /** The value that decided a passed check. Null for the other statuses and for a pass without a value. */
+ measurement?: ExperimentExposureHealthMeasurement | null;
/** The problem the check looks for, as the health panel lists it. */
name: string;
/** Why the check did not look, for the statuses not_applicable and not_checked, for example 'Runs after launch'. Null for passed and finding. */