Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(experiments): show what each health check covered (#120202)

Marcel Poelker90900919d7ba
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index 4b3fa57e7f62..dcc1d90b4e4d 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -3276,6 +3276,14 @@ snapshots:
hash: v1.k794b7964.7cb2348662f942bc00e904aca1cf74990ac3b8e25549be308b29e907c156d0eb.WQ5R8L0lbbFdH9oG3RShU2S1nHfj_SHpnz9PrjLAx6A
errortracking-volumesparkline--events-before-data-range--light:
hash: v1.k794b7964.c88c4e3b028915efba3fdbbac7107a063662d8210dfb6ba8667e96d719fa3d4f.tzwEQFa4F1f1O9PGqBkcaqSavbFVgAwPAjyh9Uf9NL8
+ experiments-healthchecksoverview--checks-that-could-not-run--dark:
+ hash: v1.k794b7964.b6036b626d31c7f4863f543dba0ca04d15c90f18c265f49baafd2f22499963cc.Sgb0OPyCSKwPWbcBulpJ21fh6HR_54N0vZ0c4gBaosw
+ experiments-healthchecksoverview--checks-that-could-not-run--light:
+ hash: v1.k794b7964.8c51277755331ab88e7b5bf4bcdc2c9a3c0c9c87fa5ea946f549b70df6c58ce6.vJaps4AlW1x8oWaPFZMzIky0VccXRtIpAw5elQ8Q424
+ experiments-healthchecksoverview--draft--dark:
+ hash: v1.k794b7964.f9bbebb78696c5c06e4f5572959db8f2874162e0e965c8db58cd4669cc7da9b2.O6UuEfwDD1O2LsejFRqibQsm_1_O3vKAK-bnRQ-J1kA
+ experiments-healthchecksoverview--draft--light:
+ hash: v1.k794b7964.b68bab92c07165ef321158a7d33578cc1916c22da7b7653e6a4d1733dc5408c7.H9wWUyEizaXI4E6yp0lk1DzoPLAKSdtVXkzmA59Z4SU
experiments-howtoreadtooltip--intervals-bayesian--dark:
hash: v1.k794b7964.a06d267cee9cb54bafc732335b35d461d3c1a59250c6daa8becace0fcb8492d2.vR5tecOozn6B51xE8LrByAFVsi03J4w-iUu-j2hQUTU
experiments-howtoreadtooltip--intervals-bayesian--light:
@@ -10739,9 +10747,9 @@ snapshots:
scenes-app-experiments--experiment-frequentist-two-variants--light:
hash: v1.k794b7964.4ddea1c7537af9d50c57b4b9a76d4bcba39f743a23b4715683c0ce59d4c24f57.0ueR8XMLqRby_u6I0EyNIiUwXkbjmWMH3vBdoFNYmII
scenes-app-experiments--experiment-health-debug-tab--dark:
- hash: v1.k794b7964.69bf5c8ee5d81e2dfcb1cfea21ca92353e0d300388424ef0654377b3a0a9b7d9.VIWed-fQfBs_yCyAD1To-hWPxDboIK-e_6RLoEUU_0Y
+ hash: v1.k794b7964.8a6f16e2e2dd9eddfcd892926ea82e83e96dc8d4d0a7a691b4bef4bd95890e80.fmB2_WJ6xKnT5hBKzZ8t2Ba1jDP0XvVrjbQitJk9FH0
scenes-app-experiments--experiment-health-debug-tab--light:
- hash: v1.k794b7964.71e39d8c8fda127eb38fb0e94ee9e2563d5aadb136c56e4303c91b39fba117fb.fUslsFx1ChV8LWyNjBK54aGTqPqRP_lCEdWBbi5aWmI
+ hash: v1.k794b7964.699a38e41fb7d0e936e5db90702e786fdde6692b1cccd8d91acc250672c5653c.b8tZKLOytCcqqZ9N3fvxU0Ztf0ELU-dbQ39SiM9sNSo
scenes-app-experiments--experiment-recordings-dropped-metric--dark:
hash: v1.k794b7964.e0e7181f2c9e61a8324eff9aa7a9230fef25b3cf37e0b24ef875780186ed25dc.9VEvZ4iVieVJwmDol0gwtAsr3wgZYW5gj2KXAGZJfiw
scenes-app-experiments--experiment-recordings-dropped-metric--light:
@@ -10863,9 +10871,9 @@ snapshots:
scenes-app-experiments--experiment-with-funnel-metric--light:
hash: v1.k794b7964.bb7e2defaafa4f1741e8da0eeba2700055a382088f7592d51bc7afe34a119aca.FI-i_7oS0vyvJQTgk_AmwDHgSn2Yq46yhWw-2X4uMMI
scenes-app-experiments--experiment-with-health-findings--dark:
- hash: v1.k794b7964.5444bb7f0d8c52bc6b999f76b8161b75b681eb0ca4226ed33ee4ac2af91def97.5RJqZf1qeIelyQZVdfLqcqGBEwjO34GgcFiPLViCikI
+ hash: v1.k794b7964.9e79d086fbb80e5dd6ea5cdd1dd5a99b751ecf25091e3f7aa35f8dd99f284fa8.2m3j7v0Fc12nbT7Z_WqaFuybkRfDJ6kyIF8W1Enjz_k
scenes-app-experiments--experiment-with-health-findings--light:
- hash: v1.k794b7964.93009a5a6f5936341f3f4d80200d2f7ff348643723927344cdcdee3a9766c734.K_mpvll3tb6dM6xQNRGYhRPcUBp0sP_4ZjjOZuhZ5-M
+ hash: v1.k794b7964.2605331e3c8c7d80218d0ca3b669d6a968f3957528daee921c1dd60a7e0cd159.0XfTu5Ki3MwyCul5S-j-LaXwMLz6STR7W6YnDb3N9LE
scenes-app-experiments--experiment-with-long-flag-key--dark:
hash: v1.k794b7964.3fbbef4f86a4d189e4bb3787512ec428e16efe1d529f7f19526a634a876abda3.cYQ-vlh6PsxZDQg2g5KTlLjzUJO0_rxE3oyGtXXJbpM
scenes-app-experiments--experiment-with-long-flag-key--light:
diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json
index 227e0f0b689f..77d2b2e5bfd1 100644
--- a/frontend/src/queries/schema.json
+++ b/frontend/src/queries/schema.json
@@ -9299,6 +9299,13 @@
"date_range": {
"$ref": "#/definitions/DateRange"
},
+ "health_checks": {
+ "description": "The result of each check behind `health_findings`, so that a check that did not run is not read as passed.",
+ "items": {
+ "$ref": "#/definitions/ExperimentExposureHealthCheck"
+ },
+ "type": "array"
+ },
"health_findings": {
"description": "Health check diagnostics that read the exposures: zero exposures, a sample ratio mismatch, and bias. Empty when every check passed.",
"items": {
@@ -25531,6 +25538,38 @@
"required": ["conversionRateInputType"],
"type": "object"
},
+ "ExperimentExposureHealthCheck": {
+ "additionalProperties": false,
+ "description": "What one health check concluded on the exposure answer. Same shape as the experiment's `health.checks`.",
+ "properties": {
+ "key": {
+ "$ref": "#/definitions/ExperimentExposureHealthCheckKey",
+ "description": "Stable identifier of the check."
+ },
+ "name": {
+ "description": "What the check looks at, as the health panel lists it.",
+ "type": "string"
+ },
+ "reason": {
+ "description": "Why the check did not look, for the statuses not_applicable and not_checked, for example 'Runs after launch'. Null for passed and finding.",
+ "type": ["string", "null"]
+ },
+ "status": {
+ "$ref": "#/definitions/ExperimentExposureHealthCheckStatus",
+ "description": "passed: no problem found. finding: a finding in `health_findings`. not_applicable: the check doesn't cover the experiment's state. not_checked: an input was missing or the check failed."
+ }
+ },
+ "required": ["key", "name", "status", "reason"],
+ "type": "object"
+ },
+ "ExperimentExposureHealthCheckKey": {
+ "enum": ["zero_exposures", "srm", "bias_risk_multiple_excluded", "multiple_variants_share"],
+ "type": "string"
+ },
+ "ExperimentExposureHealthCheckStatus": {
+ "enum": ["passed", "finding", "not_applicable", "not_checked"],
+ "type": "string"
+ },
"ExperimentExposureHealthFinding": {
"additionalProperties": false,
"description": "A problem that a health check found in the exposure answer. Same shape as the experiment's `health.findings`.",
@@ -25663,6 +25702,13 @@
"date_range": {
"$ref": "#/definitions/DateRange"
},
+ "health_checks": {
+ "description": "The result of each check behind `health_findings`, so that a check that did not run is not read as passed.",
+ "items": {
+ "$ref": "#/definitions/ExperimentExposureHealthCheck"
+ },
+ "type": "array"
+ },
"health_findings": {
"description": "Health check diagnostics that read the exposures: zero exposures, a sample ratio mismatch, and bias. Empty when every check passed.",
"items": {
@@ -43085,6 +43131,13 @@
"date_range": {
"$ref": "#/definitions/DateRange"
},
+ "health_checks": {
+ "description": "The result of each check behind `health_findings`, so that a check that did not run is not read as passed.",
+ "items": {
+ "$ref": "#/definitions/ExperimentExposureHealthCheck"
+ },
+ "type": "array"
+ },
"health_findings": {
"description": "Health check diagnostics that read the exposures: zero exposures, a sample ratio mismatch, and bias. Empty when every check passed.",
"items": {
diff --git a/frontend/src/queries/schema/schema-general.ts b/frontend/src/queries/schema/schema-general.ts
index 105cbf5fb988..5d123897baa4 100644
--- a/frontend/src/queries/schema/schema-general.ts
+++ b/frontend/src/queries/schema/schema-general.ts
@@ -6358,6 +6358,26 @@ export interface ExperimentExposureHealthFinding {
diagnostic_ref: string | null
}
+export type ExperimentExposureHealthCheckKey =
+ | 'zero_exposures'
+ | 'srm'
+ | 'bias_risk_multiple_excluded'
+ | 'multiple_variants_share'
+
+export type ExperimentExposureHealthCheckStatus = 'passed' | 'finding' | 'not_applicable' | 'not_checked'
+
+/** What one health check concluded on the exposure answer. Same shape as the experiment's `health.checks`. */
+export interface ExperimentExposureHealthCheck {
+ /** Stable identifier of the check. */
+ key: ExperimentExposureHealthCheckKey
+ /** What the check looks at, as the health panel lists it. */
+ name: string
+ /** passed: no problem found. finding: a finding in `health_findings`. not_applicable: the check doesn't cover the experiment's state. not_checked: an input was missing or the check failed. */
+ status: ExperimentExposureHealthCheckStatus
+ /** Why the check did not look, for the statuses not_applicable and not_checked, for example 'Runs after launch'. Null for passed and finding. */
+ reason: string | null
+}
+
export interface ExperimentExposureQueryResponse {
kind: NodeKind.ExperimentExposureQuery
timeseries: ExperimentExposureTimeSeries[]
@@ -6367,6 +6387,8 @@ export interface ExperimentExposureQueryResponse {
bias_risk?: BiasRisk
/** Health check diagnostics that read the exposures: zero exposures, a sample ratio mismatch, and bias. Empty when every check passed. */
health_findings?: ExperimentExposureHealthFinding[]
+ /** The result of each check behind `health_findings`, so that a check that did not run is not read as passed. */
+ health_checks?: ExperimentExposureHealthCheck[]
/** Data warehouse sync warnings — see AnalyticsQueryResponseBase.warnings for semantics. */
warnings?: DataWarehouseSyncWarning[]
}
diff --git a/frontend/src/scenes/experiments/ExperimentView/ExperimentWarningBanners.test.tsx b/frontend/src/scenes/experiments/ExperimentView/ExperimentWarningBanners.test.tsx
index 837bb8b34439..81a117ea21a0 100644
--- a/frontend/src/scenes/experiments/ExperimentView/ExperimentWarningBanners.test.tsx
+++ b/frontend/src/scenes/experiments/ExperimentView/ExperimentWarningBanners.test.tsx
@@ -68,6 +68,7 @@ describe('ExperimentWarningBanner', () => {
diagnostic_ref: null,
},
],
+ checks: [],
})
const { container } = render(
diff --git a/frontend/src/scenes/experiments/ExperimentView/healthFindingReporting.test.tsx b/frontend/src/scenes/experiments/ExperimentView/healthFindingReporting.test.tsx
index 74a80796393c..569d4c74f306 100644
--- a/frontend/src/scenes/experiments/ExperimentView/healthFindingReporting.test.tsx
+++ b/frontend/src/scenes/experiments/ExperimentView/healthFindingReporting.test.tsx
@@ -116,6 +116,7 @@ describe('health finding reporting', () => {
diagnostic_ref: 'A5',
},
],
+ checks: [],
})
logic.actions.loadExposuresSuccess(exposures)
render(
diff --git a/frontend/src/scenes/experiments/experimentLogic.test.ts b/frontend/src/scenes/experiments/experimentLogic.test.ts
index b30e054e8b8d..ea29b31a63f3 100644
--- a/frontend/src/scenes/experiments/experimentLogic.test.ts
+++ b/frontend/src/scenes/experiments/experimentLogic.test.ts
@@ -2496,13 +2496,16 @@ describe('experimentLogic', () => {
{
desc: 'server finding wins over the local rules',
overrides: { ...running, feature_flag: flag(true, multivariantFilters) },
- health: { findings: [healthFinding('flag_off_while_running', 'running_but_flag_disabled')] },
+ health: {
+ findings: [healthFinding('flag_off_while_running', 'running_but_flag_disabled')],
+ checks: [],
+ },
expected: { key: 'running_but_flag_disabled' },
},
{
desc: 'server without a finding wins over the local rules',
overrides: { ...running, feature_flag: flag(false, multivariantFilters) },
- health: { findings: [] },
+ health: { findings: [], checks: [] },
expected: null,
},
{
@@ -2515,19 +2518,20 @@ describe('experimentLogic', () => {
variant_key: 'test',
}),
],
+ checks: [],
},
expected: { key: 'running_but_single_variant_shipped', variantKey: 'test' },
},
{
desc: 'server finding of another code is no flag-state warning',
overrides: { ...running, feature_flag: flag(false, multivariantFilters) },
- health: { findings: [healthFinding('no_metric', null)] },
+ health: { findings: [healthFinding('no_metric', null)], checks: [] },
expected: null,
},
{
desc: 'server findings for a reader without the health findings flag leave the local rules',
overrides: { ...running, feature_flag: flag(false, multivariantFilters) },
- health: { findings: [] },
+ health: { findings: [], checks: [] },
healthFindingsFlag: false,
expected: { key: 'running_but_flag_disabled' },
},
@@ -2558,7 +2562,10 @@ describe('experimentLogic', () => {
{
desc: 'a server finding the local rules miss',
overrides: { ...running, ...newEngineMetrics, feature_flag: flag(true, multivariantFilters) },
- health: { findings: [healthFinding('flag_off_while_running', 'running_but_flag_disabled')] },
+ health: {
+ findings: [healthFinding('flag_off_while_running', 'running_but_flag_disabled')],
+ checks: [],
+ },
expected: {
health_ui: 'panel',
request_failed: false,
@@ -2581,25 +2588,32 @@ describe('experimentLogic', () => {
severity: 'info',
},
],
+ checks: [],
},
expected: { health_findings: ['split_changed_mid_run:not_checked:info:exposure_filters'] },
},
{
desc: 'a server subcode the page does not know',
overrides: { ...running, feature_flag: flag(true, multivariantFilters) },
- health: { findings: [healthFinding('flag_off_while_running', 'running_but_flag_archived')] },
+ health: {
+ findings: [healthFinding('flag_off_while_running', 'running_but_flag_archived')],
+ checks: [],
+ },
expected: { flag_state_browser: null, flag_state_server: 'running_but_flag_archived' },
},
{
desc: 'a reader with the flag on a legacy experiment, which shows the separate warnings',
overrides: { ...running, ...legacyMetrics, feature_flag: flag(true, multivariantFilters) },
- health: { findings: [] },
+ health: { findings: [], checks: [] },
expected: { health_ui: 'warnings', health_finding_count: 0 },
},
{
desc: 'a reader without the health findings flag, who sees the separate warnings',
overrides: { ...running, ...newEngineMetrics, feature_flag: flag(false, multivariantFilters) },
- health: { findings: [healthFinding('flag_off_while_running', 'running_but_flag_disabled')] },
+ health: {
+ findings: [healthFinding('flag_off_while_running', 'running_but_flag_disabled')],
+ checks: [],
+ },
healthFindingsFlag: false,
expected: {
health_ui: 'warnings',
@@ -2643,7 +2657,7 @@ describe('experimentLogic', () => {
)
it('loads the health with each experiment load and after a changed input, comparing only once', async () => {
- const health: ExperimentHealthApi = { findings: [] }
+ const health: ExperimentHealthApi = { findings: [], checks: [] }
useMocks({ get: { [healthUrl]: () => [200, health] } })
const keyedLogic = mountKeyedLogic()
const captureSpy = jest.spyOn(posthog, 'capture').mockReturnValue(undefined as any)
@@ -2802,7 +2816,7 @@ describe('experimentLogic', () => {
metrics_secondary: [],
saved_metrics: [],
})
- logic.actions.loadExperimentHealthSuccess({ findings: [] })
+ logic.actions.loadExperimentHealthSuccess({ findings: [], checks: [] })
logic.actions.reportHealthFindingActedOn({ code: 'bias_risk_multiple_excluded' }, 'use_first_seen_variant')
logic.actions.reportHealthFindingActedOn({ code: 'bias_risk_multiple_excluded' }, 'use_first_seen_variant')
diff --git a/frontend/src/scenes/experiments/experimentLogic.tsx b/frontend/src/scenes/experiments/experimentLogic.tsx
index 1dbeba8bfa47..60e322016e27 100644
--- a/frontend/src/scenes/experiments/experimentLogic.tsx
+++ b/frontend/src/scenes/experiments/experimentLogic.tsx
@@ -74,11 +74,13 @@ import { hasEnded, isLaunched } from 'products/experiments/frontend/experimentSt
import { experimentsHealthRetrieve, experimentsMigrateCreate } from 'products/experiments/frontend/generated/api'
import type { ExperimentHealthApi } from 'products/experiments/frontend/generated/api.schemas'
import {
+ type ExperimentHealthChecksOpenedFrom,
type ExperimentHealthFinding,
type ExperimentHealthFindingActionKind,
type ExperimentHealthFindingOpenKind,
type ExperimentHealthUi,
captureExperimentHealthChecked,
+ captureExperimentHealthChecksOpened,
captureExperimentHealthFindingActedOn,
captureExperimentHealthFindingOpened,
captureExperimentHealthFindingShown,
@@ -86,6 +88,12 @@ import {
experimentWarningFromHealth,
exposureHealthEventProperties,
} from 'products/experiments/frontend/health/experimentHealthFindingEvents'
+import {
+ type ExposureChecksState,
+ type HealthPanelCheck,
+ exposureChecksState,
+ healthPanelChecks,
+} from 'products/experiments/frontend/health/healthPanelChecks'
import { type HealthPanelFinding, healthPanelFindings } from 'products/experiments/frontend/health/healthPanelFindings'
import {
experimentsLogic,
@@ -365,6 +373,8 @@ export interface experimentLogicValues {
freezeExposureLoading: boolean
hasMetrics: boolean
hasMinimumExposureForResults: boolean
+ healthChecks: HealthPanelCheck[] | null
+ healthExposureChecks: ExposureChecksState
healthFindings: HealthPanelFinding[] | null
healthPending: boolean
healthUi: ExperimentHealthUi
@@ -745,6 +755,9 @@ export interface experimentLogicActions {
experiment: Experiment
forceRefresh: boolean
}
+ reportHealthChecksOpened: (openedFrom: ExperimentHealthChecksOpenedFrom) => {
+ openedFrom: ExperimentHealthChecksOpenedFrom
+ }
reportHealthFindingActedOn: (
finding: ExperimentHealthFinding,
actionKind: ExperimentHealthFindingActionKind
@@ -970,6 +983,16 @@ export interface experimentLogicMeta {
exposures: any,
isExperimentDraft: boolean
) => HealthPanelFinding[] | null
+ healthChecks: (
+ shownHealth: ExperimentHealthApi | null,
+ exposures: any,
+ isExperimentDraft: boolean
+ ) => HealthPanelCheck[] | null
+ healthExposureChecks: (
+ isExperimentLaunched: boolean,
+ exposures: any,
+ exposuresLoading: boolean
+ ) => ExposureChecksState
usesNewQueryRunner: (experiment: Experiment) => boolean
hasMinimumExposureForResults: (exposures: any, usesNewQueryRunner: boolean) => boolean
exposureCriteria: (experiment: Experiment) => ExperimentExposureCriteria | undefined
@@ -1059,6 +1082,7 @@ export const experimentLogic = kea<experimentLogicType>([
finding: ExperimentHealthFinding,
actionKind: ExperimentHealthFindingActionKind
) => ({ finding, actionKind }),
+ reportHealthChecksOpened: (openedFrom: ExperimentHealthChecksOpenedFrom) => ({ openedFrom }),
setExperimentMissing: true,
setExperiment: (experiment: Partial<Experiment>) => ({ experiment }),
setLaunchExperimentLoading: (loading: boolean) => ({ loading }),
@@ -1544,6 +1568,15 @@ export const experimentLogic = kea<experimentLogicType>([
reportHealthFindingActedOn: ({ finding, actionKind }) => {
captureExperimentHealthFindingActedOn(values.experiment, finding, values.healthUi, actionKind)
},
+ reportHealthChecksOpened: ({ openedFrom }) => {
+ captureExperimentHealthChecksOpened(
+ values.experiment,
+ values.healthChecks ?? [],
+ values.healthFindings?.length ?? 0,
+ values.healthUi,
+ openedFrom
+ )
+ },
setExperiment: () => {
// A save returns the whole experiment, so compare values: only a changed input makes the findings stale.
if (
@@ -2830,6 +2863,19 @@ export const experimentLogic = kea<experimentLogicType>([
): HealthPanelFinding[] | null =>
healthPending ? [] : healthPanelFindings(shownHealth, exposures, isExperimentDraft),
],
+ healthChecks: [
+ (s) => [s.shownHealth, s.exposures, s.isExperimentDraft],
+ (
+ shownHealth: ExperimentHealthApi | null,
+ exposures: any,
+ isExperimentDraft: boolean
+ ): HealthPanelCheck[] | null => healthPanelChecks(shownHealth, exposures, isExperimentDraft),
+ ],
+ healthExposureChecks: [
+ (s) => [s.isExperimentLaunched, s.exposures, s.exposuresLoading],
+ (isExperimentLaunched: boolean, exposures: any, exposuresLoading: boolean): ExposureChecksState =>
+ exposureChecksState(isExperimentLaunched, exposures, exposuresLoading),
+ ],
usesNewQueryRunner: [
(s) => [s.experiment],
(experiment: Experiment): boolean => {
diff --git a/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx b/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx
index ba3a13b4572d..e1aab1cbeaf3 100644
--- a/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx
+++ b/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx
@@ -38,6 +38,12 @@ const EXPERIMENT_HEALTH = {
diagnostic_ref: 'E3',
},
],
+ checks: [
+ { key: 'split_changed_mid_run', name: 'Variant split changes since launch', status: 'finding', reason: null },
+ { key: 'flag_state', name: 'Feature flag rollout', status: 'passed', reason: null },
+ { key: 'flag_history', name: 'Feature flag history', status: 'passed', reason: null },
+ { key: 'no_metric', name: 'Metrics', status: 'passed', reason: null },
+ ],
}
// Typed as the generated response, so a new required field on the serializer breaks the typecheck here.
diff --git a/frontend/src/scenes/experiments/stories/ExperimentWithHealthFindings.stories.tsx b/frontend/src/scenes/experiments/stories/ExperimentWithHealthFindings.stories.tsx
index 2303aa76a601..59a251f2579e 100644
--- a/frontend/src/scenes/experiments/stories/ExperimentWithHealthFindings.stories.tsx
+++ b/frontend/src/scenes/experiments/stories/ExperimentWithHealthFindings.stories.tsx
@@ -28,6 +28,13 @@ const EXPERIMENT_HEALTH = {
diagnostic_ref: 'A5',
},
],
+ checks: [
+ { key: 'flag_state', name: 'Feature flag rollout', status: 'finding', reason: null },
+ { key: 'flag_history', name: 'Feature flag history', status: 'passed', reason: null },
+ { key: 'split_changed_mid_run', name: 'Variant split changes since launch', status: 'passed', reason: null },
+ { key: 'flag_variants_mismatch', name: 'Feature flag variants', status: 'passed', reason: null },
+ { key: 'no_metric', name: 'Metrics', status: 'passed', reason: null },
+ ],
}
const EXPOSURES_WITH_BIAS_RISK = {
@@ -56,6 +63,12 @@ const EXPOSURES_WITH_BIAS_RISK = {
diagnostic_ref: 'A1',
},
],
+ health_checks: [
+ { key: 'srm', name: 'Split of users across variants', status: 'finding', reason: null },
+ { key: 'bias_risk_multiple_excluded', name: 'Bias from an uneven split', status: 'finding', reason: null },
+ { key: 'zero_exposures', name: 'Exposures', status: 'passed', reason: null },
+ { key: 'multiple_variants_share', name: 'Users exposed to multiple variants', status: 'passed', reason: null },
+ ],
}
const RECALCULATION = recalculationMocks(
diff --git a/posthog/schema.py b/posthog/schema.py
index 8151ed4df5af..5c5c5c50f602 100644
--- a/posthog/schema.py
+++ b/posthog/schema.py
@@ -124,6 +124,8 @@ from posthog.schema_enums import (
ExecutionTarget as ExecutionTarget,
ExperimentApiBreakdownAttributionType as ExperimentApiBreakdownAttributionType,
ExperimentApiPropertyBreakdownType as ExperimentApiPropertyBreakdownType,
+ ExperimentExposureHealthCheckKey as ExperimentExposureHealthCheckKey,
+ ExperimentExposureHealthCheckStatus as ExperimentExposureHealthCheckStatus,
ExperimentExposureHealthFindingActionKind as ExperimentExposureHealthFindingActionKind,
ExperimentExposureHealthFindingCode as ExperimentExposureHealthFindingCode,
ExperimentExposureHealthFindingSeverity as ExperimentExposureHealthFindingSeverity,
@@ -5808,6 +5810,30 @@ class ExperimentApiRetentionStart(BaseModel):
)
+class ExperimentExposureHealthCheck(BaseModel):
+ model_config = ConfigDict(
+ extra="forbid",
+ )
+ key: ExperimentExposureHealthCheckKey = Field(..., description="Stable identifier of the check.")
+ name: str = Field(..., description="What the check looks at, as the health panel lists it.")
+ reason: str | None = Field(
+ ...,
+ description=(
+ "Why the check did not look, for the statuses not_applicable and"
+ " not_checked, for example 'Runs after launch'. Null for passed and"
+ " finding."
+ ),
+ )
+ status: ExperimentExposureHealthCheckStatus = Field(
+ ...,
+ description=(
+ "passed: no problem found. finding: a finding in `health_findings`."
+ " not_applicable: the check doesn't cover the experiment's state."
+ " not_checked: an input was missing or the check failed."
+ ),
+ )
+
+
class ExperimentExposureHealthFinding(BaseModel):
model_config = ConfigDict(
extra="forbid",
@@ -5855,6 +5881,12 @@ class ExperimentExposureQueryResponse(BaseModel):
)
bias_risk: BiasRisk | None = None
date_range: DateRange
+ health_checks: list[ExperimentExposureHealthCheck] | None = Field(
+ default=None,
+ description=(
+ "The result of each check behind `health_findings`, so that a check that did not run is not read as passed."
+ ),
+ )
health_findings: list[ExperimentExposureHealthFinding] | None = Field(
default=None,
description=(
@@ -7505,6 +7537,12 @@ class QueryResponseAlternative19(BaseModel):
)
bias_risk: BiasRisk | None = None
date_range: DateRange
+ health_checks: list[ExperimentExposureHealthCheck] | None = Field(
+ default=None,
+ description=(
+ "The result of each check behind `health_findings`, so that a check that did not run is not read as passed."
+ ),
+ )
health_findings: list[ExperimentExposureHealthFinding] | None = Field(
default=None,
description=(
@@ -12603,6 +12641,12 @@ class CachedExperimentExposureQueryResponse(BaseModel):
description=("What triggered the calculation of the query, leave empty if user/immediate"),
)
date_range: DateRange
+ health_checks: list[ExperimentExposureHealthCheck] | None = Field(
+ default=None,
+ description=(
+ "The result of each check behind `health_findings`, so that a check that did not run is not read as passed."
+ ),
+ )
health_findings: list[ExperimentExposureHealthFinding] | None = Field(
default=None,
description=(
diff --git a/posthog/schema_enums.py b/posthog/schema_enums.py
index 71e1fef5a374..ed5a7e894d53 100644
--- a/posthog/schema_enums.py
+++ b/posthog/schema_enums.py
@@ -1429,6 +1429,20 @@ class ManualMetricType(StrEnum):
MEAN_SUM_OR_AVG = "mean_sum_or_avg"
+class ExperimentExposureHealthCheckKey(StrEnum):
+ ZERO_EXPOSURES = "zero_exposures"
+ SRM = "srm"
+ BIAS_RISK_MULTIPLE_EXCLUDED = "bias_risk_multiple_excluded"
+ MULTIPLE_VARIANTS_SHARE = "multiple_variants_share"
+
+
+class ExperimentExposureHealthCheckStatus(StrEnum):
+ PASSED = "passed"
+ FINDING = "finding"
+ NOT_APPLICABLE = "not_applicable"
+ NOT_CHECKED = "not_checked"
+
+
class ExperimentExposureHealthFindingActionKind(StrEnum):
EDIT_EXPOSURE_CRITERIA = "edit_exposure_criteria"
ADJUST_DISTRIBUTION = "adjust_distribution"
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index 8841ca53afcc..cce83222dfd1 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -9831,11 +9831,89 @@ export namespace Schemas {
*/
diagnostic_ref: string | null;
};
+ /**
+ * * `flag_state` - Flag State
+ * * `flag_history` - Flag History
+ * * `split_changed_mid_run` - Split Changed Mid Run
+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
+ * * `flag_variants_mismatch` - Flag Variants Mismatch
+ * * `forced_variant_release_condition` - Forced Variant Release Condition
+ * * `no_metric` - No Metric
+ * * `zero_exposures` - Zero Exposures
+ * * `srm` - Srm
+ * * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
+ * * `multiple_variants_share` - Multiple Variants Share
+ */
+ export type ExperimentHealthCheckKeyEnum =
+ | "flag_state"
+ | "flag_history"
+ | "split_changed_mid_run"
+ | "bucketing_changed_mid_run"
+ | "release_conditions_changed_mid_run"
+ | "flag_variants_mismatch"
+ | "forced_variant_release_condition"
+ | "no_metric"
+ | "zero_exposures"
+ | "srm"
+ | "bias_risk_multiple_excluded"
+ | "multiple_variants_share";
+ /**
+ * * `passed` - Passed
+ * * `finding` - Finding
+ * * `not_applicable` - Not Applicable
+ * * `not_checked` - Not Checked
+ */
+ export type ExperimentHealthCheckStatusEnum =
+ | "passed"
+ | "finding"
+ | "not_applicable"
+ | "not_checked";
+ export type ExperimentHealthCheckResult = {
+ /**
+ * Stable identifier of the check. A check can report several finding codes, so the key is not a finding code.
+ *
+ * * `flag_state` - Flag State
+ * * `flag_history` - Flag History
+ * * `split_changed_mid_run` - Split Changed Mid Run
+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
+ * * `flag_variants_mismatch` - Flag Variants Mismatch
+ * * `forced_variant_release_condition` - Forced Variant Release Condition
+ * * `no_metric` - No Metric
+ * * `zero_exposures` - Zero Exposures
+ * * `srm` - Srm
+ * * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
+ * * `multiple_variants_share` - Multiple Variants Share
+ */
+ key: ExperimentHealthCheckKeyEnum;
+ /**
+ * What the check looks at, for example 'Feature flag variants'.
+ */
+ name: string;
+ /**
+ * passed: the check found no problem. finding: the check reported a finding in `findings`. not_applicable: the check does not cover the experiment's current state, for example a check of the changes since launch on a draft. not_checked: the check covers the experiment, but it could not read an input or it failed, so a problem it looks for can be present without a finding.
+ *
+ * * `passed` - Passed
+ * * `finding` - Finding
+ * * `not_applicable` - Not Applicable
+ * * `not_checked` - Not Checked
+ */
+ status: ExperimentHealthCheckStatusEnum;
+ /**
+ * Why the check did not look, for the statuses not_applicable and not_checked, for example 'Runs after launch'. Null for passed and finding.
+ */
+ reason: string | null;
+ };
export type ExperimentHealth = {
/**
- * Problems that the health checks found in the experiment's configuration and its feature flag. Empty when every check passed.
+ * Problems that the health checks found in the experiment's configuration and its feature flag. Empty when no check found a problem. An empty list does not mean that every check ran: read `checks`.
*/
findings: Array<ExperimentHealthFinding>;
+ /**
+ * The result of each health check, in the order the checks run. The checks that read the exposure counts are not here: the exposure query returns them as `health_checks`.
+ */
+ checks: Array<ExperimentHealthCheckResult>;
};
/**
* Full experiment representation for the detail, create, and update endpoints.
@@ -10163,6 +10241,34 @@ export namespace Schemas {
*/
tags?: Array<string> | undefined;
};
+ export type ExperimentExposureHealthCheckKey =
+ | "zero_exposures"
+ | "srm"
+ | "bias_risk_multiple_excluded"
+ | "multiple_variants_share";
+ export type ExperimentExposureHealthCheckStatus =
+ | "passed"
+ | "finding"
+ | "not_applicable"
+ | "not_checked";
+ export type ExperimentExposureHealthCheck = {
+ /**
+ * Stable identifier of the check.
+ */
+ key: ExperimentExposureHealthCheckKey;
+ /**
+ * What the check looks at, as the health panel lists it.
+ */
+ name: string;
+ /**
+ * Why the check did not look, for the statuses not_applicable and not_checked, for example 'Runs after launch'. Null for passed and finding.
+ */
+ reason: string | null;
+ /**
+ * passed: no problem found. finding: a finding in `health_findings`. not_applicable: the check doesn't cover the experiment's state. not_checked: an input was missing or the check failed.
+ */
+ status: ExperimentExposureHealthCheckStatus;
+ };
export type ExperimentExposureHealthFindingActionKind =
| "edit_exposure_criteria"
| "adjust_distribution"
@@ -10222,6 +10328,9 @@ export namespace Schemas {
export type ExperimentExposureQueryResponse = {
bias_risk?: (BiasRisk | null) | undefined;
date_range: DateRange;
+ health_checks?:
+ | (Array<ExperimentExposureHealthCheck> | null)
+ | undefined;
health_findings?:
| (Array<ExperimentExposureHealthFinding> | null)
| undefined;
diff --git a/products/experiments/backend/experiment_service.py b/products/experiments/backend/experiment_service.py
index 03acd969c91e..507fb750d312 100644
--- a/products/experiments/backend/experiment_service.py
+++ b/products/experiments/backend/experiment_service.py
@@ -1427,7 +1427,7 @@ class ExperimentService:
try:
flag_age = timezone.now() - experiment.feature_flag.created_at
# Every launch reports the findings it leaves open, also for a launcher who has no health findings.
- findings = evaluate_health(load_health_context(experiment), EXPERIMENT_HEALTH_CHECKS)
+ findings = evaluate_health(load_health_context(experiment), EXPERIMENT_HEALTH_CHECKS).findings
self._report_lifecycle_event(
experiment,
"experiment launched",
diff --git a/products/experiments/backend/facade/api.py b/products/experiments/backend/facade/api.py
index af4067b6c6b3..f4a608de04c4 100644
--- a/products/experiments/backend/facade/api.py
+++ b/products/experiments/backend/facade/api.py
@@ -27,7 +27,7 @@ from products.experiments.backend.hogql_queries.exposure_query_logic import (
)
from products.experiments.backend.models.experiment import Experiment as ExperimentModel
-from .contracts import CreateExperimentInput, Experiment, ExperimentHealthFinding, ExperimentHealthReaderAccess
+from .contracts import CreateExperimentInput, Experiment, ExperimentHealth, ExperimentHealthReaderAccess
def create_experiment(*, team: Team, user: User, input_dto: CreateExperimentInput) -> Experiment:
@@ -127,11 +127,11 @@ def count_running_experiments_on_feature_flag_called(organization_id: UUID) -> i
return count
-def get_experiment_health_findings(
+def get_experiment_health(
*, team_id: int, experiment_id: int, reader_access: ExperimentHealthReaderAccess
-) -> list[ExperimentHealthFinding]:
- """The findings of the health checks that read the experiment, its flag, its linked metrics, and the
- changes of the flag since launch and the flag's recent calls, as far as `reader_access` allows."""
+) -> ExperimentHealth:
+ """The findings and the result of each health check that reads the experiment, its flag, its linked metrics,
+ and the changes of the flag since launch and the flag's recent calls, as far as `reader_access` allows."""
context = load_experiment_health_context(team_id=team_id, experiment_id=experiment_id, reader_access=reader_access)
return evaluate_health(context, EXPERIMENT_HEALTH_CHECKS)
diff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.py
index 3c016fd61976..15389bc22be3 100644
--- a/products/experiments/backend/facade/contracts.py
+++ b/products/experiments/backend/facade/contracts.py
@@ -179,6 +179,47 @@ class ExperimentHealthFinding:
diagnostic_ref: str | None
+class ExperimentHealthCheckKey(LabeledStrEnum):
+ # pinned: the health endpoint and the exposure query return these values, so API callers depend on them. A key
+ # names the check, not a problem: `flag_state` reports four finding codes, and `flag_history` reports its finding
+ # under `split_changed_mid_run`.
+ FLAG_STATE = "flag_state"
+ FLAG_HISTORY = "flag_history"
+ SPLIT_CHANGED_MID_RUN = "split_changed_mid_run"
+ BUCKETING_CHANGED_MID_RUN = "bucketing_changed_mid_run"
+ RELEASE_CONDITIONS_CHANGED_MID_RUN = "release_conditions_changed_mid_run"
+ FLAG_VARIANTS_MISMATCH = "flag_variants_mismatch"
+ FORCED_VARIANT_RELEASE_CONDITION = "forced_variant_release_condition"
+ NO_METRIC = "no_metric"
+ ZERO_EXPOSURES = "zero_exposures"
+ SRM = "srm"
+ BIAS_RISK_MULTIPLE_EXCLUDED = "bias_risk_multiple_excluded"
+ MULTIPLE_VARIANTS_SHARE = "multiple_variants_share"
+
+
+class ExperimentHealthCheckStatus(LabeledStrEnum):
+ PASSED = "passed"
+ FINDING = "finding"
+ NOT_APPLICABLE = "not_applicable"
+ # The check covers the experiment, but it could not read an input or it failed, so a problem can go unreported.
+ NOT_CHECKED = "not_checked"
+
+
+@frozen
+class ExperimentHealthCheckResult:
+ key: ExperimentHealthCheckKey
+ name: str
+ status: ExperimentHealthCheckStatus
+ # Why the check did not look, for the statuses not_applicable and not_checked. None otherwise.
+ reason: str | None
+
+
+@frozen
+class ExperimentHealth:
+ findings: tuple[ExperimentHealthFinding, ...]
+ checks: tuple[ExperimentHealthCheckResult, ...]
+
+
@frozen
class ExperimentHealthReaderAccess:
"""What the caller of the health checks may read, for the checks that read more than the experiment."""
diff --git a/products/experiments/backend/health/checks/bias_risk.py b/products/experiments/backend/health/checks/bias_risk.py
index 76809a40ede5..9d60eb992617 100644
--- a/products/experiments/backend/health/checks/bias_risk.py
+++ b/products/experiments/backend/health/checks/bias_risk.py
@@ -11,6 +11,13 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingSeverity,
)
from products.experiments.backend.health.context import HealthContext
+from products.experiments.backend.health.skips import (
+ NO_FLAG,
+ NOT_AFTER_END,
+ RUNS_AFTER_LAUNCH,
+ CheckSkip,
+ not_applicable,
+)
from products.experiments.backend.variant_distribution import is_evenly_distributed
MULTIPLE_VARIANT_KEY = "$multiple"
@@ -51,6 +58,16 @@ def evaluate_bias_risk(
return BiasRisk(multiple_variant_percentage=multiple_variant_percentage)
+def bias_risk_skip(ctx: HealthContext) -> CheckSkip | None:
+ if ctx.exposures is None:
+ return not_applicable(RUNS_AFTER_LAUNCH)
+ if ctx.flag is None:
+ return not_applicable(NO_FLAG)
+ if ctx.has_ended:
+ return not_applicable(NOT_AFTER_END)
+ return None
+
+
def bias_risk_multiple_excluded(ctx: HealthContext) -> ExperimentHealthFinding | None:
if ctx.exposures is None or ctx.flag is None or ctx.has_ended:
return None
diff --git a/products/experiments/backend/health/checks/flag_change_findings.py b/products/experiments/backend/health/checks/flag_change_findings.py
index 30df574e5156..7b74441db67e 100644
--- a/products/experiments/backend/health/checks/flag_change_findings.py
+++ b/products/experiments/backend/health/checks/flag_change_findings.py
@@ -26,6 +26,13 @@ from products.experiments.backend.health.context import (
FlagHistoryNotCheckedReason,
HealthContext,
)
+from products.experiments.backend.health.skips import (
+ NEEDS_VARIANTS,
+ CheckSkip,
+ flag_setup_skip,
+ not_applicable,
+ not_checked,
+)
# pinned: the subcode of a flag history that was not checked against the exposures. The page sends it as
# `finding_variant` on the health finding events.
@@ -73,6 +80,29 @@ def history_to_check(ctx: HealthContext) -> FlagHistory | FlagHistoryNotChecked
return history
+def flag_history_skip(ctx: HealthContext) -> CheckSkip | None:
+ """The states that `history_to_check` skips, in the order the reader is most likely to act on."""
+ skip = flag_setup_skip(ctx, before_launch=False)
+ if skip is not None or ctx.flag is None:
+ return skip
+ if not any(variant.key for variant in ctx.flag.variants):
+ return not_applicable(NEEDS_VARIANTS)
+ # Only the health endpoint loads the history, and it does for every state that reaches here, so None there
+ # means that the load failed.
+ if ctx.flag_history is None:
+ return not_checked("Couldn't check because the flag history failed to load")
+ return None
+
+
+def flag_change_skip(ctx: HealthContext) -> CheckSkip | None:
+ skip = flag_history_skip(ctx)
+ history = ctx.flag_history
+ # Without the history no change is known. `flag_history_not_readable` reports why.
+ if skip is None and isinstance(history, FlagHistoryNotChecked) and history.reason in _HISTORY_NOT_READ:
+ return not_checked(f"Couldn't check because {not_checked_because(history.reason)}")
+ return skip
+
+
def flag_history_not_readable(ctx: HealthContext) -> ExperimentHealthFinding | None:
"""One finding for every kind of change when the flag's history is not read."""
history = history_to_check(ctx)
diff --git a/products/experiments/backend/health/checks/flag_state.py b/products/experiments/backend/health/checks/flag_state.py
index 8b04c38caafa..79ccb8fb7d56 100644
--- a/products/experiments/backend/health/checks/flag_state.py
+++ b/products/experiments/backend/health/checks/flag_state.py
@@ -25,6 +25,7 @@ from products.experiments.backend.health.context import (
HealthContext,
SplitChange,
)
+from products.experiments.backend.health.skips import NO_FLAG, CheckSkip, not_applicable
from products.experiments.backend.health.split_changes import (
SplitChangeShape,
describe_split,
@@ -56,6 +57,10 @@ _OUTSIDE_ASSIGNMENT_SETTLED_AFTER = timedelta(days=1)
_NOT_RUNNING_BUT_EXPOSED_TITLE = "The experiment is not running, but users are exposed to multiple variants"
+def flag_state_skip(ctx: HealthContext) -> CheckSkip | None:
+ return not_applicable(NO_FLAG) if ctx.flag is None else None
+
+
def flag_state(ctx: HealthContext) -> ExperimentHealthFinding | None:
flag = ctx.flag
if flag is None:
diff --git a/products/experiments/backend/health/checks/flag_variants.py b/products/experiments/backend/health/checks/flag_variants.py
index 5a3f6b177f44..b2325d8ba2a4 100644
--- a/products/experiments/backend/health/checks/flag_variants.py
+++ b/products/experiments/backend/health/checks/flag_variants.py
@@ -12,12 +12,23 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingSeverity,
)
from products.experiments.backend.health.context import HealthContext
+from products.experiments.backend.health.skips import NEEDS_VARIANTS, CheckSkip, flag_setup_skip, not_applicable
# pinned: the subcodes are sent as `finding_variant` on the health finding events, so insights group on them.
FLAG_HAS_NO_VARIANTS = "flag_has_no_variants"
NO_CONTROL_VARIANT = "no_control_variant"
+def flag_variants_skip(ctx: HealthContext) -> CheckSkip | None:
+ skip = flag_setup_skip(ctx, before_launch=True)
+ if skip is not None or ctx.flag is None:
+ return skip
+ # A draft without variants is not set up yet, so it has no variants to compare.
+ if not ctx.is_running and not any(variant.key for variant in ctx.flag.variants):
+ return not_applicable(NEEDS_VARIANTS)
+ return None
+
+
def flag_variants_mismatch(ctx: HealthContext) -> ExperimentHealthFinding | None:
flag = ctx.flag
# A deleted flag has a finding of its own, and the flag has to be restored first. After the end the flag
diff --git a/products/experiments/backend/health/checks/forced_variant.py b/products/experiments/backend/health/checks/forced_variant.py
index a828c2ea8aef..004eceeb8305 100644
--- a/products/experiments/backend/health/checks/forced_variant.py
+++ b/products/experiments/backend/health/checks/forced_variant.py
@@ -35,6 +35,7 @@ from products.experiments.backend.health.pinned_share import (
pinned_share_severity,
pinned_variant_keys,
)
+from products.experiments.backend.health.skips import CheckSkip, flag_setup_skip
from products.experiments.backend.health.split_changes import format_change_time
# pinned: the subcodes are sent as `finding_variant` on the health finding events, so insights group on them.
@@ -66,6 +67,11 @@ def overrides_to_grade(ctx: HealthContext) -> PinnedConditions | None:
return pins.graded
+def forced_variant_skip(ctx: HealthContext) -> CheckSkip | None:
+ # The same states as `_pins`, which returns None for them.
+ return flag_setup_skip(ctx, before_launch=True)
+
+
def forced_variant_release_condition(ctx: HealthContext) -> ExperimentHealthFinding | None:
flag = ctx.flag
pins = _pins(ctx)
diff --git a/products/experiments/backend/health/checks/multiple_variants.py b/products/experiments/backend/health/checks/multiple_variants.py
index 871936979a58..637c1995c727 100644
--- a/products/experiments/backend/health/checks/multiple_variants.py
+++ b/products/experiments/backend/health/checks/multiple_variants.py
@@ -7,6 +7,7 @@ from products.experiments.backend.facade.contracts import (
)
from products.experiments.backend.health.checks.bias_risk import MULTIPLE_VARIANT_KEY
from products.experiments.backend.health.context import HealthContext
+from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckSkip, not_applicable, not_checked
# Some users get more than one ID in any experiment, for example one per device, so a low share is expected.
# The share grows as users return and log in, so a lower line fires on a large part of all experiments. The cases
@@ -17,6 +18,14 @@ MULTIPLE_VARIANTS_SHARE_THRESHOLD = 5.0 # on the 0-100 scale (5.0 = 5%)
MULTIPLE_VARIANTS_MINIMUM_COUNT = 5
+def multiple_variants_share_skip(ctx: HealthContext) -> CheckSkip | None:
+ if ctx.exposures is None:
+ return not_applicable(RUNS_AFTER_LAUNCH)
+ if ctx.exposures.multiple_variant_handling != MultipleVariantHandling.EXCLUDE:
+ return not_checked("Couldn't check because the First seen handling doesn't count these users")
+ return None
+
+
def multiple_variants_share(ctx: HealthContext) -> ExperimentHealthFinding | None:
exposures = ctx.exposures
# Under "First seen" the exposure query gives each user their first variant, so it counts none of these users.
diff --git a/products/experiments/backend/health/checks/no_metric.py b/products/experiments/backend/health/checks/no_metric.py
index 7f8f462353e8..a6cd7f44c707 100644
--- a/products/experiments/backend/health/checks/no_metric.py
+++ b/products/experiments/backend/health/checks/no_metric.py
@@ -5,6 +5,11 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingSeverity,
)
from products.experiments.backend.health.context import HealthContext
+from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckSkip, not_applicable
+
+
+def no_metric_skip(ctx: HealthContext) -> CheckSkip | None:
+ return None if ctx.is_launched else not_applicable(RUNS_AFTER_LAUNCH)
def no_metric(ctx: HealthContext) -> ExperimentHealthFinding | None:
diff --git a/products/experiments/backend/health/checks/srm.py b/products/experiments/backend/health/checks/srm.py
index 8aecf128b59c..b760ab36a4ba 100644
--- a/products/experiments/backend/health/checks/srm.py
+++ b/products/experiments/backend/health/checks/srm.py
@@ -4,10 +4,28 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingSeverity,
)
from products.experiments.backend.health.context import HealthContext
+from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckSkip, not_applicable
SAMPLE_RATIO_MISMATCH_P_VALUE = 0.001
+def srm_skip(ctx: HealthContext) -> CheckSkip | None:
+ if ctx.exposures is None:
+ return not_applicable(RUNS_AFTER_LAUNCH)
+ if ctx.exposures.sample_ratio_mismatch_p_value is None:
+ # The exposure query compares only the variants with a share of the split that the analysis keeps, so after
+ # a variant ships there is nothing to compare however many users are exposed.
+ variants_in_split = [
+ variant
+ for variant in (ctx.flag.variants if ctx.flag else ())
+ if variant.key and (variant.rollout_percentage or 0) > 0 and variant.key not in ctx.excluded_variant_keys
+ ]
+ if len(variants_in_split) < 2:
+ return not_applicable("Needs two or more variants in the split")
+ return not_applicable("Runs once enough users are exposed")
+ return None
+
+
def srm(ctx: HealthContext) -> ExperimentHealthFinding | None:
p_value = ctx.exposures.sample_ratio_mismatch_p_value if ctx.exposures else None
if p_value is None or p_value >= SAMPLE_RATIO_MISMATCH_P_VALUE:
diff --git a/products/experiments/backend/health/checks/zero_exposures.py b/products/experiments/backend/health/checks/zero_exposures.py
index 71a80da6bf33..d3b772d3b802 100644
--- a/products/experiments/backend/health/checks/zero_exposures.py
+++ b/products/experiments/backend/health/checks/zero_exposures.py
@@ -5,11 +5,21 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingSeverity,
)
from products.experiments.backend.health.context import HealthContext
+from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckSkip, not_applicable
# A new experiment often has no exposure in its first hours, so the finding waits as long as the experiments scout does.
ZERO_EXPOSURES_GRACE_HOURS = 24
+def zero_exposures_skip(ctx: HealthContext) -> CheckSkip | None:
+ exposures = ctx.exposures
+ if exposures is None or exposures.hours_since_launch is None or not ctx.is_launched:
+ return not_applicable(RUNS_AFTER_LAUNCH)
+ if exposures.hours_since_launch < ZERO_EXPOSURES_GRACE_HOURS:
+ return not_applicable(f"Runs {ZERO_EXPOSURES_GRACE_HOURS} hours after launch")
+ return None
+
+
def zero_exposures(ctx: HealthContext) -> ExperimentHealthFinding | None:
exposures = ctx.exposures
if exposures is None or exposures.hours_since_launch is None or not ctx.is_launched:
diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.py
index c8be36b2787a..d83a94d8f1fb 100644
--- a/products/experiments/backend/health/registry.py
+++ b/products/experiments/backend/health/registry.py
@@ -1,56 +1,159 @@
from collections.abc import Callable, Sequence
+from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
-from products.experiments.backend.facade.contracts import ExperimentHealthFinding
-from products.experiments.backend.health.checks.bias_risk import bias_risk_multiple_excluded
+from products.experiments.backend.facade.contracts import (
+ ExperimentHealth,
+ ExperimentHealthCheckKey,
+ ExperimentHealthCheckResult,
+ ExperimentHealthCheckStatus,
+ ExperimentHealthFinding,
+)
+from products.experiments.backend.health.checks.bias_risk import bias_risk_multiple_excluded, bias_risk_skip
from products.experiments.backend.health.checks.bucketing_changed import bucketing_changed_mid_run
from products.experiments.backend.health.checks.conditions_changed import release_conditions_changed_mid_run
-from products.experiments.backend.health.checks.flag_change_findings import flag_history_not_readable
-from products.experiments.backend.health.checks.flag_state import flag_state
-from products.experiments.backend.health.checks.flag_variants import flag_variants_mismatch
-from products.experiments.backend.health.checks.forced_variant import forced_variant_release_condition
-from products.experiments.backend.health.checks.multiple_variants import multiple_variants_share
-from products.experiments.backend.health.checks.no_metric import no_metric
+from products.experiments.backend.health.checks.flag_change_findings import (
+ flag_change_skip,
+ flag_history_not_readable,
+ flag_history_skip,
+)
+from products.experiments.backend.health.checks.flag_state import flag_state, flag_state_skip
+from products.experiments.backend.health.checks.flag_variants import flag_variants_mismatch, flag_variants_skip
+from products.experiments.backend.health.checks.forced_variant import (
+ forced_variant_release_condition,
+ forced_variant_skip,
+)
+from products.experiments.backend.health.checks.multiple_variants import (
+ multiple_variants_share,
+ multiple_variants_share_skip,
+)
+from products.experiments.backend.health.checks.no_metric import no_metric, no_metric_skip
from products.experiments.backend.health.checks.split_changed import split_changed_mid_run
-from products.experiments.backend.health.checks.srm import srm
-from products.experiments.backend.health.checks.zero_exposures import zero_exposures
+from products.experiments.backend.health.checks.srm import srm, srm_skip
+from products.experiments.backend.health.checks.zero_exposures import zero_exposures, zero_exposures_skip
from products.experiments.backend.health.context import HealthContext
+from products.experiments.backend.health.skips import CheckSkip
+
+
+@frozen
+class HealthCheck:
+ key: ExperimentHealthCheckKey
+ name: str
+ # Returns None when it finds no problem, and also when the context lacks an input it reads.
+ find: Callable[[HealthContext], ExperimentHealthFinding | None]
+ # Why the check does not look, or None when it looks. Keep in step with the guards of `find`.
+ skip: Callable[[HealthContext], CheckSkip | None]
-HealthCheck = Callable[[HealthContext], ExperimentHealthFinding | None]
EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
- flag_state,
- flag_history_not_readable,
- split_changed_mid_run,
- bucketing_changed_mid_run,
- release_conditions_changed_mid_run,
- flag_variants_mismatch,
- forced_variant_release_condition,
- no_metric,
+ HealthCheck(
+ key=ExperimentHealthCheckKey.FLAG_STATE,
+ name="Feature flag rollout",
+ find=flag_state,
+ skip=flag_state_skip,
+ ),
+ HealthCheck(
+ key=ExperimentHealthCheckKey.FLAG_HISTORY,
+ name="Feature flag history",
+ find=flag_history_not_readable,
+ skip=flag_history_skip,
+ ),
+ HealthCheck(
+ key=ExperimentHealthCheckKey.SPLIT_CHANGED_MID_RUN,
+ name="Variant split changes since launch",
+ find=split_changed_mid_run,
+ skip=flag_change_skip,
+ ),
+ HealthCheck(
+ key=ExperimentHealthCheckKey.BUCKETING_CHANGED_MID_RUN,
+ name="Flag key and bucketing changes since launch",
+ find=bucketing_changed_mid_run,
+ skip=flag_change_skip,
+ ),
+ HealthCheck(
+ key=ExperimentHealthCheckKey.RELEASE_CONDITIONS_CHANGED_MID_RUN,
+ name="Release condition changes since launch",
+ find=release_conditions_changed_mid_run,
+ skip=flag_change_skip,
+ ),
+ HealthCheck(
+ key=ExperimentHealthCheckKey.FLAG_VARIANTS_MISMATCH,
+ name="Feature flag variants",
+ find=flag_variants_mismatch,
+ skip=flag_variants_skip,
+ ),
+ HealthCheck(
+ key=ExperimentHealthCheckKey.FORCED_VARIANT_RELEASE_CONDITION,
+ name="Release conditions that force a variant",
+ find=forced_variant_release_condition,
+ skip=forced_variant_skip,
+ ),
+ HealthCheck(
+ key=ExperimentHealthCheckKey.NO_METRIC,
+ name="Metrics",
+ find=no_metric,
+ skip=no_metric_skip,
+ ),
)
# The exposure query runs these on its answer. They must not include the checks above: the exposure answer
# is cached for a day, and a flag edit would not clear their findings.
EXPOSURE_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
- zero_exposures,
- srm,
- bias_risk_multiple_excluded,
- multiple_variants_share,
+ HealthCheck(
+ key=ExperimentHealthCheckKey.ZERO_EXPOSURES,
+ name="Exposures",
+ find=zero_exposures,
+ skip=zero_exposures_skip,
+ ),
+ HealthCheck(
+ key=ExperimentHealthCheckKey.SRM,
+ name="Split of users across variants",
+ find=srm,
+ skip=srm_skip,
+ ),
+ HealthCheck(
+ key=ExperimentHealthCheckKey.BIAS_RISK_MULTIPLE_EXCLUDED,
+ name="Bias from an uneven split",
+ find=bias_risk_multiple_excluded,
+ skip=bias_risk_skip,
+ ),
+ HealthCheck(
+ key=ExperimentHealthCheckKey.MULTIPLE_VARIANTS_SHARE,
+ name="Users exposed to multiple variants",
+ find=multiple_variants_share,
+ skip=multiple_variants_share_skip,
+ ),
)
-def evaluate(ctx: HealthContext, checks: Sequence[HealthCheck]) -> list[ExperimentHealthFinding]:
- """Runs the given checks. A check returns None when the context lacks an input it reads."""
+_CHECK_FAILED = "Couldn't check because the check failed"
+
+
+def evaluate(ctx: HealthContext, checks: Sequence[HealthCheck]) -> ExperimentHealth:
findings: list[ExperimentHealthFinding] = []
+ results: list[ExperimentHealthCheckResult] = []
for check in checks:
try:
- finding = check(ctx)
+ # Every check runs whatever its skip says, so a skip that is out of step with the check's guards can't
+ # hide a finding. The skip is read only without a finding, so a failing skip can't hide one either.
+ finding = check.find(ctx)
+ skip = check.skip(ctx) if finding is None else None
except Exception as error:
# The findings ride on the health action, the exposure answer and `experiment launched`, so one
# failing check must not fail them or hide the findings of the other checks.
capture_exception(error)
+ results.append(_result(check, ExperimentHealthCheckStatus.NOT_CHECKED, _CHECK_FAILED))
continue
if finding is not None:
findings.append(finding)
- return findings
+ results.append(_result(check, ExperimentHealthCheckStatus.FINDING, None))
+ elif skip is not None:
+ results.append(_result(check, skip.status, skip.reason))
+ else:
+ results.append(_result(check, ExperimentHealthCheckStatus.PASSED, None))
+ return ExperimentHealth(findings=tuple(findings), checks=tuple(results))
+
+
+def _result(check: HealthCheck, status: ExperimentHealthCheckStatus, reason: str | None) -> ExperimentHealthCheckResult:
+ return ExperimentHealthCheckResult(key=check.key, name=check.name, status=status, reason=reason)
diff --git a/products/experiments/backend/health/skips.py b/products/experiments/backend/health/skips.py
new file mode 100644
index 000000000000..b08ec8c26e89
--- /dev/null
+++ b/products/experiments/backend/health/skips.py
@@ -0,0 +1,44 @@
+from typing import Literal
+
+from posthog.dataclasses import frozen
+
+from products.experiments.backend.facade.contracts import ExperimentHealthCheckStatus
+from products.experiments.backend.health.context import HealthContext
+
+
+@frozen
+class CheckSkip:
+ status: Literal[ExperimentHealthCheckStatus.NOT_APPLICABLE, ExperimentHealthCheckStatus.NOT_CHECKED]
+ reason: str
+
+
+RUNS_AFTER_LAUNCH = "Runs after launch"
+NOT_AFTER_END = "Doesn't run after the experiment ends"
+NOT_ON_ARCHIVED = "Doesn't run on archived experiments"
+NO_FLAG = "The experiment has no feature flag"
+FLAG_DELETED = "The feature flag is deleted"
+NEEDS_VARIANTS = "Runs once the feature flag has variants"
+
+
+def not_applicable(reason: str) -> CheckSkip:
+ return CheckSkip(status=ExperimentHealthCheckStatus.NOT_APPLICABLE, reason=reason)
+
+
+def not_checked(reason: str) -> CheckSkip:
+ return CheckSkip(status=ExperimentHealthCheckStatus.NOT_CHECKED, reason=reason)
+
+
+def flag_setup_skip(ctx: HealthContext, *, before_launch: bool) -> CheckSkip | None:
+ """The skip of a check of the flag's setup. After the end the flag often holds the shipped decision, not the
+ setup of the run, so these checks stop there."""
+ if ctx.archived:
+ return not_applicable(NOT_ON_ARCHIVED)
+ if ctx.flag is None:
+ return not_applicable(NO_FLAG)
+ if ctx.flag.deleted:
+ return not_applicable(FLAG_DELETED)
+ if ctx.has_ended:
+ return not_applicable(NOT_AFTER_END)
+ if not before_launch and not ctx.is_launched:
+ return not_applicable(RUNS_AFTER_LAUNCH)
+ return None
diff --git a/products/experiments/backend/health/test/test_registry.py b/products/experiments/backend/health/test/test_registry.py
index 9524f515901d..6402aa73756c 100644
--- a/products/experiments/backend/health/test/test_registry.py
+++ b/products/experiments/backend/health/test/test_registry.py
@@ -1,4 +1,5 @@
from dataclasses import replace
+from datetime import UTC, datetime
from typing import Any
from unittest import TestCase
@@ -8,13 +9,26 @@ from parameterized import parameterized
from posthog.schema import MultipleVariantHandling
-from products.experiments.backend.facade.contracts import ExperimentHealthFindingCode
+from products.experiments.backend.facade.contracts import (
+ ExperimentHealthCheckKey,
+ ExperimentHealthCheckStatus,
+ ExperimentHealthFindingCode,
+)
from products.experiments.backend.health import registry
-from products.experiments.backend.health.checks.bias_risk import bias_risk_multiple_excluded
-from products.experiments.backend.health.checks.multiple_variants import multiple_variants_share
-from products.experiments.backend.health.context import ExposureTotals, FlagState, FlagVariant, HealthContext
-from products.experiments.backend.health.registry import EXPOSURE_HEALTH_CHECKS, evaluate
-from products.experiments.backend.health.test.factories import RUNNING_CONTEXT
+from products.experiments.backend.health.context import (
+ ExposureTotals,
+ FlagHistoryNotCheckedReason,
+ FlagState,
+ FlagVariant,
+ HealthContext,
+)
+from products.experiments.backend.health.registry import (
+ EXPERIMENT_HEALTH_CHECKS,
+ EXPOSURE_HEALTH_CHECKS,
+ HealthCheck,
+ evaluate,
+)
+from products.experiments.backend.health.test.factories import DRAFT, RUNNING_CONTEXT, flag_history
UNEVEN_SPLIT_TOTALS = ExposureTotals(
total_exposures={"control": 800, "test": 200, "$multiple": 20},
@@ -24,23 +38,27 @@ UNEVEN_SPLIT_TOTALS = ExposureTotals(
counts_groups=False,
)
-RUNNING_UNEVEN_SPLIT = replace(
- RUNNING_CONTEXT,
- flag=FlagState(
- active=True,
- deleted=False,
- release_groups=(),
- variants=(
- FlagVariant(key="control", rollout_percentage=80),
- FlagVariant(key="test", rollout_percentage=20),
- ),
- reachable_conditions=(),
+UNEVEN_FLAG = FlagState(
+ active=True,
+ deleted=False,
+ release_groups=(),
+ variants=(
+ FlagVariant(key="control", rollout_percentage=80),
+ FlagVariant(key="test", rollout_percentage=20),
),
- exposures=UNEVEN_SPLIT_TOTALS,
+ reachable_conditions=(),
)
+RUNNING_UNEVEN_SPLIT = replace(RUNNING_CONTEXT, flag=UNEVEN_FLAG, exposures=UNEVEN_SPLIT_TOTALS)
+
NO_EXPOSURES = replace(UNEVEN_SPLIT_TOTALS, total_exposures={"control": 0, "test": 0})
ENDED: dict[str, Any] = {"is_running": False, "has_ended": True}
+LAUNCHED_AT = datetime(2026, 10, 1, tzinfo=UTC)
+
+Key = ExperimentHealthCheckKey
+Status = ExperimentHealthCheckStatus
+NOT_LOOKED = (Status.NOT_APPLICABLE, Status.NOT_CHECKED)
+FLAG_CHANGE_KEYS = (Key.SPLIT_CHANGED_MID_RUN, Key.BUCKETING_CHANGED_MID_RUN, Key.RELEASE_CONDITIONS_CHANGED_MID_RUN)
EVEN_SPLIT_FLAG = FlagState(
active=True,
deleted=False,
@@ -62,6 +80,10 @@ def _raise(_ctx: HealthContext) -> Any:
raise ValueError("malformed")
+MULTIPLE_VARIANTS_CHECK = next(check for check in EXPOSURE_HEALTH_CHECKS if check.key == Key.MULTIPLE_VARIANTS_SHARE)
+FAILING_CHECK = HealthCheck(key=Key.FLAG_STATE, name="Failing", find=_raise, skip=lambda _ctx: None)
+
+
class TestEvaluate(TestCase):
@parameterized.expand(
[
@@ -119,9 +141,93 @@ class TestEvaluate(TestCase):
def test_exposure_checks(
self, _name: str, changes: dict[str, Any], expected: list[ExperimentHealthFindingCode]
) -> None:
- findings = evaluate(replace(RUNNING_UNEVEN_SPLIT, **changes), EXPOSURE_HEALTH_CHECKS)
+ health = evaluate(replace(RUNNING_UNEVEN_SPLIT, **changes), EXPOSURE_HEALTH_CHECKS)
+
+ self.assertEqual([finding.code for finding in health.findings], expected)
+
+ @parameterized.expand(
+ [
+ (
+ "a_draft_is_not_checked_for_what_needs_a_launch",
+ {**DRAFT, "flag": replace(UNEVEN_FLAG, active=False), "exposures": None},
+ {
+ Key.FLAG_STATE: Status.PASSED,
+ Key.FLAG_VARIANTS_MISMATCH: Status.PASSED,
+ Key.FLAG_HISTORY: Status.NOT_APPLICABLE,
+ **dict.fromkeys(FLAG_CHANGE_KEYS, Status.NOT_APPLICABLE),
+ Key.NO_METRIC: Status.NOT_APPLICABLE,
+ Key.ZERO_EXPOSURES: Status.NOT_APPLICABLE,
+ Key.SRM: Status.NOT_APPLICABLE,
+ Key.MULTIPLE_VARIANTS_SHARE: Status.NOT_APPLICABLE,
+ },
+ ),
+ (
+ "flag_changes_are_not_checked_without_the_history",
+ {
+ "flag_history": flag_history(
+ [], launched_at=LAUNCHED_AT, not_checked=FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS
+ )
+ },
+ {Key.FLAG_HISTORY: Status.FINDING, **dict.fromkeys(FLAG_CHANGE_KEYS, Status.NOT_CHECKED)},
+ ),
+ (
+ "flag_changes_pass_with_a_history_without_changes",
+ {"flag_history": flag_history([], launched_at=LAUNCHED_AT)},
+ {Key.FLAG_HISTORY: Status.PASSED, **dict.fromkeys(FLAG_CHANGE_KEYS, Status.PASSED)},
+ ),
+ (
+ "zero_exposures_is_not_checked_in_the_first_day",
+ {"exposures": replace(NO_EXPOSURES, hours_since_launch=23.9)},
+ {Key.ZERO_EXPOSURES: Status.NOT_APPLICABLE, Key.SRM: Status.NOT_APPLICABLE},
+ ),
+ (
+ "multiple_variants_are_not_counted_under_first_seen",
+ {
+ "exposures": replace(
+ _totals_with_multiple(60, 1000), multiple_variant_handling=MultipleVariantHandling.FIRST_SEEN
+ )
+ },
+ {Key.MULTIPLE_VARIANTS_SHARE: Status.NOT_CHECKED, Key.BIAS_RISK_MULTIPLE_EXCLUDED: Status.PASSED},
+ ),
+ (
+ "zero_exposures_finds_a_day_after_launch",
+ {"exposures": replace(NO_EXPOSURES, hours_since_launch=24)},
+ {Key.ZERO_EXPOSURES: Status.FINDING},
+ ),
+ ]
+ )
+ def test_check_statuses(self, _name: str, changes: dict[str, Any], expected: dict[Key, Status]) -> None:
+ health = evaluate(
+ replace(RUNNING_UNEVEN_SPLIT, **changes), (*EXPERIMENT_HEALTH_CHECKS, *EXPOSURE_HEALTH_CHECKS)
+ )
+
+ statuses = {check.key: check.status for check in health.checks}
+ self.assertEqual({key: statuses[key] for key in expected}, expected)
+ self.assertEqual(
+ [check.key for check in health.checks if (check.reason is None) == (check.status in NOT_LOOKED)], []
+ )
+
+ @parameterized.expand(
+ [
+ ("too_few_exposures", UNEVEN_FLAG, "Runs once enough users are exposed"),
+ (
+ "one_variant_in_the_split_after_a_ship",
+ replace(
+ UNEVEN_FLAG,
+ variants=(
+ FlagVariant(key="control", rollout_percentage=0),
+ FlagVariant(key="test", rollout_percentage=100),
+ ),
+ ),
+ "Needs two or more variants in the split",
+ ),
+ ]
+ )
+ def test_srm_says_why_it_did_not_run(self, _name: str, flag: FlagState, reason: str) -> None:
+ health = evaluate(replace(RUNNING_UNEVEN_SPLIT, flag=flag), EXPOSURE_HEALTH_CHECKS)
- self.assertEqual([finding.code for finding in findings], expected)
+ srm = next(check for check in health.checks if check.key == Key.SRM)
+ self.assertEqual((srm.status, srm.reason), (Status.NOT_APPLICABLE, reason))
@parameterized.expand(
[
@@ -131,15 +237,16 @@ class TestEvaluate(TestCase):
)
def test_multiple_variants_share_names_the_counted_unit(self, _name: str, counts_groups: bool, title: str) -> None:
exposures = replace(_totals_with_multiple(60, 1000), counts_groups=counts_groups)
- findings = evaluate(replace(RUNNING_UNEVEN_SPLIT, **ENDED, exposures=exposures), (multiple_variants_share,))
+ health = evaluate(replace(RUNNING_UNEVEN_SPLIT, **ENDED, exposures=exposures), (MULTIPLE_VARIANTS_CHECK,))
- self.assertEqual([finding.title for finding in findings], [title])
+ self.assertEqual([finding.title for finding in health.findings], [title])
def test_a_failing_check_does_not_hide_the_other_findings(self) -> None:
with patch.object(registry, "capture_exception") as capture:
- findings = evaluate(RUNNING_UNEVEN_SPLIT, (_raise, bias_risk_multiple_excluded))
+ health = evaluate(RUNNING_UNEVEN_SPLIT, (FAILING_CHECK, *EXPOSURE_HEALTH_CHECKS))
self.assertEqual(
- [finding.code for finding in findings], [ExperimentHealthFindingCode.BIAS_RISK_MULTIPLE_EXCLUDED]
+ [finding.code for finding in health.findings], [ExperimentHealthFindingCode.BIAS_RISK_MULTIPLE_EXCLUDED]
)
+ self.assertEqual(health.checks[0].status, Status.NOT_CHECKED)
capture.assert_called_once()
diff --git a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
index 5a764d491e84..8590a4ace473 100644
--- a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
+++ b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
@@ -9,6 +9,9 @@ from posthog.schema import (
BiasRisk,
CachedExperimentExposureQueryResponse,
DateRange,
+ ExperimentExposureHealthCheck,
+ ExperimentExposureHealthCheckKey,
+ ExperimentExposureHealthCheckStatus,
ExperimentExposureHealthFinding,
ExperimentExposureHealthFindingActionKind,
ExperimentExposureHealthFindingCode,
@@ -32,6 +35,7 @@ from posthog.hogql_queries.utils.query_date_range import QueryDateRange
from posthog.models.team.extensions import get_or_create_team_extension
from products.analytics_platform.backend.lazy_computation.lazy_computation_executor import LazyComputationResult
+from products.experiments.backend.facade.contracts import ExperimentHealth
from products.experiments.backend.health.checks.bias_risk import evaluate_bias_risk
from products.experiments.backend.health.context import ExposureTotals, load_health_context
from products.experiments.backend.health.registry import (
@@ -305,8 +309,11 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
total_exposures=total_exposures,
)
- def _evaluate_health(self, exposures: ExposureTotals) -> list[ExperimentExposureHealthFinding]:
- findings = evaluate_health(load_health_context(self.experiment, exposures), EXPOSURE_HEALTH_CHECKS)
+ def _evaluate_health(self, exposures: ExposureTotals) -> ExperimentHealth:
+ return evaluate_health(load_health_context(self.experiment, exposures), EXPOSURE_HEALTH_CHECKS)
+
+ @staticmethod
+ def _health_findings(health: ExperimentHealth) -> list[ExperimentExposureHealthFinding]:
return [
ExperimentExposureHealthFinding(
code=ExperimentExposureHealthFindingCode(finding.code.value),
@@ -318,7 +325,19 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
actions=[ExperimentExposureHealthFindingActionKind(action.value) for action in finding.actions],
diagnostic_ref=finding.diagnostic_ref,
)
- for finding in findings
+ for finding in health.findings
+ ]
+
+ @staticmethod
+ def _health_checks(health: ExperimentHealth) -> list[ExperimentExposureHealthCheck]:
+ return [
+ ExperimentExposureHealthCheck(
+ key=ExperimentExposureHealthCheckKey(check.key.value),
+ name=check.name,
+ status=ExperimentExposureHealthCheckStatus(check.status.value),
+ reason=check.reason,
+ )
+ for check in health.checks
]
@experiment_error_handler
@@ -390,7 +409,7 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
).multiple_variant_handling
sample_ratio_mismatch = self._calculate_srm(total_exposures)
bias_risk = self._evaluate_bias_risk(total_exposures, multiple_variant_handling)
- health_findings = self._evaluate_health(
+ health = self._evaluate_health(
ExposureTotals(
total_exposures=total_exposures,
multiple_variant_handling=multiple_variant_handling,
@@ -406,7 +425,8 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
date_range=self.date_range,
sample_ratio_mismatch=sample_ratio_mismatch,
bias_risk=bias_risk,
- health_findings=health_findings,
+ health_findings=self._health_findings(health),
+ health_checks=self._health_checks(health),
)
def to_query(self) -> ast.SelectQuery:
diff --git a/products/experiments/backend/presentation/serializers.py b/products/experiments/backend/presentation/serializers.py
index f1f2fd91b7e7..23c1935474e6 100644
--- a/products/experiments/backend/presentation/serializers.py
+++ b/products/experiments/backend/presentation/serializers.py
@@ -45,6 +45,8 @@ from products.ai_observability.backend.models.llm_prompt import LLMPrompt
from products.experiments.backend.experiment_service import ExperimentService
from products.experiments.backend.facade.contracts import (
CreateExperimentInput,
+ ExperimentHealthCheckKey,
+ ExperimentHealthCheckStatus,
ExperimentHealthFindingActionKind,
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
@@ -242,12 +244,46 @@ class ExperimentHealthFindingSerializer(serializers.Serializer):
)
+class ExperimentHealthCheckResultSerializer(serializers.Serializer):
+ key = serializers.ChoiceField(
+ choices=ExperimentHealthCheckKey.choices,
+ help_text=(
+ "Stable identifier of the check. A check can report several finding codes, so the key is not a "
+ "finding code."
+ ),
+ )
+ name = serializers.CharField(help_text="What the check looks at, for example 'Feature flag variants'.")
+ status = serializers.ChoiceField(
+ choices=ExperimentHealthCheckStatus.choices,
+ help_text=(
+ "passed: the check found no problem. finding: the check reported a finding in `findings`. "
+ "not_applicable: the check does not cover the experiment's current state, for example a check of the "
+ "changes since launch on a draft. not_checked: the check covers the experiment, but it could not read "
+ "an input or it failed, so a problem it looks for can be present without a finding."
+ ),
+ )
+ reason = serializers.CharField(
+ allow_null=True,
+ help_text=(
+ "Why the check did not look, for the statuses not_applicable and not_checked, for example "
+ "'Runs after launch'. Null for passed and finding."
+ ),
+ )
+
+
class ExperimentHealthSerializer(serializers.Serializer):
findings = ExperimentHealthFindingSerializer(
many=True,
help_text=(
"Problems that the health checks found in the experiment's configuration and its feature flag. "
- "Empty when every check passed."
+ "Empty when no check found a problem. An empty list does not mean that every check ran: read `checks`."
+ ),
+ )
+ checks = ExperimentHealthCheckResultSerializer(
+ many=True,
+ help_text=(
+ "The result of each health check, in the order the checks run. The checks that read the exposure "
+ "counts are not here: the exposure query returns them as `health_checks`."
),
)
diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.py
index 865a2db80f5f..2b7c4cc83b54 100644
--- a/products/experiments/backend/presentation/views.py
+++ b/products/experiments/backend/presentation/views.py
@@ -78,7 +78,7 @@ from products.access_control.backend.facade.user_access_control import (
from products.access_control.backend.presentation.access_control import AccessControlViewSetMixin
from products.approvals.backend.mixins import ApprovalHandlingMixin
from products.experiments.backend.experiment_service import ExperimentService, ExperimentVersionConflict
-from products.experiments.backend.facade.api import get_experiment_health_findings, get_experiment_health_inputs
+from products.experiments.backend.facade.api import get_experiment_health, get_experiment_health_inputs
from products.experiments.backend.facade.contracts import ExperimentHealthReaderAccess
from products.experiments.backend.facade.legacy_migration import (
LegacyMigrationError,
@@ -911,7 +911,9 @@ class EnterpriseExperimentsViewSet(
def health(self, request: Request, *args: Any, **kwargs: Any) -> Response:
"""
Health check findings for this experiment: problems in its configuration and its feature flag
- that keep it from producing trustworthy results, each with a fix.
+ that keep it from producing trustworthy results, each with a fix. `checks` gives the result of
+ each check, so a check that found no problem can be told from one that did not apply or could
+ not read an input.
The checks here read the experiment, its flag, its linked metrics, and the changes of the
flag's variant split since launch. The checks that read exposure counts come with the
@@ -944,12 +946,12 @@ class EnterpriseExperimentsViewSet(
`assigned_by_property` replaces `all_conditions_pinned`.
"""
experiment: Experiment = self.get_object()
- findings = get_experiment_health_findings(
+ health = get_experiment_health(
team_id=self.team_id,
experiment_id=experiment.id,
reader_access=self._health_reader_access(request, experiment),
)
- return Response(ExperimentHealthSerializer({"findings": findings}).data)
+ return Response(ExperimentHealthSerializer(health).data)
@extend_schema(
request=None,
diff --git a/products/experiments/backend/test/test_presentation_api.py b/products/experiments/backend/test/test_presentation_api.py
index 7077df71ce41..14da9bf28115 100644
--- a/products/experiments/backend/test/test_presentation_api.py
+++ b/products/experiments/backend/test/test_presentation_api.py
@@ -844,6 +844,7 @@ class TestExperimentCRUD(_HoistFlagConfigClientMixin, APILicensedTest):
self.assertEqual([(finding["code"], finding["subcode"]) for finding in health["findings"]], expected)
self.assertEqual(health["findings"][0]["actions"], ["open_feature_flag"])
self.assertEqual(health["findings"][0]["diagnostic_ref"], "A5")
+ self.assertEqual((health["checks"][0]["key"], health["checks"][0]["status"]), ("flag_state", "finding"))
@parameterized.expand(
[
diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.ts
index ed704780760b..4de5c0e99585 100644
--- a/products/experiments/frontend/generated/api.schemas.ts
+++ b/products/experiments/frontend/generated/api.schemas.ts
@@ -2385,9 +2385,91 @@ export interface ExperimentHealthFindingApi {
diagnostic_ref: string | null
}
+/**
+ * * `flag_state` - Flag State
+ * * `flag_history` - Flag History
+ * * `split_changed_mid_run` - Split Changed Mid Run
+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
+ * * `flag_variants_mismatch` - Flag Variants Mismatch
+ * * `forced_variant_release_condition` - Forced Variant Release Condition
+ * * `no_metric` - No Metric
+ * * `zero_exposures` - Zero Exposures
+ * * `srm` - Srm
+ * * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
+ * * `multiple_variants_share` - Multiple Variants Share
+ */
+export type ExperimentHealthCheckKeyEnumApi =
+ (typeof ExperimentHealthCheckKeyEnumApi)[keyof typeof ExperimentHealthCheckKeyEnumApi]
+
+export const ExperimentHealthCheckKeyEnumApi = {
+ FlagState: 'flag_state',
+ FlagHistory: 'flag_history',
+ SplitChangedMidRun: 'split_changed_mid_run',
+ BucketingChangedMidRun: 'bucketing_changed_mid_run',
+ ReleaseConditionsChangedMidRun: 'release_conditions_changed_mid_run',
+ FlagVariantsMismatch: 'flag_variants_mismatch',
+ ForcedVariantReleaseCondition: 'forced_variant_release_condition',
+ NoMetric: 'no_metric',
+ ZeroExposures: 'zero_exposures',
+ Srm: 'srm',
+ BiasRiskMultipleExcluded: 'bias_risk_multiple_excluded',
+ MultipleVariantsShare: 'multiple_variants_share',
+} as const
+
+/**
+ * * `passed` - Passed
+ * * `finding` - Finding
+ * * `not_applicable` - Not Applicable
+ * * `not_checked` - Not Checked
+ */
+export type ExperimentHealthCheckStatusEnumApi =
+ (typeof ExperimentHealthCheckStatusEnumApi)[keyof typeof ExperimentHealthCheckStatusEnumApi]
+
+export const ExperimentHealthCheckStatusEnumApi = {
+ Passed: 'passed',
+ Finding: 'finding',
+ NotApplicable: 'not_applicable',
+ NotChecked: 'not_checked',
+} as const
+
+export interface ExperimentHealthCheckResultApi {
+ /** Stable identifier of the check. A check can report several finding codes, so the key is not a finding code.
+ *
+ * * `flag_state` - Flag State
+ * * `flag_history` - Flag History
+ * * `split_changed_mid_run` - Split Changed Mid Run
+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
+ * * `flag_variants_mismatch` - Flag Variants Mismatch
+ * * `forced_variant_release_condition` - Forced Variant Release Condition
+ * * `no_metric` - No Metric
+ * * `zero_exposures` - Zero Exposures
+ * * `srm` - Srm
+ * * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
+ * * `multiple_variants_share` - Multiple Variants Share */
+ key: ExperimentHealthCheckKeyEnumApi
+ /** What the check looks at, for example 'Feature flag variants'. */
+ name: string
+ /** passed: the check found no problem. finding: the check reported a finding in `findings`. not_applicable: the check does not cover the experiment's current state, for example a check of the changes since launch on a draft. not_checked: the check covers the experiment, but it could not read an input or it failed, so a problem it looks for can be present without a finding.
+ *
+ * * `passed` - Passed
+ * * `finding` - Finding
+ * * `not_applicable` - Not Applicable
+ * * `not_checked` - Not Checked */
+ status: ExperimentHealthCheckStatusEnumApi
+ /**
+ * Why the check did not look, for the statuses not_applicable and not_checked, for example 'Runs after launch'. Null for passed and finding.
+ * @nullable
+ */
+ reason: string | null
+}
+
export interface ExperimentHealthApi {
- /** Problems that the health checks found in the experiment's configuration and its feature flag. Empty when every check passed. */
+ /** Problems that the health checks found in the experiment's configuration and its feature flag. Empty when no check found a problem. An empty list does not mean that every check ran: read `checks`. */
findings: ExperimentHealthFindingApi[]
+ /** The result of each health check, in the order the checks run. The checks that read the exposure counts are not here: the exposure query returns them as `health_checks`. */
+ checks: ExperimentHealthCheckResultApi[]
}
/**
diff --git a/products/experiments/frontend/generated/api.ts b/products/experiments/frontend/generated/api.ts
index 561b7fc9a74a..4dc0178da9b2 100644
--- a/products/experiments/frontend/generated/api.ts
+++ b/products/experiments/frontend/generated/api.ts
@@ -699,7 +699,9 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =
/**
* Health check findings for this experiment: problems in its configuration and its feature flag
- * that keep it from producing trustworthy results, each with a fix.
+ * that keep it from producing trustworthy results, each with a fix. `checks` gives the result of
+ * each check, so a check that found no problem can be told from one that did not apply or could
+ * not read an input.
*
* The checks here read the experiment, its flag, its linked metrics, and the changes of the
* flag's variant split since launch. The checks that read exposure counts come with the
diff --git a/products/experiments/frontend/health/ExperimentHealthChip.tsx b/products/experiments/frontend/health/ExperimentHealthChip.tsx
index 2d5b49ddb1b2..17c4002ce6e0 100644
--- a/products/experiments/frontend/health/ExperimentHealthChip.tsx
+++ b/products/experiments/frontend/health/ExperimentHealthChip.tsx
@@ -1,36 +1,75 @@
-import { useValues } from 'kea'
+import { useActions, useValues } from 'kea'
import { IconWarning } from '@posthog/icons'
-import { LemonButton } from '@posthog/lemon-ui'
+import { LemonButton, LemonDropdown, Spinner } from '@posthog/lemon-ui'
import { pluralize } from 'lib/utils/strings'
import { experimentLogic } from 'scenes/experiments/experimentLogic'
import { EXPERIMENT_HEALTH_PANEL_ELEMENT_ID } from '../constants'
-import { healthIssues } from './healthPanelFindings'
+import { HealthChecksOverview } from './HealthChecksOverview'
+import { healthChipState } from './healthPanelChecks'
export function ExperimentHealthChip(): JSX.Element | null {
- const { healthFindings } = useValues(experimentLogic)
- const issues = healthIssues(healthFindings)
+ const { healthFindings, healthChecks, healthExposureChecks, isExperimentDraft } = useValues(experimentLogic)
+ const { reportHealthChecksOpened } = useActions(experimentLogic)
+ const state = healthChipState(healthFindings, healthChecks, healthExposureChecks)
- if (!issues.length) {
- return null
+ switch (state.kind) {
+ case 'hidden':
+ return null
+ case 'issues':
+ return (
+ <LemonButton
+ type="secondary"
+ size="xsmall"
+ icon={<IconWarning className={state.hasCritical ? 'text-danger' : 'text-warning'} />}
+ onClick={() =>
+ document
+ .getElementById(EXPERIMENT_HEALTH_PANEL_ELEMENT_ID)
+ ?.scrollIntoView({ block: 'nearest' })
+ }
+ tooltip="Show the health checks"
+ data-attr="experiment-health-chip"
+ >
+ {pluralize(state.count, 'issue')}
+ </LemonButton>
+ )
+ case 'pending':
+ return (
+ <span className="flex items-center gap-1 text-secondary" data-attr="experiment-health-chip-pending">
+ <Spinner className="text-sm" />
+ <span>Checking health</span>
+ </span>
+ )
+ case 'checks':
+ return (
+ <LemonDropdown
+ overlay={
+ <HealthChecksOverview
+ checks={state.checks}
+ notes={state.notes}
+ isDraft={isExperimentDraft}
+ exposureChecksMissing={healthExposureChecks === 'missing'}
+ />
+ }
+ closeOnClickInside={false}
+ placement="bottom-start"
+ onVisibilityChange={(visible) => {
+ if (visible) {
+ reportHealthChecksOpened('meta_bar')
+ }
+ }}
+ >
+ <LemonButton
+ type="tertiary"
+ size="xsmall"
+ tooltip="Health checks"
+ data-attr="experiment-health-checks-button"
+ >
+ <span className="font-normal">{state.label}</span>
+ </LemonButton>
+ </LemonDropdown>
+ )
}
-
- const hasCritical = issues.some((finding) => finding.severity === 'critical')
-
- return (
- <LemonButton
- type="secondary"
- size="xsmall"
- icon={<IconWarning className={hasCritical ? 'text-danger' : 'text-warning'} />}
- onClick={() =>
- document.getElementById(EXPERIMENT_HEALTH_PANEL_ELEMENT_ID)?.scrollIntoView({ block: 'nearest' })
- }
- tooltip="Show the health checks"
- data-attr="experiment-health-chip"
- >
- {pluralize(issues.length, 'issue')}
- </LemonButton>
- )
}
diff --git a/products/experiments/frontend/health/ExperimentHealthPanel.tsx b/products/experiments/frontend/health/ExperimentHealthPanel.tsx
index 1562008b7bbf..104ee98e5433 100644
--- a/products/experiments/frontend/health/ExperimentHealthPanel.tsx
+++ b/products/experiments/frontend/health/ExperimentHealthPanel.tsx
@@ -1,18 +1,29 @@
-import { useValues } from 'kea'
+import { useActions, useValues } from 'kea'
+import { useId, useState } from 'react'
-import { LemonCard } from '@posthog/lemon-ui'
+import { IconChevronDown, IconChevronRight } from '@posthog/icons'
+import { LemonButton, LemonCard, LemonTag } from '@posthog/lemon-ui'
import { experimentLogic } from 'scenes/experiments/experimentLogic'
import { EXPERIMENT_HEALTH_PANEL_ELEMENT_ID } from '../constants'
+import { HealthCheckList } from './HealthCheckList'
import { HealthFindingRow } from './HealthFindingRow'
+import { listedHealthChecks } from './healthPanelChecks'
+import { healthIssues } from './healthPanelFindings'
export function ExperimentHealthPanel(): JSX.Element | null {
- const { healthFindings } = useValues(experimentLogic)
+ const { healthFindings, healthChecks, healthExposureChecks } = useValues(experimentLogic)
+ const { reportHealthChecksOpened } = useActions(experimentLogic)
+ const [isCheckListOpen, setIsCheckListOpen] = useState(false)
+ const checkListId = useId()
- if (!healthFindings?.length) {
+ // Without an issue the chip next to the experiment's status holds the checks, so the page keeps its space.
+ if (healthFindings === null || healthChecks === null || healthIssues(healthFindings).length === 0) {
return null
}
+ const listedChecks = listedHealthChecks(healthChecks)
+ const exposureChecksMissing = healthExposureChecks === 'missing'
return (
<section
@@ -21,17 +32,46 @@ export function ExperimentHealthPanel(): JSX.Element | null {
data-attr="experiment-health-panel"
>
<LemonCard hoverEffect={false} className="p-0">
- <h3
- id={`${EXPERIMENT_HEALTH_PANEL_ELEMENT_ID}-title`}
- className="m-0 px-3 py-2 border-b text-sm font-semibold"
- >
- Health checks
- </h3>
+ <div className="flex items-center gap-2 px-3 py-2 border-b">
+ <h3 id={`${EXPERIMENT_HEALTH_PANEL_ELEMENT_ID}-title`} className="m-0 text-sm font-semibold">
+ Health checks
+ </h3>
+ <LemonTag type="completion">Beta</LemonTag>
+ </div>
<div className="divide-y">
{healthFindings.map((finding) => (
<HealthFindingRow key={`${finding.code}:${finding.subcode ?? ''}`} finding={finding} />
))}
</div>
+ {(listedChecks.length > 0 || exposureChecksMissing) && (
+ <>
+ <div className="px-1 py-1 border-t">
+ <LemonButton
+ size="xsmall"
+ type="tertiary"
+ sideIcon={isCheckListOpen ? <IconChevronDown /> : <IconChevronRight />}
+ onClick={() => {
+ if (!isCheckListOpen) {
+ reportHealthChecksOpened('panel')
+ }
+ setIsCheckListOpen(!isCheckListOpen)
+ }}
+ aria-expanded={isCheckListOpen}
+ aria-controls={checkListId}
+ data-attr="experiment-health-check-list-toggle"
+ >
+ Other checks
+ </LemonButton>
+ </div>
+ {isCheckListOpen && (
+ <HealthCheckList
+ id={checkListId}
+ checks={listedChecks}
+ exposureChecksMissing={exposureChecksMissing}
+ />
+ )}
+ </>
+ )}
</LemonCard>
</section>
)
diff --git a/products/experiments/frontend/health/HealthCheckList.tsx b/products/experiments/frontend/health/HealthCheckList.tsx
new file mode 100644
index 000000000000..59a8f487a9d7
--- /dev/null
+++ b/products/experiments/frontend/health/HealthCheckList.tsx
@@ -0,0 +1,70 @@
+import { IconCheck, IconMinus, IconQuestion } from '@posthog/icons'
+
+import { cn } from 'lib/utils/css-classes'
+
+import type { ListedHealthCheck } from './healthPanelChecks'
+
+type ListedStatus = ListedHealthCheck['status']
+
+const STATUS_DISPLAY: Record<ListedStatus, { icon: JSX.Element; label: string }> = {
+ not_checked: { icon: <IconQuestion className="text-secondary" />, label: "Couldn't check" },
+ passed: { icon: <IconCheck className="text-success" />, label: 'No problem found' },
+ not_applicable: { icon: <IconMinus className="text-tertiary" />, label: "Doesn't apply" },
+}
+
+interface CheckGroup {
+ label: string
+ status: ListedStatus
+ rows: { key: string; name: string }[]
+}
+
+// Checks that share an outcome share a heading, so a reason such as "Runs after launch" shows once.
+function groupByOutcome(checks: ListedHealthCheck[], exposureChecksMissing: boolean): CheckGroup[] {
+ const groups = new Map<string, CheckGroup>()
+ if (exposureChecksMissing) {
+ // The exposure checks come with the exposure answer, so without one the page has no names for them.
+ groups.set('exposure_checks_missing', {
+ label: "Couldn't check. Refresh the results to run these checks.",
+ status: 'not_checked',
+ rows: [{ key: 'exposure_checks', name: 'Exposure checks' }],
+ })
+ }
+ for (const check of checks) {
+ const label = check.reason ?? STATUS_DISPLAY[check.status].label
+ const key = `${check.status}:${label}`
+ const group = groups.get(key) ?? { label, status: check.status, rows: [] }
+ group.rows.push({ key: check.key, name: check.name })
+ groups.set(key, group)
+ }
+ return [...groups.values()]
+}
+
+export function HealthCheckList({
+ id,
+ checks,
+ exposureChecksMissing,
+}: {
+ id?: string
+ checks: ListedHealthCheck[]
+ exposureChecksMissing: boolean
+}): JSX.Element {
+ return (
+ <div id={id} className="flex flex-col gap-2 px-3 py-2" data-attr="experiment-health-check-list">
+ {groupByOutcome(checks, exposureChecksMissing).map((group) => (
+ <div key={`${group.status}:${group.label}`} className="flex flex-col gap-1">
+ <span className="text-xs font-semibold text-secondary">{group.label}</span>
+ <ul className="m-0 p-0 list-none flex flex-col gap-1">
+ {group.rows.map((row) => (
+ <li key={row.key} className="flex items-center gap-2">
+ <span className="flex shrink-0 text-base">{STATUS_DISPLAY[group.status].icon}</span>
+ <span className={cn(group.status === 'not_applicable' && 'text-secondary')}>
+ {row.name}
+ </span>
+ </li>
+ ))}
+ </ul>
+ </div>
+ ))}
+ </div>
+ )
+}
diff --git a/products/experiments/frontend/health/HealthChecksOverview.stories.tsx b/products/experiments/frontend/health/HealthChecksOverview.stories.tsx
new file mode 100644
index 000000000000..59d0e39e40a7
--- /dev/null
+++ b/products/experiments/frontend/health/HealthChecksOverview.stories.tsx
@@ -0,0 +1,73 @@
+import type { Meta, StoryObj } from '@storybook/react'
+
+import { HealthChecksOverview } from './HealthChecksOverview'
+import type { HealthPanelCheck } from './healthPanelChecks'
+
+const passed = (key: HealthPanelCheck['key'], name: string): HealthPanelCheck => ({
+ key,
+ name,
+ status: 'passed',
+ reason: null,
+})
+
+const skipped = (
+ key: HealthPanelCheck['key'],
+ name: string,
+ status: 'not_applicable' | 'not_checked',
+ reason: string
+): HealthPanelCheck => ({ key, name, status, reason })
+
+const FLAG_SETUP_CHECKS = [
+ passed('flag_state', 'Feature flag rollout'),
+ passed('flag_variants_mismatch', 'Feature flag variants'),
+ passed('forced_variant_release_condition', 'Release conditions that force a variant'),
+]
+
+const meta: Meta<typeof HealthChecksOverview> = {
+ title: 'Experiments/HealthChecksOverview',
+ component: HealthChecksOverview,
+ parameters: {
+ layout: 'centered',
+ testOptions: { snapshotBrowsers: ['chromium'] },
+ },
+}
+export default meta
+
+type Story = StoryObj<typeof HealthChecksOverview>
+
+export const Draft: Story = {
+ args: {
+ checks: [
+ ...FLAG_SETUP_CHECKS,
+ skipped('flag_history', 'Feature flag history', 'not_applicable', 'Runs after launch'),
+ skipped(
+ 'split_changed_mid_run',
+ 'Variant split changes since launch',
+ 'not_applicable',
+ 'Runs after launch'
+ ),
+ skipped('no_metric', 'Metrics', 'not_applicable', 'Runs after launch'),
+ ],
+ notes: [],
+ isDraft: true,
+ exposureChecksMissing: false,
+ },
+}
+
+export const ChecksThatCouldNotRun: Story = {
+ args: {
+ checks: [
+ ...FLAG_SETUP_CHECKS,
+ skipped(
+ 'split_changed_mid_run',
+ 'Variant split changes since launch',
+ 'not_checked',
+ "Couldn't check because you can't view the linked feature flag's history"
+ ),
+ passed('no_metric', 'Metrics'),
+ ],
+ notes: [],
+ isDraft: false,
+ exposureChecksMissing: true,
+ },
+}
diff --git a/products/experiments/frontend/health/HealthChecksOverview.tsx b/products/experiments/frontend/health/HealthChecksOverview.tsx
new file mode 100644
index 000000000000..a9c478b141d8
--- /dev/null
+++ b/products/experiments/frontend/health/HealthChecksOverview.tsx
@@ -0,0 +1,46 @@
+import { LemonTag } from '@posthog/lemon-ui'
+
+import { HealthCheckList } from './HealthCheckList'
+import { HealthChecksSummary } from './HealthChecksSummary'
+import { HealthFindingRow } from './HealthFindingRow'
+import { type HealthPanelCheck, listedHealthChecks } from './healthPanelChecks'
+import type { HealthPanelFinding } from './healthPanelFindings'
+
+export function HealthChecksOverview({
+ checks,
+ notes,
+ isDraft,
+ exposureChecksMissing,
+}: {
+ checks: HealthPanelCheck[]
+ notes: HealthPanelFinding[]
+ isDraft: boolean
+ exposureChecksMissing: boolean
+}): JSX.Element {
+ const listedChecks = listedHealthChecks(checks)
+
+ return (
+ <div className="flex flex-col w-96 max-w-full" data-attr="experiment-health-overview">
+ <div className="flex flex-col gap-1 px-3 py-2">
+ <div className="flex items-center gap-2">
+ <span className="text-sm font-semibold">Health checks</span>
+ <LemonTag type="completion">Beta</LemonTag>
+ </div>
+ <HealthChecksSummary checks={checks} isDraft={isDraft} />
+ </div>
+ {notes.length > 0 && (
+ // A note mounts, and so reports as shown, only when the reader opens this overview.
+ <div className="divide-y border-t">
+ {notes.map((finding) => (
+ <HealthFindingRow key={`${finding.code}:${finding.subcode ?? ''}`} finding={finding} />
+ ))}
+ </div>
+ )}
+ {(listedChecks.length > 0 || exposureChecksMissing) && (
+ <div className="border-t">
+ <HealthCheckList checks={listedChecks} exposureChecksMissing={exposureChecksMissing} />
+ </div>
+ )}
+ </div>
+ )
+}
diff --git a/products/experiments/frontend/health/HealthChecksSummary.tsx b/products/experiments/frontend/health/HealthChecksSummary.tsx
new file mode 100644
index 000000000000..16c31384cfe6
--- /dev/null
+++ b/products/experiments/frontend/health/HealthChecksSummary.tsx
@@ -0,0 +1,29 @@
+import { pluralize } from 'lib/utils/strings'
+
+import { type HealthPanelCheck, countHealthChecks } from './healthPanelChecks'
+
+export function HealthChecksSummary({
+ checks,
+ isDraft,
+}: {
+ checks: HealthPanelCheck[]
+ isDraft: boolean
+}): JSX.Element {
+ const passedCount = countHealthChecks(checks, 'passed')
+ const notCheckedCount = countHealthChecks(checks, 'not_checked')
+
+ return (
+ <div className="flex flex-col gap-0.5" data-attr="experiment-health-summary">
+ <span className="font-semibold">
+ {passedCount > 0
+ ? `No problems found by ${pluralize(passedCount, 'check')}`
+ : 'No checks apply to this experiment yet'}
+ </span>
+ <p className="m-0 text-secondary">
+ {notCheckedCount > 0 && <span>{pluralize(notCheckedCount, 'check')} couldn't run. </span>}
+ {isDraft && <span>More checks run after launch. </span>}
+ <span>We're still adding checks, so they don't cover every problem yet.</span>
+ </p>
+ </div>
+ )
+}
diff --git a/products/experiments/frontend/health/debug/healthDebugReport.test.ts b/products/experiments/frontend/health/debug/healthDebugReport.test.ts
index 98850a445b54..c2a5f908466c 100644
--- a/products/experiments/frontend/health/debug/healthDebugReport.test.ts
+++ b/products/experiments/frontend/health/debug/healthDebugReport.test.ts
@@ -15,7 +15,7 @@ const PAUSED: ExperimentHealthFindingApi = {
}
const input = (overrides: Partial<HealthDebugInput>): HealthDebugInput => ({
- health: { findings: [] },
+ health: { findings: [], checks: [] },
browserWarning: null,
browserNoMetricsWarning: false,
isExperimentDraft: false,
@@ -37,35 +37,35 @@ describe('buildHealthDebugChecks', () => {
}>([
{
name: 'browser rules agree with the server',
- health: { findings: [PAUSED] },
+ health: { findings: [PAUSED], checks: [] },
browserWarning: { key: 'running_but_flag_disabled' },
browserNoMetricsWarning: false,
expected: [],
},
{
name: 'a flag-state warning only the server finds',
- health: { findings: [PAUSED] },
+ health: { findings: [PAUSED], checks: [] },
browserWarning: null,
browserNoMetricsWarning: false,
expected: ['flag_state:disagrees'],
},
{
name: 'a flag-state subcode the page does not know, while the browser finds nothing',
- health: { findings: [{ ...PAUSED, subcode: 'running_but_unknown_case' }] },
+ health: { findings: [{ ...PAUSED, subcode: 'running_but_unknown_case' }], checks: [] },
browserWarning: null,
browserNoMetricsWarning: false,
expected: ['flag_state:disagrees'],
},
{
name: 'no metrics on the page while the server counts one',
- health: { findings: [] },
+ health: { findings: [], checks: [] },
browserWarning: null,
browserNoMetricsWarning: true,
expected: ['no_metric:disagrees'],
},
{
name: "an SRM finding in the server's exposure answer only",
- health: { findings: [] },
+ health: { findings: [], checks: [] },
browserWarning: null,
browserNoMetricsWarning: false,
extra: {
diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
index 677f6b3074e1..1ea48802d4d3 100644
--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts
+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
@@ -10,6 +10,7 @@ import { EXPERIMENT_VARIANT_MULTIPLE } from '../constants'
import { getExperimentStatus } from '../experimentStatus'
import type { ExperimentHealthApi, ExperimentHealthFindingApi } from '../generated/api.schemas'
import { getTotalExposures, hasSampleRatioMismatch } from './exposureHealth'
+import { type HealthPanelCheck, countHealthChecks } from './healthPanelChecks'
// pinned: `finding_code` property values. Insights group and filter on them, so a rename breaks those insights.
export type ExperimentHealthFindingCode =
@@ -63,7 +64,12 @@ export const FLAG_STATE_FINDING_CODES: ReadonlySet<ExperimentHealthFindingCode>
Object.values(EXPERIMENT_WARNING_FINDING_CODES)
)
-// pinned: `health_ui` property values. Insights compare the health panel with the separate warnings on them.
+// pinned: `opened_from` property values of `experiment health checks opened`.
+export type ExperimentHealthChecksOpenedFrom = 'meta_bar' | 'panel'
+
+// pinned: `health_ui` property values. Insights compare the server's findings with the separate warnings on them.
+// 'panel' means the page shows the server's findings: in the health panel when one is an issue, otherwise in the
+// health chip next to the experiment's status.
export type ExperimentHealthUi = 'panel' | 'warnings'
export function experimentHealthUi(experiment: Experiment, showsServerHealth: boolean): ExperimentHealthUi {
@@ -165,6 +171,28 @@ export function captureExperimentHealthFindingActedOn(
})
}
+export function captureExperimentHealthChecksOpened(
+ experiment: Experiment,
+ checks: HealthPanelCheck[],
+ findingCount: number,
+ healthUi: ExperimentHealthUi,
+ openedFrom: ExperimentHealthChecksOpenedFrom
+): void {
+ // pinned: analytics event name, so renaming it breaks insights
+ posthog.capture('experiment health checks opened', {
+ experiment_id: experiment.id,
+ experiment_status: getExperimentStatus(experiment),
+ health_ui: healthUi,
+ health_finding_count: findingCount,
+ checks_passed: countHealthChecks(checks, 'passed'),
+ checks_finding: countHealthChecks(checks, 'finding'),
+ checks_not_applicable: countHealthChecks(checks, 'not_applicable'),
+ checks_not_checked: countHealthChecks(checks, 'not_checked'),
+ opened_from: openedFrom,
+ surface: 'experiment_page',
+ })
+}
+
/**
* Compares the page's own rules with the server's checks for one experiment load. A `_browser` value is the
* result of the page's rule, and a `_server` value is the result of the server's check.
diff --git a/products/experiments/frontend/health/healthPanelChecks.test.ts b/products/experiments/frontend/health/healthPanelChecks.test.ts
new file mode 100644
index 000000000000..aff7bae5bb6c
--- /dev/null
+++ b/products/experiments/frontend/health/healthPanelChecks.test.ts
@@ -0,0 +1,163 @@
+import { type ExperimentExposureQueryResponse, NodeKind } from '~/queries/schema/schema-general'
+
+import type { ExperimentHealthApi, ExperimentHealthCheckResultApi } from '../generated/api.schemas'
+import {
+ type ExposureChecksState,
+ type HealthChipState,
+ type HealthPanelCheck,
+ exposureChecksState,
+ healthChipState,
+ healthPanelChecks,
+} from './healthPanelChecks'
+import type { HealthPanelFinding } from './healthPanelFindings'
+
+const check = (
+ key: ExperimentHealthCheckResultApi['key'],
+ status: ExperimentHealthCheckResultApi['status']
+): HealthPanelCheck => ({ key, name: key, status, reason: null })
+
+const finding = (severity: HealthPanelFinding['severity']): HealthPanelFinding => ({
+ code: 'no_metric',
+ subcode: null,
+ severity,
+ title: 'No metrics defined',
+ detail: '',
+ evidence: {},
+ actions: [],
+})
+
+const EXPOSURES = {
+ kind: NodeKind.ExperimentExposureQuery,
+ timeseries: [],
+ total_exposures: { control: 600, test: 400 },
+ date_range: { date_from: '2026-10-01', date_to: '2026-10-02' },
+ health_findings: [],
+ health_checks: [
+ { key: 'zero_exposures', name: 'Exposures', status: 'passed', reason: null },
+ { key: 'srm', name: 'Split of users across variants', status: 'finding', reason: null },
+ ],
+} satisfies ExperimentExposureQueryResponse
+
+const EXPOSURES_WITHOUT_CHECKS = { ...EXPOSURES, health_checks: undefined }
+
+describe('healthPanelChecks', () => {
+ test.each<{ name: string; isExperimentDraft: boolean; expected: string[] }>([
+ {
+ name: 'the checks of the exposure answer join those of the experiment read, problems first',
+ isExperimentDraft: false,
+ expected: ['srm:finding', 'flag_history:not_checked', 'no_metric:passed', 'zero_exposures:passed'],
+ },
+ {
+ name: 'a draft lists no exposure check that a reset left in the page',
+ isExperimentDraft: true,
+ expected: ['flag_history:not_checked', 'no_metric:passed'],
+ },
+ ])('$name', ({ isExperimentDraft, expected }) => {
+ const health = { findings: [], checks: [check('no_metric', 'passed'), check('flag_history', 'not_checked')] }
+ expect(
+ healthPanelChecks(health, EXPOSURES, isExperimentDraft)?.map(({ key, status }) => `${key}:${status}`)
+ ).toEqual(expected)
+ })
+
+ it('lists the exposure checks when the health response has no checks', () => {
+ const healthWithoutChecks = { findings: [] } as unknown as ExperimentHealthApi
+
+ expect(healthPanelChecks(healthWithoutChecks, EXPOSURES, false)?.map(({ key }) => key)).toEqual([
+ 'srm',
+ 'zero_exposures',
+ ])
+ })
+
+ test.each<{
+ name: string
+ isLaunched: boolean
+ exposures: ExperimentExposureQueryResponse | null
+ loading: boolean
+ expected: ExposureChecksState
+ }>([
+ {
+ name: 'a draft has no exposure checks',
+ isLaunched: false,
+ exposures: null,
+ loading: false,
+ expected: 'not_launched',
+ },
+ {
+ name: 'an answer with checks is ready while it reloads',
+ isLaunched: true,
+ exposures: EXPOSURES,
+ loading: true,
+ expected: 'ready',
+ },
+ { name: 'the first load is pending', isLaunched: true, exposures: null, loading: true, expected: 'pending' },
+ {
+ name: 'a failed load misses the checks',
+ isLaunched: true,
+ exposures: null,
+ loading: false,
+ expected: 'missing',
+ },
+ {
+ name: 'an answer cached before the checks misses them',
+ isLaunched: true,
+ exposures: EXPOSURES_WITHOUT_CHECKS,
+ loading: false,
+ expected: 'missing',
+ },
+ ])('$name', ({ isLaunched, exposures, loading, expected }) => {
+ expect(exposureChecksState(isLaunched, exposures, loading)).toEqual(expected)
+ })
+
+ test.each<{
+ name: string
+ findings: HealthPanelFinding[] | null
+ checks: HealthPanelCheck[] | null
+ exposureChecks: ExposureChecksState
+ expected: Partial<HealthChipState>
+ }>([
+ {
+ name: 'no chip without server findings',
+ findings: null,
+ checks: null,
+ exposureChecks: 'ready',
+ expected: { kind: 'hidden' },
+ },
+ {
+ name: 'issues lead to the panel and count no info finding',
+ findings: [finding('critical'), finding('warning'), finding('info')],
+ checks: [],
+ exposureChecks: 'ready',
+ expected: { kind: 'issues', count: 2, hasCritical: true },
+ },
+ {
+ name: 'an issue shows while the exposure checks run',
+ findings: [finding('warning')],
+ checks: [],
+ exposureChecks: 'pending',
+ expected: { kind: 'issues', count: 1, hasCritical: false },
+ },
+ {
+ name: 'no verdict while the exposure checks run',
+ findings: [],
+ checks: [check('no_metric', 'passed')],
+ exposureChecks: 'pending',
+ expected: { kind: 'pending' },
+ },
+ {
+ name: 'the label counts the passed checks and the notes',
+ findings: [finding('info')],
+ checks: [check('no_metric', 'passed'), check('flag_state', 'passed'), check('srm', 'not_applicable')],
+ exposureChecks: 'ready',
+ expected: { kind: 'checks', label: '2\u00a0checks passed · 1\u00a0note' },
+ },
+ {
+ name: 'the label says so when no check applies',
+ findings: [],
+ checks: [check('no_metric', 'not_applicable')],
+ exposureChecks: 'not_launched',
+ expected: { kind: 'checks', label: 'No checks apply yet' },
+ },
+ ])('$name', ({ findings, checks, exposureChecks, expected }) => {
+ expect(healthChipState(findings, checks, exposureChecks)).toMatchObject(expected)
+ })
+})
diff --git a/products/experiments/frontend/health/healthPanelChecks.ts b/products/experiments/frontend/health/healthPanelChecks.ts
new file mode 100644
index 000000000000..b3739feeb70e
--- /dev/null
+++ b/products/experiments/frontend/health/healthPanelChecks.ts
@@ -0,0 +1,96 @@
+import { pluralize } from 'lib/utils/strings'
+
+import type { ExperimentExposureQueryResponse } from '~/queries/schema/schema-general'
+
+import type { ExperimentHealthApi, ExperimentHealthCheckResultApi } from '../generated/api.schemas'
+import { type HealthPanelFinding, healthIssues } from './healthPanelFindings'
+
+export type HealthPanelCheck = ExperimentHealthCheckResultApi
+
+/** A check in the list of checks. A check that reported a finding shows as that finding instead. */
+export type ListedHealthCheck = HealthPanelCheck & { status: Exclude<HealthPanelCheck['status'], 'finding'> }
+
+export type ExposureChecksState = 'not_launched' | 'pending' | 'missing' | 'ready'
+
+export type HealthChipState =
+ | { kind: 'hidden' }
+ | { kind: 'issues'; count: number; hasCritical: boolean }
+ | { kind: 'pending' }
+ | { kind: 'checks'; label: string; checks: HealthPanelCheck[]; notes: HealthPanelFinding[] }
+
+const CHECK_STATUS_ORDER: Record<HealthPanelCheck['status'], number> = {
+ finding: 0,
+ not_checked: 1,
+ passed: 2,
+ not_applicable: 3,
+}
+
+export function healthPanelChecks(
+ health: ExperimentHealthApi | null,
+ exposures: ExperimentExposureQueryResponse | null | undefined,
+ isExperimentDraft: boolean
+): HealthPanelCheck[] | null {
+ if (!health) {
+ return null
+ }
+ // A reset to draft keeps the previous exposure answer in the page, as for the findings.
+ const exposureChecks = isExperimentDraft ? [] : (exposures?.health_checks ?? [])
+ // A server from before the checks were reported sends none, for example while a deploy rolls out.
+ return [...(health.checks ?? []), ...exposureChecks].sort(
+ (a, b) => CHECK_STATUS_ORDER[a.status] - CHECK_STATUS_ORDER[b.status]
+ )
+}
+
+export function exposureChecksState(
+ isExperimentLaunched: boolean,
+ exposures: ExperimentExposureQueryResponse | null | undefined,
+ exposuresLoading: boolean
+): ExposureChecksState {
+ if (!isExperimentLaunched) {
+ return 'not_launched'
+ }
+ if (exposures?.health_checks) {
+ return 'ready'
+ }
+ // A failed load, or an answer cached before the server reported the checks.
+ return exposuresLoading ? 'pending' : 'missing'
+}
+
+export function listedHealthChecks(checks: HealthPanelCheck[]): ListedHealthCheck[] {
+ return checks.filter((check): check is ListedHealthCheck => check.status !== 'finding')
+}
+
+export function countHealthChecks(checks: HealthPanelCheck[], status: HealthPanelCheck['status']): number {
+ return checks.filter((check) => check.status === status).length
+}
+
+export function healthChipState(
+ findings: HealthPanelFinding[] | null,
+ checks: HealthPanelCheck[] | null,
+ exposureChecks: ExposureChecksState
+): HealthChipState {
+ if (findings === null || checks === null) {
+ return { kind: 'hidden' }
+ }
+ const issues = healthIssues(findings)
+ if (issues.length > 0) {
+ return {
+ kind: 'issues',
+ count: issues.length,
+ hasCritical: issues.some((finding) => finding.severity === 'critical'),
+ }
+ }
+ // A verdict before the exposure checks answer could turn into an issue a moment later.
+ if (exposureChecks === 'pending') {
+ return { kind: 'pending' }
+ }
+ // Without an issue, every finding is an info finding.
+ const passedCount = countHealthChecks(checks, 'passed')
+ const label = [
+ passedCount > 0 ? `${pluralize(passedCount, 'check')} passed` : null,
+ findings.length > 0 ? pluralize(findings.length, 'note') : null,
+ ]
+ .filter(Boolean)
+ .join(' · ')
+ return { kind: 'checks', label: label || 'No checks apply yet', checks, notes: findings }
+}
diff --git a/products/experiments/frontend/health/healthPanelFindings.test.ts b/products/experiments/frontend/health/healthPanelFindings.test.ts
index 6b9343007b4d..349b3def0fb9 100644
--- a/products/experiments/frontend/health/healthPanelFindings.test.ts
+++ b/products/experiments/frontend/health/healthPanelFindings.test.ts
@@ -61,21 +61,21 @@ describe('healthPanelFindings', () => {
},
{
name: 'no exposure finding before the exposures load',
- health: { findings: [serverFinding('no_metric')] },
+ health: { findings: [serverFinding('no_metric')], checks: [] },
exposures: null,
isExperimentDraft: false,
expected: ['no_metric'],
},
{
name: 'the findings of the exposure answer join those of the experiment read',
- health: { findings: [serverFinding('no_metric')] },
+ health: { findings: [serverFinding('no_metric')], checks: [] },
exposures: UNEVEN_EXPOSURES,
isExperimentDraft: false,
expected: ['no_metric', 'srm', 'bias_risk_multiple_excluded'],
},
{
name: 'a draft shows no finding of the exposure answer that a reset left in the page',
- health: { findings: [serverFinding('flag_live_before_launch')] },
+ health: { findings: [serverFinding('flag_live_before_launch')], checks: [] },
exposures: UNEVEN_EXPOSURES,
isExperimentDraft: true,
expected: ['flag_live_before_launch'],
@@ -84,6 +84,7 @@ describe('healthPanelFindings', () => {
name: 'an info finding sorts below the warnings of both sources',
health: {
findings: [serverFinding('forced_variant_release_condition', 'info'), serverFinding('no_metric')],
+ checks: [],
},
exposures: UNEVEN_EXPOSURES,
isExperimentDraft: false,
@@ -91,7 +92,7 @@ describe('healthPanelFindings', () => {
},
{
name: 'a code the experiment read sent is not added again from the exposure answer',
- health: { findings: [serverFinding('bias_risk_multiple_excluded')] },
+ health: { findings: [serverFinding('bias_risk_multiple_excluded')], checks: [] },
exposures: UNEVEN_EXPOSURES,
isExperimentDraft: false,
expected: ['bias_risk_multiple_excluded', 'srm'],
@@ -114,6 +115,8 @@ describe('healthPanelFindings', () => {
expected: ['no_metric'],
},
])('$name', ({ findings, expected }) => {
- expect(healthIssues(healthPanelFindings({ findings }, null, false)).map(({ code }) => code)).toEqual(expected)
+ expect(
+ healthIssues(healthPanelFindings({ findings, checks: [] }, null, false)).map(({ code }) => code)
+ ).toEqual(expected)
})
})
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 02ac7f484a37..8b9ac004ad98 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -44720,9 +44720,91 @@ export namespace Schemas {
diagnostic_ref: string | null;
}
+ /**
+ * * `flag_state` - Flag State
+ * * `flag_history` - Flag History
+ * * `split_changed_mid_run` - Split Changed Mid Run
+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
+ * * `flag_variants_mismatch` - Flag Variants Mismatch
+ * * `forced_variant_release_condition` - Forced Variant Release Condition
+ * * `no_metric` - No Metric
+ * * `zero_exposures` - Zero Exposures
+ * * `srm` - Srm
+ * * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
+ * * `multiple_variants_share` - Multiple Variants Share
+ */
+ export type ExperimentHealthCheckKeyEnum = typeof ExperimentHealthCheckKeyEnum[keyof typeof ExperimentHealthCheckKeyEnum];
+
+
+ export const ExperimentHealthCheckKeyEnum = {
+ FlagState: 'flag_state',
+ FlagHistory: 'flag_history',
+ SplitChangedMidRun: 'split_changed_mid_run',
+ BucketingChangedMidRun: 'bucketing_changed_mid_run',
+ ReleaseConditionsChangedMidRun: 'release_conditions_changed_mid_run',
+ FlagVariantsMismatch: 'flag_variants_mismatch',
+ ForcedVariantReleaseCondition: 'forced_variant_release_condition',
+ NoMetric: 'no_metric',
+ ZeroExposures: 'zero_exposures',
+ Srm: 'srm',
+ BiasRiskMultipleExcluded: 'bias_risk_multiple_excluded',
+ MultipleVariantsShare: 'multiple_variants_share',
+ } as const;
+
+ /**
+ * * `passed` - Passed
+ * * `finding` - Finding
+ * * `not_applicable` - Not Applicable
+ * * `not_checked` - Not Checked
+ */
+ export type ExperimentHealthCheckStatusEnum = typeof ExperimentHealthCheckStatusEnum[keyof typeof ExperimentHealthCheckStatusEnum];
+
+
+ export const ExperimentHealthCheckStatusEnum = {
+ Passed: 'passed',
+ Finding: 'finding',
+ NotApplicable: 'not_applicable',
+ NotChecked: 'not_checked',
+ } as const;
+
+ export interface ExperimentHealthCheckResult {
+ /** Stable identifier of the check. A check can report several finding codes, so the key is not a finding code.
+ *
+ * * `flag_state` - Flag State
+ * * `flag_history` - Flag History
+ * * `split_changed_mid_run` - Split Changed Mid Run
+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
+ * * `flag_variants_mismatch` - Flag Variants Mismatch
+ * * `forced_variant_release_condition` - Forced Variant Release Condition
+ * * `no_metric` - No Metric
+ * * `zero_exposures` - Zero Exposures
+ * * `srm` - Srm
+ * * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
+ * * `multiple_variants_share` - Multiple Variants Share */
+ key: ExperimentHealthCheckKeyEnum;
+ /** What the check looks at, for example 'Feature flag variants'. */
+ name: string;
+ /** passed: the check found no problem. finding: the check reported a finding in `findings`. not_applicable: the check does not cover the experiment's current state, for example a check of the changes since launch on a draft. not_checked: the check covers the experiment, but it could not read an input or it failed, so a problem it looks for can be present without a finding.
+ *
+ * * `passed` - Passed
+ * * `finding` - Finding
+ * * `not_applicable` - Not Applicable
+ * * `not_checked` - Not Checked */
+ status: ExperimentHealthCheckStatusEnum;
+ /**
+ * Why the check did not look, for the statuses not_applicable and not_checked, for example 'Runs after launch'. Null for passed and finding.
+ * @nullable
+ */
+ reason: string | null;
+ }
+
export interface ExperimentHealth {
- /** Problems that the health checks found in the experiment's configuration and its feature flag. Empty when every check passed. */
+ /** Problems that the health checks found in the experiment's configuration and its feature flag. Empty when no check found a problem. An empty list does not mean that every check ran: read `checks`. */
findings: ExperimentHealthFinding[];
+ /** The result of each health check, in the order the checks run. The checks that read the exposure counts are not here: the exposure query returns them as `health_checks`. */
+ checks: ExperimentHealthCheckResult[];
}
/**
@@ -44971,6 +45053,37 @@ export namespace Schemas {
multiple_variant_handling?: MultipleVariantHandling | null;
}
+ export type ExperimentExposureHealthCheckKey = typeof ExperimentExposureHealthCheckKey[keyof typeof ExperimentExposureHealthCheckKey];
+
+
+ export const ExperimentExposureHealthCheckKey = {
+ ZeroExposures: 'zero_exposures',
+ Srm: 'srm',
+ BiasRiskMultipleExcluded: 'bias_risk_multiple_excluded',
+ MultipleVariantsShare: 'multiple_variants_share',
+ } as const;
+
+ export type ExperimentExposureHealthCheckStatus = typeof ExperimentExposureHealthCheckStatus[keyof typeof ExperimentExposureHealthCheckStatus];
+
+
+ export const ExperimentExposureHealthCheckStatus = {
+ Passed: 'passed',
+ Finding: 'finding',
+ NotApplicable: 'not_applicable',
+ NotChecked: 'not_checked',
+ } as const;
+
+ export interface ExperimentExposureHealthCheck {
+ /** Stable identifier of the check. */
+ key: ExperimentExposureHealthCheckKey;
+ /** What the check looks at, as the health panel lists it. */
+ name: string;
+ /** Why the check did not look, for the statuses not_applicable and not_checked, for example 'Runs after launch'. Null for passed and finding. */
+ reason: string | null;
+ /** passed: no problem found. finding: a finding in `health_findings`. not_applicable: the check doesn't cover the experiment's state. not_checked: an input was missing or the check failed. */
+ status: ExperimentExposureHealthCheckStatus;
+ }
+
/**
* The values behind the finding, such as the p-value of the sample ratio test. The keys depend on the code.
*/
@@ -45069,6 +45182,8 @@ export namespace Schemas {
export interface ExperimentExposureQueryResponse {
bias_risk?: BiasRisk | null;
date_range: DateRange;
+ /** The result of each check behind `health_findings`, so that a check that did not run is not read as passed. */
+ health_checks?: ExperimentExposureHealthCheck[] | null;
/** Health check diagnostics that read the exposures: zero exposures, a sample ratio mismatch, and bias. Empty when every check passed. */
health_findings?: ExperimentExposureHealthFinding[] | null;
kind?: 'ExperimentExposureQuery';
@@ -94088,6 +94203,8 @@ export namespace Schemas {
export interface QueryResponseAlternative19 {
bias_risk?: BiasRisk | null;
date_range: DateRange;
+ /** The result of each check behind `health_findings`, so that a check that did not run is not read as passed. */
+ health_checks?: ExperimentExposureHealthCheck[] | null;
/** Health check diagnostics that read the exposures: zero exposures, a sample ratio mismatch, and bias. Empty when every check passed. */
health_findings?: ExperimentExposureHealthFinding[] | null;
kind?: 'ExperimentExposureQuery';