Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(experiments): clearer health check names and a working issue chip (#124693)

Marcel Poelker40a2a1a67ffb
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index 66be35291586..ffca986a89ef 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -3329,21 +3329,25 @@ snapshots:
errortracking-volumesparkline--events-before-data-range--light:
hash: v1.k794b7964.c88c4e3b028915efba3fdbbac7107a063662d8210dfb6ba8667e96d719fa3d4f.tzwEQFa4F1f1O9PGqBkcaqSavbFVgAwPAjyh9Uf9NL8
experiments-healthchecksoverview--checks-that-could-not-run--dark:
- hash: v1.k794b7964.b6036b626d31c7f4863f543dba0ca04d15c90f18c265f49baafd2f22499963cc.Sgb0OPyCSKwPWbcBulpJ21fh6HR_54N0vZ0c4gBaosw
+ hash: v1.k794b7964.812abddceca7cccf98524576a651f1a5e39826c130f93f8b8229fb509cb726ec.fbTQPszA7Of4H8fHgkioDk9ZSf6QzUTa-WDv733Ie5U
experiments-healthchecksoverview--checks-that-could-not-run--light:
- hash: v1.k794b7964.8c51277755331ab88e7b5bf4bcdc2c9a3c0c9c87fa5ea946f549b70df6c58ce6.vJaps4AlW1x8oWaPFZMzIky0VccXRtIpAw5elQ8Q424
+ hash: v1.k794b7964.1727dcbe43bb519bf22fa837716404283cc730151b237a45c098b44a0d753c76.2nNVgElqCL8SDJ6i2TZZ3h_1baUpeR_6nkfgNG7AjrQ
experiments-healthchecksoverview--draft--dark:
- hash: v1.k794b7964.998711a45ae74b0bc74066f1d70b2b58c809e9d9e0bd7d4d0b541698a312cad3.IPJlc0qZzHtFDsZ5Gb06zukOf9kk_JnT6-OU2V-Pwa0
+ hash: v1.k794b7964.2d850eef5214f620c272b8391b847b24bd7a7e54db105b82fb2092edca5a749b.hkbIChET-cnHtxRGPXq7mINCKqre6micZp4vGHkZPqg
experiments-healthchecksoverview--draft--light:
- hash: v1.k794b7964.5b098ee13c120e815fdf13e18a59d6234a4c904e747a7cc83fec0e1406cd3491.YrNkI7P5QfXa-TnjviRwcQIIML09K1-POg889feICRg
+ hash: v1.k794b7964.6602e11819404ed4d40aa502a6acecd02e50e277bf6433a7094806d42aae273a.4zUBdKR311LuUiFR-oJrHG2wSkKtGqHSTmTu1TggsD8
experiments-healthchecksoverview--draft-with-events-not-received--dark:
- hash: v1.k794b7964.8fc4d4ec055de978ea383b15138251df99dafb87f0713218485a74da67b31634.XBTJmRJvnfs_9gv6zfC10-5JQAAGTRfTctWZc7T852s
+ hash: v1.k794b7964.dd26de28a16b7d2b41c1a766c72b2cc15024ef94eedd1298d44ee9ac4f407a84.RPGW8Ele3QwdlSX_3rEN6tVAHSnKGqEd5c1Rlyh-pPc
experiments-healthchecksoverview--draft-with-events-not-received--light:
- hash: v1.k794b7964.207d3f925f5972dbbc0825b2d7f6e9026c362c4b513b2db68b70303715bcd866.J3uCNeI2NrU5MV9AWMYFCI4ftj9WCM-4d0Vvjp_7K-k
+ hash: v1.k794b7964.9ffbba359b8dca6f948edccdde02acf1539a4e2af552b9005de4377bbf876c1c.VlXBwUZKPYbSBWabKTIoqjp66EheAgMdqTDbaWOexP8
+ experiments-healthchecksoverview--running-with-an-issue--dark:
+ hash: v1.k794b7964.c513495899160638ead3ad13632d4ca59a62c8ea0fb2c86fa1ea17719d46a2fa.8ApNUhlTxYF7nrUqO1zsTZte04NzMXf7hmjJz3P4rPE
+ experiments-healthchecksoverview--running-with-an-issue--light:
+ hash: v1.k794b7964.3cbd4355330c14f1afadb886cd182879c5b436b5143fe8f136771027bd56cc7f.YApFmE4xWr1dZ_tNUv89PeWQW5QEDxuJwxaSW7siajc
experiments-healthchecksoverview--running-with-significance-on-a-small-sample--dark:
- hash: v1.k794b7964.4b8d903f39c6569a954ff966fae2d7e305b3a136578aed58d9205dabe5739c26.jglF4ElvqVB73DiCtuc3bZAv5ESatCaxg3Z1LU9Q-iQ
+ hash: v1.k794b7964.409d7eccff93e0d58ad5f9272a09937b807b66863ffa41bce1946aeb716f5ab3.UgfqDxyCdNSrMh_WE_Cf9KXyjPelEnkBYw3FzAWgkeY
experiments-healthchecksoverview--running-with-significance-on-a-small-sample--light:
- hash: v1.k794b7964.d8d89699c9a340a82b74fb99e9c6b890facb29d2c7cad15fcdf251d9e70a2854.G1q2SJtAvjRnHoJIEAwyTQfa0W5DT8KTI25oVpdesI8
+ hash: v1.k794b7964.22d9ffdacb87c01038bdea719d48ac9007740ae874c600cd07a6b1ee9c733c79.NOZW2_XUZpp2wjCx6S5-onjs3VltBKvbwyiyRMCSKgY
experiments-howtoreadtooltip--intervals-bayesian--dark:
hash: v1.k794b7964.a06d267cee9cb54bafc732335b35d461d3c1a59250c6daa8becace0fcb8492d2.vR5tecOozn6B51xE8LrByAFVsi03J4w-iUu-j2hQUTU
experiments-howtoreadtooltip--intervals-bayesian--light:
@@ -11179,9 +11183,9 @@ snapshots:
scenes-app-experiments--experiment-frequentist-two-variants--light:
hash: v1.k794b7964.4ddea1c7537af9d50c57b4b9a76d4bcba39f743a23b4715683c0ce59d4c24f57.0ueR8XMLqRby_u6I0EyNIiUwXkbjmWMH3vBdoFNYmII
scenes-app-experiments--experiment-health-debug-tab--dark:
- hash: v1.k794b7964.07a80c2033b866a07cebe59831d8db3bb9d3ac9495c547023dd3a4f23102cf76.5EvtNcty-1BR6fQcptymV7gZMB3hGt9Z44k-kLns33g
+ hash: v1.k794b7964.b6915ebfac93068655e213d55254ba4a8a904dbe004aec55dfb5ba15e853afb9.SdUNTU-MTUTz_F2r4fTML3a2MIyFn7hag3Pqx16ge_g
scenes-app-experiments--experiment-health-debug-tab--light:
- hash: v1.k794b7964.f0ea28894142a01a25cf3ea19ae46943fb58a3ee6ace693ebd4fd7a9dfea1d28.vCFW8ATe64fjDJDITqHBkQJveadbaLPrD_4v2m_sXv4
+ hash: v1.k794b7964.f43d72c3525d5c7dc207b4ee30664dd895faec19a9966dd113bffebf09998bba.zY0dq5GZK_KGHnKFbl24Hf4VfWzRyapUy1DtAv-Wt7g
scenes-app-experiments--experiment-recordings-dropped-metric--dark:
hash: v1.k794b7964.e0e7181f2c9e61a8324eff9aa7a9230fef25b3cf37e0b24ef875780186ed25dc.9VEvZ4iVieVJwmDol0gwtAsr3wgZYW5gj2KXAGZJfiw
scenes-app-experiments--experiment-recordings-dropped-metric--light:
@@ -11311,9 +11315,9 @@ snapshots:
scenes-app-experiments--experiment-with-funnel-metric--light:
hash: v1.k794b7964.fd105a9918bab77db9fb47b351ffa32065556aae7f46736d002f5edbf4b7eb2a.kz6qe5r99nmq8xrOvZhI2qeQLpYwhaRXTTQZEyWb9oU
scenes-app-experiments--experiment-with-health-findings--dark:
- hash: v1.k794b7964.5213e921ee249f3eb6cef2b7a9c8f13cabb3915512176017523cdb8b435f7077.6U2v8vo0X1jjiHr4yEV_uf8_9iZWP2iwDD8s65f8O2M
+ hash: v1.k794b7964.0e6c443c539e742e42e4d46604810c76b934acefbb12e610dd59403a9852406d.EL4WPQY3idMVDczVAea7lHhCOYuI1ATfdmoFZ691SEo
scenes-app-experiments--experiment-with-health-findings--light:
- hash: v1.k794b7964.31e3f4ca0bdf4d909564322fb2a2fd1a8508558652531c5468ba38169c19638f.DtxQIjgf_5c5mi4GFDIZb4m8_NjfWohQM-pPq1k5Qp4
+ hash: v1.k794b7964.8c72c764757a385da276373c4df108ea8f309820467cd77e624a11db5b50e405.ScKZnCot1howsO9pqK1Em_QrS8u_AQNFM2xDWJ7u18k
scenes-app-experiments--experiment-with-long-flag-key--dark:
hash: v1.k794b7964.72ab9fcbc06f35556a7d96e0fe3bd56e8a2796ed5653d6b279ec05c8a9cce9de.1HcvRm0AN_gzYBTpwvzGMG55FWmg2E7lQZD_w3f_xJc
scenes-app-experiments--experiment-with-long-flag-key--light:
diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json
index 136ffb9c10be..244d2e082906 100644
--- a/frontend/src/queries/schema.json
+++ b/frontend/src/queries/schema.json
@@ -26004,7 +26004,7 @@
"description": "Stable identifier of the check."
},
"name": {
- "description": "What the check looks at, as the health panel lists it.",
+ "description": "The problem the check looks for, as the health panel lists it.",
"type": "string"
},
"reason": {
diff --git a/frontend/src/queries/schema/schema-general.ts b/frontend/src/queries/schema/schema-general.ts
index 33e87364a7db..01ab5cece4d3 100644
--- a/frontend/src/queries/schema/schema-general.ts
+++ b/frontend/src/queries/schema/schema-general.ts
@@ -6493,7 +6493,7 @@ export type ExperimentExposureHealthCheckStatus = 'passed' | 'finding' | 'not_ap
export interface ExperimentExposureHealthCheck {
/** Stable identifier of the check. */
key: ExperimentExposureHealthCheckKey
- /** What the check looks at, as the health panel lists it. */
+ /** The problem the check looks for, as the health panel lists it. */
name: string
/** passed: no problem found. finding: a finding in `health_findings`. not_applicable: the check doesn't cover the experiment's state. not_checked: an input was missing or the check failed. */
status: ExperimentExposureHealthCheckStatus
diff --git a/frontend/src/scenes/experiments/ExperimentView/MultiVariantBiasWarning.tsx b/frontend/src/scenes/experiments/ExperimentView/MultiVariantBiasWarning.tsx
index 7093ce71c9d5..7b32dfbe7822 100644
--- a/frontend/src/scenes/experiments/ExperimentView/MultiVariantBiasWarning.tsx
+++ b/frontend/src/scenes/experiments/ExperimentView/MultiVariantBiasWarning.tsx
@@ -41,7 +41,7 @@ export function MultiVariantBiasWarning(): JSX.Element | null {
<LemonBanner type="warning" className="mt-4">
<div className="flex items-start justify-between gap-4 flex-wrap">
<div className="flex-1 min-w-[300px]">
- <div className="font-semibold">Setup likely introduced bias</div>
+ <div className="font-semibold">The uneven split may bias the results</div>
<p className="m-0">
<strong>{risk.multiple_variant_percentage.toFixed(1)}%</strong> of users were exposed to
multiple variants. With your uneven variant split and the current <strong>Exclude</strong>{' '}
diff --git a/frontend/src/scenes/experiments/ExperimentView/healthFindingReporting.test.tsx b/frontend/src/scenes/experiments/ExperimentView/healthFindingReporting.test.tsx
index 569d4c74f306..af5f76187945 100644
--- a/frontend/src/scenes/experiments/ExperimentView/healthFindingReporting.test.tsx
+++ b/frontend/src/scenes/experiments/ExperimentView/healthFindingReporting.test.tsx
@@ -53,7 +53,7 @@ const UNEVEN_EXPOSURES = {
bias_risk: { multiple_variant_percentage: 5 },
health_findings: [
exposureFinding('srm', 'Users are not split across variants as configured', []),
- exposureFinding('bias_risk_multiple_excluded', 'Setup likely introduced bias', [
+ exposureFinding('bias_risk_multiple_excluded', 'The uneven split may bias the results', [
'adjust_distribution',
'use_first_seen_variant',
]),
diff --git a/frontend/src/scenes/experiments/experimentLogic.tsx b/frontend/src/scenes/experiments/experimentLogic.tsx
index e896f1e9487b..db03f4db3d3b 100644
--- a/frontend/src/scenes/experiments/experimentLogic.tsx
+++ b/frontend/src/scenes/experiments/experimentLogic.tsx
@@ -755,7 +755,7 @@ export interface experimentLogicActions {
forceRefresh: boolean
}
reportHealthChecksOpened: (openedFrom: ExperimentHealthChecksOpenedFrom) => {
- openedFrom: ExperimentHealthChecksOpenedFrom
+ openedFrom: 'meta_bar'
}
reportHealthFindingActedOn: (
finding: ExperimentHealthFinding,
diff --git a/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx b/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx
index f446ad2fc5ad..7c17df644c30 100644
--- a/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx
+++ b/frontend/src/scenes/experiments/stories/ExperimentHealthDebugTab.stories.tsx
@@ -40,9 +40,9 @@ const EXPERIMENT_HEALTH = {
],
checks: [
{ key: 'split_changed_mid_run', name: 'Variant split changes since launch', status: 'finding', reason: null },
- { key: 'flag_state', name: 'Feature flag rollout', status: 'passed', reason: null },
- { key: 'flag_history', name: 'Feature flag history', status: 'passed', reason: null },
- { key: 'no_metric', name: 'Metrics', status: 'passed', reason: null },
+ { key: 'flag_state', name: "Flag state that doesn't match the experiment", status: 'passed', reason: null },
+ { key: 'flag_history', name: 'Unreadable flag history', status: 'passed', reason: null },
+ { key: 'no_metric', name: 'Metrics not set up', status: 'passed', reason: null },
],
}
diff --git a/frontend/src/scenes/experiments/stories/ExperimentWithHealthFindings.stories.tsx b/frontend/src/scenes/experiments/stories/ExperimentWithHealthFindings.stories.tsx
index 59a251f2579e..cb3ed26bb789 100644
--- a/frontend/src/scenes/experiments/stories/ExperimentWithHealthFindings.stories.tsx
+++ b/frontend/src/scenes/experiments/stories/ExperimentWithHealthFindings.stories.tsx
@@ -29,11 +29,16 @@ const EXPERIMENT_HEALTH = {
},
],
checks: [
- { key: 'flag_state', name: 'Feature flag rollout', status: 'finding', reason: null },
- { key: 'flag_history', name: 'Feature flag history', status: 'passed', reason: null },
+ { key: 'flag_state', name: "Flag state that doesn't match the experiment", status: 'finding', reason: null },
+ { key: 'flag_history', name: 'Unreadable flag history', status: 'passed', reason: null },
{ key: 'split_changed_mid_run', name: 'Variant split changes since launch', status: 'passed', reason: null },
- { key: 'flag_variants_mismatch', name: 'Feature flag variants', status: 'passed', reason: null },
- { key: 'no_metric', name: 'Metrics', status: 'passed', reason: null },
+ {
+ key: 'flag_variants_mismatch',
+ name: 'Flag variants not set up',
+ status: 'passed',
+ reason: null,
+ },
+ { key: 'no_metric', name: 'Metrics not set up', status: 'passed', reason: null },
],
}
@@ -56,7 +61,7 @@ const EXPOSURES_WITH_BIAS_RISK = {
code: 'bias_risk_multiple_excluded',
subcode: null,
severity: 'warning',
- title: 'Setup likely introduced bias',
+ title: 'The uneven split may bias the results',
detail: '4.2% of users were exposed to multiple variants. With an uneven variant split and the Exclude handling, these users were dropped more often from the smaller variant, so its metrics can be biased. Use an even split and control exposure with the overall rollout, or switch the handling to First seen.',
evidence: { multiple_variant_percentage: 4.2 },
actions: ['adjust_distribution', 'use_first_seen_variant'],
@@ -66,7 +71,7 @@ const EXPOSURES_WITH_BIAS_RISK = {
health_checks: [
{ key: 'srm', name: 'Split of users across variants', status: 'finding', reason: null },
{ key: 'bias_risk_multiple_excluded', name: 'Bias from an uneven split', status: 'finding', reason: null },
- { key: 'zero_exposures', name: 'Exposures', status: 'passed', reason: null },
+ { key: 'zero_exposures', name: 'Exposures not received since launch', status: 'passed', reason: null },
{ key: 'multiple_variants_share', name: 'Users exposed to multiple variants', status: 'passed', reason: null },
],
}
diff --git a/posthog/schema.py b/posthog/schema.py
index 04f8ec707b4f..0a56df74cda9 100644
--- a/posthog/schema.py
+++ b/posthog/schema.py
@@ -5797,7 +5797,10 @@ class ExperimentExposureHealthCheck(BaseModel):
extra="forbid",
)
key: ExperimentExposureHealthCheckKey = Field(..., description="Stable identifier of the check.")
- name: str = Field(..., description="What the check looks at, as the health panel lists it.")
+ name: str = Field(
+ ...,
+ description="The problem the check looks for, as the health panel lists it.",
+ )
reason: str | None = Field(
...,
description=(
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index 4065613725f0..e20c6beafd1a 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -10071,7 +10071,7 @@ export namespace Schemas {
*/
key: ExperimentHealthCheckKeyEnum;
/**
- * What the check looks at, for example 'Feature flag variants'.
+ * The problem the check looks for, for example 'Metrics not set up'.
*/
name: string;
/**
@@ -10440,7 +10440,7 @@ export namespace Schemas {
*/
key: ExperimentExposureHealthCheckKey;
/**
- * What the check looks at, as the health panel lists it.
+ * The problem the check looks for, as the health panel lists it.
*/
name: string;
/**
diff --git a/products/experiments/backend/health/checks/bias_risk.py b/products/experiments/backend/health/checks/bias_risk.py
index d37d235ed42b..ab60060954de 100644
--- a/products/experiments/backend/health/checks/bias_risk.py
+++ b/products/experiments/backend/health/checks/bias_risk.py
@@ -76,7 +76,7 @@ def bias_risk_multiple_excluded(ctx: HealthContext) -> CheckOutcome:
code=ExperimentHealthFindingCode.BIAS_RISK_MULTIPLE_EXCLUDED,
subcode=None,
severity=ExperimentHealthFindingSeverity.WARNING,
- title="Setup likely introduced bias",
+ title="The uneven split may bias the results",
detail=(
f"{risk.multiple_variant_percentage:.1f}% of users were exposed to multiple variants. "
"With an uneven variant split and the Exclude handling, these users were dropped more often "
diff --git a/products/experiments/backend/health/checks/metric_calculation_failed.py b/products/experiments/backend/health/checks/metric_calculation_failed.py
index cfc2a1246d68..ce76ea4bc5ec 100644
--- a/products/experiments/backend/health/checks/metric_calculation_failed.py
+++ b/products/experiments/backend/health/checks/metric_calculation_failed.py
@@ -156,7 +156,7 @@ def metric_calculation_failed(ctx: HealthContext) -> CheckOutcome:
if not calculation.failures:
return None
compiles = ctx.metric_compiles
- # A metric whose query doesn't compile fails every calculation, and the metric queries check words that better.
+ # A metric whose query doesn't compile fails every calculation, and `metric_cannot_compile` words that better.
not_compiling = {failure.metric_id for failure in compiles.failures} if compiles else set()
exposure_query_fails = compiles is not None and compiles.exposure_query_fails
worded = [
@@ -165,7 +165,7 @@ def metric_calculation_failed(ctx: HealthContext) -> CheckOutcome:
if failure.metric_id not in not_compiling
]
if not worded:
- return not_applicable("The metric queries check reports the metric that failed")
+ return not_applicable('Reported under "Invalid metric definitions"')
worded.sort(key=lambda item: (item[0].role != "primary", item[1].kind not in _NEEDS_A_FIX))
count = len(worded)
return ExperimentHealthFinding(
@@ -313,7 +313,7 @@ def _settings(failure: MetricCalculationFailure) -> _Wording | None:
if not failure.can_quote_evaluation_rules:
wording = word_compile_error(ExperimentHealthMetricCompileErrorKind.QUERY, message, failure.definition)
if wording.error != MetricCompileError.OTHER:
- # Unlike the metric queries check, which compiles as the reader, the evidence leaves out HogQL's text.
+ # Unlike `metric_cannot_compile`, which compiles as the reader, the evidence leaves out HogQL's text.
return _setting(wording.error.value, wording.explanation)
if failure.error_type == "validation_error":
return _setting(
diff --git a/products/experiments/backend/health/checks/multiple_variants.py b/products/experiments/backend/health/checks/multiple_variants.py
index 34994f8cf937..f2690773387c 100644
--- a/products/experiments/backend/health/checks/multiple_variants.py
+++ b/products/experiments/backend/health/checks/multiple_variants.py
@@ -37,24 +37,25 @@ def multiple_variants_share(ctx: HealthContext) -> CheckOutcome:
title = "Many groups were exposed to multiple variants"
detail = (
f"{percentage:.1f}% of groups ({multiple}) were exposed to multiple variants, so the results leave them "
- "out. The exposures table lists them as $multiple. The flag gives every member of a group the same "
- "variant, so this usually means that the flag changed after launch, for example with a new variant split "
- "or a change from matching users to matching groups. It also happens when a person switches to another "
- "group and your code reads the flag before the flags reload: the event then carries the new group with "
- "the variant of the old group. Check the flag's history, and after a group switch, read the flag only "
- "once the flags reload."
+ "out. The exposures table lists them as $multiple.\n\n"
+ "The flag gives every member of a group the same variant, so this usually means that the flag changed "
+ "after launch, for example with a new variant split or a change from matching users to matching groups. "
+ "It also happens when a person switches to another group and your code reads the flag before the flags "
+ "reload: the event then carries the new group with the variant of the old group.\n\n"
+ "Check the flag's history, and after a group switch, read the flag only once the flags reload."
)
else:
title = "Many users were exposed to multiple variants"
detail = (
f"{percentage:.1f}% of users ({multiple}) were exposed to multiple variants, so the results leave them "
- "out. The exposures table lists them as $multiple. This usually means that one person has more than one "
- "ID: an anonymous ID before login and another after, an ID for each device, a new ID after reset(), or a "
- "different ID on your server than in the browser. It also happens when a bootstrapped flag value differs "
- "from the value PostHog evaluates, or when the flag changes after launch, for example with a new variant "
- "split. If a variant changes how often users log in, each variant loses a different group of users, and "
- "the results can be biased. Identify users before your code evaluates the flag, call reset() only at "
- "logout, and send the same ID from your server."
+ "out. The exposures table lists them as $multiple. If a variant changes how often users log in, each "
+ "variant loses a different group of users, and the results can be biased.\n\n"
+ "This usually means that one person has more than one ID: an anonymous ID before login and another after, "
+ "an ID for each device, a new ID after reset(), or a different ID on your server than in the browser. It "
+ "also happens when a bootstrapped flag value differs from the value PostHog evaluates, or when the flag "
+ "changes after launch, for example with a new variant split.\n\n"
+ "Identify users before your code evaluates the flag, call reset() only at logout, and send the same ID "
+ "from your server."
)
return ExperimentHealthFinding(
code=ExperimentHealthFindingCode.MULTIPLE_VARIANTS_SHARE,
diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.py
index 7269c9d8be5d..74fc70365799 100644
--- a/products/experiments/backend/health/registry.py
+++ b/products/experiments/backend/health/registry.py
@@ -44,12 +44,12 @@ class HealthCheck:
EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
HealthCheck(
key=ExperimentHealthCheckKey.FLAG_STATE,
- name="Feature flag rollout",
+ name="Flag state that doesn't match the experiment",
run=flag_state,
),
HealthCheck(
key=ExperimentHealthCheckKey.FLAG_HISTORY,
- name="Feature flag history",
+ name="Unreadable flag history",
run=flag_history_not_readable,
),
HealthCheck(
@@ -59,7 +59,7 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
),
HealthCheck(
key=ExperimentHealthCheckKey.BUCKETING_CHANGED_MID_RUN,
- name="Flag key and bucketing changes since launch",
+ name="Variant assignment changes since launch",
run=bucketing_changed_mid_run,
),
HealthCheck(
@@ -69,7 +69,7 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
),
HealthCheck(
key=ExperimentHealthCheckKey.FLAG_VARIANTS_MISMATCH,
- name="Feature flag variants",
+ name="Flag variants not set up",
run=flag_variants_mismatch,
),
HealthCheck(
@@ -79,42 +79,42 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
),
HealthCheck(
key=ExperimentHealthCheckKey.NEW_FLAG_NO_TRAFFIC,
- name="Feature flag traffic",
+ name="Flag calls not received yet",
run=new_flag_no_traffic,
),
HealthCheck(
key=ExperimentHealthCheckKey.NO_METRIC,
- name="Metrics",
+ name="Metrics not set up",
run=no_metric,
),
HealthCheck(
key=ExperimentHealthCheckKey.METRIC_EVENT_NOT_INGESTED,
- name="Metric events",
+ name="Metric events not received yet",
run=metric_event_not_ingested,
),
HealthCheck(
key=ExperimentHealthCheckKey.METRIC_CANNOT_COMPILE,
- name="Metric queries",
+ name="Invalid metric definitions",
run=metric_cannot_compile,
),
HealthCheck(
key=ExperimentHealthCheckKey.METRIC_CALCULATION_FAILED,
- name="Metric results",
+ name="Failed metric calculations",
run=metric_calculation_failed,
),
HealthCheck(
key=ExperimentHealthCheckKey.EXPOSURE_CRITERIA_CANNOT_MATCH,
- name="Exposure criteria",
+ name="Exposure criteria that can't match",
run=exposure_criteria_cannot_match,
),
HealthCheck(
key=ExperimentHealthCheckKey.EXPOSURE_FILTERS,
- name="Cohorts and event filters in the exposure criteria",
+ name="Exposure filters that count unexpected users",
run=exposure_filters,
),
HealthCheck(
key=ExperimentHealthCheckKey.EARLY_SIGNIFICANCE,
- name="Sample size behind significant results",
+ name="Significance on a small sample",
run=early_significance,
),
)
@@ -125,7 +125,7 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
EXPOSURE_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
HealthCheck(
key=ExperimentHealthCheckKey.ZERO_EXPOSURES,
- name="Exposures",
+ name="Exposures not received since launch",
run=zero_exposures,
),
HealthCheck(
diff --git a/products/experiments/backend/health/test/test_metric_calculation_failed.py b/products/experiments/backend/health/test/test_metric_calculation_failed.py
index 68df1dd433fb..1bf40e74416f 100644
--- a/products/experiments/backend/health/test/test_metric_calculation_failed.py
+++ b/products/experiments/backend/health/test/test_metric_calculation_failed.py
@@ -359,7 +359,7 @@ class TestMetricCalculationFailed(TestCase):
outcome,
CheckSkip(
status=ExperimentHealthCheckStatus.NOT_APPLICABLE,
- reason="The metric queries check reports the metric that failed",
+ reason='Reported under "Invalid metric definitions"',
),
)
diff --git a/products/experiments/backend/presentation/serializers.py b/products/experiments/backend/presentation/serializers.py
index 51d0df9128a3..03e9d871cb11 100644
--- a/products/experiments/backend/presentation/serializers.py
+++ b/products/experiments/backend/presentation/serializers.py
@@ -256,7 +256,7 @@ class ExperimentHealthCheckResultSerializer(serializers.Serializer):
"finding code."
),
)
- name = serializers.CharField(help_text="What the check looks at, for example 'Feature flag variants'.")
+ name = serializers.CharField(help_text="The problem the check looks for, for example 'Metrics not set up'.")
status = serializers.ChoiceField(
choices=ExperimentHealthCheckStatus.choices,
help_text=(
@@ -3620,9 +3620,9 @@ class ExperimentHealthInputsSerializer(serializers.Serializer):
metric_compiles = ExperimentHealthInputsMetricCompilesSerializer(
allow_null=True,
help_text=(
- "The compile of each metric's query without running it, as the reader, which the metric queries check "
- "reads. A metric that compiles is cached per reader for a day, and a failing metric compiles again on "
- "every request. Null when the compile itself failed."
+ "The compile of each metric's query without running it, as the reader, which the 'Invalid metric "
+ "definitions' check reads. A metric that compiles is cached per reader for a day, and a failing metric "
+ "compiles again on every request. Null when the compile itself failed."
),
)
latest_calculation = ExperimentHealthInputsLatestCalculationSerializer(
diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.ts
index 34cac444a5da..012dcb5eba19 100644
--- a/products/experiments/frontend/generated/api.schemas.ts
+++ b/products/experiments/frontend/generated/api.schemas.ts
@@ -2527,7 +2527,7 @@ export interface ExperimentHealthCheckResultApi {
* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
* * `multiple_variants_share` - Multiple Variants Share */
key: ExperimentHealthCheckKeyEnumApi
- /** What the check looks at, for example 'Feature flag variants'. */
+ /** The problem the check looks for, for example 'Metrics not set up'. */
name: string
/** passed: the check found no problem. finding: the check reported a finding in `findings`. not_applicable: the check does not cover the experiment's current state, for example a check of the changes since launch on a draft. not_checked: the check covers the experiment, but it could not read an input or it failed, so a problem it looks for can be present without a finding.
*
@@ -3751,7 +3751,7 @@ export interface ExperimentHealthInputsApi {
exposure_events: ExperimentHealthInputsExposureEventsApi | null
/** Whether the project has received each event that the metrics and the exposure criteria name, by event name. A name that ingestion keeps no record of is absent. */
events_received: ExperimentHealthInputsApiEventsReceived
- /** The compile of each metric's query without running it, as the reader, which the metric queries check reads. A metric that compiles is cached per reader for a day, and a failing metric compiles again on every request. Null when the compile itself failed. */
+ /** The compile of each metric's query without running it, as the reader, which the 'Invalid metric definitions' check reads. A metric that compiles is cached per reader for a day, and a failing metric compiles again on every request. Null when the compile itself failed. */
metric_compiles: ExperimentHealthInputsMetricCompilesApi | null
/** The latest finished calculation of the metrics, which the metric results check reads. Null before launch, on an archived experiment, or when the read failed. */
latest_calculation: ExperimentHealthInputsLatestCalculationApi | null
diff --git a/products/experiments/frontend/health/ExperimentHealthChip.tsx b/products/experiments/frontend/health/ExperimentHealthChip.tsx
index 17c4002ce6e0..209164a19a6a 100644
--- a/products/experiments/frontend/health/ExperimentHealthChip.tsx
+++ b/products/experiments/frontend/health/ExperimentHealthChip.tsx
@@ -1,4 +1,5 @@
import { useActions, useValues } from 'kea'
+import { useState } from 'react'
import { IconWarning } from '@posthog/icons'
import { LemonButton, LemonDropdown, Spinner } from '@posthog/lemon-ui'
@@ -6,70 +7,69 @@ import { LemonButton, LemonDropdown, Spinner } from '@posthog/lemon-ui'
import { pluralize } from 'lib/utils/strings'
import { experimentLogic } from 'scenes/experiments/experimentLogic'
-import { EXPERIMENT_HEALTH_PANEL_ELEMENT_ID } from '../constants'
import { HealthChecksOverview } from './HealthChecksOverview'
import { healthChipState } from './healthPanelChecks'
export function ExperimentHealthChip(): JSX.Element | null {
const { healthFindings, healthChecks, healthExposureChecks, isExperimentDraft } = useValues(experimentLogic)
const { reportHealthChecksOpened } = useActions(experimentLogic)
+ const [isOverviewOpen, setIsOverviewOpen] = useState(false)
const state = healthChipState(healthFindings, healthChecks, healthExposureChecks)
- switch (state.kind) {
- case 'hidden':
- return null
- case 'issues':
- return (
+ if (state.kind === 'hidden') {
+ return null
+ }
+ if (state.kind === 'pending') {
+ return (
+ <span className="flex items-center gap-1 text-secondary" data-attr="experiment-health-chip-pending">
+ <Spinner className="text-sm" />
+ <span>Checking health</span>
+ </span>
+ )
+ }
+ // The same overview opens with and without issues, so the reader finds the checks behind either label.
+ return (
+ <LemonDropdown
+ overlay={
+ <HealthChecksOverview
+ checks={state.checks}
+ findings={state.findings}
+ isDraft={isExperimentDraft}
+ exposureChecksMissing={healthExposureChecks === 'missing'}
+ // Most actions open a modal, and a modal renders below a popover, so the overview closes first.
+ onFindingActedOn={() => setIsOverviewOpen(false)}
+ />
+ }
+ visible={isOverviewOpen}
+ closeOnClickInside={false}
+ placement="bottom-start"
+ onVisibilityChange={(visible) => {
+ setIsOverviewOpen(visible)
+ if (visible) {
+ reportHealthChecksOpened('meta_bar')
+ }
+ }}
+ >
+ {state.kind === 'issues' ? (
<LemonButton
type="secondary"
size="xsmall"
icon={<IconWarning className={state.hasCritical ? 'text-danger' : 'text-warning'} />}
- onClick={() =>
- document
- .getElementById(EXPERIMENT_HEALTH_PANEL_ELEMENT_ID)
- ?.scrollIntoView({ block: 'nearest' })
- }
tooltip="Show the health checks"
data-attr="experiment-health-chip"
>
{pluralize(state.count, 'issue')}
</LemonButton>
- )
- case 'pending':
- return (
- <span className="flex items-center gap-1 text-secondary" data-attr="experiment-health-chip-pending">
- <Spinner className="text-sm" />
- <span>Checking health</span>
- </span>
- )
- case 'checks':
- return (
- <LemonDropdown
- overlay={
- <HealthChecksOverview
- checks={state.checks}
- notes={state.notes}
- isDraft={isExperimentDraft}
- exposureChecksMissing={healthExposureChecks === 'missing'}
- />
- }
- closeOnClickInside={false}
- placement="bottom-start"
- onVisibilityChange={(visible) => {
- if (visible) {
- reportHealthChecksOpened('meta_bar')
- }
- }}
+ ) : (
+ <LemonButton
+ type="tertiary"
+ size="xsmall"
+ tooltip="Health checks"
+ data-attr="experiment-health-checks-button"
>
- <LemonButton
- type="tertiary"
- size="xsmall"
- tooltip="Health checks"
- data-attr="experiment-health-checks-button"
- >
- <span className="font-normal">{state.label}</span>
- </LemonButton>
- </LemonDropdown>
- )
- }
+ <span className="font-normal">{state.label}</span>
+ </LemonButton>
+ )}
+ </LemonDropdown>
+ )
}
diff --git a/products/experiments/frontend/health/ExperimentHealthPanel.tsx b/products/experiments/frontend/health/ExperimentHealthPanel.tsx
index 104ee98e5433..50bea1a97cdf 100644
--- a/products/experiments/frontend/health/ExperimentHealthPanel.tsx
+++ b/products/experiments/frontend/health/ExperimentHealthPanel.tsx
@@ -1,29 +1,20 @@
-import { useActions, useValues } from 'kea'
-import { useId, useState } from 'react'
+import { useValues } from 'kea'
-import { IconChevronDown, IconChevronRight } from '@posthog/icons'
-import { LemonButton, LemonCard, LemonTag } from '@posthog/lemon-ui'
+import { LemonCard, LemonTag } from '@posthog/lemon-ui'
import { experimentLogic } from 'scenes/experiments/experimentLogic'
import { EXPERIMENT_HEALTH_PANEL_ELEMENT_ID } from '../constants'
-import { HealthCheckList } from './HealthCheckList'
import { HealthFindingRow } from './HealthFindingRow'
-import { listedHealthChecks } from './healthPanelChecks'
import { healthIssues } from './healthPanelFindings'
export function ExperimentHealthPanel(): JSX.Element | null {
- const { healthFindings, healthChecks, healthExposureChecks } = useValues(experimentLogic)
- const { reportHealthChecksOpened } = useActions(experimentLogic)
- const [isCheckListOpen, setIsCheckListOpen] = useState(false)
- const checkListId = useId()
+ const { healthFindings } = useValues(experimentLogic)
- // Without an issue the chip next to the experiment's status holds the checks, so the page keeps its space.
- if (healthFindings === null || healthChecks === null || healthIssues(healthFindings).length === 0) {
+ // The chip next to the experiment's status lists every check, so the panel shows the findings, and only with an issue.
+ if (healthFindings === null || healthIssues(healthFindings).length === 0) {
return null
}
- const listedChecks = listedHealthChecks(healthChecks)
- const exposureChecksMissing = healthExposureChecks === 'missing'
return (
<section
@@ -43,35 +34,6 @@ export function ExperimentHealthPanel(): JSX.Element | null {
<HealthFindingRow key={`${finding.code}:${finding.subcode ?? ''}`} finding={finding} />
))}
</div>
- {(listedChecks.length > 0 || exposureChecksMissing) && (
- <>
- <div className="px-1 py-1 border-t">
- <LemonButton
- size="xsmall"
- type="tertiary"
- sideIcon={isCheckListOpen ? <IconChevronDown /> : <IconChevronRight />}
- onClick={() => {
- if (!isCheckListOpen) {
- reportHealthChecksOpened('panel')
- }
- setIsCheckListOpen(!isCheckListOpen)
- }}
- aria-expanded={isCheckListOpen}
- aria-controls={checkListId}
- data-attr="experiment-health-check-list-toggle"
- >
- Other checks
- </LemonButton>
- </div>
- {isCheckListOpen && (
- <HealthCheckList
- id={checkListId}
- checks={listedChecks}
- exposureChecksMissing={exposureChecksMissing}
- />
- )}
- </>
- )}
</LemonCard>
</section>
)
diff --git a/products/experiments/frontend/health/HealthCheckList.tsx b/products/experiments/frontend/health/HealthCheckList.tsx
index 59a8f487a9d7..de2046a05fcb 100644
--- a/products/experiments/frontend/health/HealthCheckList.tsx
+++ b/products/experiments/frontend/health/HealthCheckList.tsx
@@ -8,7 +8,7 @@ type ListedStatus = ListedHealthCheck['status']
const STATUS_DISPLAY: Record<ListedStatus, { icon: JSX.Element; label: string }> = {
not_checked: { icon: <IconQuestion className="text-secondary" />, label: "Couldn't check" },
- passed: { icon: <IconCheck className="text-success" />, label: 'No problem found' },
+ passed: { icon: <IconCheck className="text-success" />, label: 'Looked for, not found' },
not_applicable: { icon: <IconMinus className="text-tertiary" />, label: "Doesn't apply" },
}
@@ -40,16 +40,14 @@ function groupByOutcome(checks: ListedHealthCheck[], exposureChecksMissing: bool
}
export function HealthCheckList({
- id,
checks,
exposureChecksMissing,
}: {
- id?: string
checks: ListedHealthCheck[]
exposureChecksMissing: boolean
}): JSX.Element {
return (
- <div id={id} className="flex flex-col gap-2 px-3 py-2" data-attr="experiment-health-check-list">
+ <div className="flex flex-col gap-2 px-3 py-2" data-attr="experiment-health-check-list">
{groupByOutcome(checks, exposureChecksMissing).map((group) => (
<div key={`${group.status}:${group.label}`} className="flex flex-col gap-1">
<span className="text-xs font-semibold text-secondary">{group.label}</span>
diff --git a/products/experiments/frontend/health/HealthChecksOverview.stories.tsx b/products/experiments/frontend/health/HealthChecksOverview.stories.tsx
index 57b088b9a8b6..aa2332d69341 100644
--- a/products/experiments/frontend/health/HealthChecksOverview.stories.tsx
+++ b/products/experiments/frontend/health/HealthChecksOverview.stories.tsx
@@ -18,8 +18,8 @@ const skipped = (
): HealthPanelCheck => ({ key, name, status, reason })
const FLAG_SETUP_CHECKS = [
- passed('flag_state', 'Feature flag rollout'),
- passed('flag_variants_mismatch', 'Feature flag variants'),
+ passed('flag_state', "Flag state that doesn't match the experiment"),
+ passed('flag_variants_mismatch', 'Flag variants not set up'),
passed('forced_variant_release_condition', 'Release conditions that force a variant'),
]
@@ -30,6 +30,7 @@ const meta: Meta<typeof HealthChecksOverview> = {
layout: 'centered',
testOptions: { snapshotBrowsers: ['chromium'] },
},
+ args: { onFindingActedOn: () => {} },
}
export default meta
@@ -39,16 +40,15 @@ export const Draft: Story = {
args: {
checks: [
...FLAG_SETUP_CHECKS,
- skipped('flag_history', 'Feature flag history', 'not_applicable', 'Runs after launch'),
skipped(
'split_changed_mid_run',
'Variant split changes since launch',
'not_applicable',
'Runs after launch'
),
- skipped('no_metric', 'Metrics', 'not_applicable', 'Runs after launch'),
+ skipped('no_metric', 'Metrics not set up', 'not_applicable', 'Runs after launch'),
],
- notes: [
+ findings: [
{
code: 'new_flag_no_traffic',
subcode: null,
@@ -75,11 +75,11 @@ export const DraftWithEventsNotReceived: Story = {
args: {
checks: [
...FLAG_SETUP_CHECKS,
- skipped('no_metric', 'Metrics', 'not_applicable', 'Runs after launch'),
- found('metric_event_not_ingested', 'Metric events'),
- found('exposure_criteria_cannot_match', 'Exposure criteria'),
+ skipped('no_metric', 'Metrics not set up', 'not_applicable', 'Runs after launch'),
+ found('metric_event_not_ingested', 'Metric events not received yet'),
+ found('exposure_criteria_cannot_match', "Exposure criteria that can't match"),
],
- notes: [
+ findings: [
{
code: 'metric_event_not_ingested',
subcode: null,
@@ -112,10 +112,10 @@ export const RunningWithSignificanceOnASmallSample: Story = {
args: {
checks: [
...FLAG_SETUP_CHECKS,
- passed('no_metric', 'Metrics'),
- found('early_significance', 'Sample size behind significant results'),
+ passed('no_metric', 'Metrics not set up'),
+ found('early_significance', 'Significance on a small sample'),
],
- notes: [
+ findings: [
{
code: 'early_significance',
subcode: 'below_detectable_effect',
@@ -148,10 +148,33 @@ export const ChecksThatCouldNotRun: Story = {
'not_checked',
"Couldn't check because you can't view the linked feature flag's history"
),
- passed('no_metric', 'Metrics'),
+ passed('no_metric', 'Metrics not set up'),
],
- notes: [],
+ findings: [],
isDraft: false,
exposureChecksMissing: true,
},
}
+
+export const RunningWithAnIssue: Story = {
+ args: {
+ checks: [
+ ...FLAG_SETUP_CHECKS,
+ passed('no_metric', 'Metrics not set up'),
+ found('srm', 'Split of users across variants'),
+ ],
+ findings: [
+ {
+ code: 'srm',
+ subcode: null,
+ severity: 'warning',
+ title: 'Users are not split across variants as configured',
+ detail: "The distribution of users across variants doesn't match your configured rollout percentages (p < 0.001). This may indicate issues with randomization or data collection.",
+ evidence: { p_value: 0.0002 },
+ actions: [],
+ },
+ ],
+ isDraft: false,
+ exposureChecksMissing: false,
+ },
+}
diff --git a/products/experiments/frontend/health/HealthChecksOverview.tsx b/products/experiments/frontend/health/HealthChecksOverview.tsx
index a9c478b141d8..e44f0948577f 100644
--- a/products/experiments/frontend/health/HealthChecksOverview.tsx
+++ b/products/experiments/frontend/health/HealthChecksOverview.tsx
@@ -4,18 +4,21 @@ import { HealthCheckList } from './HealthCheckList'
import { HealthChecksSummary } from './HealthChecksSummary'
import { HealthFindingRow } from './HealthFindingRow'
import { type HealthPanelCheck, listedHealthChecks } from './healthPanelChecks'
-import type { HealthPanelFinding } from './healthPanelFindings'
+import { type HealthPanelFinding, healthIssues } from './healthPanelFindings'
export function HealthChecksOverview({
checks,
- notes,
+ findings,
isDraft,
exposureChecksMissing,
+ onFindingActedOn,
}: {
checks: HealthPanelCheck[]
- notes: HealthPanelFinding[]
+ /** Issues first, as `healthPanelFindings` sorts them. */
+ findings: HealthPanelFinding[]
isDraft: boolean
exposureChecksMissing: boolean
+ onFindingActedOn: () => void
}): JSX.Element {
const listedChecks = listedHealthChecks(checks)
@@ -26,13 +29,18 @@ export function HealthChecksOverview({
<span className="text-sm font-semibold">Health checks</span>
<LemonTag type="completion">Beta</LemonTag>
</div>
- <HealthChecksSummary checks={checks} isDraft={isDraft} />
+ <HealthChecksSummary checks={checks} issueCount={healthIssues(findings).length} isDraft={isDraft} />
</div>
- {notes.length > 0 && (
- // A note mounts, and so reports as shown, only when the reader opens this overview.
+ {findings.length > 0 && (
+ // A note mounts, and so reports as shown, only when the reader opens this overview. The panel reports the
+ // issues on its own.
<div className="divide-y border-t">
- {notes.map((finding) => (
- <HealthFindingRow key={`${finding.code}:${finding.subcode ?? ''}`} finding={finding} />
+ {findings.map((finding) => (
+ <HealthFindingRow
+ key={`${finding.code}:${finding.subcode ?? ''}`}
+ finding={finding}
+ onActedOn={onFindingActedOn}
+ />
))}
</div>
)}
diff --git a/products/experiments/frontend/health/HealthChecksSummary.tsx b/products/experiments/frontend/health/HealthChecksSummary.tsx
index 16c31384cfe6..5616cd2e56e1 100644
--- a/products/experiments/frontend/health/HealthChecksSummary.tsx
+++ b/products/experiments/frontend/health/HealthChecksSummary.tsx
@@ -4,9 +4,11 @@ import { type HealthPanelCheck, countHealthChecks } from './healthPanelChecks'
export function HealthChecksSummary({
checks,
+ issueCount,
isDraft,
}: {
checks: HealthPanelCheck[]
+ issueCount: number
isDraft: boolean
}): JSX.Element {
const passedCount = countHealthChecks(checks, 'passed')
@@ -15,9 +17,11 @@ export function HealthChecksSummary({
return (
<div className="flex flex-col gap-0.5" data-attr="experiment-health-summary">
<span className="font-semibold">
- {passedCount > 0
- ? `No problems found by ${pluralize(passedCount, 'check')}`
- : 'No checks apply to this experiment yet'}
+ {issueCount > 0
+ ? `${pluralize(issueCount, 'issue')} found`
+ : passedCount > 0
+ ? `No problems found by ${pluralize(passedCount, 'check')}`
+ : 'No checks apply to this experiment yet'}
</span>
<p className="m-0 text-secondary">
{notCheckedCount > 0 && <span>{pluralize(notCheckedCount, 'check')} couldn't run. </span>}
diff --git a/products/experiments/frontend/health/HealthFindingRow.tsx b/products/experiments/frontend/health/HealthFindingRow.tsx
index 8467fa0eca87..440d0449a8bf 100644
--- a/products/experiments/frontend/health/HealthFindingRow.tsx
+++ b/products/experiments/frontend/health/HealthFindingRow.tsx
@@ -1,7 +1,7 @@
import { useId, useState } from 'react'
import { IconChevronDown, IconChevronRight, IconInfo, IconWarning } from '@posthog/icons'
-import { LemonButton } from '@posthog/lemon-ui'
+import { LemonButton, Tooltip } from '@posthog/lemon-ui'
import { cn } from 'lib/utils/css-classes'
@@ -10,7 +10,20 @@ import { HealthFindingActionButton } from './HealthFindingActionButton'
import type { HealthPanelFinding } from './healthPanelFindings'
import { useHealthFindingReporting } from './useHealthFindingReporting'
-export function HealthFindingRow({ finding }: { finding: HealthPanelFinding }): JSX.Element {
+// The icons differ in shape and color only, so each says its severity in words too.
+const SEVERITY_LABELS: Record<HealthPanelFinding['severity'], string> = {
+ info: 'Note',
+ warning: 'Warning',
+ critical: 'Critical',
+}
+
+export function HealthFindingRow({
+ finding,
+ onActedOn,
+}: {
+ finding: HealthPanelFinding
+ onActedOn?: () => void
+}): JSX.Element {
const [isDetailOpen, setIsDetailOpen] = useState(false)
const detailId = useId()
const { reportOpened, reportActedOn } = useHealthFindingReporting({
@@ -24,16 +37,20 @@ export function HealthFindingRow({ finding }: { finding: HealthPanelFinding }):
<div className="flex flex-col gap-1 px-3 py-2" data-attr="experiment-health-finding">
<div className="flex flex-wrap items-center justify-between gap-2">
<div className="flex items-center gap-2 min-w-0">
- {finding.severity === 'info' ? (
- <IconInfo className="text-secondary text-lg shrink-0" />
- ) : (
- <IconWarning
- className={cn(
- 'text-lg shrink-0',
- finding.severity === 'critical' ? 'text-danger' : 'text-warning'
+ <Tooltip title={SEVERITY_LABELS[finding.severity]}>
+ <span role="img" aria-label={SEVERITY_LABELS[finding.severity]} className="flex shrink-0">
+ {finding.severity === 'info' ? (
+ <IconInfo className="text-secondary text-lg" />
+ ) : (
+ <IconWarning
+ className={cn(
+ 'text-lg',
+ finding.severity === 'critical' ? 'text-danger' : 'text-warning'
+ )}
+ />
)}
- />
- )}
+ </span>
+ </Tooltip>
<span className="font-semibold">{finding.title}</span>
<LemonButton
size="xsmall"
@@ -58,14 +75,17 @@ export function HealthFindingRow({ finding }: { finding: HealthPanelFinding }):
<HealthFindingActionButton
key={actionKind}
actionKind={actionKind}
- onClick={() => reportActedOn(actionKind)}
+ onClick={() => {
+ reportActedOn(actionKind)
+ onActedOn?.()
+ }}
/>
))}
</div>
)}
</div>
{isDetailOpen && (
- <p id={detailId} className="m-0 pl-7 text-secondary">
+ <p id={detailId} className="m-0 pl-7 text-secondary whitespace-pre-line">
{finding.detail}
</p>
)}
diff --git a/products/experiments/frontend/health/debug/healthDebugInputs.ts b/products/experiments/frontend/health/debug/healthDebugInputs.ts
index f6ea002826b0..30e876bfc920 100644
--- a/products/experiments/frontend/health/debug/healthDebugInputs.ts
+++ b/products/experiments/frontend/health/debug/healthDebugInputs.ts
@@ -316,7 +316,7 @@ export function buildMetricFacts(
{
label: 'Metric queries',
value: serverValue(state, (inputs) => describeMetricCompiles(inputs.metric_compiles)),
- help: "The metric queries check compiles each metric's query as you, without running it. A metric that compiles is cached for a day. A failing metric compiles again on every load.",
+ help: "The 'Invalid metric definitions' check compiles each metric's query as you, without running it. A metric that compiles is cached for a day. A failing metric compiles again on every load.",
},
...(state.inputs?.metric_compiles?.failures ?? []).map((failure) => ({
label: `Doesn't compile: ${failure.name} (${failure.role})`,
diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
index 8dcf6a38595e..3d296663ea16 100644
--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts
+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
@@ -73,7 +73,7 @@ export const FLAG_STATE_FINDING_CODES: ReadonlySet<ExperimentHealthFindingCode>
)
// pinned: `opened_from` property values of `experiment health checks opened`.
-export type ExperimentHealthChecksOpenedFrom = 'meta_bar' | 'panel'
+export type ExperimentHealthChecksOpenedFrom = 'meta_bar'
// pinned: `health_ui` property values. Insights compare the server's findings with the separate warnings on them.
// 'panel' means the page shows the server's findings: in the health panel when one is an issue, otherwise in the
diff --git a/products/experiments/frontend/health/healthPanelChecks.test.ts b/products/experiments/frontend/health/healthPanelChecks.test.ts
index aff7bae5bb6c..0c356ae1764f 100644
--- a/products/experiments/frontend/health/healthPanelChecks.test.ts
+++ b/products/experiments/frontend/health/healthPanelChecks.test.ts
@@ -33,7 +33,7 @@ const EXPOSURES = {
date_range: { date_from: '2026-10-01', date_to: '2026-10-02' },
health_findings: [],
health_checks: [
- { key: 'zero_exposures', name: 'Exposures', status: 'passed', reason: null },
+ { key: 'zero_exposures', name: 'Exposures not received since launch', status: 'passed', reason: null },
{ key: 'srm', name: 'Split of users across variants', status: 'finding', reason: null },
],
} satisfies ExperimentExposureQueryResponse
@@ -45,15 +45,23 @@ describe('healthPanelChecks', () => {
{
name: 'the checks of the exposure answer join those of the experiment read, problems first',
isExperimentDraft: false,
- expected: ['srm:finding', 'flag_history:not_checked', 'no_metric:passed', 'zero_exposures:passed'],
+ expected: ['srm:finding', 'split_changed_mid_run:not_checked', 'no_metric:passed', 'zero_exposures:passed'],
},
{
name: 'a draft lists no exposure check that a reset left in the page',
isExperimentDraft: true,
- expected: ['flag_history:not_checked', 'no_metric:passed'],
+ expected: ['split_changed_mid_run:not_checked', 'no_metric:passed'],
},
])('$name', ({ isExperimentDraft, expected }) => {
- const health = { findings: [], checks: [check('no_metric', 'passed'), check('flag_history', 'not_checked')] }
+ // The flag history check is left out: its note and the checks it serves report when it can't read the history.
+ const health = {
+ findings: [],
+ checks: [
+ check('no_metric', 'passed'),
+ check('split_changed_mid_run', 'not_checked'),
+ check('flag_history', 'passed'),
+ ],
+ }
expect(
healthPanelChecks(health, EXPOSURES, isExperimentDraft)?.map(({ key, status }) => `${key}:${status}`)
).toEqual(expected)
@@ -123,7 +131,7 @@ describe('healthPanelChecks', () => {
expected: { kind: 'hidden' },
},
{
- name: 'issues lead to the panel and count no info finding',
+ name: 'issues count no info finding',
findings: [finding('critical'), finding('warning'), finding('info')],
checks: [],
exposureChecks: 'ready',
@@ -148,7 +156,7 @@ describe('healthPanelChecks', () => {
findings: [finding('info')],
checks: [check('no_metric', 'passed'), check('flag_state', 'passed'), check('srm', 'not_applicable')],
exposureChecks: 'ready',
- expected: { kind: 'checks', label: '2\u00a0checks passed · 1\u00a0note' },
+ expected: { kind: 'checks', label: 'No problems found (2\u00a0checks) · 1\u00a0note' },
},
{
name: 'the label says so when no check applies',
diff --git a/products/experiments/frontend/health/healthPanelChecks.ts b/products/experiments/frontend/health/healthPanelChecks.ts
index b3739feeb70e..0a1506f7a3b0 100644
--- a/products/experiments/frontend/health/healthPanelChecks.ts
+++ b/products/experiments/frontend/health/healthPanelChecks.ts
@@ -14,9 +14,19 @@ export type ExposureChecksState = 'not_launched' | 'pending' | 'missing' | 'read
export type HealthChipState =
| { kind: 'hidden' }
- | { kind: 'issues'; count: number; hasCritical: boolean }
+ | {
+ kind: 'issues'
+ count: number
+ hasCritical: boolean
+ checks: HealthPanelCheck[]
+ findings: HealthPanelFinding[]
+ }
| { kind: 'pending' }
- | { kind: 'checks'; label: string; checks: HealthPanelCheck[]; notes: HealthPanelFinding[] }
+ | { kind: 'checks'; label: string; checks: HealthPanelCheck[]; findings: HealthPanelFinding[] }
+
+// The flag history check only lets the checks of changes since launch look. When it can't read the history, they say
+// they couldn't check and its note says why, so its own row would tell the reader nothing.
+const UNLISTED_CHECKS: ReadonlySet<HealthPanelCheck['key']> = new Set(['flag_history'])
const CHECK_STATUS_ORDER: Record<HealthPanelCheck['status'], number> = {
finding: 0,
@@ -36,9 +46,9 @@ export function healthPanelChecks(
// A reset to draft keeps the previous exposure answer in the page, as for the findings.
const exposureChecks = isExperimentDraft ? [] : (exposures?.health_checks ?? [])
// A server from before the checks were reported sends none, for example while a deploy rolls out.
- return [...(health.checks ?? []), ...exposureChecks].sort(
- (a, b) => CHECK_STATUS_ORDER[a.status] - CHECK_STATUS_ORDER[b.status]
- )
+ return [...(health.checks ?? []), ...exposureChecks]
+ .filter((check) => !UNLISTED_CHECKS.has(check.key))
+ .sort((a, b) => CHECK_STATUS_ORDER[a.status] - CHECK_STATUS_ORDER[b.status])
}
export function exposureChecksState(
@@ -78,6 +88,8 @@ export function healthChipState(
kind: 'issues',
count: issues.length,
hasCritical: issues.some((finding) => finding.severity === 'critical'),
+ checks,
+ findings,
}
}
// A verdict before the exposure checks answer could turn into an issue a moment later.
@@ -87,10 +99,10 @@ export function healthChipState(
// Without an issue, every finding is an info finding.
const passedCount = countHealthChecks(checks, 'passed')
const label = [
- passedCount > 0 ? `${pluralize(passedCount, 'check')} passed` : null,
+ passedCount > 0 ? `No problems found (${pluralize(passedCount, 'check')})` : null,
findings.length > 0 ? pluralize(findings.length, 'note') : null,
]
.filter(Boolean)
.join(' · ')
- return { kind: 'checks', label: label || 'No checks apply yet', checks, notes: findings }
+ return { kind: 'checks', label: label || 'No checks apply yet', checks, findings }
}
diff --git a/products/experiments/skills/debugging-experiments/references/customer-reply.md b/products/experiments/skills/debugging-experiments/references/customer-reply.md
index 13c8211a72cb..d184f4c34804 100644
--- a/products/experiments/skills/debugging-experiments/references/customer-reply.md
+++ b/products/experiments/skills/debugging-experiments/references/customer-reply.md
@@ -72,7 +72,7 @@ anything random, and it's a quick fix.
**The problem:** your experiment runs an 80/20 split, and "Multiple variant handling" is set
to **Exclude from analysis**. About 2% of your users were exposed to more than one variant,
and excluding them hits the smaller (20%) variant harder, which is why it looks worse than it
-should. That's what the "Setup likely introduced bias" banner is telling you.
+should. That's what the warning "The uneven split may bias the results" is telling you.
**The fix:**
1. **Switch "Multiple variant handling" to Use first seen variant.** That keeps those users in
diff --git a/products/experiments/skills/diagnosing-experiment-health/references/bias-and-skew.md b/products/experiments/skills/diagnosing-experiment-health/references/bias-and-skew.md
index a809673815f3..cddfa93c7a52 100644
--- a/products/experiments/skills/diagnosing-experiment-health/references/bias-and-skew.md
+++ b/products/experiments/skills/diagnosing-experiment-health/references/bias-and-skew.md
@@ -61,7 +61,7 @@ On an experiment with a custom exposure event these queries read the flag's call
## A1 — Multi-variant exclusion bias on uneven split [HIGH]
-This is the signal behind the in-app banner "Setup likely introduced bias".
+This is the signal behind the in-app warning "The uneven split may bias the results".
**Evidence.** The banner's check reports a finding when **all four** hold:
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 7c8011fae9ce..3fe2ea03e745 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -47736,7 +47736,7 @@ export namespace Schemas {
* * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
* * `multiple_variants_share` - Multiple Variants Share */
key: ExperimentHealthCheckKeyEnum;
- /** What the check looks at, for example 'Feature flag variants'. */
+ /** The problem the check looks for, for example 'Metrics not set up'. */
name: string;
/** passed: the check found no problem. finding: the check reported a finding in `findings`. not_applicable: the check does not cover the experiment's current state, for example a check of the changes since launch on a draft. not_checked: the check covers the experiment, but it could not read an input or it failed, so a problem it looks for can be present without a finding.
*
@@ -48028,7 +48028,7 @@ export namespace Schemas {
export interface ExperimentExposureHealthCheck {
/** Stable identifier of the check. */
key: ExperimentExposureHealthCheckKey;
- /** What the check looks at, as the health panel lists it. */
+ /** The problem the check looks for, as the health panel lists it. */
name: string;
/** Why the check did not look, for the statuses not_applicable and not_checked, for example 'Runs after launch'. Null for passed and finding. */
reason: string | null;
@@ -48898,7 +48898,7 @@ export namespace Schemas {
exposure_events: ExperimentHealthInputsExposureEvents | null;
/** Whether the project has received each event that the metrics and the exposure criteria name, by event name. A name that ingestion keeps no record of is absent. */
events_received: ExperimentHealthInputsEventsReceived;
- /** The compile of each metric's query without running it, as the reader, which the metric queries check reads. A metric that compiles is cached per reader for a day, and a failing metric compiles again on every request. Null when the compile itself failed. */
+ /** The compile of each metric's query without running it, as the reader, which the 'Invalid metric definitions' check reads. A metric that compiles is cached per reader for a day, and a failing metric compiles again on every request. Null when the compile itself failed. */
metric_compiles: ExperimentHealthInputsMetricCompiles | null;
/** The latest finished calculation of the metrics, which the metric results check reads. Null before launch, on an archived experiment, or when the read failed. */
latest_calculation: ExperimentHealthInputsLatestCalculation | null;