Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(experiments): show what each health check read (#128988)

Marcel Poelker7d7067ce1eb4
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index fc95f6f392ef..e4d5265c7db0 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -3404,42 +3404,50 @@ snapshots:
hash: v1.k794b7964.7cb2348662f942bc00e904aca1cf74990ac3b8e25549be308b29e907c156d0eb.WQ5R8L0lbbFdH9oG3RShU2S1nHfj_SHpnz9PrjLAx6A
errortracking-volumesparkline--events-before-data-range--light:
hash: v1.k794b7964.c88c4e3b028915efba3fdbbac7107a063662d8210dfb6ba8667e96d719fa3d4f.tzwEQFa4F1f1O9PGqBkcaqSavbFVgAwPAjyh9Uf9NL8
+ experiments-healthchecksoverview--check-details-from-an-answer-cached-before-the-values--dark:
+ hash: v1.k794b7964.8f06822792edb869dc302b1b55548e750711a680282af87becf23fa05ff88d0d.d5sAlitIKlk05A9pvaewVVf5bmgf-Jxn0WhagfbAam4
+ experiments-healthchecksoverview--check-details-from-an-answer-cached-before-the-values--light:
+ hash: v1.k794b7964.3e6c929f9052c3e07b0a179bddf874b6f3389a6cc7a2cf13b3dea38eeb53411f.LV5Czi7t5fgxV4apeVQOOYZqBBxBH4SVLVQ5bcmLzZo
experiments-healthchecksoverview--check-details-of-a-check-that-does-not-apply--dark:
- hash: v1.k794b7964.ac9691ae7fc3d5474c510b5e55920aec536a30642aab1c9fd7aff05357b410b5.q2-oBgQnvLkgb0wTyPc79z7c9XIwv3KeWXcJMfCmyH8
+ hash: v1.k794b7964.f9a702503f7f187702a14ddd3de2e6e87d1a2a9eeb93e5c0944b008f60fc2406.OfmUY-ruWZYUwkuauRF1-UvnqXLIVIkNISJloTzaa5I
experiments-healthchecksoverview--check-details-of-a-check-that-does-not-apply--light:
- hash: v1.k794b7964.d624e91300f29ae3764d991314bca061ee3f4ce82e95b4e80d47c9486e949428.ZeSDib1epm7V-v0za6cgt1fOQrbzdbyJfPKpgo7LOKQ
- experiments-healthchecksoverview--check-details-with-a-measurement--dark:
- hash: v1.k794b7964.5c0fe861f003d8d40ef6fb2c1ef865e4e33a99c00c7806f927c4c1d124a4da5b.g8YZDTqe7h9wK_zfgCwKK3rWtBmxe4dPY7H1kZ56Rqk
- experiments-healthchecksoverview--check-details-with-a-measurement--light:
- hash: v1.k794b7964.2285a46e750551fcf1517e608aa44c2e0ff21063073163b5cc3cdfe11299fe19.RGf6GJMCnDe9524DnNmaXbiwfJTzKC5VPhtSwoxLNjk
+ hash: v1.k794b7964.1ad45af88c4579b2bca14bd94ca1e8db84fb9d1b0dd930972a718c49db52d3bb.JA0TofrwELYzgN6_hJRbKtoUgq9d7e6iA8gFkntEl-A
+ experiments-healthchecksoverview--check-details-with-what-it-read--dark:
+ hash: v1.k794b7964.15654ee956b9a9d101396b1b7970b7a5864812b6ddeceb943d122996334ed09a.V-Gvw6sFRFzPGS4jTAAvqqo9pi3pEOh3Z-iuU5wjVyg
+ experiments-healthchecksoverview--check-details-with-what-it-read--light:
+ hash: v1.k794b7964.1e88a7a4ee790962b68b1be6e6ba05335df73a2440c17800339c1facf04c6cef.bP3feLTH_LmNroRUhcey63jaQ2rbbawbccTEhC1RhiM
experiments-healthchecksoverview--checks-that-could-not-run--dark:
- hash: v1.k794b7964.4f9a1d79253a510d61a042b769ea713fe81a0dfd69f637793a3b4885767b4ebf.KEXIDF1qOULpEjp9trhlH1Q2NyDsE-Ts0L_xuRMqETg
+ hash: v1.k794b7964.d5ff75d2345d4acad2dd357cbe3cb0eac67c0a2e91cb936e72775a4d2d77198b.3UZCkvoHukOJrcqPAJECf262qjn1mVcGr9bjEHUdcc8
experiments-healthchecksoverview--checks-that-could-not-run--light:
- hash: v1.k794b7964.6a3b2955e394047ece6233bf17b3a12ed0c969aab5b514b511879f2289b11f4e.txKJCMAIXncg726hJ4ev6YS3xDPue5hXCrHrjeoXFZU
+ hash: v1.k794b7964.6446f8babaebdd8b81ddd828f1031a1ae80f26f85a1aaa181c3b4eaf9a93c547.KqHa1VyB5KyRLkIbZEfDHPEw0lophWgsO9uok7Zgp8Y
experiments-healthchecksoverview--draft--dark:
- hash: v1.k794b7964.227cc0afccb826a9b2e2be4f71f568298b3d71274cbaeb7c6efca8ac1f1ad781.ndRiDlo2tQLo6JYkTZiZvKkzdw_4ZZLugU2Dz8yPfBY
+ hash: v1.k794b7964.80d8b34eaf6e2b119692d44a3b9dc07fb0292001d41b8a0acc1623a4fed8bb10.4iISmQeCOJBV6HenWEbD9iLVn80J9O0kWz_QtGwkXoc
experiments-healthchecksoverview--draft--light:
- hash: v1.k794b7964.c67c443f646df3902b4b1d7f29abdb44e2d4daba0f8316176be44999ec5ec06d.JCtVFPHsbvHcrPf_jx5vCQbZJyZVU2pec7z9Kl3MsZE
+ hash: v1.k794b7964.678e7486ead3fd5e8476124bd525fbf1dbefb6340bde0318f6fda206fbe2d150.wKVKdMEYziHlPLyjMxi9JyyJVnKcRTHDqUCTLAJXJ9c
experiments-healthchecksoverview--draft-with-events-not-received--dark:
- hash: v1.k794b7964.dd76a5cb3a731fff2c1cc9f27273daf6d9d35d7dd6c5afab250c63b7a92251b8.YyL-Hh9vEkAP2WqWWCcV2YQA53bHaH3GoxMO1zU8w0k
+ hash: v1.k794b7964.6993e570e66f28385d6f90c5fd7327238c1f8cb33e14a0f0d114142a2d3cf415.q5LY1guwiqXaN_A6t6AKeUbOySr3n6gVyCF65URiAbk
experiments-healthchecksoverview--draft-with-events-not-received--light:
- hash: v1.k794b7964.e7f16fc830a5d3fb7f7ec2a62373cac995c145471ccee8fca6a32887ae65b844.9hBRYTORg7jy_A9qVN96BU6f2MY-EmkyHUhzVRnWA3k
- experiments-healthchecksoverview--narrow-with-a-check-picked--dark:
- hash: v1.k794b7964.5506c13a990e9a69b2d2fdeeba5ef49e0ec36e910ebd687abb78b8032bfbfc36.jFkmbmtjk6TUMg6zK5zU59pUmyKDaoghH9nFl90RKmM
- experiments-healthchecksoverview--narrow-with-a-check-picked--light:
- hash: v1.k794b7964.4f7b9d85c2056e42771ceedfc5ebfbd54ea7277b749e05bbaf1652162f82e775.LC8RwdJpkXnwWYuas947fm8syybLy3QDAFAbf8ZXWRs
+ hash: v1.k794b7964.1cf599b962a0359cc2ddedb5aca6d265cf1e8e61e542688f642427c03cb6f867.hykUIYStN2_H2jq0WxAfvDpFAR8xR5RslDKlR0aDo8s
+ experiments-healthchecksoverview--narrow-with-an-issue--dark:
+ hash: v1.k794b7964.9718730e1a0d477004c4bb34c5faf5a2763f67864e6231b5c0852c7bfbd0ae07.JSr581FVrLGjyPO3svu7NNijgQGwkEf2ANN-Wkm5g2o
+ experiments-healthchecksoverview--narrow-with-an-issue--light:
+ hash: v1.k794b7964.d8deed8c590e874802ae03f8cbd5868dd05fe39a7f8e08234b073df8420f5ba6.i4Sxti3ECbsyi78FEv9DpEyUdtqRThBEn6dK579IDGc
+ experiments-healthchecksoverview--nothing-to-fix--dark:
+ hash: v1.k794b7964.b5775825d9c3c2cb42f08e9d924b53e72349ad819f0b39059c59612c0c5c3c38.ILV2tLsPBEFrkCSTe2V_154Ygy3SIU3t02tT8HxMk18
+ experiments-healthchecksoverview--nothing-to-fix--light:
+ hash: v1.k794b7964.cb2740830df02f246e5e583a5a932b89675a18d372bb9ee950189bd4ba5b587c.FIiF22sl59xysGx9kRsn56UCjjs04StNSgBCd-dZqEA
experiments-healthchecksoverview--running-with-an-issue--dark:
- hash: v1.k794b7964.fe8cf445d9e5ed4f406c794bf62357aa827742beb8c53464e2925f75843abc9b.Kbi8IXqEetITzgs_VhDwPQYsxEvJawQwHEoKpGhdCLY
+ hash: v1.k794b7964.cfeab0bb22f3c73d6a3cc9d03badffaaee9cc8f6ba57a7f5ae5b8a12db5dbf9b.qPfC-Q6cR-Et4ycTR5ATF1gDFW6_Zyo40kqy_D-xdFo
experiments-healthchecksoverview--running-with-an-issue--light:
- hash: v1.k794b7964.844ee662239ead860dbbfe305ab35133e43419fae91b7fcd6fc2e784817bb9d3.pm1SyPteJF0wjcsY89B_yhp63O9rZrqX3uueA9Nv9yk
+ hash: v1.k794b7964.e0b528bdab2ca1dd5732c5e6862f9aa5e8d50609e8de1b69410258a615a153e2.LVlExPVbLi5Pp_iCgEPyfiUQvj-qL-2KmilHtk8ruBE
experiments-healthchecksoverview--running-with-metric-windows-not-complete--dark:
- hash: v1.k794b7964.2aa9d15debf2e7a5c3e89c97544307912994e5068cd11411b22b69703f861d6b.V8kz3w27dBeWJAyP0Wd9VEBZz7tQpYJAtQ3nYlP0kqI
+ hash: v1.k794b7964.d57e30783abd1f0d6448f28461131248858a0e43d99444dd1ab5a65966a46842.KSkJ9WFy6ed_Xey7QR8R8mrYY9KSjO1IdXkNUw9eW-o
experiments-healthchecksoverview--running-with-metric-windows-not-complete--light:
- hash: v1.k794b7964.aab9171c323a4de21008508fe6c03bd64c4b84b8603c934e1bb2b494992155ba.U4_d1SnqQ3VhuW6H43277T0YpsflKV8lpfSu1txOnKY
+ hash: v1.k794b7964.5084584e83f8fe4c09292423b875b194212ab07f6facff466ebb09a31840fba8.lB0STpZwwj3obAUJM6DtowBvXEWQebWoqE8XUxiWA5M
experiments-healthchecksoverview--running-with-significance-on-a-small-sample--dark:
- hash: v1.k794b7964.780b197f87a7ed50a5e0da73e9e66e6de2b86308e8127fd581c0c01b741080ad.vdVUcA3q7eLsQKRrTfuiXkkVGnkHWvq89a-mQYRE0b0
+ hash: v1.k794b7964.f5000f107e4b86ada149997b6857bb26fbac21ff340dac43f639529afd0b2e16.rGkjvMCD2W33zjvTcOOhnqBBZfCajq7UBWYXXpwglsc
experiments-healthchecksoverview--running-with-significance-on-a-small-sample--light:
- hash: v1.k794b7964.b048631809f04945042086b7ddc687ebd3744d4077cd5ad3d4a198b61581d8b2.QeR91ugmGnSAhNZ7BB35BKu-2mN0kU0H3aYsXsk4Wr0
+ hash: v1.k794b7964.85c6232b61d332c1027575be6d240043f8b146d1f5aab7392935024396aeef1d.2gjoE0_YzfcmmBAnySN4x_3cFZQZyvhUCG2YGE3iK4g
experiments-howtoreadtooltip--intervals-bayesian--dark:
hash: v1.k794b7964.a06d267cee9cb54bafc732335b35d461d3c1a59250c6daa8becace0fcb8492d2.vR5tecOozn6B51xE8LrByAFVsi03J4w-iUu-j2hQUTU
experiments-howtoreadtooltip--intervals-bayesian--light:
diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json
index 46e49a87507e..6a49fdc5a0a6 100644
--- a/frontend/src/queries/schema.json
+++ b/frontend/src/queries/schema.json
@@ -26465,17 +26465,6 @@
"$ref": "#/definitions/ExperimentExposureHealthCheckKey",
"description": "Stable identifier of the check."
},
- "measurement": {
- "anyOf": [
- {
- "$ref": "#/definitions/ExperimentExposureHealthMeasurement"
- },
- {
- "type": "null"
- }
- ],
- "description": "The value that decided a passed check. Null for the other statuses and for a pass without a value."
- },
"name": {
"description": "The problem the check looks for, as the health panel lists it.",
"type": "string"
@@ -26484,9 +26473,27 @@
"description": "Why the check did not look, for the statuses not_applicable and not_checked, for example 'Runs after launch'. Null for passed and finding.",
"type": ["string", "null"]
},
+ "rule": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/ExperimentExposureHealthCheckRule"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "How the check decides, the same for every experiment. Null for a check without a numeric rule. Missing on an answer cached before the field existed."
+ },
"status": {
"$ref": "#/definitions/ExperimentExposureHealthCheckStatus",
"description": "passed: no problem found. finding: a finding in `health_findings`. not_applicable: the check doesn't cover the experiment's state. not_checked: an input was missing or the check failed."
+ },
+ "values": {
+ "additionalProperties": {
+ "type": ["string", "number", "boolean", "null"]
+ },
+ "description": "What the check read on the experiment, for every status. The suffix of a key gives the unit: _count is a count, _percentage is on the 0-100 scale, _hours is hours. A key without a suffix is a setting. Missing on an answer cached before the field existed.",
+ "type": "object"
}
},
"required": ["key", "name", "status", "reason"],
@@ -26502,6 +26509,48 @@
],
"type": "string"
},
+ "ExperimentExposureHealthCheckRule": {
+ "additionalProperties": false,
+ "description": "How an exposure check decides. Same shape as the `rule` of the experiment's `health.checks`.",
+ "properties": {
+ "minimum": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/ExperimentExposureHealthRuleMinimum"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "What the check needs before it judges. Null when the check judges from the first value."
+ },
+ "test": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/ExperimentExposureHealthRuleTest"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "The statistical test that computes the value. Null for a count or a share."
+ },
+ "threshold": {
+ "description": "The threshold, in the unit that the suffix of the key gives.",
+ "type": "number"
+ },
+ "value": {
+ "$ref": "#/definitions/ExperimentExposureHealthValueKey",
+ "description": "The key in `values` that the check compares with the threshold."
+ },
+ "warns": {
+ "$ref": "#/definitions/ExperimentExposureHealthRuleSide",
+ "description": "above: the check reports a finding when the value is above the threshold. below: when the value is below it. A value at the threshold passes."
+ }
+ },
+ "required": ["value", "threshold", "warns", "test", "minimum"],
+ "type": "object"
+ },
"ExperimentExposureHealthCheckStatus": {
"enum": ["passed", "finding", "not_applicable", "not_checked"],
"type": "string"
@@ -26517,6 +26566,10 @@
},
"type": "array"
},
+ "check": {
+ "$ref": "#/definitions/ExperimentExposureHealthCheckKey",
+ "description": "The key of the check in `health_checks` that reported the finding. Missing on an answer cached before the field existed."
+ },
"code": {
"$ref": "#/definitions/ExperimentExposureHealthFindingCode",
"description": "Stable identifier of the problem. Each code has one meaning across every surface that reports it."
@@ -26584,36 +26637,43 @@
"enum": ["critical", "warning", "info"],
"type": "string"
},
- "ExperimentExposureHealthMeasurement": {
+ "ExperimentExposureHealthRuleMinimum": {
"additionalProperties": false,
- "description": "The value that decided a passed exposure check, against its threshold.",
+ "description": "What an exposure check needs before it judges.",
"properties": {
"threshold": {
- "description": "The value at which the check reports a finding.",
+ "description": "The value that the key must reach. Below it, the check reports no finding.",
"type": "number"
},
- "unit": {
- "$ref": "#/definitions/ExperimentExposureHealthMeasurementUnit",
- "description": "percentage: a share of the exposed users or groups, on the 0-100 scale. p_value: the p-value of a statistical test."
- },
"value": {
- "description": "The value that the check measured on the experiment.",
- "type": "number"
- },
- "warns": {
- "$ref": "#/definitions/ExperimentExposureHealthMeasurementSide",
- "description": "The side of the threshold that reports a finding."
+ "$ref": "#/definitions/ExperimentExposureHealthValueKey",
+ "description": "The key in `values` that must reach the threshold before the check judges."
}
},
- "required": ["value", "threshold", "unit", "warns"],
+ "required": ["value", "threshold"],
"type": "object"
},
- "ExperimentExposureHealthMeasurementSide": {
+ "ExperimentExposureHealthRuleSide": {
"enum": ["above", "below"],
"type": "string"
},
- "ExperimentExposureHealthMeasurementUnit": {
- "enum": ["percentage", "p_value"],
+ "ExperimentExposureHealthRuleTest": {
+ "const": "chi_squared",
+ "type": "string"
+ },
+ "ExperimentExposureHealthValueKey": {
+ "description": "The keys in the `values` of an exposure check. Same meaning as in the experiment's `health.checks`.",
+ "enum": [
+ "exposed_count",
+ "compared_count",
+ "multiple_variant_count",
+ "multiple_variant_percentage",
+ "multiple_variant_handling",
+ "p_value",
+ "since_launch_hours",
+ "exposed_without_test_account_filter_count",
+ "kept_by_test_account_filter_percentage"
+ ],
"type": "string"
},
"ExperimentExposureNode": {
diff --git a/frontend/src/queries/schema/schema-general.ts b/frontend/src/queries/schema/schema-general.ts
index 46d0ebd89720..0fa03849bb08 100644
--- a/frontend/src/queries/schema/schema-general.ts
+++ b/frontend/src/queries/schema/schema-general.ts
@@ -6604,6 +6604,8 @@ export interface ExperimentExposureHealthFinding {
actions: ExperimentExposureHealthFindingActionKind[]
/** The id of the matching diagnostic in the diagnosing-experiment-health skill, for example 'A2'. Null when the skill has none. */
diagnostic_ref: string | null
+ /** The key of the check in `health_checks` that reported the finding. Missing on an answer cached before the field existed. */
+ check?: ExperimentExposureHealthCheckKey
}
export type ExperimentExposureHealthCheckKey =
@@ -6615,20 +6617,42 @@ export type ExperimentExposureHealthCheckKey =
export type ExperimentExposureHealthCheckStatus = 'passed' | 'finding' | 'not_applicable' | 'not_checked'
-export type ExperimentExposureHealthMeasurementUnit = 'percentage' | 'p_value'
-
-export type ExperimentExposureHealthMeasurementSide = 'above' | 'below'
+/** The keys in the `values` of an exposure check. Same meaning as in the experiment's `health.checks`. */
+export type ExperimentExposureHealthValueKey =
+ | 'exposed_count'
+ | 'compared_count'
+ | 'multiple_variant_count'
+ | 'multiple_variant_percentage'
+ | 'multiple_variant_handling'
+ | 'p_value'
+ | 'since_launch_hours'
+ | 'exposed_without_test_account_filter_count'
+ | 'kept_by_test_account_filter_percentage'
+
+export type ExperimentExposureHealthRuleSide = 'above' | 'below'
+
+export type ExperimentExposureHealthRuleTest = 'chi_squared'
+
+/** What an exposure check needs before it judges. */
+export interface ExperimentExposureHealthRuleMinimum {
+ /** The key in `values` that must reach the threshold before the check judges. */
+ value: ExperimentExposureHealthValueKey
+ /** The value that the key must reach. Below it, the check reports no finding. */
+ threshold: number
+}
-/** The value that decided a passed exposure check, against its threshold. */
-export interface ExperimentExposureHealthMeasurement {
- /** The value that the check measured on the experiment. */
- value: number
- /** The value at which the check reports a finding. */
+/** How an exposure check decides. Same shape as the `rule` of the experiment's `health.checks`. */
+export interface ExperimentExposureHealthCheckRule {
+ /** The key in `values` that the check compares with the threshold. */
+ value: ExperimentExposureHealthValueKey
+ /** The threshold, in the unit that the suffix of the key gives. */
threshold: number
- /** percentage: a share of the exposed users or groups, on the 0-100 scale. p_value: the p-value of a statistical test. */
- unit: ExperimentExposureHealthMeasurementUnit
- /** The side of the threshold that reports a finding. */
- warns: ExperimentExposureHealthMeasurementSide
+ /** above: the check reports a finding when the value is above the threshold. below: when the value is below it. A value at the threshold passes. */
+ warns: ExperimentExposureHealthRuleSide
+ /** The statistical test that computes the value. Null for a count or a share. */
+ test: ExperimentExposureHealthRuleTest | null
+ /** What the check needs before it judges. Null when the check judges from the first value. */
+ minimum: ExperimentExposureHealthRuleMinimum | null
}
/** What one health check concluded on the exposure answer. Same shape as the experiment's `health.checks`. */
@@ -6643,8 +6667,10 @@ export interface ExperimentExposureHealthCheck {
status: ExperimentExposureHealthCheckStatus
/** Why the check did not look, for the statuses not_applicable and not_checked, for example 'Runs after launch'. Null for passed and finding. */
reason: string | null
- /** The value that decided a passed check. Null for the other statuses and for a pass without a value. */
- measurement?: ExperimentExposureHealthMeasurement | null
+ /** What the check read on the experiment, for every status. The suffix of a key gives the unit: _count is a count, _percentage is on the 0-100 scale, _hours is hours. A key without a suffix is a setting. Missing on an answer cached before the field existed. */
+ values?: Record<string, string | number | boolean | null>
+ /** How the check decides, the same for every experiment. Null for a check without a numeric rule. Missing on an answer cached before the field existed. */
+ rule?: ExperimentExposureHealthCheckRule | null
}
export interface ExperimentExposureQueryResponse {
diff --git a/frontend/src/scenes/experiments/ExperimentView/ExperimentWarningBanners.test.tsx b/frontend/src/scenes/experiments/ExperimentView/ExperimentWarningBanners.test.tsx
index 6258902595fc..452387e2b97c 100644
--- a/frontend/src/scenes/experiments/ExperimentView/ExperimentWarningBanners.test.tsx
+++ b/frontend/src/scenes/experiments/ExperimentView/ExperimentWarningBanners.test.tsx
@@ -64,6 +64,7 @@ describe('ExperimentWarningBanner', () => {
severity: 'warning',
title: '',
detail: '',
+ check: 'flag_state',
evidence: { variant_key: variantKey },
actions: [],
diagnostic_ref: null,
diff --git a/frontend/src/scenes/experiments/ExperimentView/healthFindingReporting.test.tsx b/frontend/src/scenes/experiments/ExperimentView/healthFindingReporting.test.tsx
index 5f649a21e3ab..7898933d97b2 100644
--- a/frontend/src/scenes/experiments/ExperimentView/healthFindingReporting.test.tsx
+++ b/frontend/src/scenes/experiments/ExperimentView/healthFindingReporting.test.tsx
@@ -52,6 +52,7 @@ const UNEVEN_EXPOSURES = {
{ variant: 'test', days: ['2026-01-01', '2026-01-02'], exposure_counts: [200, 400] },
],
total_exposures: { control: 600, test: 400 },
+ date_range: { date_from: '2026-01-01', date_to: '2026-01-02' },
sample_ratio_mismatch: { expected: { control: 500, test: 500 }, p_value: 0.0001 },
bias_risk: { multiple_variant_percentage: 5 },
health_findings: [
@@ -121,6 +122,7 @@ describe('health finding reporting', () => {
evidence: {},
actions: ['open_feature_flag'],
diagnostic_ref: 'A5',
+ check: 'flag_state',
},
],
checks: [
@@ -130,7 +132,8 @@ describe('health finding reporting', () => {
description: 'Checks that a launched experiment has at least one metric.',
status: 'passed',
reason: null,
- measurement: null,
+ values: {},
+ rule: null,
},
],
})
@@ -266,6 +269,7 @@ describe('health finding reporting', () => {
expected: [
['experiment health finding opened', 'srm'],
['experiment health check expanded', 'no_metric'],
+ ['experiment health summary opened', undefined],
],
},
{
@@ -276,10 +280,12 @@ describe('health finding reporting', () => {
['experiment health finding opened', 'srm'],
['experiment health finding opened', 'srm'],
['experiment health check expanded', 'no_metric'],
+ ['experiment health summary opened', undefined],
+ ['experiment health summary opened', undefined],
],
},
])(
- 'reports each finding or check a person opens on the health tab in a $container container',
+ 'reports each finding, check or summary a person opens on the health tab in a $container container',
async ({ isNarrow, expected }) => {
// jsdom applies no container query, so the detail stays beside the list until `hidden` takes effect,
// as it does below the breakpoint.
@@ -291,9 +297,11 @@ describe('health finding reporting', () => {
const pickEvents = (): (string | undefined)[][] =>
captureSpy.mock.calls
.filter(([event]) =>
- ['experiment health finding opened', 'experiment health check expanded'].includes(
- String(event)
- )
+ [
+ 'experiment health finding opened',
+ 'experiment health check expanded',
+ 'experiment health summary opened',
+ ].includes(String(event))
)
.map(([event, properties]) => [event, properties.finding_code ?? properties.check_key])
expect(pickEvents()).toEqual([])
@@ -304,6 +312,8 @@ describe('health finding reporting', () => {
'Users are not split across variants as configured',
'Users are not split across variants as configured',
'Metrics not set up',
+ 'Health checks',
+ 'Health checks',
]) {
await pickFromList(title)
if (isNarrow) {
diff --git a/frontend/src/scenes/experiments/experimentLogic.test.ts b/frontend/src/scenes/experiments/experimentLogic.test.ts
index 2ae4241a316b..dac3384cb9cf 100644
--- a/frontend/src/scenes/experiments/experimentLogic.test.ts
+++ b/frontend/src/scenes/experiments/experimentLogic.test.ts
@@ -2398,6 +2398,7 @@ describe('experimentLogic', () => {
evidence,
actions: [],
diagnostic_ref: null,
+ check: 'flag_state',
})
const flag = (
diff --git a/frontend/src/scenes/experiments/experimentLogic.tsx b/frontend/src/scenes/experiments/experimentLogic.tsx
index c16d378b9c00..a6b173d4397a 100644
--- a/frontend/src/scenes/experiments/experimentLogic.tsx
+++ b/frontend/src/scenes/experiments/experimentLogic.tsx
@@ -84,6 +84,7 @@ import {
type ExperimentHealthUi,
captureExperimentHealthChecked,
captureExperimentHealthCheckExpanded,
+ captureExperimentHealthSummaryOpened,
captureExperimentHealthChecksOpened,
captureExperimentHealthFindingActedOn,
captureExperimentHealthFindingOpened,
@@ -783,6 +784,9 @@ export interface experimentLogicActions {
reportHealthFindingShown: (finding: ExperimentHealthFinding) => {
finding: ExperimentHealthFinding
}
+ reportHealthSummaryOpened: () => {
+ value: true
+ }
resetRunningExperiment: () => {
value: true
}
@@ -1094,6 +1098,7 @@ export const experimentLogic = kea<experimentLogicType>([
) => ({ finding, actionKind }),
reportHealthChecksOpened: (openedFrom: ExperimentHealthChecksOpenedFrom) => ({ openedFrom }),
reportHealthCheckExpanded: (check: HealthPanelCheck) => ({ check }),
+ reportHealthSummaryOpened: true,
setExperimentMissing: true,
setExperiment: (experiment: Partial<Experiment>) => ({ experiment }),
setLaunchExperimentLoading: (loading: boolean) => ({ loading }),
@@ -1582,6 +1587,14 @@ export const experimentLogic = kea<experimentLogicType>([
reportHealthCheckExpanded: ({ check }) => {
captureExperimentHealthCheckExpanded(values.experiment, check, values.healthUi)
},
+ reportHealthSummaryOpened: () => {
+ captureExperimentHealthSummaryOpened(
+ values.experiment,
+ values.healthChecks ?? [],
+ values.healthFindings?.length ?? 0,
+ values.healthUi
+ )
+ },
reportHealthChecksOpened: ({ openedFrom }) => {
captureExperimentHealthChecksOpened(
values.experiment,
diff --git a/frontend/src/scenes/experiments/stories/ExperimentHealthTab.stories.tsx b/frontend/src/scenes/experiments/stories/ExperimentHealthTab.stories.tsx
index 61db28d3a99a..47ed5988eb31 100644
--- a/frontend/src/scenes/experiments/stories/ExperimentHealthTab.stories.tsx
+++ b/frontend/src/scenes/experiments/stories/ExperimentHealthTab.stories.tsx
@@ -19,7 +19,8 @@ type Check = {
description: string
status: string
reason: string | null
- measurement: { value: number; threshold: number; unit: string; warns: string } | null
+ values: Record<string, string | number | boolean | null>
+ rule: object | null
}
const DESCRIPTION = 'Looks for a problem in the setup of the experiment that can make its results wrong.'
@@ -30,7 +31,8 @@ const passed = (key: string, name: string): Check => ({
description: DESCRIPTION,
status: 'passed',
reason: null,
- measurement: null,
+ values: {},
+ rule: null,
})
const EXPERIMENT_CHECKS: Check[] = [
@@ -62,7 +64,14 @@ const EXPOSURE_CHECKS: Check[] = [
passed('test_account_filter_removes_exposures', 'Test account filters that leave out most users'),
{
...passed('srm', 'Split of users across variants'),
- measurement: { value: 0.71, threshold: 0.001, unit: 'p_value', warns: 'below' },
+ values: { p_value: 0.62, compared_count: 3000, multiple_variant_count: 18 },
+ rule: {
+ value: 'p_value',
+ threshold: 0.001,
+ warns: 'below',
+ test: 'chi_squared',
+ minimum: { value: 'compared_count', threshold: 100 },
+ },
},
{
...passed('bias_risk_multiple_excluded', 'Bias from an uneven split'),
diff --git a/posthog/schema.py b/posthog/schema.py
index 22f6f763c2da..165ebfaa8e1d 100644
--- a/posthog/schema.py
+++ b/posthog/schema.py
@@ -133,8 +133,8 @@ from posthog.schema_enums import (
ExperimentExposureHealthFindingActionKind as ExperimentExposureHealthFindingActionKind,
ExperimentExposureHealthFindingCode as ExperimentExposureHealthFindingCode,
ExperimentExposureHealthFindingSeverity as ExperimentExposureHealthFindingSeverity,
- ExperimentExposureHealthMeasurementSide as ExperimentExposureHealthMeasurementSide,
- ExperimentExposureHealthMeasurementUnit as ExperimentExposureHealthMeasurementUnit,
+ ExperimentExposureHealthRuleSide as ExperimentExposureHealthRuleSide,
+ ExperimentExposureHealthValueKey as ExperimentExposureHealthValueKey,
ExperimentMetricGoal as ExperimentMetricGoal,
ExperimentMetricMathType as ExperimentMetricMathType,
ExperimentMetricType as ExperimentMetricType,
@@ -5854,6 +5854,13 @@ class ExperimentExposureHealthFinding(BaseModel):
actions: list[ExperimentExposureHealthFindingActionKind] = Field(
..., description="The actions that fix the problem, in order of preference."
)
+ check: ExperimentExposureHealthCheckKey | None = Field(
+ default=None,
+ description=(
+ "The key of the check in `health_checks` that reported the finding. Missing"
+ " on an answer cached before the field existed."
+ ),
+ )
code: ExperimentExposureHealthFindingCode = Field(
...,
description=(
@@ -5895,21 +5902,17 @@ class ExperimentExposureHealthFinding(BaseModel):
title: str = Field(..., description="One-line summary of the problem.")
-class ExperimentExposureHealthMeasurement(BaseModel):
+class ExperimentExposureHealthRuleMinimum(BaseModel):
model_config = ConfigDict(
extra="forbid",
)
- threshold: float = Field(..., description="The value at which the check reports a finding.")
- unit: ExperimentExposureHealthMeasurementUnit = Field(
+ threshold: float = Field(
...,
- description=(
- "percentage: a share of the exposed users or groups, on the 0-100 scale."
- " p_value: the p-value of a statistical test."
- ),
+ description=("The value that the key must reach. Below it, the check reports no finding."),
)
- value: float = Field(..., description="The value that the check measured on the experiment.")
- warns: ExperimentExposureHealthMeasurementSide = Field(
- ..., description="The side of the threshold that reports a finding."
+ value: ExperimentExposureHealthValueKey = Field(
+ ...,
+ description=("The key in `values` that must reach the threshold before the check judges."),
)
@@ -19408,72 +19411,30 @@ class ExperimentBreakdownResult(BaseModel):
)
-class ExperimentExposureHealthCheck(BaseModel):
+class ExperimentExposureHealthCheckRule(BaseModel):
model_config = ConfigDict(
extra="forbid",
)
- description: str | None = Field(
- default=None,
- description=(
- "What the check looks for and why it matters. Missing on an answer cached before the field existed."
- ),
- )
- key: ExperimentExposureHealthCheckKey = Field(..., description="Stable identifier of the check.")
- measurement: ExperimentExposureHealthMeasurement | None = Field(
- default=None,
- description=(
- "The value that decided a passed check. Null for the other statuses and for a pass without a value."
- ),
- )
- name: str = Field(
+ minimum: ExperimentExposureHealthRuleMinimum | None = Field(
...,
- description="The problem the check looks for, as the health panel lists it.",
+ description=("What the check needs before it judges. Null when the check judges from the first value."),
)
- reason: str | None = Field(
+ test: Literal["chi_squared"] | None = Field(
...,
- description=(
- "Why the check did not look, for the statuses not_applicable and"
- " not_checked, for example 'Runs after launch'. Null for passed and"
- " finding."
- ),
+ description=("The statistical test that computes the value. Null for a count or a share."),
)
- status: ExperimentExposureHealthCheckStatus = Field(
+ threshold: float = Field(..., description="The threshold, in the unit that the suffix of the key gives.")
+ value: ExperimentExposureHealthValueKey = Field(
...,
- description=(
- "passed: no problem found. finding: a finding in `health_findings`."
- " not_applicable: the check doesn't cover the experiment's state."
- " not_checked: an input was missing or the check failed."
- ),
+ description="The key in `values` that the check compares with the threshold.",
)
-
-
-class ExperimentExposureQueryResponse(BaseModel):
- model_config = ConfigDict(
- extra="forbid",
- )
- bias_risk: BiasRisk | None = None
- date_range: DateRange
- health_checks: list[ExperimentExposureHealthCheck] | None = Field(
- default=None,
- description=(
- "The result of each check behind `health_findings`, so that a check that did not run is not read as passed."
- ),
- )
- health_findings: list[ExperimentExposureHealthFinding] | None = Field(
- default=None,
+ warns: ExperimentExposureHealthRuleSide = Field(
+ ...,
description=(
- "Health check diagnostics that read the exposures: zero exposures, a sample"
- " ratio mismatch, and bias. Empty when every check passed."
+ "above: the check reports a finding when the value is above the threshold."
+ " below: when the value is below it. A value at the threshold passes."
),
)
- kind: Literal["ExperimentExposureQuery"] = "ExperimentExposureQuery"
- sample_ratio_mismatch: SampleRatioMismatch | None = None
- timeseries: list[ExperimentExposureTimeSeries]
- total_exposures: dict[str, float]
- warnings: list[DataWarehouseSyncWarning] | None = Field(
- default=None,
- description=("Data warehouse sync warnings — see AnalyticsQueryResponseBase.warnings for semantics."),
- )
class ExperimentQueryResponse(BaseModel):
@@ -22387,35 +22348,6 @@ class QueryResponseAlternative19(BaseModel):
)
-class QueryResponseAlternative20(BaseModel):
- model_config = ConfigDict(
- extra="forbid",
- )
- bias_risk: BiasRisk | None = None
- date_range: DateRange
- health_checks: list[ExperimentExposureHealthCheck] | None = Field(
- default=None,
- description=(
- "The result of each check behind `health_findings`, so that a check that did not run is not read as passed."
- ),
- )
- health_findings: list[ExperimentExposureHealthFinding] | None = Field(
- default=None,
- description=(
- "Health check diagnostics that read the exposures: zero exposures, a sample"
- " ratio mismatch, and bias. Empty when every check passed."
- ),
- )
- kind: Literal["ExperimentExposureQuery"] = "ExperimentExposureQuery"
- sample_ratio_mismatch: SampleRatioMismatch | None = None
- timeseries: list[ExperimentExposureTimeSeries]
- total_exposures: dict[str, float]
- warnings: list[DataWarehouseSyncWarning] | None = Field(
- default=None,
- description=("Data warehouse sync warnings — see AnalyticsQueryResponseBase.warnings for semantics."),
- )
-
-
class QueryResponseAlternative21(BaseModel):
model_config = ConfigDict(
extra="forbid",
@@ -28376,54 +28308,6 @@ class CachedErrorTrackingQueryResponse(BaseModel):
)
-class CachedExperimentExposureQueryResponse(BaseModel):
- model_config = ConfigDict(
- extra="forbid",
- )
- bias_risk: BiasRisk | None = None
- cache_key: str
- cache_target_age: AwareDatetime | None = None
- calculation_trigger: str | None = Field(
- default=None,
- description=("What triggered the calculation of the query, leave empty if user/immediate"),
- )
- date_range: DateRange
- health_checks: list[ExperimentExposureHealthCheck] | None = Field(
- default=None,
- description=(
- "The result of each check behind `health_findings`, so that a check that did not run is not read as passed."
- ),
- )
- health_findings: list[ExperimentExposureHealthFinding] | None = Field(
- default=None,
- description=(
- "Health check diagnostics that read the exposures: zero exposures, a sample"
- " ratio mismatch, and bias. Empty when every check passed."
- ),
- )
- is_cached: bool
- kind: Literal["ExperimentExposureQuery"] = "ExperimentExposureQuery"
- last_refresh: AwareDatetime
- next_allowed_client_refresh: AwareDatetime
- query_metadata: dict[str, Any] | None = None
- query_scan: QueryScanSummary | None = Field(
- default=None,
- description=("The rows and time of the run that produced these results, with its analysis once it is stored."),
- )
- query_status: QueryStatus | None = Field(
- default=None,
- description=("Query status indicates whether next to the provided data, a query is still running."),
- )
- sample_ratio_mismatch: SampleRatioMismatch | None = None
- timeseries: list[ExperimentExposureTimeSeries]
- timezone: str
- total_exposures: dict[str, float]
- warnings: list[DataWarehouseSyncWarning] | None = Field(
- default=None,
- description=("Data warehouse sync warnings — see AnalyticsQueryResponseBase.warnings for semantics."),
- )
-
-
class CachedExperimentQueryResponse(BaseModel):
model_config = ConfigDict(
extra="forbid",
@@ -29445,6 +29329,85 @@ class ExperimentEventExposureConfig(BaseModel):
version: float | None = Field(default=None, description="version of the node, used for schema migrations")
+class ExperimentExposureHealthCheck(BaseModel):
+ model_config = ConfigDict(
+ extra="forbid",
+ )
+ description: str | None = Field(
+ default=None,
+ description=(
+ "What the check looks for and why it matters. Missing on an answer cached before the field existed."
+ ),
+ )
+ key: ExperimentExposureHealthCheckKey = Field(..., description="Stable identifier of the check.")
+ name: str = Field(
+ ...,
+ description="The problem the check looks for, as the health panel lists it.",
+ )
+ reason: str | None = Field(
+ ...,
+ description=(
+ "Why the check did not look, for the statuses not_applicable and"
+ " not_checked, for example 'Runs after launch'. Null for passed and"
+ " finding."
+ ),
+ )
+ rule: ExperimentExposureHealthCheckRule | None = Field(
+ default=None,
+ description=(
+ "How the check decides, the same for every experiment. Null for a check"
+ " without a numeric rule. Missing on an answer cached before the field"
+ " existed."
+ ),
+ )
+ status: ExperimentExposureHealthCheckStatus = Field(
+ ...,
+ description=(
+ "passed: no problem found. finding: a finding in `health_findings`."
+ " not_applicable: the check doesn't cover the experiment's state."
+ " not_checked: an input was missing or the check failed."
+ ),
+ )
+ values: dict[str, str | float | bool | None] | None = Field(
+ default=None,
+ description=(
+ "What the check read on the experiment, for every status. The suffix of a"
+ " key gives the unit: _count is a count, _percentage is on the 0-100 scale,"
+ " _hours is hours. A key without a suffix is a setting. Missing on an"
+ " answer cached before the field existed."
+ ),
+ )
+
+
+class ExperimentExposureQueryResponse(BaseModel):
+ model_config = ConfigDict(
+ extra="forbid",
+ )
+ bias_risk: BiasRisk | None = None
+ date_range: DateRange
+ health_checks: list[ExperimentExposureHealthCheck] | None = Field(
+ default=None,
+ description=(
+ "The result of each check behind `health_findings`, so that a check that did not run is not read as passed."
+ ),
+ )
+ health_findings: list[ExperimentExposureHealthFinding] | None = Field(
+ default=None,
+ description=(
+ "Health check diagnostics that read the exposures: zero exposures, a sample"
+ " ratio mismatch, and bias. Empty when every check passed."
+ ),
+ )
+ kind: Literal["ExperimentExposureQuery"] = "ExperimentExposureQuery"
+ sample_ratio_mismatch: SampleRatioMismatch | None = None
+ timeseries: list[ExperimentExposureTimeSeries]
+ total_exposures: dict[str, float]
+ warnings: list[DataWarehouseSyncWarning] | None = Field(
+ default=None,
+ description=("Data warehouse sync warnings — see AnalyticsQueryResponseBase.warnings for semantics."),
+ )
+
+
class ExperimentMetricTimeseries(BaseModel):
model_config = ConfigDict(
extra="forbid",
@@ -30962,6 +30925,35 @@ class QueryResponseAlternative17(BaseModel):
)
+class QueryResponseAlternative20(BaseModel):
+ model_config = ConfigDict(
+ extra="forbid",
+ )
+ bias_risk: BiasRisk | None = None
+ date_range: DateRange
+ health_checks: list[ExperimentExposureHealthCheck] | None = Field(
+ default=None,
+ description=(
+ "The result of each check behind `health_findings`, so that a check that did not run is not read as passed."
+ ),
+ )
+ health_findings: list[ExperimentExposureHealthFinding] | None = Field(
+ default=None,
+ description=(
+ "Health check diagnostics that read the exposures: zero exposures, a sample"
+ " ratio mismatch, and bias. Empty when every check passed."
+ ),
+ )
+ kind: Literal["ExperimentExposureQuery"] = "ExperimentExposureQuery"
+ sample_ratio_mismatch: SampleRatioMismatch | None = None
+ timeseries: list[ExperimentExposureTimeSeries]
+ total_exposures: dict[str, float]
+ warnings: list[DataWarehouseSyncWarning] | None = Field(
+ default=None,
+ description=("Data warehouse sync warnings — see AnalyticsQueryResponseBase.warnings for semantics."),
+ )
+
+
class QueryResponseAlternative43(BaseModel):
model_config = ConfigDict(
extra="forbid",
@@ -31753,6 +31745,54 @@ class CachedErrorTrackingIssueCorrelationQueryResponse(BaseModel):
)
+class CachedExperimentExposureQueryResponse(BaseModel):
+ model_config = ConfigDict(
+ extra="forbid",
+ )
+ bias_risk: BiasRisk | None = None
+ cache_key: str
+ cache_target_age: AwareDatetime | None = None
+ calculation_trigger: str | None = Field(
+ default=None,
+ description=("What triggered the calculation of the query, leave empty if user/immediate"),
+ )
+ date_range: DateRange
+ health_checks: list[ExperimentExposureHealthCheck] | None = Field(
+ default=None,
+ description=(
+ "The result of each check behind `health_findings`, so that a check that did not run is not read as passed."
+ ),
+ )
+ health_findings: list[ExperimentExposureHealthFinding] | None = Field(
+ default=None,
+ description=(
+ "Health check diagnostics that read the exposures: zero exposures, a sample"
+ " ratio mismatch, and bias. Empty when every check passed."
+ ),
+ )
+ is_cached: bool
+ kind: Literal["ExperimentExposureQuery"] = "ExperimentExposureQuery"
+ last_refresh: AwareDatetime
+ next_allowed_client_refresh: AwareDatetime
+ query_metadata: dict[str, Any] | None = None
+ query_scan: QueryScanSummary | None = Field(
+ default=None,
+ description=("The rows and time of the run that produced these results, with its analysis once it is stored."),
+ )
+ query_status: QueryStatus | None = Field(
+ default=None,
+ description=("Query status indicates whether next to the provided data, a query is still running."),
+ )
+ sample_ratio_mismatch: SampleRatioMismatch | None = None
+ timeseries: list[ExperimentExposureTimeSeries]
+ timezone: str
+ total_exposures: dict[str, float]
+ warnings: list[DataWarehouseSyncWarning] | None = Field(
+ default=None,
+ description=("Data warehouse sync warnings — see AnalyticsQueryResponseBase.warnings for semantics."),
+ )
+
+
class CachedHogQLQueryResponse(BaseModel):
model_config = ConfigDict(
extra="forbid",
diff --git a/posthog/schema_enums.py b/posthog/schema_enums.py
index 2224f3340240..bc304d494e36 100644
--- a/posthog/schema_enums.py
+++ b/posthog/schema_enums.py
@@ -1504,14 +1504,21 @@ class ExperimentExposureHealthFindingSeverity(StrEnum):
INFO = "info"
-class ExperimentExposureHealthMeasurementSide(StrEnum):
+class ExperimentExposureHealthRuleSide(StrEnum):
ABOVE = "above"
BELOW = "below"
-class ExperimentExposureHealthMeasurementUnit(StrEnum):
- PERCENTAGE = "percentage"
+class ExperimentExposureHealthValueKey(StrEnum):
+ EXPOSED_COUNT = "exposed_count"
+ COMPARED_COUNT = "compared_count"
+ MULTIPLE_VARIANT_COUNT = "multiple_variant_count"
+ MULTIPLE_VARIANT_PERCENTAGE = "multiple_variant_percentage"
+ MULTIPLE_VARIANT_HANDLING = "multiple_variant_handling"
P_VALUE = "p_value"
+ SINCE_LAUNCH_HOURS = "since_launch_hours"
+ EXPOSED_WITHOUT_TEST_ACCOUNT_FILTER_COUNT = "exposed_without_test_account_filter_count"
+ KEPT_BY_TEST_ACCOUNT_FILTER_PERCENTAGE = "kept_by_test_account_filter_percentage"
class ExperimentMetricGoal(StrEnum):
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index f67096c3d5b9..ad2491b5fd95 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -10077,6 +10077,55 @@ export namespace Schemas {
| "add_secondary_metric"
| "edit_exposure_criteria"
| "edit_release_conditions";
+ /**
+ * * `flag_state` - Flag State
+ * * `flag_shared` - Flag Shared
+ * * `flag_history` - Flag History
+ * * `split_changed_mid_run` - Split Changed Mid Run
+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
+ * * `flag_variants_mismatch` - Flag Variants Mismatch
+ * * `forced_variant_release_condition` - Forced Variant Release Condition
+ * * `new_flag_no_traffic` - New Flag No Traffic
+ * * `no_metric` - No Metric
+ * * `metric_event_not_ingested` - Metric Event Not Ingested
+ * * `metric_cannot_compile` - Metric Cannot Compile
+ * * `metric_calculation_failed` - Metric Calculation Failed
+ * * `metric_all_zero` - Metric All Zero
+ * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
+ * * `exposure_filters` - Exposure Filters
+ * * `early_significance` - Early Significance
+ * * `metrics_not_matured` - Metrics Not Matured
+ * * `zero_exposures` - Zero Exposures
+ * * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposures
+ * * `srm` - Srm
+ * * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
+ * * `multiple_variants_share` - Multiple Variants Share
+ */
+ export type ExperimentHealthCheckKeyEnum =
+ | "flag_state"
+ | "flag_shared"
+ | "flag_history"
+ | "split_changed_mid_run"
+ | "bucketing_changed_mid_run"
+ | "release_conditions_changed_mid_run"
+ | "flag_variants_mismatch"
+ | "forced_variant_release_condition"
+ | "new_flag_no_traffic"
+ | "no_metric"
+ | "metric_event_not_ingested"
+ | "metric_cannot_compile"
+ | "metric_calculation_failed"
+ | "metric_all_zero"
+ | "exposure_criteria_cannot_match"
+ | "exposure_filters"
+ | "early_significance"
+ | "metrics_not_matured"
+ | "zero_exposures"
+ | "test_account_filter_removes_exposures"
+ | "srm"
+ | "bias_risk_multiple_excluded"
+ | "multiple_variants_share";
export type ExperimentHealthFinding = {
/**
* Stable identifier of the problem. Each code has one meaning across every surface that reports it.
@@ -10144,56 +10193,35 @@ export namespace Schemas {
* The id of the matching diagnostic in the diagnosing-experiment-health skill, for example 'A5'. Null when the skill has none.
*/
diagnostic_ref: string | null;
+ /**
+ * The key of the check in `checks` that reported the finding. A check can report several codes, so the code doesn't name the check.
+ *
+ * * `flag_state` - Flag State
+ * * `flag_shared` - Flag Shared
+ * * `flag_history` - Flag History
+ * * `split_changed_mid_run` - Split Changed Mid Run
+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
+ * * `flag_variants_mismatch` - Flag Variants Mismatch
+ * * `forced_variant_release_condition` - Forced Variant Release Condition
+ * * `new_flag_no_traffic` - New Flag No Traffic
+ * * `no_metric` - No Metric
+ * * `metric_event_not_ingested` - Metric Event Not Ingested
+ * * `metric_cannot_compile` - Metric Cannot Compile
+ * * `metric_calculation_failed` - Metric Calculation Failed
+ * * `metric_all_zero` - Metric All Zero
+ * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
+ * * `exposure_filters` - Exposure Filters
+ * * `early_significance` - Early Significance
+ * * `metrics_not_matured` - Metrics Not Matured
+ * * `zero_exposures` - Zero Exposures
+ * * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposures
+ * * `srm` - Srm
+ * * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
+ * * `multiple_variants_share` - Multiple Variants Share
+ */
+ check: ExperimentHealthCheckKeyEnum;
};
- /**
- * * `flag_state` - Flag State
- * * `flag_shared` - Flag Shared
- * * `flag_history` - Flag History
- * * `split_changed_mid_run` - Split Changed Mid Run
- * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
- * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
- * * `flag_variants_mismatch` - Flag Variants Mismatch
- * * `forced_variant_release_condition` - Forced Variant Release Condition
- * * `new_flag_no_traffic` - New Flag No Traffic
- * * `no_metric` - No Metric
- * * `metric_event_not_ingested` - Metric Event Not Ingested
- * * `metric_cannot_compile` - Metric Cannot Compile
- * * `metric_calculation_failed` - Metric Calculation Failed
- * * `metric_all_zero` - Metric All Zero
- * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
- * * `exposure_filters` - Exposure Filters
- * * `early_significance` - Early Significance
- * * `metrics_not_matured` - Metrics Not Matured
- * * `zero_exposures` - Zero Exposures
- * * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposures
- * * `srm` - Srm
- * * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
- * * `multiple_variants_share` - Multiple Variants Share
- */
- export type ExperimentHealthCheckKeyEnum =
- | "flag_state"
- | "flag_shared"
- | "flag_history"
- | "split_changed_mid_run"
- | "bucketing_changed_mid_run"
- | "release_conditions_changed_mid_run"
- | "flag_variants_mismatch"
- | "forced_variant_release_condition"
- | "new_flag_no_traffic"
- | "no_metric"
- | "metric_event_not_ingested"
- | "metric_cannot_compile"
- | "metric_calculation_failed"
- | "metric_all_zero"
- | "exposure_criteria_cannot_match"
- | "exposure_filters"
- | "early_significance"
- | "metrics_not_matured"
- | "zero_exposures"
- | "test_account_filter_removes_exposures"
- | "srm"
- | "bias_risk_multiple_excluded"
- | "multiple_variants_share";
/**
* * `passed` - Passed
* * `finding` - Finding
@@ -10206,38 +10234,135 @@ export namespace Schemas {
| "not_applicable"
| "not_checked";
/**
- * * `percentage` - Percentage of exposed units
- * * `p_value` - P-value of a statistical test
- */
- export type ExperimentHealthMeasurementUnitEnum = "percentage" | "p_value";
+ * * `exposed_count` - Users or groups exposed since launch, as the exposure criteria count them, including those exposed to multiple variants
+ * * `compared_count` - Exposed users or groups in the variants that the sample ratio test compares, without those exposed to multiple variants
+ * * `multiple_variant_count` - Users or groups exposed to more than one variant
+ * * `multiple_variant_percentage` - multiple_variant_count as a share of exposed_count
+ * * `multiple_variant_handling` - What the results do with users exposed to more than one variant: exclude or first_seen
+ * * `p_value` - P-value of the sample ratio test
+ * * `since_launch_hours` - Hours since launch
+ * * `exposed_without_test_account_filter_count` - Users or groups that the exposure criteria count without the test account filters
+ * * `kept_by_test_account_filter_percentage` - exposed_count as a share of exposed_without_test_account_filter_count
+ * * `only_count_matured_users` - Whether "Require completed conversion or retention window" is on
+ * * `primary_metric_count` - Primary metrics that the calculation computes
+ * * `waiting_metric_count` - Of the primary metrics, those whose conversion or retention window hasn't passed for any user yet
+ * * `metric_count` - Metrics that the calculation computes
+ * * `completed_metric_count` - Metrics whose result of the latest calculation the page shows
+ * * `failed_metric_count` - Metrics whose latest calculation failed
+ * * `calculated_at` - When the latest calculation of the metrics finished
+ * * `flag_active` - Whether the feature flag is enabled
+ * * `flag_deleted` - Whether the feature flag is deleted
+ * * `flag_rolled_out` - Whether a release condition of the feature flag rolls out to more than 0%
+ * * `shipped_variant` - The key of the variant that the feature flag serves to everyone, or null when it serves more than one
+ */
+ export type ExperimentHealthValueKeyEnum =
+ | "exposed_count"
+ | "compared_count"
+ | "multiple_variant_count"
+ | "multiple_variant_percentage"
+ | "multiple_variant_handling"
+ | "p_value"
+ | "since_launch_hours"
+ | "exposed_without_test_account_filter_count"
+ | "kept_by_test_account_filter_percentage"
+ | "only_count_matured_users"
+ | "primary_metric_count"
+ | "waiting_metric_count"
+ | "metric_count"
+ | "completed_metric_count"
+ | "failed_metric_count"
+ | "calculated_at"
+ | "flag_active"
+ | "flag_deleted"
+ | "flag_rolled_out"
+ | "shipped_variant";
/**
* * `above` - Finding above the threshold
* * `below` - Finding below the threshold
*/
- export type ExperimentHealthMeasurementSideEnum = "above" | "below";
- export type ExperimentHealthMeasurement = {
- /**
- * The value that the check measured on the experiment.
- */
- value: number;
+ export type ExperimentHealthRuleSideEnum = "above" | "below";
+ /**
+ * * `chi_squared` - Chi-squared goodness-of-fit test
+ */
+ export type ExperimentHealthRuleTestEnum = "chi_squared";
+ export type ExperimentHealthRuleMinimum = {
/**
- * The value at which the check reports a finding.
+ * The key in `values` that must reach the threshold before the check judges.
+ *
+ * * `exposed_count` - Users or groups exposed since launch, as the exposure criteria count them, including those exposed to multiple variants
+ * * `compared_count` - Exposed users or groups in the variants that the sample ratio test compares, without those exposed to multiple variants
+ * * `multiple_variant_count` - Users or groups exposed to more than one variant
+ * * `multiple_variant_percentage` - multiple_variant_count as a share of exposed_count
+ * * `multiple_variant_handling` - What the results do with users exposed to more than one variant: exclude or first_seen
+ * * `p_value` - P-value of the sample ratio test
+ * * `since_launch_hours` - Hours since launch
+ * * `exposed_without_test_account_filter_count` - Users or groups that the exposure criteria count without the test account filters
+ * * `kept_by_test_account_filter_percentage` - exposed_count as a share of exposed_without_test_account_filter_count
+ * * `only_count_matured_users` - Whether "Require completed conversion or retention window" is on
+ * * `primary_metric_count` - Primary metrics that the calculation computes
+ * * `waiting_metric_count` - Of the primary metrics, those whose conversion or retention window hasn't passed for any user yet
+ * * `metric_count` - Metrics that the calculation computes
+ * * `completed_metric_count` - Metrics whose result of the latest calculation the page shows
+ * * `failed_metric_count` - Metrics whose latest calculation failed
+ * * `calculated_at` - When the latest calculation of the metrics finished
+ * * `flag_active` - Whether the feature flag is enabled
+ * * `flag_deleted` - Whether the feature flag is deleted
+ * * `flag_rolled_out` - Whether a release condition of the feature flag rolls out to more than 0%
+ * * `shipped_variant` - The key of the variant that the feature flag serves to everyone, or null when it serves more than one
+ */
+ value: ExperimentHealthValueKeyEnum;
+ /**
+ * The value that the key must reach. Below it, the check reports no finding.
*/
threshold: number;
+ };
+ export type ExperimentHealthCheckRule = {
/**
- * percentage: a share of the exposed users or groups, on the 0-100 scale. p_value: the p-value of a statistical test.
+ * The key in `values` that the check compares with the threshold.
*
- * * `percentage` - Percentage of exposed units
- * * `p_value` - P-value of a statistical test
+ * * `exposed_count` - Users or groups exposed since launch, as the exposure criteria count them, including those exposed to multiple variants
+ * * `compared_count` - Exposed users or groups in the variants that the sample ratio test compares, without those exposed to multiple variants
+ * * `multiple_variant_count` - Users or groups exposed to more than one variant
+ * * `multiple_variant_percentage` - multiple_variant_count as a share of exposed_count
+ * * `multiple_variant_handling` - What the results do with users exposed to more than one variant: exclude or first_seen
+ * * `p_value` - P-value of the sample ratio test
+ * * `since_launch_hours` - Hours since launch
+ * * `exposed_without_test_account_filter_count` - Users or groups that the exposure criteria count without the test account filters
+ * * `kept_by_test_account_filter_percentage` - exposed_count as a share of exposed_without_test_account_filter_count
+ * * `only_count_matured_users` - Whether "Require completed conversion or retention window" is on
+ * * `primary_metric_count` - Primary metrics that the calculation computes
+ * * `waiting_metric_count` - Of the primary metrics, those whose conversion or retention window hasn't passed for any user yet
+ * * `metric_count` - Metrics that the calculation computes
+ * * `completed_metric_count` - Metrics whose result of the latest calculation the page shows
+ * * `failed_metric_count` - Metrics whose latest calculation failed
+ * * `calculated_at` - When the latest calculation of the metrics finished
+ * * `flag_active` - Whether the feature flag is enabled
+ * * `flag_deleted` - Whether the feature flag is deleted
+ * * `flag_rolled_out` - Whether a release condition of the feature flag rolls out to more than 0%
+ * * `shipped_variant` - The key of the variant that the feature flag serves to everyone, or null when it serves more than one
+ */
+ value: ExperimentHealthValueKeyEnum;
+ /**
+ * The threshold, in the unit that the suffix of the key gives.
*/
- unit: ExperimentHealthMeasurementUnitEnum;
+ threshold: number;
/**
- * The side of the threshold that reports a finding: above or below.
+ * above: the check reports a finding when the value is above the threshold. below: when the value is below it. A value at the threshold passes.
*
* * `above` - Finding above the threshold
* * `below` - Finding below the threshold
*/
- warns: ExperimentHealthMeasurementSideEnum;
+ warns: ExperimentHealthRuleSideEnum;
+ /**
+ * The statistical test that computes the value, for example chi_squared. Null for a count or a share.
+ *
+ * * `chi_squared` - Chi-squared goodness-of-fit test
+ */
+ test: ExperimentHealthRuleTestEnum | NullEnum;
+ /**
+ * What the check needs before it judges. Null when the check judges from the first value.
+ */
+ minimum: ExperimentHealthRuleMinimum | null;
};
export type ExperimentHealthCheckResult = {
/**
@@ -10290,9 +10415,13 @@ export namespace Schemas {
*/
reason: string | null;
/**
- * The value that decided a passed check, against its threshold. Null for the other statuses, and for a pass without a value. A finding carries its values in `evidence`.
+ * What the check read on the experiment, for every status, as numbers, strings, booleans or null. Empty when the check read nothing or shows nothing. The suffix of a key gives the unit: _count is a count, _percentage is on the 0-100 scale, _hours is hours and _at is an ISO 8601 time in UTC. A key without a suffix is a setting or a state. A key means the same thing in every check. `rule.value` names the key that decides.
*/
- measurement: ExperimentHealthMeasurement | null;
+ values: Record<string, string | number | boolean | null>;
+ /**
+ * How the check decides. The same for every experiment and every status. Null for a check without a numeric rule.
+ */
+ rule: ExperimentHealthCheckRule | null;
};
export type ExperimentHealth = {
/**
@@ -10636,27 +10765,48 @@ export namespace Schemas {
| "bias_risk_multiple_excluded"
| "multiple_variants_share"
| "test_account_filter_removes_exposures";
- export type ExperimentExposureHealthMeasurementUnit =
- | "percentage"
- | "p_value";
- export type ExperimentExposureHealthMeasurementSide = "above" | "below";
- export type ExperimentExposureHealthMeasurement = {
- /**
- * The value at which the check reports a finding.
+ export type ExperimentExposureHealthValueKey =
+ | "exposed_count"
+ | "compared_count"
+ | "multiple_variant_count"
+ | "multiple_variant_percentage"
+ | "multiple_variant_handling"
+ | "p_value"
+ | "since_launch_hours"
+ | "exposed_without_test_account_filter_count"
+ | "kept_by_test_account_filter_percentage";
+ export type ExperimentExposureHealthRuleMinimum = {
+ /**
+ * The value that the key must reach. Below it, the check reports no finding.
*/
threshold: number;
/**
- * percentage: a share of the exposed users or groups, on the 0-100 scale. p_value: the p-value of a statistical test.
+ * The key in `values` that must reach the threshold before the check judges.
*/
- unit: ExperimentExposureHealthMeasurementUnit;
+ value: ExperimentExposureHealthValueKey;
+ };
+ export type ExperimentExposureHealthRuleSide = "above" | "below";
+ export type ExperimentExposureHealthCheckRule = {
/**
- * The value that the check measured on the experiment.
+ * What the check needs before it judges. Null when the check judges from the first value.
*/
- value: number;
+ minimum: ExperimentExposureHealthRuleMinimum | null;
+ /**
+ * The statistical test that computes the value. Null for a count or a share.
+ */
+ test: string | null;
/**
- * The side of the threshold that reports a finding.
+ * The threshold, in the unit that the suffix of the key gives.
*/
- warns: ExperimentExposureHealthMeasurementSide;
+ threshold: number;
+ /**
+ * The key in `values` that the check compares with the threshold.
+ */
+ value: ExperimentExposureHealthValueKey;
+ /**
+ * above: the check reports a finding when the value is above the threshold. below: when the value is below it. A value at the threshold passes.
+ */
+ warns: ExperimentExposureHealthRuleSide;
};
export type ExperimentExposureHealthCheckStatus =
| "passed"
@@ -10669,7 +10819,6 @@ export namespace Schemas {
* Stable identifier of the check.
*/
key: ExperimentExposureHealthCheckKey;
- measurement?: (ExperimentExposureHealthMeasurement | null) | undefined;
/**
* The problem the check looks for, as the health panel lists it.
*/
@@ -10678,10 +10827,14 @@ export namespace Schemas {
* Why the check did not look, for the statuses not_applicable and not_checked, for example 'Runs after launch'. Null for passed and finding.
*/
reason: string | null;
+ rule?: (ExperimentExposureHealthCheckRule | null) | undefined;
/**
* passed: no problem found. finding: a finding in `health_findings`. not_applicable: the check doesn't cover the experiment's state. not_checked: an input was missing or the check failed.
*/
status: ExperimentExposureHealthCheckStatus;
+ values?:
+ | (Record<string, string | number | boolean | null> | null)
+ | undefined;
};
export type ExperimentExposureHealthFindingActionKind =
| "open_feature_flag"
@@ -10703,6 +10856,7 @@ export namespace Schemas {
* The actions that fix the problem, in order of preference.
*/
actions: Array<ExperimentExposureHealthFindingActionKind>;
+ check?: (ExperimentExposureHealthCheckKey | null) | undefined;
/**
* Stable identifier of the problem. Each code has one meaning across every surface that reports it.
*/
diff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.py
index 3b279db905d0..bd7afaacce15 100644
--- a/products/experiments/backend/facade/contracts.py
+++ b/products/experiments/backend/facade/contracts.py
@@ -316,22 +316,6 @@ class ExperimentHealthFindingActionKind(LabeledStrEnum):
EDIT_RELEASE_CONDITIONS = "edit_release_conditions"
-@frozen
-class ExperimentHealthFinding:
- """One problem that a health check found in an experiment, with what fixes it."""
-
- code: ExperimentHealthFindingCode
- subcode: str | None
- # pinned: the page sends it as `finding_reason` on the health finding events. Never customer text.
- reason: str | None = None
- severity: ExperimentHealthFindingSeverity
- title: str
- detail: str
- evidence: Mapping[str, str | int | float | bool | None]
- actions: tuple[ExperimentHealthFindingActionKind, ...]
- diagnostic_ref: str | None
-
-
class ExperimentHealthCheckKey(LabeledStrEnum):
# pinned: the health endpoint and the exposure query return these values, so API callers depend on them. A key
# names the check, not a problem: `flag_state` reports four finding codes, and `flag_history` reports its finding
@@ -361,6 +345,25 @@ class ExperimentHealthCheckKey(LabeledStrEnum):
MULTIPLE_VARIANTS_SHARE = "multiple_variants_share"
+@frozen
+class ExperimentHealthFinding:
+ """One problem that a health check found in an experiment, with what fixes it."""
+
+ code: ExperimentHealthFindingCode
+ subcode: str | None
+ # pinned: the page sends it as `finding_reason` on the health finding events. Never customer text.
+ reason: str | None = None
+ severity: ExperimentHealthFindingSeverity
+ title: str
+ detail: str
+ evidence: Mapping[str, str | int | float | bool | None]
+ actions: tuple[ExperimentHealthFindingActionKind, ...]
+ diagnostic_ref: str | None
+ # The check that reported the finding. A code doesn't name its check, because one check can report several codes.
+ # The registry sets it, so a check leaves it out.
+ check: ExperimentHealthCheckKey | None = None
+
+
class ExperimentHealthCheckStatus(LabeledStrEnum):
PASSED = "passed"
FINDING = "finding"
@@ -369,26 +372,90 @@ class ExperimentHealthCheckStatus(LabeledStrEnum):
NOT_CHECKED = "not_checked"
-class ExperimentHealthMeasurementUnit(LabeledStrEnum):
- # On the 0-100 scale.
- PERCENTAGE = "percentage", "Percentage of exposed units"
- P_VALUE = "p_value", "P-value of a statistical test"
-
-
-class ExperimentHealthMeasurementSide(LabeledStrEnum):
+class ExperimentHealthValueKey(LabeledStrEnum):
+ # API callers read these keys in a check's `values` and its `rule`. The suffix gives the unit: `_count` is a
+ # count, `_percentage` is on the 0-100 scale, `_hours` is hours and `_at` is an ISO 8601 time in UTC. A key without a
+ # unit suffix is a setting or a state, as a string or a boolean. A key means the same thing in every check, so two
+ # values that differ in meaning get two keys.
+ EXPOSED_COUNT = (
+ "exposed_count",
+ "Users or groups exposed since launch, as the exposure criteria count them, including those exposed to "
+ "multiple variants",
+ )
+ COMPARED_COUNT = (
+ "compared_count",
+ "Exposed users or groups in the variants that the sample ratio test compares, without those exposed to "
+ "multiple variants",
+ )
+ MULTIPLE_VARIANT_COUNT = "multiple_variant_count", "Users or groups exposed to more than one variant"
+ MULTIPLE_VARIANT_PERCENTAGE = "multiple_variant_percentage", "multiple_variant_count as a share of exposed_count"
+ MULTIPLE_VARIANT_HANDLING = (
+ "multiple_variant_handling",
+ "What the results do with users exposed to more than one variant: exclude or first_seen",
+ )
+ P_VALUE = "p_value", "P-value of the sample ratio test"
+ SINCE_LAUNCH_HOURS = "since_launch_hours", "Hours since launch"
+ EXPOSED_WITHOUT_TEST_ACCOUNT_FILTER_COUNT = (
+ "exposed_without_test_account_filter_count",
+ "Users or groups that the exposure criteria count without the test account filters",
+ )
+ KEPT_BY_TEST_ACCOUNT_FILTER_PERCENTAGE = (
+ "kept_by_test_account_filter_percentage",
+ "exposed_count as a share of exposed_without_test_account_filter_count",
+ )
+ ONLY_COUNT_MATURED_USERS = (
+ "only_count_matured_users",
+ 'Whether "Require completed conversion or retention window" is on',
+ )
+ PRIMARY_METRIC_COUNT = "primary_metric_count", "Primary metrics that the calculation computes"
+ WAITING_METRIC_COUNT = (
+ "waiting_metric_count",
+ "Of the primary metrics, those whose conversion or retention window hasn't passed for any user yet",
+ )
+ METRIC_COUNT = "metric_count", "Metrics that the calculation computes"
+ COMPLETED_METRIC_COUNT = "completed_metric_count", "Metrics whose result of the latest calculation the page shows"
+ FAILED_METRIC_COUNT = "failed_metric_count", "Metrics whose latest calculation failed"
+ CALCULATED_AT = "calculated_at", "When the latest calculation of the metrics finished"
+ FLAG_ACTIVE = "flag_active", "Whether the feature flag is enabled"
+ FLAG_DELETED = "flag_deleted", "Whether the feature flag is deleted"
+ FLAG_ROLLED_OUT = "flag_rolled_out", "Whether a release condition of the feature flag rolls out to more than 0%"
+ SHIPPED_VARIANT = (
+ "shipped_variant",
+ "The key of the variant that the feature flag serves to everyone, or null when it serves more than one",
+ )
+
+
+ExperimentHealthValue = str | int | float | bool | None
+
+
+class ExperimentHealthRuleSide(LabeledStrEnum):
ABOVE = "above", "Finding above the threshold"
BELOW = "below", "Finding below the threshold"
+class ExperimentHealthRuleTest(LabeledStrEnum):
+ CHI_SQUARED = "chi_squared", "Chi-squared goodness-of-fit test"
+
+
+@frozen
+class ExperimentHealthRuleMinimum:
+ """What a check needs before it judges: below the threshold, it doesn't report a finding."""
+
+ value: ExperimentHealthValueKey
+ threshold: float
+
+
@frozen
-class ExperimentHealthMeasurement:
- """The value that decided a passed check, so a reader sees how far it is from a finding."""
+class ExperimentHealthCheckRule:
+ """How a check decides, as data. It is the same for every experiment, so the values of a run say how close the
+ check is to a finding."""
- value: float
+ value: ExperimentHealthValueKey
threshold: float
- unit: ExperimentHealthMeasurementUnit
- # The side of the threshold that reports a finding.
- warns: ExperimentHealthMeasurementSide
+ # The side of the threshold that reports a finding. A value at the threshold doesn't.
+ warns: ExperimentHealthRuleSide
+ test: ExperimentHealthRuleTest | None = None
+ minimum: ExperimentHealthRuleMinimum | None = None
@frozen
@@ -399,8 +466,9 @@ class ExperimentHealthCheckResult:
status: ExperimentHealthCheckStatus
# Why the check did not look, for the statuses not_applicable and not_checked. None otherwise.
reason: str | None
- # Only for the status passed, when the check shows the value that decided the pass.
- measurement: ExperimentHealthMeasurement | None
+ # What the check read on the experiment, for every status. Empty when it read nothing or shows nothing.
+ values: Mapping[ExperimentHealthValueKey, ExperimentHealthValue]
+ rule: ExperimentHealthCheckRule | None
@frozen
diff --git a/products/experiments/backend/health/checks/bias_risk.py b/products/experiments/backend/health/checks/bias_risk.py
index c471cc5f0808..b2e3e59bbeaa 100644
--- a/products/experiments/backend/health/checks/bias_risk.py
+++ b/products/experiments/backend/health/checks/bias_risk.py
@@ -9,12 +9,10 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingActionKind,
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
- ExperimentHealthMeasurement,
- ExperimentHealthMeasurementSide,
- ExperimentHealthMeasurementUnit,
+ ExperimentHealthValueKey,
)
from products.experiments.backend.health.checks.forced_variant import assigns_nobody_by_split
-from products.experiments.backend.health.context import HealthContext
+from products.experiments.backend.health.context import ExposureTotals, HealthContext
from products.experiments.backend.health.skips import (
NEEDS_EXPOSURES,
NEEDS_SPLIT_VARIANTS,
@@ -23,8 +21,8 @@ from products.experiments.backend.health.skips import (
RUNS_AFTER_LAUNCH,
SPLIT_ASSIGNS_NOBODY,
CheckOutcome,
- CheckPassed,
CheckSkip,
+ CheckValues,
not_applicable,
)
from products.feature_flags.backend.facade.edits import FlagVariant
@@ -87,6 +85,33 @@ def evaluate_bias_risk(
return BiasRisk(multiple_variant_percentage=share)
+def multiple_variant_count(exposures: ExposureTotals) -> int | None:
+ """The users exposed to more than one variant, or None under "First seen". There the exposure query gives each user
+ their first variant, so it counts none of these users."""
+ if exposures.multiple_variant_handling != MultipleVariantHandling.EXCLUDE:
+ return None
+ return exposures.total_exposures.get(MULTIPLE_VARIANT_KEY, 0)
+
+
+def multiple_variant_values(exposures: ExposureTotals) -> CheckValues:
+ """The users exposed to more than one variant, which the bias and the multiple variants checks read."""
+ values: dict[ExperimentHealthValueKey, int | float | str | None] = {
+ ExperimentHealthValueKey.EXPOSED_COUNT: exposures.total,
+ ExperimentHealthValueKey.MULTIPLE_VARIANT_HANDLING: exposures.multiple_variant_handling.value,
+ }
+ multiple = multiple_variant_count(exposures)
+ if multiple is not None:
+ values[ExperimentHealthValueKey.MULTIPLE_VARIANT_COUNT] = multiple
+ values[ExperimentHealthValueKey.MULTIPLE_VARIANT_PERCENTAGE] = (
+ multiple / exposures.total * 100 if exposures.total else None
+ )
+ return values
+
+
+def bias_risk_values(ctx: HealthContext) -> CheckValues:
+ return multiple_variant_values(ctx.exposures) if ctx.exposures else {}
+
+
def bias_risk_multiple_excluded(ctx: HealthContext) -> CheckOutcome:
if ctx.exposures is None:
return not_applicable(RUNS_AFTER_LAUNCH)
@@ -105,17 +130,7 @@ def bias_risk_multiple_excluded(ctx: HealthContext) -> CheckOutcome:
)
if isinstance(share, CheckSkip):
return share
- if share <= MULTIPLE_VARIANT_BIAS_THRESHOLD:
- return CheckPassed(
- measurement=ExperimentHealthMeasurement(
- value=share,
- threshold=MULTIPLE_VARIANT_BIAS_THRESHOLD,
- unit=ExperimentHealthMeasurementUnit.PERCENTAGE,
- warns=ExperimentHealthMeasurementSide.ABOVE,
- )
- )
- # The minimum count decides this pass, not the threshold, so it shows no measurement.
- if _too_few_to_warn(ctx.exposures.total_exposures):
+ if share <= MULTIPLE_VARIANT_BIAS_THRESHOLD or _too_few_to_warn(ctx.exposures.total_exposures):
return None
return ExperimentHealthFinding(
code=ExperimentHealthFindingCode.BIAS_RISK_MULTIPLE_EXCLUDED,
diff --git a/products/experiments/backend/health/checks/filtered_exposures.py b/products/experiments/backend/health/checks/filtered_exposures.py
index e3eb5ee2096e..96d396e2d4a1 100644
--- a/products/experiments/backend/health/checks/filtered_exposures.py
+++ b/products/experiments/backend/health/checks/filtered_exposures.py
@@ -3,9 +3,7 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingActionKind,
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
- ExperimentHealthMeasurement,
- ExperimentHealthMeasurementSide,
- ExperimentHealthMeasurementUnit,
+ ExperimentHealthValueKey,
)
from products.experiments.backend.health.context import ExposureTotals, HealthContext
from products.experiments.backend.health.delays import ZERO_EXPOSURES_GRACE_HOURS
@@ -13,8 +11,8 @@ from products.experiments.backend.health.skips import (
NEEDS_EXPOSURES,
RUNS_AFTER_LAUNCH,
CheckOutcome,
- CheckPassed,
CheckSkip,
+ CheckValues,
not_applicable,
not_checked,
)
@@ -35,6 +33,22 @@ def reads_share_kept_by_test_account_filter(ctx: HealthContext) -> bool:
return ctx.exposures is not None and _skip(ctx, ctx.exposures) is None
+def test_account_filter_values(ctx: HealthContext) -> CheckValues:
+ exposures = ctx.exposures
+ if exposures is None:
+ return {}
+ values: dict[ExperimentHealthValueKey, int | float | None] = {
+ ExperimentHealthValueKey.EXPOSED_COUNT: exposures.total
+ }
+ exposed = exposures.exposed_without_test_account_filter
+ if exposed is not None:
+ values[ExperimentHealthValueKey.EXPOSED_WITHOUT_TEST_ACCOUNT_FILTER_COUNT] = exposed
+ values[ExperimentHealthValueKey.KEPT_BY_TEST_ACCOUNT_FILTER_PERCENTAGE] = (
+ exposures.total / exposed * 100 if exposed else None
+ )
+ return values
+
+
def exposures_removed_by_test_account_filter(ctx: HealthContext) -> CheckOutcome:
exposures = ctx.exposures
if exposures is None:
@@ -51,14 +65,7 @@ def exposures_removed_by_test_account_filter(ctx: HealthContext) -> CheckOutcome
if exposed < MINIMUM_EXPOSED_WITHOUT_FILTER:
return not_applicable(NEEDS_EXPOSURES)
if counted * 100 >= exposed * KEPT_SHARE_THRESHOLD:
- return CheckPassed(
- measurement=ExperimentHealthMeasurement(
- value=counted / exposed * 100,
- threshold=KEPT_SHARE_THRESHOLD,
- unit=ExperimentHealthMeasurementUnit.PERCENTAGE,
- warns=ExperimentHealthMeasurementSide.BELOW,
- )
- )
+ return None
return ExperimentHealthFinding(
code=ExperimentHealthFindingCode.TEST_ACCOUNT_FILTER_REMOVES_EXPOSURES,
subcode=None,
diff --git a/products/experiments/backend/health/checks/flag_state.py b/products/experiments/backend/health/checks/flag_state.py
index daaa0e44ae0e..f95323ff12cb 100644
--- a/products/experiments/backend/health/checks/flag_state.py
+++ b/products/experiments/backend/health/checks/flag_state.py
@@ -27,6 +27,7 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingActionKind,
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
+ ExperimentHealthValueKey,
)
from products.experiments.backend.health.context import (
ConditionSetTraffic,
@@ -37,7 +38,7 @@ from products.experiments.backend.health.context import (
PauseHistory,
)
from products.experiments.backend.health.delays import ENDED_FLAG_QUIET_PERIOD
-from products.experiments.backend.health.skips import NO_FLAG, CheckOutcome, CheckSkip, not_applicable
+from products.experiments.backend.health.skips import NO_FLAG, CheckOutcome, CheckSkip, CheckValues, not_applicable
from products.experiments.backend.health.split_changes import (
SplitChangeShape,
describe_split,
@@ -80,6 +81,20 @@ _NOT_RUNNING_BUT_EXPOSED_TITLE = "The experiment is not running, but users are e
OTHER_EXPERIMENT_DECIDES = "Another experiment on the same feature flag decides the flag's state"
+def flag_state_values(ctx: HealthContext) -> CheckValues:
+ flag = ctx.flag
+ if flag is None:
+ return {}
+ return {
+ ExperimentHealthValueKey.FLAG_ACTIVE: flag.active,
+ ExperimentHealthValueKey.FLAG_DELETED: flag.deleted,
+ ExperimentHealthValueKey.FLAG_ROLLED_OUT: not _has_zero_rollout(flag),
+ ExperimentHealthValueKey.SHIPPED_VARIANT: shipped_variant_key(flag)
+ if is_single_variant_shipped(flag)
+ else None,
+ }
+
+
def flag_state(ctx: HealthContext) -> CheckOutcome:
flag = ctx.flag
if flag is None:
diff --git a/products/experiments/backend/health/checks/metric_calculation_failed.py b/products/experiments/backend/health/checks/metric_calculation_failed.py
index 7fed8f8edbb5..f34ebdd60056 100644
--- a/products/experiments/backend/health/checks/metric_calculation_failed.py
+++ b/products/experiments/backend/health/checks/metric_calculation_failed.py
@@ -10,6 +10,7 @@ reader's warehouse access, or the text of other exceptions, which is written for
import re
from dataclasses import replace
+from datetime import UTC
from enum import StrEnum
from posthog.dataclasses import frozen
@@ -21,6 +22,7 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
ExperimentHealthMetricCompileErrorKind,
+ ExperimentHealthValueKey,
)
from products.experiments.backend.facade.enums import ExposuresPath
from products.experiments.backend.health.checks.events_received import joined
@@ -36,6 +38,7 @@ from products.experiments.backend.health.skips import (
NOT_ON_ARCHIVED,
RUNS_AFTER_LAUNCH,
CheckOutcome,
+ CheckValues,
no_calculated_metric,
not_applicable,
not_checked,
@@ -165,6 +168,21 @@ _EXPOSURE_QUERY = _Wording(
)
+def metric_calculation_values(ctx: HealthContext) -> CheckValues:
+ calculation = ctx.latest_calculation
+ if calculation is None:
+ return {}
+ finished_at = calculation.finished_at
+ return {
+ ExperimentHealthValueKey.METRIC_COUNT: calculation.metric_count,
+ ExperimentHealthValueKey.COMPLETED_METRIC_COUNT: calculation.completed_count,
+ ExperimentHealthValueKey.FAILED_METRIC_COUNT: len(calculation.failures),
+ ExperimentHealthValueKey.CALCULATED_AT: None
+ if finished_at is None
+ else finished_at.astimezone(UTC).isoformat(),
+ }
+
+
def metric_calculation_failed(ctx: HealthContext) -> CheckOutcome:
if ctx.archived:
return not_applicable(NOT_ON_ARCHIVED)
diff --git a/products/experiments/backend/health/checks/metrics_not_matured.py b/products/experiments/backend/health/checks/metrics_not_matured.py
index 6c8b52ebb5a2..3d7a1aa619bc 100644
--- a/products/experiments/backend/health/checks/metrics_not_matured.py
+++ b/products/experiments/backend/health/checks/metrics_not_matured.py
@@ -12,6 +12,7 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFinding,
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
+ ExperimentHealthValueKey,
)
from products.experiments.backend.health.context import HealthContext, MetricWindow
from products.experiments.backend.health.skips import (
@@ -20,6 +21,7 @@ from products.experiments.backend.health.skips import (
NOT_ON_ARCHIVED,
RUNS_AFTER_LAUNCH,
CheckOutcome,
+ CheckValues,
not_applicable,
not_checked,
)
@@ -29,6 +31,21 @@ _SETTING_LABEL = '"Require completed conversion or retention window"'
_SECONDS_PER_HOUR = 3600
+def metrics_not_matured_values(ctx: HealthContext) -> CheckValues:
+ values: dict[ExperimentHealthValueKey, bool | int] = {
+ ExperimentHealthValueKey.ONLY_COUNT_MATURED_USERS: ctx.only_count_matured_users
+ }
+ windows = ctx.primary_metric_windows
+ hours_since_launch = ctx.hours_since_launch
+ # The windows load only while the setting is on, so with it off the setting is all the check knows.
+ if windows is not None and hours_since_launch is not None:
+ values[ExperimentHealthValueKey.PRIMARY_METRIC_COUNT] = len(windows)
+ values[ExperimentHealthValueKey.WAITING_METRIC_COUNT] = sum(
+ 1 for window in windows if _holds_back_every_user(window, hours_since_launch)
+ )
+ return values
+
+
def metrics_not_matured(ctx: HealthContext) -> CheckOutcome:
if ctx.archived:
return not_applicable(NOT_ON_ARCHIVED)
diff --git a/products/experiments/backend/health/checks/multiple_variants.py b/products/experiments/backend/health/checks/multiple_variants.py
index 669180f1947c..b59a773f6b54 100644
--- a/products/experiments/backend/health/checks/multiple_variants.py
+++ b/products/experiments/backend/health/checks/multiple_variants.py
@@ -4,16 +4,13 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFinding,
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
- ExperimentHealthMeasurement,
- ExperimentHealthMeasurementSide,
- ExperimentHealthMeasurementUnit,
)
-from products.experiments.backend.health.checks.bias_risk import MULTIPLE_VARIANT_KEY
+from products.experiments.backend.health.checks.bias_risk import MULTIPLE_VARIANT_KEY, multiple_variant_values
from products.experiments.backend.health.context import HealthContext
from products.experiments.backend.health.skips import (
RUNS_AFTER_LAUNCH,
CheckOutcome,
- CheckPassed,
+ CheckValues,
not_applicable,
not_checked,
)
@@ -27,6 +24,10 @@ MULTIPLE_VARIANTS_SHARE_THRESHOLD = 5.0 # on the 0-100 scale (5.0 = 5%)
MULTIPLE_VARIANTS_MINIMUM_COUNT = 5
+def multiple_variants_values(ctx: HealthContext) -> CheckValues:
+ return multiple_variant_values(ctx.exposures) if ctx.exposures else {}
+
+
def multiple_variants_share(ctx: HealthContext) -> CheckOutcome:
exposures = ctx.exposures
if exposures is None:
@@ -40,14 +41,7 @@ def multiple_variants_share(ctx: HealthContext) -> CheckOutcome:
return None
percentage = multiple / total * 100
if percentage <= MULTIPLE_VARIANTS_SHARE_THRESHOLD:
- return CheckPassed(
- measurement=ExperimentHealthMeasurement(
- value=percentage,
- threshold=MULTIPLE_VARIANTS_SHARE_THRESHOLD,
- unit=ExperimentHealthMeasurementUnit.PERCENTAGE,
- warns=ExperimentHealthMeasurementSide.ABOVE,
- )
- )
+ return None
# The flag gives every member of a group the same variant, so the causes of a person's several IDs do not apply.
if exposures.counts_groups:
title = "Many groups were exposed to multiple variants"
diff --git a/products/experiments/backend/health/checks/srm.py b/products/experiments/backend/health/checks/srm.py
index 52a95488ae7d..3a03c2788c24 100644
--- a/products/experiments/backend/health/checks/srm.py
+++ b/products/experiments/backend/health/checks/srm.py
@@ -17,10 +17,9 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFinding,
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
- ExperimentHealthMeasurement,
- ExperimentHealthMeasurementSide,
- ExperimentHealthMeasurementUnit,
+ ExperimentHealthValueKey,
)
+from products.experiments.backend.health.checks.bias_risk import multiple_variant_count
from products.experiments.backend.health.checks.forced_variant import (
SOME_CONDITIONS_PINNED,
assigns_nobody_by_split,
@@ -42,7 +41,7 @@ from products.experiments.backend.health.skips import (
RUNS_AFTER_LAUNCH,
SPLIT_ASSIGNS_NOBODY,
CheckOutcome,
- CheckPassed,
+ CheckValues,
not_applicable,
)
from products.experiments.backend.hogql_queries.experiment_exposure_query_builder import ExposureSegmentDimension
@@ -136,6 +135,32 @@ def segments_first_exposed_after(ctx: HealthContext) -> date | None:
return _last_split_change_day(history)
+def srm_values(ctx: HealthContext) -> CheckValues:
+ exposures = ctx.exposures
+ if exposures is None:
+ return {}
+ # The test compares the variants of `sample_ratio_expected`. Below its minimum the test doesn't run and leaves that
+ # empty, so the variants come from the split, as the exposure query picks them.
+ compared = exposures.sample_ratio_expected.keys() or _variants_in_split(ctx)
+ values: dict[ExperimentHealthValueKey, int | float | None] = {
+ ExperimentHealthValueKey.P_VALUE: exposures.sample_ratio_mismatch_p_value,
+ ExperimentHealthValueKey.COMPARED_COUNT: sum(exposures.total_exposures.get(key, 0) for key in compared),
+ }
+ multiple = multiple_variant_count(exposures)
+ if multiple is not None:
+ values[ExperimentHealthValueKey.MULTIPLE_VARIANT_COUNT] = multiple
+ return values
+
+
+def _variants_in_split(ctx: HealthContext) -> list[str]:
+ """The variants with a share of the split that the analysis keeps, which the exposure query compares."""
+ return [
+ variant.key
+ for variant in (ctx.flag.variants if ctx.flag else ())
+ if variant.key and (variant.rollout_percentage or 0) > 0 and variant.key not in ctx.excluded_variant_keys
+ ]
+
+
def srm(ctx: HealthContext) -> CheckOutcome:
exposures = ctx.exposures
if exposures is None:
@@ -148,23 +173,11 @@ def srm(ctx: HealthContext) -> CheckOutcome:
if p_value is None:
# The exposure query compares only the variants with a share of the split that the analysis keeps, so after
# a variant ships there is nothing to compare however many users are exposed.
- variants_in_split = [
- variant
- for variant in (ctx.flag.variants if ctx.flag else ())
- if variant.key and (variant.rollout_percentage or 0) > 0 and variant.key not in ctx.excluded_variant_keys
- ]
- if len(variants_in_split) < 2:
+ if len(_variants_in_split(ctx)) < 2:
return not_applicable(NEEDS_SPLIT_VARIANTS)
return not_applicable(NEEDS_EXPOSURES)
if p_value >= SAMPLE_RATIO_MISMATCH_P_VALUE:
- return CheckPassed(
- measurement=ExperimentHealthMeasurement(
- value=p_value,
- threshold=SAMPLE_RATIO_MISMATCH_P_VALUE,
- unit=ExperimentHealthMeasurementUnit.P_VALUE,
- warns=ExperimentHealthMeasurementSide.BELOW,
- )
- )
+ return None
share_off_split = _share_off_split(exposures.total_exposures, exposures.sample_ratio_expected)
split_change = _split_change_warning(ctx)
history = ctx.flag_history
diff --git a/products/experiments/backend/health/checks/zero_exposures.py b/products/experiments/backend/health/checks/zero_exposures.py
index c6301235e540..d89c8c78c597 100644
--- a/products/experiments/backend/health/checks/zero_exposures.py
+++ b/products/experiments/backend/health/checks/zero_exposures.py
@@ -3,11 +3,12 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingActionKind,
ExperimentHealthFindingCode,
ExperimentHealthFindingSeverity,
+ ExperimentHealthValueKey,
)
from products.experiments.backend.health.checks.exposure_criteria import exposure_criteria_cannot_match
from products.experiments.backend.health.context import HealthContext
from products.experiments.backend.health.delays import LAST_CALLED_AT_LAG, ZERO_EXPOSURES_GRACE_HOURS
-from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckOutcome, not_applicable
+from products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckOutcome, CheckValues, not_applicable
# pinned: the finding's subcodes. The page sends them as `finding_subcode` on the health finding events.
EXPERIMENT_PAUSED = "experiment_paused"
@@ -26,6 +27,17 @@ def reads_exposed_without_test_account_filter(ctx: HealthContext) -> bool:
return isinstance(outcome, ExperimentHealthFinding) and outcome.subcode == CALLS_NOT_COUNTED
+def zero_exposures_values(ctx: HealthContext) -> CheckValues:
+ exposures = ctx.exposures
+ if exposures is None:
+ return {}
+ hours = exposures.hours_since_launch
+ return {
+ ExperimentHealthValueKey.EXPOSED_COUNT: exposures.total,
+ ExperimentHealthValueKey.SINCE_LAUNCH_HOURS: None if hours is None else round(hours, 1),
+ }
+
+
def zero_exposures(ctx: HealthContext) -> CheckOutcome:
exposures = ctx.exposures
if exposures is None or exposures.hours_since_launch is None or not ctx.is_launched:
diff --git a/products/experiments/backend/health/registry.py b/products/experiments/backend/health/registry.py
index cb4b293ab82b..d90acdb70cfe 100644
--- a/products/experiments/backend/health/registry.py
+++ b/products/experiments/backend/health/registry.py
@@ -1,4 +1,5 @@
from collections.abc import Callable, Sequence
+from dataclasses import replace
from typing import assert_never
from posthog.dataclasses import frozen
@@ -8,35 +9,66 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealth,
ExperimentHealthCheckKey,
ExperimentHealthCheckResult,
+ ExperimentHealthCheckRule,
ExperimentHealthCheckStatus,
ExperimentHealthFinding,
- ExperimentHealthMeasurement,
+ ExperimentHealthRuleMinimum,
+ ExperimentHealthRuleSide,
+ ExperimentHealthRuleTest,
+ ExperimentHealthValueKey,
+)
+from products.experiments.backend.health.checks.bias_risk import (
+ MULTIPLE_VARIANT_BIAS_MINIMUM_COUNT,
+ MULTIPLE_VARIANT_BIAS_THRESHOLD,
+ bias_risk_multiple_excluded,
+ bias_risk_values,
)
-from products.experiments.backend.health.checks.bias_risk import bias_risk_multiple_excluded
from products.experiments.backend.health.checks.bucketing_changed import bucketing_changed_mid_run
from products.experiments.backend.health.checks.conditions_changed import release_conditions_changed_mid_run
from products.experiments.backend.health.checks.early_significance import early_significance
from products.experiments.backend.health.checks.exposure_criteria import exposure_criteria_cannot_match
from products.experiments.backend.health.checks.exposure_filters import exposure_filters
-from products.experiments.backend.health.checks.filtered_exposures import exposures_removed_by_test_account_filter
+from products.experiments.backend.health.checks.filtered_exposures import (
+ KEPT_SHARE_THRESHOLD,
+ MINIMUM_EXPOSED_WITHOUT_FILTER,
+ exposures_removed_by_test_account_filter,
+ test_account_filter_values,
+)
from products.experiments.backend.health.checks.flag_change_findings import flag_history_not_readable
-from products.experiments.backend.health.checks.flag_state import flag_state
+from products.experiments.backend.health.checks.flag_state import flag_state, flag_state_values
from products.experiments.backend.health.checks.flag_variants import flag_variants_mismatch
from products.experiments.backend.health.checks.forced_variant import forced_variant_release_condition
from products.experiments.backend.health.checks.metric_all_zero import metric_all_zero
-from products.experiments.backend.health.checks.metric_calculation_failed import metric_calculation_failed
+from products.experiments.backend.health.checks.metric_calculation_failed import (
+ metric_calculation_failed,
+ metric_calculation_values,
+)
from products.experiments.backend.health.checks.metric_cannot_compile import metric_cannot_compile
from products.experiments.backend.health.checks.metric_event_not_ingested import metric_event_not_ingested
-from products.experiments.backend.health.checks.metrics_not_matured import metrics_not_matured
-from products.experiments.backend.health.checks.multiple_variants import multiple_variants_share
+from products.experiments.backend.health.checks.metrics_not_matured import (
+ metrics_not_matured,
+ metrics_not_matured_values,
+)
+from products.experiments.backend.health.checks.multiple_variants import (
+ MULTIPLE_VARIANTS_MINIMUM_COUNT,
+ MULTIPLE_VARIANTS_SHARE_THRESHOLD,
+ multiple_variants_share,
+ multiple_variants_values,
+)
from products.experiments.backend.health.checks.new_flag_no_traffic import new_flag_no_traffic
from products.experiments.backend.health.checks.no_metric import no_metric
from products.experiments.backend.health.checks.shared_flag import flag_shared
from products.experiments.backend.health.checks.split_changed import split_changed_mid_run
-from products.experiments.backend.health.checks.srm import srm
-from products.experiments.backend.health.checks.zero_exposures import zero_exposures
+from products.experiments.backend.health.checks.srm import (
+ SAMPLE_RATIO_MISMATCH_P_VALUE,
+ SRM_MINIMUM_SAMPLE_SIZE,
+ srm,
+ srm_values,
+)
+from products.experiments.backend.health.checks.zero_exposures import zero_exposures, zero_exposures_values
from products.experiments.backend.health.context import HealthContext
-from products.experiments.backend.health.skips import CheckOutcome, CheckPassed, CheckSkip
+from products.experiments.backend.health.delays import ZERO_EXPOSURES_GRACE_HOURS
+from products.experiments.backend.health.skips import CheckOutcome, CheckSkip, CheckValues
@frozen
@@ -46,6 +78,10 @@ class HealthCheck:
# What the check looks for and why it matters. The tab shows it for every status, so it holds for every experiment.
description: str
run: Callable[[HealthContext], CheckOutcome]
+ # What the check read, for every status. It doesn't run when `run` raised.
+ read: Callable[[HealthContext], CheckValues] | None = None
+ # How the check decides, for every status. None for a check without a numeric rule.
+ rule: ExperimentHealthCheckRule | None = None
EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
@@ -58,6 +94,7 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
"serves variants shows them to users that the results don't count."
),
run=flag_state,
+ read=flag_state_values,
),
HealthCheck(
key=ExperimentHealthCheckKey.FLAG_SHARED,
@@ -168,6 +205,7 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
"results or none, and some failures repeat until you change the metric or a setting."
),
run=metric_calculation_failed,
+ read=metric_calculation_values,
),
HealthCheck(
key=ExperimentHealthCheckKey.METRIC_ALL_ZERO,
@@ -216,6 +254,7 @@ EXPERIMENT_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
"while nothing is wrong."
),
run=metrics_not_matured,
+ read=metrics_not_matured_values,
),
)
@@ -232,6 +271,15 @@ EXPOSURE_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
"means that your code doesn't evaluate the flag or that no event matches the exposure criteria."
),
run=zero_exposures,
+ read=zero_exposures_values,
+ rule=ExperimentHealthCheckRule(
+ value=ExperimentHealthValueKey.EXPOSED_COUNT,
+ threshold=1,
+ warns=ExperimentHealthRuleSide.BELOW,
+ minimum=ExperimentHealthRuleMinimum(
+ value=ExperimentHealthValueKey.SINCE_LAUNCH_HOURS, threshold=ZERO_EXPOSURES_GRACE_HOURS
+ ),
+ ),
),
HealthCheck(
key=ExperimentHealthCheckKey.TEST_ACCOUNT_FILTER_REMOVES_EXPOSURES,
@@ -241,6 +289,19 @@ EXPOSURE_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
"often match real users by mistake, so the results measure only a small part of your users."
),
run=exposures_removed_by_test_account_filter,
+ read=test_account_filter_values,
+ # The check also waits for `ZERO_EXPOSURES_GRACE_HOURS` after launch and stops at `FEW_EXPOSURES_LIMIT` exposed
+ # users, because its count scans the exposure events. Those limits decide whether it runs, not what it finds,
+ # so its reason states them and the rule leaves them out.
+ rule=ExperimentHealthCheckRule(
+ value=ExperimentHealthValueKey.KEPT_BY_TEST_ACCOUNT_FILTER_PERCENTAGE,
+ threshold=KEPT_SHARE_THRESHOLD,
+ warns=ExperimentHealthRuleSide.BELOW,
+ minimum=ExperimentHealthRuleMinimum(
+ value=ExperimentHealthValueKey.EXPOSED_WITHOUT_TEST_ACCOUNT_FILTER_COUNT,
+ threshold=MINIMUM_EXPOSED_WITHOUT_FILTER,
+ ),
+ ),
),
HealthCheck(
key=ExperimentHealthCheckKey.SRM,
@@ -251,6 +312,16 @@ EXPOSURE_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
"one variant, which can bias the results."
),
run=srm,
+ read=srm_values,
+ rule=ExperimentHealthCheckRule(
+ value=ExperimentHealthValueKey.P_VALUE,
+ threshold=SAMPLE_RATIO_MISMATCH_P_VALUE,
+ warns=ExperimentHealthRuleSide.BELOW,
+ test=ExperimentHealthRuleTest.CHI_SQUARED,
+ minimum=ExperimentHealthRuleMinimum(
+ value=ExperimentHealthValueKey.COMPARED_COUNT, threshold=SRM_MINIMUM_SAMPLE_SIZE
+ ),
+ ),
),
HealthCheck(
key=ExperimentHealthCheckKey.BIAS_RISK_MULTIPLE_EXCLUDED,
@@ -260,6 +331,15 @@ EXPOSURE_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
"users out, and the smaller variant loses a larger share of its users, which biases the comparison."
),
run=bias_risk_multiple_excluded,
+ read=bias_risk_values,
+ rule=ExperimentHealthCheckRule(
+ value=ExperimentHealthValueKey.MULTIPLE_VARIANT_PERCENTAGE,
+ threshold=MULTIPLE_VARIANT_BIAS_THRESHOLD,
+ warns=ExperimentHealthRuleSide.ABOVE,
+ minimum=ExperimentHealthRuleMinimum(
+ value=ExperimentHealthValueKey.MULTIPLE_VARIANT_COUNT, threshold=MULTIPLE_VARIANT_BIAS_MINIMUM_COUNT
+ ),
+ ),
),
HealthCheck(
key=ExperimentHealthCheckKey.MULTIPLE_VARIANTS_SHARE,
@@ -270,6 +350,15 @@ EXPOSURE_HEALTH_CHECKS: tuple[HealthCheck, ...] = (
"results."
),
run=multiple_variants_share,
+ read=multiple_variants_values,
+ rule=ExperimentHealthCheckRule(
+ value=ExperimentHealthValueKey.MULTIPLE_VARIANT_PERCENTAGE,
+ threshold=MULTIPLE_VARIANTS_SHARE_THRESHOLD,
+ warns=ExperimentHealthRuleSide.ABOVE,
+ minimum=ExperimentHealthRuleMinimum(
+ value=ExperimentHealthValueKey.MULTIPLE_VARIANT_COUNT, threshold=MULTIPLE_VARIANTS_MINIMUM_COUNT
+ ),
+ ),
),
)
@@ -287,28 +376,38 @@ def evaluate(ctx: HealthContext, checks: Sequence[HealthCheck]) -> ExperimentHea
# The findings ride on the health action, the exposure answer and `experiment launched`, so one
# failing check must not fail them or hide the findings of the other checks.
capture_exception(error)
- results.append(_result(check, ExperimentHealthCheckStatus.NOT_CHECKED, _CHECK_FAILED))
+ results.append(_result(check, ExperimentHealthCheckStatus.NOT_CHECKED, _CHECK_FAILED, {}))
continue
+ values = _read(check, ctx)
match outcome:
case CheckSkip(status=status, reason=reason):
- results.append(_result(check, status, reason))
+ results.append(_result(check, status, reason, values))
case ExperimentHealthFinding():
- findings.append(outcome)
- results.append(_result(check, ExperimentHealthCheckStatus.FINDING, None))
- case CheckPassed(measurement=measurement):
- results.append(_result(check, ExperimentHealthCheckStatus.PASSED, None, measurement))
+ findings.append(replace(outcome, check=check.key))
+ results.append(_result(check, ExperimentHealthCheckStatus.FINDING, None, values))
case None:
- results.append(_result(check, ExperimentHealthCheckStatus.PASSED, None))
+ results.append(_result(check, ExperimentHealthCheckStatus.PASSED, None, values))
case _:
assert_never(outcome)
return ExperimentHealth(findings=tuple(findings), checks=tuple(results))
+def _read(check: HealthCheck, ctx: HealthContext) -> CheckValues:
+ if check.read is None:
+ return {}
+ try:
+ return check.read(ctx)
+ except Exception as error:
+ # The values only explain the status, so a failed read keeps the status and shows no values.
+ capture_exception(error)
+ return {}
+
+
def _result(
check: HealthCheck,
status: ExperimentHealthCheckStatus,
reason: str | None,
- measurement: ExperimentHealthMeasurement | None = None,
+ values: CheckValues,
) -> ExperimentHealthCheckResult:
return ExperimentHealthCheckResult(
key=check.key,
@@ -316,5 +415,6 @@ def _result(
description=check.description,
status=status,
reason=reason,
- measurement=measurement,
+ values=values,
+ rule=check.rule,
)
diff --git a/products/experiments/backend/health/skips.py b/products/experiments/backend/health/skips.py
index 12e32c48d371..2d0cca158e79 100644
--- a/products/experiments/backend/health/skips.py
+++ b/products/experiments/backend/health/skips.py
@@ -1,3 +1,4 @@
+from collections.abc import Mapping
from typing import Literal
from posthog.dataclasses import frozen
@@ -5,7 +6,8 @@ from posthog.dataclasses import frozen
from products.experiments.backend.facade.contracts import (
ExperimentHealthCheckStatus,
ExperimentHealthFinding,
- ExperimentHealthMeasurement,
+ ExperimentHealthValue,
+ ExperimentHealthValueKey,
)
from products.experiments.backend.health.context import FlagState, HealthContext
@@ -16,14 +18,10 @@ class CheckSkip:
reason: str
-@frozen
-class CheckPassed:
- measurement: ExperimentHealthMeasurement
-
-
-# What a check returns: a finding, why the check doesn't look, or a pass. CheckPassed is a pass that shows the value
-# that decided it against its threshold, and None is a pass without a value.
-CheckOutcome = ExperimentHealthFinding | CheckSkip | CheckPassed | None
+# What a check returns: a finding, why the check doesn't look, or None for a pass.
+CheckOutcome = ExperimentHealthFinding | CheckSkip | None
+# What a check read on the experiment, for every status.
+CheckValues = Mapping[ExperimentHealthValueKey, ExperimentHealthValue]
RUNS_AFTER_LAUNCH = "Runs after launch"
RUNS_BEFORE_LAUNCH = "Runs only before launch"
diff --git a/products/experiments/backend/health/test/test_registry.py b/products/experiments/backend/health/test/test_registry.py
index 930d7e358b03..7f60a0a94c30 100644
--- a/products/experiments/backend/health/test/test_registry.py
+++ b/products/experiments/backend/health/test/test_registry.py
@@ -1,6 +1,7 @@
from dataclasses import replace
from datetime import UTC, datetime
from typing import Any
+from zoneinfo import ZoneInfo
from unittest import TestCase
from unittest.mock import patch
@@ -13,9 +14,7 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthCheckKey,
ExperimentHealthCheckStatus,
ExperimentHealthFindingCode,
- ExperimentHealthMeasurement,
- ExperimentHealthMeasurementSide,
- ExperimentHealthMeasurementUnit,
+ ExperimentHealthValueKey,
)
from products.experiments.backend.health import registry
from products.experiments.backend.health.checks.filtered_exposures import reads_share_kept_by_test_account_filter
@@ -24,9 +23,12 @@ from products.experiments.backend.health.context import (
ExposureTotals,
FlagHistoryNotCheckedReason,
FlagKeyFilter,
+ FlagReleaseGroup,
FlagState,
HealthContext,
+ LatestCalculation,
MetricEvents,
+ MetricWindow,
ReachableCondition,
)
from products.experiments.backend.health.registry import (
@@ -36,6 +38,7 @@ from products.experiments.backend.health.registry import (
evaluate,
)
from products.experiments.backend.health.test.factories import DRAFT, RUNNING_CONTEXT, flag_history
+from products.experiments.backend.hogql_queries.experiment_exposures_query_runner import ExperimentExposuresQueryRunner
from products.feature_flags.backend.facade.edits import FlagVariant
UNEVEN_SPLIT_TOTALS = ExposureTotals(
@@ -68,6 +71,7 @@ ENDED: dict[str, Any] = {"is_running": False, "has_ended": True}
LAUNCHED_AT = datetime(2026, 10, 1, tzinfo=UTC)
Key = ExperimentHealthCheckKey
+Value = ExperimentHealthValueKey
Status = ExperimentHealthCheckStatus
NOT_LOOKED = (Status.NOT_APPLICABLE, Status.NOT_CHECKED)
FLAG_CHANGE_KEYS = (Key.SPLIT_CHANGED_MID_RUN, Key.BUCKETING_CHANGED_MID_RUN, Key.RELEASE_CONDITIONS_CHANGED_MID_RUN)
@@ -316,6 +320,10 @@ class TestEvaluate(TestCase):
self.assertEqual(
[check.key for check in health.checks if (check.reason is None) == (check.status in NOT_LOOKED)], []
)
+ self.assertEqual(
+ sorted(finding.check for finding in health.findings if finding.check),
+ sorted(check.key for check in health.checks if check.status == Status.FINDING),
+ )
@parameterized.expand(
[
@@ -379,81 +387,195 @@ class TestEvaluate(TestCase):
@parameterized.expand(
[
(
- "srm_p_value",
+ "srm_compares_the_variants_without_multiple",
Key.SRM,
- replace(UNEVEN_SPLIT_TOTALS, sample_ratio_mismatch_p_value=0.71),
- ExperimentHealthMeasurement(
- value=0.71,
- threshold=0.001,
- unit=ExperimentHealthMeasurementUnit.P_VALUE,
- warns=ExperimentHealthMeasurementSide.BELOW,
- ),
+ {"exposures": replace(UNEVEN_SPLIT_TOTALS, sample_ratio_mismatch_p_value=0.71)},
+ Status.PASSED,
+ {Value.P_VALUE: 0.71, Value.COMPARED_COUNT: 1000, Value.MULTIPLE_VARIANT_COUNT: 20},
+ ),
+ (
+ "srm_leaves_out_the_multiple_count_under_first_seen",
+ Key.SRM,
+ {
+ "exposures": replace(
+ UNEVEN_SPLIT_TOTALS,
+ total_exposures={"control": 800, "test": 200},
+ multiple_variant_handling=MultipleVariantHandling.FIRST_SEEN,
+ sample_ratio_mismatch_p_value=0.71,
+ )
+ },
+ Status.PASSED,
+ {Value.P_VALUE: 0.71, Value.COMPARED_COUNT: 1000},
+ ),
+ (
+ "bias_risk_divides_by_every_exposure",
+ Key.BIAS_RISK_MULTIPLE_EXCLUDED,
+ {},
+ Status.FINDING,
+ {
+ Value.EXPOSED_COUNT: 1020,
+ Value.MULTIPLE_VARIANT_COUNT: 20,
+ Value.MULTIPLE_VARIANT_PERCENTAGE: 20 / 1020 * 100,
+ Value.MULTIPLE_VARIANT_HANDLING: "exclude",
+ },
),
(
"multiple_variants_share",
Key.MULTIPLE_VARIANTS_SHARE,
- _totals_with_multiple(30, 1000),
- ExperimentHealthMeasurement(
- value=3.0,
- threshold=5.0,
- unit=ExperimentHealthMeasurementUnit.PERCENTAGE,
- warns=ExperimentHealthMeasurementSide.ABOVE,
- ),
+ {"exposures": _totals_with_multiple(30, 1000)},
+ Status.PASSED,
+ {
+ Value.EXPOSED_COUNT: 1000,
+ Value.MULTIPLE_VARIANT_COUNT: 30,
+ Value.MULTIPLE_VARIANT_PERCENTAGE: 3.0,
+ Value.MULTIPLE_VARIANT_HANDLING: "exclude",
+ },
),
(
- "multiple_variants_below_the_minimum_count",
+ "multiple_variants_are_not_counted_under_first_seen",
Key.MULTIPLE_VARIANTS_SHARE,
- _totals_with_multiple(4, 20),
- None,
+ {
+ "exposures": replace(
+ _totals_with_multiple(30, 1000), multiple_variant_handling=MultipleVariantHandling.FIRST_SEEN
+ )
+ },
+ Status.NOT_CHECKED,
+ {Value.EXPOSED_COUNT: 1000, Value.MULTIPLE_VARIANT_HANDLING: "first_seen"},
),
(
"kept_share",
Key.TEST_ACCOUNT_FILTER_REMOVES_EXPOSURES,
- _filtered_totals(400, 1000),
- ExperimentHealthMeasurement(
- value=40.0,
- threshold=10.0,
- unit=ExperimentHealthMeasurementUnit.PERCENTAGE,
- warns=ExperimentHealthMeasurementSide.BELOW,
- ),
+ {"exposures": _filtered_totals(400, 1000)},
+ Status.PASSED,
+ {
+ Value.EXPOSED_COUNT: 400,
+ Value.EXPOSED_WITHOUT_TEST_ACCOUNT_FILTER_COUNT: 1000,
+ Value.KEPT_BY_TEST_ACCOUNT_FILTER_PERCENTAGE: 40.0,
+ },
),
(
- "kept_share_at_the_threshold",
+ "kept_share_without_the_count_without_the_filter",
Key.TEST_ACCOUNT_FILTER_REMOVES_EXPOSURES,
- _filtered_totals(100, 1000),
- ExperimentHealthMeasurement(
- value=10.0,
- threshold=10.0,
- unit=ExperimentHealthMeasurementUnit.PERCENTAGE,
- warns=ExperimentHealthMeasurementSide.BELOW,
- ),
+ {"exposures": _filtered_totals(400, None)},
+ Status.NOT_CHECKED,
+ {Value.EXPOSED_COUNT: 400},
),
(
- "bias_risk_share",
- Key.BIAS_RISK_MULTIPLE_EXCLUDED,
- _totals_with_multiple(0, 1000),
- ExperimentHealthMeasurement(
- value=0.0,
- threshold=0.1,
- unit=ExperimentHealthMeasurementUnit.PERCENTAGE,
- warns=ExperimentHealthMeasurementSide.ABOVE,
- ),
+ "zero_exposures_in_the_first_day",
+ Key.ZERO_EXPOSURES,
+ {"exposures": replace(NO_EXPOSURES, hours_since_launch=23.94)},
+ Status.NOT_APPLICABLE,
+ {Value.EXPOSED_COUNT: 0, Value.SINCE_LAUNCH_HOURS: 23.9},
),
(
- "bias_risk_share_above_the_threshold_from_too_few_users",
- Key.BIAS_RISK_MULTIPLE_EXCLUDED,
- _totals_with_multiple(3, 1000),
- None,
+ "metrics_not_matured_with_the_setting_off",
+ Key.METRICS_NOT_MATURED,
+ {},
+ Status.NOT_APPLICABLE,
+ {Value.ONLY_COUNT_MATURED_USERS: False},
+ ),
+ (
+ "metrics_not_matured_counts_the_metrics_without_users",
+ Key.METRICS_NOT_MATURED,
+ {
+ "only_count_matured_users": True,
+ "primary_metric_windows": (
+ MetricWindow(name="Day", window_seconds=86_400, first_users_count_at=LAUNCHED_AT),
+ MetricWindow(name="Week", window_seconds=7 * 86_400, first_users_count_at=LAUNCHED_AT),
+ ),
+ },
+ Status.PASSED,
+ {Value.ONLY_COUNT_MATURED_USERS: True, Value.PRIMARY_METRIC_COUNT: 2, Value.WAITING_METRIC_COUNT: 1},
+ ),
+ (
+ "metric_calculation",
+ Key.METRIC_CALCULATION_FAILED,
+ {
+ "latest_calculation": LatestCalculation(
+ metric_count=3,
+ finished_at=datetime(2026, 10, 2, 12, tzinfo=ZoneInfo("Europe/Berlin")),
+ failures=(),
+ completed_count=3,
+ completed_exposures_paths=frozenset(),
+ )
+ },
+ Status.PASSED,
+ {
+ Value.METRIC_COUNT: 3,
+ Value.COMPLETED_METRIC_COUNT: 3,
+ Value.FAILED_METRIC_COUNT: 0,
+ Value.CALCULATED_AT: "2026-10-02T10:00:00+00:00",
+ },
+ ),
+ (
+ "flag_state_names_a_shipped_variant",
+ Key.FLAG_STATE,
+ {
+ "flag": replace(
+ UNEVEN_FLAG,
+ release_groups=(FlagReleaseGroup(rollout_percentage=100, property_count=0),),
+ variants=(
+ FlagVariant(key="control", rollout_percentage=0),
+ FlagVariant(key="test", rollout_percentage=100),
+ ),
+ )
+ },
+ Status.FINDING,
+ {
+ Value.FLAG_ACTIVE: True,
+ Value.FLAG_DELETED: False,
+ Value.FLAG_ROLLED_OUT: True,
+ Value.SHIPPED_VARIANT: "test",
+ },
),
]
)
- def test_a_pass_reports_its_value_only_when_the_threshold_decided_it(
- self, _name: str, key: ExperimentHealthCheckKey, exposures: ExposureTotals, measurement: Any
+ def test_a_check_reports_what_it_read(
+ self, _name: str, key: Key, changes: dict[str, Any], status: Status, values: dict[Value, Any]
) -> None:
- health = evaluate(replace(RUNNING_UNEVEN_SPLIT, exposures=exposures), EXPOSURE_HEALTH_CHECKS)
+ health = evaluate(
+ replace(RUNNING_UNEVEN_SPLIT, **changes), (*EXPERIMENT_HEALTH_CHECKS, *EXPOSURE_HEALTH_CHECKS)
+ )
check = next(check for check in health.checks if check.key == key)
- self.assertEqual((check.status, check.measurement), (Status.PASSED, measurement))
+ self.assertEqual((check.status, dict(check.values)), (status, values))
+
+ @parameterized.expand(
+ [
+ (Key.SRM, {"exposures": replace(UNEVEN_SPLIT_TOTALS, sample_ratio_mismatch_p_value=0.71)}),
+ (Key.BIAS_RISK_MULTIPLE_EXCLUDED, {"exposures": _totals_with_multiple(30, 1000)}),
+ (Key.MULTIPLE_VARIANTS_SHARE, {"exposures": _totals_with_multiple(30, 1000)}),
+ (Key.TEST_ACCOUNT_FILTER_REMOVES_EXPOSURES, {"exposures": _filtered_totals(400, 1000)}),
+ (Key.ZERO_EXPOSURES, {}),
+ ]
+ )
+ def test_a_rule_names_values_that_its_check_reads(self, key: Key, changes: dict[str, Any]) -> None:
+ health = evaluate(replace(RUNNING_UNEVEN_SPLIT, **changes), EXPOSURE_HEALTH_CHECKS)
+
+ check = next(check for check in health.checks if check.key == key)
+ assert check.rule is not None and check.rule.minimum is not None
+ self.assertEqual(
+ {check.rule.value, check.rule.minimum.value} - check.values.keys(), set(), (check.status, check.values)
+ )
+
+ @parameterized.expand([(check.key.value, check) for check in EXPOSURE_HEALTH_CHECKS if check.rule])
+ def test_an_exposure_rule_converts_to_the_exposure_schema(self, _name: str, check: HealthCheck) -> None:
+ assert check.rule is not None
+
+ rule = ExperimentExposuresQueryRunner._health_check_rule(check.rule)
+
+ self.assertEqual(rule.value.value, check.rule.value.value)
+
+ def test_a_failing_read_keeps_the_status(self) -> None:
+ check = replace(MULTIPLE_VARIANTS_CHECK, read=_raise)
+
+ with patch.object(registry, "capture_exception") as capture:
+ health = evaluate(replace(RUNNING_UNEVEN_SPLIT, exposures=_totals_with_multiple(60, 1000)), (check,))
+
+ self.assertEqual(
+ (health.checks[0].status, dict(health.checks[0].values), len(health.findings)), (Status.FINDING, {}, 1)
+ )
+ capture.assert_called_once()
@parameterized.expand(
[
diff --git a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
index 145f3b17601b..ba8031444be7 100644
--- a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
+++ b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py
@@ -2,7 +2,7 @@ import uuid
from dataclasses import replace
from datetime import UTC, date, datetime, timedelta
from functools import cached_property
-from typing import Any
+from typing import Any, Literal
import structlog
from rest_framework.exceptions import ValidationError
@@ -13,14 +13,15 @@ from posthog.schema import (
DateRange,
ExperimentExposureHealthCheck,
ExperimentExposureHealthCheckKey,
+ ExperimentExposureHealthCheckRule,
ExperimentExposureHealthCheckStatus,
ExperimentExposureHealthFinding,
ExperimentExposureHealthFindingActionKind,
ExperimentExposureHealthFindingCode,
ExperimentExposureHealthFindingSeverity,
- ExperimentExposureHealthMeasurement,
- ExperimentExposureHealthMeasurementSide,
- ExperimentExposureHealthMeasurementUnit,
+ ExperimentExposureHealthRuleMinimum,
+ ExperimentExposureHealthRuleSide,
+ ExperimentExposureHealthValueKey,
ExperimentExposureQuery,
ExperimentExposureQueryResponse,
ExperimentExposureTimeSeries,
@@ -48,7 +49,12 @@ from products.access_control.backend.facade.api import (
)
from products.access_control.backend.facade.user_access_control import UserAccessControl
from products.analytics_platform.backend.lazy_computation.lazy_computation_executor import LazyComputationResult
-from products.experiments.backend.facade.contracts import ExperimentHealth, ExperimentHealthReaderAccess
+from products.experiments.backend.facade.contracts import (
+ ExperimentHealth,
+ ExperimentHealthCheckRule,
+ ExperimentHealthReaderAccess,
+ ExperimentHealthRuleTest,
+)
from products.experiments.backend.health.checks.bias_risk import evaluate_bias_risk
from products.experiments.backend.health.checks.filtered_exposures import reads_share_kept_by_test_account_filter
from products.experiments.backend.health.checks.srm import (
@@ -463,6 +469,7 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
evidence=dict(finding.evidence),
actions=[ExperimentExposureHealthFindingActionKind(action.value) for action in finding.actions],
diagnostic_ref=finding.diagnostic_ref,
+ check=ExperimentExposureHealthCheckKey(finding.check.value) if finding.check else None,
)
for finding in health.findings
]
@@ -476,18 +483,26 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
description=check.description,
status=ExperimentExposureHealthCheckStatus(check.status.value),
reason=check.reason,
- measurement=ExperimentExposureHealthMeasurement(
- value=check.measurement.value,
- threshold=check.measurement.threshold,
- unit=ExperimentExposureHealthMeasurementUnit(check.measurement.unit.value),
- warns=ExperimentExposureHealthMeasurementSide(check.measurement.warns.value),
- )
- if check.measurement
- else None,
+ values={key.value: value for key, value in check.values.items()},
+ rule=ExperimentExposuresQueryRunner._health_check_rule(check.rule) if check.rule else None,
)
for check in health.checks
]
+ @staticmethod
+ def _health_check_rule(rule: ExperimentHealthCheckRule) -> ExperimentExposureHealthCheckRule:
+ return ExperimentExposureHealthCheckRule(
+ value=ExperimentExposureHealthValueKey(rule.value.value),
+ threshold=rule.threshold,
+ warns=ExperimentExposureHealthRuleSide(rule.warns.value),
+ test=_RULE_TESTS[rule.test] if rule.test else None,
+ minimum=ExperimentExposureHealthRuleMinimum(
+ value=ExperimentExposureHealthValueKey(rule.minimum.value.value), threshold=rule.minimum.threshold
+ )
+ if rule.minimum
+ else None,
+ )
+
@experiment_error_handler
def _calculate(self) -> ExperimentExposureQueryResponse:
tag_queries(
@@ -631,3 +646,9 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):
if applies_test_account_filters(self.team, self.exposure_criteria):
payload["test_account_filters"] = self.team.test_account_filters
return payload
+
+
+# The exposure schema types a single choice as a literal rather than an enum.
+_RULE_TESTS: dict[ExperimentHealthRuleTest, Literal["chi_squared"]] = {
+ ExperimentHealthRuleTest.CHI_SQUARED: "chi_squared"
+}
diff --git a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py
index d670500fdfea..073d7e3ae069 100644
--- a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py
+++ b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py
@@ -17,6 +17,7 @@ from posthog.schema import (
ActionsNode,
ExperimentEventExposureConfig,
ExperimentExposureCriteria,
+ ExperimentExposureHealthCheckKey,
ExperimentExposureHealthFindingCode,
ExperimentExposureQuery,
ExperimentWarehouseNativeExposureConfig,
@@ -1507,7 +1508,22 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):
# (well below the 0.001 threshold for SRM detection)
self.assertLess(response.sample_ratio_mismatch.p_value, 0.001)
self.assertEqual(
- [finding.code for finding in response.health_findings or []], [ExperimentExposureHealthFindingCode.SRM]
+ [(finding.code, finding.check) for finding in response.health_findings or []],
+ [(ExperimentExposureHealthFindingCode.SRM, ExperimentExposureHealthCheckKey.SRM)],
+ )
+ srm_check = next(check for check in response.health_checks or [] if check.key == "srm")
+ self.assertEqual(
+ (srm_check.values, srm_check.rule and srm_check.rule.model_dump(mode="json")),
+ (
+ {"p_value": response.sample_ratio_mismatch.p_value, "compared_count": 100, "multiple_variant_count": 0},
+ {
+ "value": "p_value",
+ "threshold": 0.001,
+ "warns": "below",
+ "test": "chi_squared",
+ "minimum": {"value": "compared_count", "threshold": 100},
+ },
+ ),
)
# Expected counts should be 50/50 of total (100)
diff --git a/products/experiments/backend/presentation/serializers.py b/products/experiments/backend/presentation/serializers.py
index 80802d1ac9dc..df7f573431f6 100644
--- a/products/experiments/backend/presentation/serializers.py
+++ b/products/experiments/backend/presentation/serializers.py
@@ -53,9 +53,10 @@ from products.experiments.backend.facade.contracts import (
ExperimentHealthFindingSeverity,
ExperimentHealthFlagChangeKind,
ExperimentHealthInputsTrafficStatus,
- ExperimentHealthMeasurementSide,
- ExperimentHealthMeasurementUnit,
ExperimentHealthMetricCompileErrorKind,
+ ExperimentHealthRuleSide,
+ ExperimentHealthRuleTest,
+ ExperimentHealthValueKey,
ExperimentValidationError,
HealthExposureInputsStatus,
RunningTimeMode,
@@ -210,6 +211,19 @@ class ExperimentHealthEvidenceField(serializers.DictField):
pass
+@extend_schema_field(
+ {
+ "type": "object",
+ "additionalProperties": {
+ "oneOf": [{"type": "string"}, {"type": "number"}, {"type": "boolean"}],
+ "nullable": True,
+ },
+ }
+)
+class ExperimentHealthCheckValuesField(serializers.DictField):
+ pass
+
+
class ExperimentHealthFindingSerializer(serializers.Serializer):
code = serializers.ChoiceField(
choices=ExperimentHealthFindingCode.choices,
@@ -257,21 +271,46 @@ class ExperimentHealthFindingSerializer(serializers.Serializer):
"Null when the skill has none."
),
)
+ check = serializers.ChoiceField(
+ choices=ExperimentHealthCheckKey.choices,
+ help_text=(
+ "The key of the check in `checks` that reported the finding. A check can report several codes, so the "
+ "code doesn't name the check."
+ ),
+ )
+
+class ExperimentHealthRuleMinimumSerializer(serializers.Serializer):
+ value = serializers.ChoiceField(
+ choices=ExperimentHealthValueKey.choices,
+ help_text="The key in `values` that must reach the threshold before the check judges.",
+ )
+ threshold = serializers.FloatField(
+ help_text="The value that the key must reach. Below it, the check reports no finding."
+ )
-class ExperimentHealthMeasurementSerializer(serializers.Serializer):
- value = serializers.FloatField(help_text="The value that the check measured on the experiment.")
- threshold = serializers.FloatField(help_text="The value at which the check reports a finding.")
- unit = serializers.ChoiceField(
- choices=ExperimentHealthMeasurementUnit.choices,
+
+class ExperimentHealthCheckRuleSerializer(serializers.Serializer):
+ value = serializers.ChoiceField(
+ choices=ExperimentHealthValueKey.choices,
+ help_text="The key in `values` that the check compares with the threshold.",
+ )
+ threshold = serializers.FloatField(help_text="The threshold, in the unit that the suffix of the key gives.")
+ warns = serializers.ChoiceField(
+ choices=ExperimentHealthRuleSide.choices,
help_text=(
- "percentage: a share of the exposed users or groups, on the 0-100 scale. p_value: the p-value of a "
- "statistical test."
+ "above: the check reports a finding when the value is above the threshold. below: when the value is "
+ "below it. A value at the threshold passes."
),
)
- warns = serializers.ChoiceField(
- choices=ExperimentHealthMeasurementSide.choices,
- help_text="The side of the threshold that reports a finding: above or below.",
+ test = serializers.ChoiceField(
+ choices=ExperimentHealthRuleTest.choices,
+ allow_null=True,
+ help_text="The statistical test that computes the value, for example chi_squared. Null for a count or a share.",
+ )
+ minimum = ExperimentHealthRuleMinimumSerializer(
+ allow_null=True,
+ help_text="What the check needs before it judges. Null when the check judges from the first value.",
)
@@ -303,11 +342,20 @@ class ExperimentHealthCheckResultSerializer(serializers.Serializer):
"'Runs after launch'. Null for passed and finding."
),
)
- measurement = ExperimentHealthMeasurementSerializer(
+ values = ExperimentHealthCheckValuesField(
+ help_text=(
+ "What the check read on the experiment, for every status, as numbers, strings, booleans or null. Empty "
+ "when the check read nothing or shows nothing. The suffix of a key gives the unit: _count is a count, "
+ "_percentage is on the 0-100 scale, _hours is hours and _at is an ISO 8601 time in UTC. A key without a "
+ "suffix is a setting or a state. A key means the same thing in every check. `rule.value` names the key "
+ "that decides."
+ ),
+ )
+ rule = ExperimentHealthCheckRuleSerializer(
allow_null=True,
help_text=(
- "The value that decided a passed check, against its threshold. Null for the other statuses, and for a "
- "pass without a value. A finding carries its values in `evidence`."
+ "How the check decides. The same for every experiment and every status. Null for a check without a "
+ "numeric rule."
),
)
diff --git a/products/experiments/backend/test/test_presentation_api.py b/products/experiments/backend/test/test_presentation_api.py
index c1ba3f1a2f1a..7ef9c5097195 100644
--- a/products/experiments/backend/test/test_presentation_api.py
+++ b/products/experiments/backend/test/test_presentation_api.py
@@ -848,7 +848,21 @@ class TestExperimentCRUD(_HoistFlagConfigClientMixin, APILicensedTest):
self.assertEqual([(finding["code"], finding["subcode"]) for finding in health["findings"]], expected)
self.assertEqual(health["findings"][0]["actions"], ["open_feature_flag"])
self.assertEqual(health["findings"][0]["diagnostic_ref"], "A5")
- self.assertEqual((health["checks"][0]["key"], health["checks"][0]["status"]), ("flag_state", "finding"))
+ self.assertEqual(health["findings"][0]["check"], "flag_state")
+ self.assertEqual(
+ {key: health["checks"][0][key] for key in ("key", "status", "values", "rule")},
+ {
+ "key": "flag_state",
+ "status": "finding",
+ "values": {
+ "flag_active": False,
+ "flag_deleted": False,
+ "flag_rolled_out": True,
+ "shipped_variant": None,
+ },
+ "rule": None,
+ },
+ )
@parameterized.expand(
[
diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.ts
index 26e400380c5e..7e6ef03896f0 100644
--- a/products/experiments/frontend/generated/api.schemas.ts
+++ b/products/experiments/frontend/generated/api.schemas.ts
@@ -2399,6 +2399,60 @@ export const ExperimentHealthFindingActionKindEnumApi = {
EditReleaseConditions: 'edit_release_conditions',
} as const
+/**
+ * * `flag_state` - Flag State
+ * * `flag_shared` - Flag Shared
+ * * `flag_history` - Flag History
+ * * `split_changed_mid_run` - Split Changed Mid Run
+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
+ * * `flag_variants_mismatch` - Flag Variants Mismatch
+ * * `forced_variant_release_condition` - Forced Variant Release Condition
+ * * `new_flag_no_traffic` - New Flag No Traffic
+ * * `no_metric` - No Metric
+ * * `metric_event_not_ingested` - Metric Event Not Ingested
+ * * `metric_cannot_compile` - Metric Cannot Compile
+ * * `metric_calculation_failed` - Metric Calculation Failed
+ * * `metric_all_zero` - Metric All Zero
+ * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
+ * * `exposure_filters` - Exposure Filters
+ * * `early_significance` - Early Significance
+ * * `metrics_not_matured` - Metrics Not Matured
+ * * `zero_exposures` - Zero Exposures
+ * * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposures
+ * * `srm` - Srm
+ * * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
+ * * `multiple_variants_share` - Multiple Variants Share
+ */
+export type ExperimentHealthCheckKeyEnumApi =
+ (typeof ExperimentHealthCheckKeyEnumApi)[keyof typeof ExperimentHealthCheckKeyEnumApi]
+
+export const ExperimentHealthCheckKeyEnumApi = {
+ FlagState: 'flag_state',
+ FlagShared: 'flag_shared',
+ FlagHistory: 'flag_history',
+ SplitChangedMidRun: 'split_changed_mid_run',
+ BucketingChangedMidRun: 'bucketing_changed_mid_run',
+ ReleaseConditionsChangedMidRun: 'release_conditions_changed_mid_run',
+ FlagVariantsMismatch: 'flag_variants_mismatch',
+ ForcedVariantReleaseCondition: 'forced_variant_release_condition',
+ NewFlagNoTraffic: 'new_flag_no_traffic',
+ NoMetric: 'no_metric',
+ MetricEventNotIngested: 'metric_event_not_ingested',
+ MetricCannotCompile: 'metric_cannot_compile',
+ MetricCalculationFailed: 'metric_calculation_failed',
+ MetricAllZero: 'metric_all_zero',
+ ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',
+ ExposureFilters: 'exposure_filters',
+ EarlySignificance: 'early_significance',
+ MetricsNotMatured: 'metrics_not_matured',
+ ZeroExposures: 'zero_exposures',
+ TestAccountFilterRemovesExposures: 'test_account_filter_removes_exposures',
+ Srm: 'srm',
+ BiasRiskMultipleExcluded: 'bias_risk_multiple_excluded',
+ MultipleVariantsShare: 'multiple_variants_share',
+} as const
+
/**
* The values behind the finding, such as the key of a shipped variant or the share of users exposed to multiple variants. The keys depend on the code.
*/
@@ -2462,62 +2516,34 @@ export interface ExperimentHealthFindingApi {
* @nullable
*/
diagnostic_ref: string | null
+ /** The key of the check in `checks` that reported the finding. A check can report several codes, so the code doesn't name the check.
+ *
+ * * `flag_state` - Flag State
+ * * `flag_shared` - Flag Shared
+ * * `flag_history` - Flag History
+ * * `split_changed_mid_run` - Split Changed Mid Run
+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
+ * * `flag_variants_mismatch` - Flag Variants Mismatch
+ * * `forced_variant_release_condition` - Forced Variant Release Condition
+ * * `new_flag_no_traffic` - New Flag No Traffic
+ * * `no_metric` - No Metric
+ * * `metric_event_not_ingested` - Metric Event Not Ingested
+ * * `metric_cannot_compile` - Metric Cannot Compile
+ * * `metric_calculation_failed` - Metric Calculation Failed
+ * * `metric_all_zero` - Metric All Zero
+ * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
+ * * `exposure_filters` - Exposure Filters
+ * * `early_significance` - Early Significance
+ * * `metrics_not_matured` - Metrics Not Matured
+ * * `zero_exposures` - Zero Exposures
+ * * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposures
+ * * `srm` - Srm
+ * * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
+ * * `multiple_variants_share` - Multiple Variants Share */
+ check: ExperimentHealthCheckKeyEnumApi
}
-/**
- * * `flag_state` - Flag State
- * * `flag_shared` - Flag Shared
- * * `flag_history` - Flag History
- * * `split_changed_mid_run` - Split Changed Mid Run
- * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
- * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
- * * `flag_variants_mismatch` - Flag Variants Mismatch
- * * `forced_variant_release_condition` - Forced Variant Release Condition
- * * `new_flag_no_traffic` - New Flag No Traffic
- * * `no_metric` - No Metric
- * * `metric_event_not_ingested` - Metric Event Not Ingested
- * * `metric_cannot_compile` - Metric Cannot Compile
- * * `metric_calculation_failed` - Metric Calculation Failed
- * * `metric_all_zero` - Metric All Zero
- * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
- * * `exposure_filters` - Exposure Filters
- * * `early_significance` - Early Significance
- * * `metrics_not_matured` - Metrics Not Matured
- * * `zero_exposures` - Zero Exposures
- * * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposures
- * * `srm` - Srm
- * * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
- * * `multiple_variants_share` - Multiple Variants Share
- */
-export type ExperimentHealthCheckKeyEnumApi =
- (typeof ExperimentHealthCheckKeyEnumApi)[keyof typeof ExperimentHealthCheckKeyEnumApi]
-
-export const ExperimentHealthCheckKeyEnumApi = {
- FlagState: 'flag_state',
- FlagShared: 'flag_shared',
- FlagHistory: 'flag_history',
- SplitChangedMidRun: 'split_changed_mid_run',
- BucketingChangedMidRun: 'bucketing_changed_mid_run',
- ReleaseConditionsChangedMidRun: 'release_conditions_changed_mid_run',
- FlagVariantsMismatch: 'flag_variants_mismatch',
- ForcedVariantReleaseCondition: 'forced_variant_release_condition',
- NewFlagNoTraffic: 'new_flag_no_traffic',
- NoMetric: 'no_metric',
- MetricEventNotIngested: 'metric_event_not_ingested',
- MetricCannotCompile: 'metric_cannot_compile',
- MetricCalculationFailed: 'metric_calculation_failed',
- MetricAllZero: 'metric_all_zero',
- ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',
- ExposureFilters: 'exposure_filters',
- EarlySignificance: 'early_significance',
- MetricsNotMatured: 'metrics_not_matured',
- ZeroExposures: 'zero_exposures',
- TestAccountFilterRemovesExposures: 'test_account_filter_removes_exposures',
- Srm: 'srm',
- BiasRiskMultipleExcluded: 'bias_risk_multiple_excluded',
- MultipleVariantsShare: 'multiple_variants_share',
-} as const
-
/**
* * `passed` - Passed
* * `finding` - Finding
@@ -2535,46 +2561,147 @@ export const ExperimentHealthCheckStatusEnumApi = {
} as const
/**
- * * `percentage` - Percentage of exposed units
- * * `p_value` - P-value of a statistical test
+ * * `exposed_count` - Users or groups exposed since launch, as the exposure criteria count them, including those exposed to multiple variants
+ * * `compared_count` - Exposed users or groups in the variants that the sample ratio test compares, without those exposed to multiple variants
+ * * `multiple_variant_count` - Users or groups exposed to more than one variant
+ * * `multiple_variant_percentage` - multiple_variant_count as a share of exposed_count
+ * * `multiple_variant_handling` - What the results do with users exposed to more than one variant: exclude or first_seen
+ * * `p_value` - P-value of the sample ratio test
+ * * `since_launch_hours` - Hours since launch
+ * * `exposed_without_test_account_filter_count` - Users or groups that the exposure criteria count without the test account filters
+ * * `kept_by_test_account_filter_percentage` - exposed_count as a share of exposed_without_test_account_filter_count
+ * * `only_count_matured_users` - Whether "Require completed conversion or retention window" is on
+ * * `primary_metric_count` - Primary metrics that the calculation computes
+ * * `waiting_metric_count` - Of the primary metrics, those whose conversion or retention window hasn't passed for any user yet
+ * * `metric_count` - Metrics that the calculation computes
+ * * `completed_metric_count` - Metrics whose result of the latest calculation the page shows
+ * * `failed_metric_count` - Metrics whose latest calculation failed
+ * * `calculated_at` - When the latest calculation of the metrics finished
+ * * `flag_active` - Whether the feature flag is enabled
+ * * `flag_deleted` - Whether the feature flag is deleted
+ * * `flag_rolled_out` - Whether a release condition of the feature flag rolls out to more than 0%
+ * * `shipped_variant` - The key of the variant that the feature flag serves to everyone, or null when it serves more than one
*/
-export type ExperimentHealthMeasurementUnitEnumApi =
- (typeof ExperimentHealthMeasurementUnitEnumApi)[keyof typeof ExperimentHealthMeasurementUnitEnumApi]
-
-export const ExperimentHealthMeasurementUnitEnumApi = {
- Percentage: 'percentage',
+export type ExperimentHealthValueKeyEnumApi =
+ (typeof ExperimentHealthValueKeyEnumApi)[keyof typeof ExperimentHealthValueKeyEnumApi]
+
+export const ExperimentHealthValueKeyEnumApi = {
+ ExposedCount: 'exposed_count',
+ ComparedCount: 'compared_count',
+ MultipleVariantCount: 'multiple_variant_count',
+ MultipleVariantPercentage: 'multiple_variant_percentage',
+ MultipleVariantHandling: 'multiple_variant_handling',
PValue: 'p_value',
+ SinceLaunchHours: 'since_launch_hours',
+ ExposedWithoutTestAccountFilterCount: 'exposed_without_test_account_filter_count',
+ KeptByTestAccountFilterPercentage: 'kept_by_test_account_filter_percentage',
+ OnlyCountMaturedUsers: 'only_count_matured_users',
+ PrimaryMetricCount: 'primary_metric_count',
+ WaitingMetricCount: 'waiting_metric_count',
+ MetricCount: 'metric_count',
+ CompletedMetricCount: 'completed_metric_count',
+ FailedMetricCount: 'failed_metric_count',
+ CalculatedAt: 'calculated_at',
+ FlagActive: 'flag_active',
+ FlagDeleted: 'flag_deleted',
+ FlagRolledOut: 'flag_rolled_out',
+ ShippedVariant: 'shipped_variant',
} as const
/**
* * `above` - Finding above the threshold
* * `below` - Finding below the threshold
*/
-export type ExperimentHealthMeasurementSideEnumApi =
- (typeof ExperimentHealthMeasurementSideEnumApi)[keyof typeof ExperimentHealthMeasurementSideEnumApi]
+export type ExperimentHealthRuleSideEnumApi =
+ (typeof ExperimentHealthRuleSideEnumApi)[keyof typeof ExperimentHealthRuleSideEnumApi]
-export const ExperimentHealthMeasurementSideEnumApi = {
+export const ExperimentHealthRuleSideEnumApi = {
Above: 'above',
Below: 'below',
} as const
-export interface ExperimentHealthMeasurementApi {
- /** The value that the check measured on the experiment. */
- value: number
- /** The value at which the check reports a finding. */
+/**
+ * * `chi_squared` - Chi-squared goodness-of-fit test
+ */
+export type ExperimentHealthRuleTestEnumApi =
+ (typeof ExperimentHealthRuleTestEnumApi)[keyof typeof ExperimentHealthRuleTestEnumApi]
+
+export const ExperimentHealthRuleTestEnumApi = {
+ ChiSquared: 'chi_squared',
+} as const
+
+export interface ExperimentHealthRuleMinimumApi {
+ /** The key in `values` that must reach the threshold before the check judges.
+ *
+ * * `exposed_count` - Users or groups exposed since launch, as the exposure criteria count them, including those exposed to multiple variants
+ * * `compared_count` - Exposed users or groups in the variants that the sample ratio test compares, without those exposed to multiple variants
+ * * `multiple_variant_count` - Users or groups exposed to more than one variant
+ * * `multiple_variant_percentage` - multiple_variant_count as a share of exposed_count
+ * * `multiple_variant_handling` - What the results do with users exposed to more than one variant: exclude or first_seen
+ * * `p_value` - P-value of the sample ratio test
+ * * `since_launch_hours` - Hours since launch
+ * * `exposed_without_test_account_filter_count` - Users or groups that the exposure criteria count without the test account filters
+ * * `kept_by_test_account_filter_percentage` - exposed_count as a share of exposed_without_test_account_filter_count
+ * * `only_count_matured_users` - Whether "Require completed conversion or retention window" is on
+ * * `primary_metric_count` - Primary metrics that the calculation computes
+ * * `waiting_metric_count` - Of the primary metrics, those whose conversion or retention window hasn't passed for any user yet
+ * * `metric_count` - Metrics that the calculation computes
+ * * `completed_metric_count` - Metrics whose result of the latest calculation the page shows
+ * * `failed_metric_count` - Metrics whose latest calculation failed
+ * * `calculated_at` - When the latest calculation of the metrics finished
+ * * `flag_active` - Whether the feature flag is enabled
+ * * `flag_deleted` - Whether the feature flag is deleted
+ * * `flag_rolled_out` - Whether a release condition of the feature flag rolls out to more than 0%
+ * * `shipped_variant` - The key of the variant that the feature flag serves to everyone, or null when it serves more than one */
+ value: ExperimentHealthValueKeyEnumApi
+ /** The value that the key must reach. Below it, the check reports no finding. */
threshold: number
- /** percentage: a share of the exposed users or groups, on the 0-100 scale. p_value: the p-value of a statistical test.
+}
+
+export interface ExperimentHealthCheckRuleApi {
+ /** The key in `values` that the check compares with the threshold.
*
- * * `percentage` - Percentage of exposed units
- * * `p_value` - P-value of a statistical test */
- unit: ExperimentHealthMeasurementUnitEnumApi
- /** The side of the threshold that reports a finding: above or below.
+ * * `exposed_count` - Users or groups exposed since launch, as the exposure criteria count them, including those exposed to multiple variants
+ * * `compared_count` - Exposed users or groups in the variants that the sample ratio test compares, without those exposed to multiple variants
+ * * `multiple_variant_count` - Users or groups exposed to more than one variant
+ * * `multiple_variant_percentage` - multiple_variant_count as a share of exposed_count
+ * * `multiple_variant_handling` - What the results do with users exposed to more than one variant: exclude or first_seen
+ * * `p_value` - P-value of the sample ratio test
+ * * `since_launch_hours` - Hours since launch
+ * * `exposed_without_test_account_filter_count` - Users or groups that the exposure criteria count without the test account filters
+ * * `kept_by_test_account_filter_percentage` - exposed_count as a share of exposed_without_test_account_filter_count
+ * * `only_count_matured_users` - Whether "Require completed conversion or retention window" is on
+ * * `primary_metric_count` - Primary metrics that the calculation computes
+ * * `waiting_metric_count` - Of the primary metrics, those whose conversion or retention window hasn't passed for any user yet
+ * * `metric_count` - Metrics that the calculation computes
+ * * `completed_metric_count` - Metrics whose result of the latest calculation the page shows
+ * * `failed_metric_count` - Metrics whose latest calculation failed
+ * * `calculated_at` - When the latest calculation of the metrics finished
+ * * `flag_active` - Whether the feature flag is enabled
+ * * `flag_deleted` - Whether the feature flag is deleted
+ * * `flag_rolled_out` - Whether a release condition of the feature flag rolls out to more than 0%
+ * * `shipped_variant` - The key of the variant that the feature flag serves to everyone, or null when it serves more than one */
+ value: ExperimentHealthValueKeyEnumApi
+ /** The threshold, in the unit that the suffix of the key gives. */
+ threshold: number
+ /** above: the check reports a finding when the value is above the threshold. below: when the value is below it. A value at the threshold passes.
*
* * `above` - Finding above the threshold
* * `below` - Finding below the threshold */
- warns: ExperimentHealthMeasurementSideEnumApi
+ warns: ExperimentHealthRuleSideEnumApi
+ /** The statistical test that computes the value, for example chi_squared. Null for a count or a share.
+ *
+ * * `chi_squared` - Chi-squared goodness-of-fit test */
+ test: ExperimentHealthRuleTestEnumApi | null
+ /** What the check needs before it judges. Null when the check judges from the first value. */
+ minimum: ExperimentHealthRuleMinimumApi | null
}
+/**
+ * What the check read on the experiment, for every status, as numbers, strings, booleans or null. Empty when the check read nothing or shows nothing. The suffix of a key gives the unit: _count is a count, _percentage is on the 0-100 scale, _hours is hours and _at is an ISO 8601 time in UTC. A key without a suffix is a setting or a state. A key means the same thing in every check. `rule.value` names the key that decides.
+ */
+export type ExperimentHealthCheckResultApiValues = { [key: string]: string | number | boolean | null }
+
export interface ExperimentHealthCheckResultApi {
/** Stable identifier of the check. A check can report several finding codes, so the key is not a finding code.
*
@@ -2618,8 +2745,10 @@ export interface ExperimentHealthCheckResultApi {
* @nullable
*/
reason: string | null
- /** The value that decided a passed check, against its threshold. Null for the other statuses, and for a pass without a value. A finding carries its values in `evidence`. */
- measurement: ExperimentHealthMeasurementApi | null
+ /** What the check read on the experiment, for every status, as numbers, strings, booleans or null. Empty when the check read nothing or shows nothing. The suffix of a key gives the unit: _count is a count, _percentage is on the 0-100 scale, _hours is hours and _at is an ISO 8601 time in UTC. A key without a suffix is a setting or a state. A key means the same thing in every check. `rule.value` names the key that decides. */
+ values: ExperimentHealthCheckResultApiValues
+ /** How the check decides. The same for every experiment and every status. Null for a check without a numeric rule. */
+ rule: ExperimentHealthCheckRuleApi | null
}
export interface ExperimentHealthApi {
diff --git a/products/experiments/frontend/health/ExperimentHealthTab.tsx b/products/experiments/frontend/health/ExperimentHealthTab.tsx
index 5b956d781ed6..7e1e91a47491 100644
--- a/products/experiments/frontend/health/ExperimentHealthTab.tsx
+++ b/products/experiments/frontend/health/ExperimentHealthTab.tsx
@@ -7,8 +7,14 @@ import { experimentLogic } from 'scenes/experiments/experimentLogic'
import { HealthChecksOverview } from './HealthChecksOverview'
export function ExperimentHealthTab(): JSX.Element {
- const { healthFindings, healthChecks, healthExposureChecks, isExperimentDraft, experimentHealthLoading } =
- useValues(experimentLogic)
+ const {
+ healthFindings,
+ healthChecks,
+ healthExposureChecks,
+ isExperimentDraft,
+ experimentHealthLoading,
+ exposures,
+ } = useValues(experimentLogic)
if (healthFindings === null || healthChecks === null) {
return experimentHealthLoading ? (
@@ -27,6 +33,8 @@ export function ExperimentHealthTab(): JSX.Element {
<HealthChecksOverview
checks={healthChecks}
findings={healthFindings}
+ // A reset to draft keeps the previous exposure answer in the page, as for the checks.
+ exposures={isExperimentDraft ? null : (exposures ?? null)}
isDraft={isExperimentDraft}
exposureChecksMissing={healthExposureChecks === 'missing'}
/>
diff --git a/products/experiments/frontend/health/HealthCheckDetail.tsx b/products/experiments/frontend/health/HealthCheckDetail.tsx
index 71100bf69ef6..f43c0abdb7dd 100644
--- a/products/experiments/frontend/health/HealthCheckDetail.tsx
+++ b/products/experiments/frontend/health/HealthCheckDetail.tsx
@@ -1,7 +1,22 @@
+import type { ExperimentExposureQueryResponse } from '~/queries/schema/schema-general'
+
+import { HealthCheckReadSection } from './HealthCheckReadSection'
+import { HealthCheckRuleSection } from './HealthCheckRuleSection'
import { HEALTH_CHECK_STATUS_DISPLAY, healthCheckStatusText } from './healthCheckStatusDisplay'
-import { type ListedHealthCheck, healthMeasurementText } from './healthPanelChecks'
+import type { ListedHealthCheck } from './healthPanelChecks'
+import { HealthPanelSection } from './HealthPanelSection'
-export function HealthCheckDetail({ check }: { check: ListedHealthCheck }): JSX.Element {
+export function HealthCheckDetail({
+ check,
+ units,
+ exposures,
+ checkedAt,
+}: {
+ check: ListedHealthCheck
+ units: string
+ exposures: ExperimentExposureQueryResponse | null
+ checkedAt: string | null
+}): JSX.Element {
return (
<div className="flex flex-col gap-4" data-attr="experiment-health-check-detail">
<div className="flex flex-col gap-1">
@@ -12,17 +27,12 @@ export function HealthCheckDetail({ check }: { check: ListedHealthCheck }): JSX.
<span className="pl-6 text-xs text-secondary">{healthCheckStatusText(check)}</span>
</div>
{check.description && (
- <section className="flex flex-col gap-1">
- <h4 className="m-0 text-xs font-semibold text-secondary">What it looks for</h4>
+ <HealthPanelSection title="What it looks for">
<p className="m-0 max-w-prose">{check.description}</p>
- </section>
- )}
- {check.measurement && (
- <section className="flex flex-col gap-1">
- <h4 className="m-0 text-xs font-semibold text-secondary">Result</h4>
- <p className="m-0 max-w-prose">{healthMeasurementText(check.measurement)}</p>
- </section>
+ </HealthPanelSection>
)}
+ <HealthCheckReadSection check={check} units={units} exposures={exposures} />
+ <HealthCheckRuleSection check={check} units={units} checkedAt={checkedAt} />
</div>
)
}
diff --git a/products/experiments/frontend/health/HealthCheckReadSection.tsx b/products/experiments/frontend/health/HealthCheckReadSection.tsx
new file mode 100644
index 000000000000..847e1a0e4e7f
--- /dev/null
+++ b/products/experiments/frontend/health/HealthCheckReadSection.tsx
@@ -0,0 +1,38 @@
+import type { ExperimentExposureQueryResponse } from '~/queries/schema/schema-general'
+
+import { type HealthReading, healthCheckReadings, healthExposureSplit } from './healthCheckValues'
+import { HealthExposureSplitTable } from './HealthExposureSplitTable'
+import type { HealthPanelCheck } from './healthPanelChecks'
+import { HealthPanelSection } from './HealthPanelSection'
+import { HealthReadingList } from './HealthReadingList'
+
+// These checks compare the variants, so they show the split that the exposure answer counted.
+const CHECKS_WITH_SPLIT: ReadonlySet<HealthPanelCheck['key']> = new Set(['srm', 'bias_risk_multiple_excluded'])
+
+export function HealthCheckReadSection({
+ check,
+ units,
+ exposures,
+}: {
+ check: HealthPanelCheck
+ units: string
+ exposures: ExperimentExposureQueryResponse | null
+}): JSX.Element | null {
+ // An exposure answer cached before the server added the values keeps the layout without them.
+ if (!check.values || Object.keys(check.values).length === 0) {
+ return null
+ }
+ const split = CHECKS_WITH_SPLIT.has(check.key) ? healthExposureSplit(exposures) : null
+ const countedFrom = split ? exposures?.date_range.date_from : null
+ const readings: HealthReading[] = [
+ ...healthCheckReadings(check.values, units),
+ ...(countedFrom ? [{ label: 'Exposures counted from', time: countedFrom, asDate: true }] : []),
+ ]
+
+ return (
+ <HealthPanelSection title="What it read">
+ {split && <HealthExposureSplitTable rows={split} units={units} />}
+ {readings.length > 0 && <HealthReadingList readings={readings} />}
+ </HealthPanelSection>
+ )
+}
diff --git a/products/experiments/frontend/health/HealthCheckRuleSection.tsx b/products/experiments/frontend/health/HealthCheckRuleSection.tsx
new file mode 100644
index 000000000000..912e6aa3e9cd
--- /dev/null
+++ b/products/experiments/frontend/health/HealthCheckRuleSection.tsx
@@ -0,0 +1,29 @@
+import { healthRuleReadings } from './healthCheckValues'
+import type { HealthPanelCheck } from './healthPanelChecks'
+import { HealthPanelSection } from './HealthPanelSection'
+import { HealthReadingList } from './HealthReadingList'
+
+export function HealthCheckRuleSection({
+ check,
+ units,
+ checkedAt,
+}: {
+ check: HealthPanelCheck
+ units: string
+ /** When the check ran, for a check that the page reads from a cached answer. */
+ checkedAt: string | null
+}): JSX.Element | null {
+ if (!check.rule) {
+ return null
+ }
+ return (
+ <HealthPanelSection title="How it decides">
+ <HealthReadingList
+ readings={[
+ ...healthRuleReadings(check, check.rule, units),
+ ...(checkedAt ? [{ label: 'Checked', time: checkedAt }] : []),
+ ]}
+ />
+ </HealthPanelSection>
+ )
+}
diff --git a/products/experiments/frontend/health/HealthChecksOverview.stories.tsx b/products/experiments/frontend/health/HealthChecksOverview.stories.tsx
index debfb47da52b..6dca5e7a9d2e 100644
--- a/products/experiments/frontend/health/HealthChecksOverview.stories.tsx
+++ b/products/experiments/frontend/health/HealthChecksOverview.stories.tsx
@@ -1,10 +1,14 @@
import type { Meta, StoryObj } from '@storybook/react'
import { within } from '@testing-library/dom'
+import { type ExperimentExposureQueryResponse, NodeKind } from '~/queries/schema/schema-general'
+
import { userEvent } from 'storybook/test'
+import type { ExperimentHealthCheckResultApi, ExperimentHealthCheckRuleApi } from '../generated/api.schemas'
import { HealthChecksOverview } from './HealthChecksOverview'
import type { HealthPanelCheck } from './healthPanelChecks'
+import type { HealthPanelFinding } from './healthPanelFindings'
const DESCRIPTIONS: Partial<Record<HealthPanelCheck['key'], string>> = {
flag_state:
@@ -21,13 +25,14 @@ const DESCRIPTIONS: Partial<Record<HealthPanelCheck['key'], string>> = {
const description = (key: HealthPanelCheck['key']): string =>
DESCRIPTIONS[key] ?? 'Looks for a problem in the setup of the experiment that can make its results wrong.'
-const passed = (key: HealthPanelCheck['key'], name: string): HealthPanelCheck => ({
+const passed = (key: HealthPanelCheck['key'], name: string): ExperimentHealthCheckResultApi => ({
key,
name,
description: description(key),
status: 'passed',
reason: null,
- measurement: null,
+ values: {},
+ rule: null,
})
const skipped = (
@@ -35,7 +40,15 @@ const skipped = (
name: string,
status: 'not_applicable' | 'not_checked',
reason: string
-): HealthPanelCheck => ({ key, name, description: description(key), status, reason, measurement: null })
+): ExperimentHealthCheckResultApi => ({
+ key,
+ name,
+ description: description(key),
+ status,
+ reason,
+ values: {},
+ rule: null,
+})
const FLAG_SETUP_CHECKS = [
passed('flag_state', "Flag state that doesn't match the experiment"),
@@ -48,9 +61,10 @@ const meta: Meta<typeof HealthChecksOverview> = {
component: HealthChecksOverview,
parameters: {
layout: 'centered',
+ mockDate: '2026-11-03 12:00:00',
testOptions: { snapshotBrowsers: ['chromium'] },
},
- args: { onFindingActedOn: () => {} },
+ args: { onFindingActedOn: () => {}, exposures: null },
decorators: [
(Story) => (
<div className="w-[60rem]">
@@ -92,13 +106,14 @@ export const Draft: Story = {
},
}
-const found = (key: HealthPanelCheck['key'], name: string): HealthPanelCheck => ({
+const found = (key: HealthPanelCheck['key'], name: string): ExperimentHealthCheckResultApi => ({
key,
name,
description: description(key),
status: 'finding',
reason: null,
- measurement: null,
+ values: {},
+ rule: null,
})
export const DraftWithEventsNotReceived: Story = {
@@ -213,79 +228,175 @@ export const ChecksThatCouldNotRun: Story = {
},
}
+const SRM_RULE: ExperimentHealthCheckRuleApi = {
+ value: 'p_value',
+ threshold: 0.001,
+ warns: 'below',
+ test: 'chi_squared',
+ minimum: { value: 'compared_count', threshold: 100 },
+}
+
+const BIAS_RULE: ExperimentHealthCheckRuleApi = {
+ value: 'multiple_variant_percentage',
+ threshold: 0.1,
+ warns: 'above',
+ test: null,
+ minimum: { value: 'multiple_variant_count', threshold: 5 },
+}
+
+const MULTIPLE_VARIANTS_RULE: ExperimentHealthCheckRuleApi = { ...BIAS_RULE, threshold: 5 }
+
+const UNEVEN_SPLIT_EXPOSURES: ExperimentExposureQueryResponse = {
+ kind: NodeKind.ExperimentExposureQuery,
+ timeseries: [],
+ total_exposures: { control: 4790, test: 2420, 'test-2': 2392, $multiple: 403 },
+ date_range: { date_from: '2026-10-19T00:00:00Z' },
+ sample_ratio_mismatch: { expected: { control: 4801, test: 2400.5, 'test-2': 2400.5 }, p_value: 0.71 },
+ health_checks: [
+ { key: 'srm', name: 'Split of users across variants', status: 'passed', reason: null },
+ { key: 'bias_risk_multiple_excluded', name: 'Bias from an uneven split', status: 'finding', reason: null },
+ { key: 'multiple_variants_share', name: 'Users exposed to multiple variants', status: 'passed', reason: null },
+ ],
+ last_refresh: '2026-11-03T10:00:00Z',
+} as ExperimentExposureQueryResponse
+
+const MULTIPLE_VALUES = {
+ exposed_count: 10005,
+ multiple_variant_count: 403,
+ multiple_variant_percentage: 4.03,
+ multiple_variant_handling: 'exclude',
+}
+
+const READ_CHECKS: HealthPanelCheck[] = [
+ {
+ ...passed('flag_state', "Flag state that doesn't match the experiment"),
+ values: { flag_active: true, flag_deleted: false, flag_rolled_out: true, shipped_variant: null },
+ },
+ passed('flag_variants_mismatch', 'Flag variants not set up'),
+ {
+ ...passed('metric_calculation_failed', 'Failed metric calculations'),
+ values: {
+ metric_count: 3,
+ completed_metric_count: 3,
+ failed_metric_count: 0,
+ calculated_at: '2026-11-03T08:00:00Z',
+ },
+ },
+ {
+ ...passed('zero_exposures', 'Exposures not received since launch'),
+ values: { exposed_count: 10005, since_launch_hours: 348 },
+ rule: {
+ value: 'exposed_count',
+ threshold: 1,
+ warns: 'below',
+ test: null,
+ minimum: { value: 'since_launch_hours', threshold: 24 },
+ },
+ },
+ {
+ ...passed('srm', 'Split of users across variants'),
+ values: { p_value: 0.71, compared_count: 9602, multiple_variant_count: 403 },
+ rule: SRM_RULE,
+ },
+ {
+ ...passed('multiple_variants_share', 'Users exposed to multiple variants'),
+ values: MULTIPLE_VALUES,
+ rule: MULTIPLE_VARIANTS_RULE,
+ },
+ {
+ ...skipped(
+ 'metrics_not_matured',
+ 'Metric windows not complete yet',
+ 'not_applicable',
+ 'Runs when "Require completed conversion or retention window" is on'
+ ),
+ values: { only_count_matured_users: false },
+ },
+]
+
+const BIAS_FINDING: HealthPanelFinding = {
+ code: 'bias_risk_multiple_excluded',
+ subcode: null,
+ reason: null,
+ severity: 'warning',
+ title: 'The uneven split may bias the results',
+ detail: '4.0% of users were exposed to multiple variants. With an uneven variant split and the Exclude handling, these users were dropped more often from the smaller variant, so its metrics can be biased. Use an even split and control exposure with the overall rollout, or switch the handling to First seen.',
+ evidence: { multiple_variant_percentage: 4.03 },
+ actions: ['adjust_distribution', 'use_first_seen_variant'],
+ check: 'bias_risk_multiple_excluded',
+}
+
export const RunningWithAnIssue: Story = {
args: {
checks: [
- ...FLAG_SETUP_CHECKS,
- passed('no_metric', 'Metrics not set up'),
- found('srm', 'Split of users across variants'),
- ],
- findings: [
+ ...READ_CHECKS,
{
- code: 'srm',
- subcode: null,
- reason: null,
- severity: 'warning',
- title: 'Users are not split across variants as configured',
- detail: "The distribution of users across variants doesn't match your configured rollout percentages (p < 0.001). This may indicate issues with randomization or data collection.",
- evidence: { p_value: 0.0002 },
- actions: [],
+ ...found('bias_risk_multiple_excluded', 'Bias from an uneven split'),
+ values: MULTIPLE_VALUES,
+ rule: BIAS_RULE,
},
],
+ findings: [BIAS_FINDING],
+ exposures: UNEVEN_SPLIT_EXPOSURES,
isDraft: false,
exposureChecksMissing: false,
},
}
-export const CheckDetailsWithAMeasurement: Story = {
+export const NothingToFix: Story = {
args: {
- checks: [
- ...FLAG_SETUP_CHECKS,
- passed('no_metric', 'Metrics not set up'),
- {
- ...passed('srm', 'Split of users across variants'),
- measurement: { value: 0.71, threshold: 0.001, unit: 'p_value', warns: 'below' },
- },
- {
- ...passed('multiple_variants_share', 'Users exposed to multiple variants'),
- measurement: { value: 4.2, threshold: 5, unit: 'percentage', warns: 'above' },
- },
- ],
+ checks: READ_CHECKS,
findings: [],
+ exposures: UNEVEN_SPLIT_EXPOSURES,
isDraft: false,
exposureChecksMissing: false,
},
+}
+
+export const CheckDetailsWithWhatItRead: Story = {
+ args: NothingToFix.args,
play: async ({ canvasElement }) => {
await userEvent.click(
- await within(canvasElement).findByRole('button', { name: /Users exposed to multiple variants/ })
+ await within(canvasElement).findByRole('button', { name: /Split of users across variants/ })
)
},
}
export const CheckDetailsOfACheckThatDoesNotApply: Story = {
+ args: NothingToFix.args,
+ play: async ({ canvasElement }) => {
+ await userEvent.click(
+ await within(canvasElement).findByRole('button', { name: /Metric windows not complete yet/ })
+ )
+ },
+}
+
+export const CheckDetailsFromAnAnswerCachedBeforeTheValues: Story = {
args: {
checks: [
...FLAG_SETUP_CHECKS,
- skipped(
- 'split_changed_mid_run',
- 'Variant split changes since launch',
- 'not_applicable',
- 'Runs after launch'
- ),
+ {
+ key: 'srm',
+ name: 'Split of users across variants',
+ description: description('srm'),
+ status: 'passed',
+ reason: null,
+ },
],
findings: [],
- isDraft: true,
+ exposures: UNEVEN_SPLIT_EXPOSURES,
+ isDraft: false,
exposureChecksMissing: false,
},
play: async ({ canvasElement }) => {
await userEvent.click(
- await within(canvasElement).findByRole('button', { name: /Variant split changes since launch/ })
+ await within(canvasElement).findByRole('button', { name: /Split of users across variants/ })
)
},
}
-export const NarrowWithACheckPicked: Story = {
- args: CheckDetailsWithAMeasurement.args,
+export const NarrowWithAnIssue: Story = {
+ args: RunningWithAnIssue.args,
decorators: [
(Story) => (
<div className="w-[32rem]">
@@ -295,7 +406,7 @@ export const NarrowWithACheckPicked: Story = {
],
play: async ({ canvasElement }) => {
await userEvent.click(
- await within(canvasElement).findByRole('button', { name: /Split of users across variants/ })
+ await within(canvasElement).findByRole('button', { name: /The uneven split may bias the results/ })
)
},
}
diff --git a/products/experiments/frontend/health/HealthChecksOverview.tsx b/products/experiments/frontend/health/HealthChecksOverview.tsx
index eb171e71b563..8132daa2c069 100644
--- a/products/experiments/frontend/health/HealthChecksOverview.tsx
+++ b/products/experiments/frontend/health/HealthChecksOverview.tsx
@@ -1,4 +1,4 @@
-import { useActions } from 'kea'
+import { useActions, useValues } from 'kea'
import { useRef, useState } from 'react'
import { IconChevronLeft } from '@posthog/icons'
@@ -7,6 +7,10 @@ import { LemonButton, LemonTag } from '@posthog/lemon-ui'
import { cn } from 'lib/utils/css-classes'
import { experimentLogic } from 'scenes/experiments/experimentLogic'
+import { groupsModel } from '~/models/groupsModel'
+import type { ExperimentExposureQueryResponse } from '~/queries/schema/schema-general'
+
+import { exposureAnswerComputedAt } from './exposureHealth'
import { HealthCheckDetail } from './HealthCheckDetail'
import { HealthCheckList } from './HealthCheckList'
import { HealthChecksSummary } from './HealthChecksSummary'
@@ -20,6 +24,7 @@ import { type HealthPanelSelection, healthFindingKey, resolveHealthPanelSelectio
export function HealthChecksOverview({
checks,
findings,
+ exposures,
isDraft,
exposureChecksMissing,
onFindingActedOn,
@@ -27,6 +32,8 @@ export function HealthChecksOverview({
checks: HealthPanelCheck[]
/** Issues first, as `healthPanelFindings` sorts them. */
findings: HealthPanelFinding[]
+ /** The exposure answer that the exposure checks come from. Null for a draft and before it loads. */
+ exposures: ExperimentExposureQueryResponse | null
isDraft: boolean
exposureChecksMissing: boolean
onFindingActedOn?: () => void
@@ -34,10 +41,18 @@ export function HealthChecksOverview({
const [pickedSelection, setPickedSelection] = useState<HealthPanelSelection | null>(null)
// A narrow container shows the list or the detail, not both. Wide containers ignore this.
const [isDetailOpenWhenNarrow, setIsDetailOpenWhenNarrow] = useState(false)
- const { reportHealthCheckExpanded } = useActions(experimentLogic)
+ const { experiment } = useValues(experimentLogic)
+ const { aggregationLabel } = useValues(groupsModel)
+ const { reportHealthCheckExpanded, reportHealthSummaryOpened } = useActions(experimentLogic)
const listedChecks = listedHealthChecks(checks)
const issueCount = healthIssues(findings).length
const selection = resolveHealthPanelSelection(pickedSelection, findings, listedChecks)
+ const units = aggregationLabel(experiment.feature_flag?.filters?.aggregation_group_type_index, true).plural
+ const exposureCheckKeys = new Set<string>(exposures?.health_checks?.map(({ key }) => key))
+ const exposureCheckedAt = exposureAnswerComputedAt(exposures)
+ // The checks of the experiment run on each load, so only the exposure checks say when they ran.
+ const checkedAt = (key: HealthPanelCheck['key']): string | null =>
+ exposureCheckKeys.has(key) ? exposureCheckedAt : null
const detailRef = useRef<HTMLElement>(null)
@@ -61,12 +76,26 @@ export function HealthChecksOverview({
isDetailOpenWhenNarrow ? 'hidden' : 'flex'
)}
>
- <div className="flex flex-col gap-1 px-3 py-2">
- <div className="flex items-center gap-2">
- <span className="text-sm font-semibold">Health checks</span>
- <LemonTag type="completion">Beta</LemonTag>
- </div>
- <HealthChecksSummary checks={checks} issueCount={issueCount} isDraft={isDraft} />
+ <div className="px-1 py-1">
+ <LemonButton
+ fullWidth
+ active={selection.kind === 'summary'}
+ onClick={() => {
+ if (select({ kind: 'summary' }, selection.kind === 'summary')) {
+ reportHealthSummaryOpened()
+ }
+ }}
+ aria-current={selection.kind === 'summary' ? 'true' : undefined}
+ data-attr="experiment-health-summary-header"
+ >
+ <span className="flex flex-col gap-1 py-1 text-left font-normal">
+ <span className="flex items-center gap-2">
+ <span className="text-sm font-semibold">Health checks</span>
+ <LemonTag type="completion">Beta</LemonTag>
+ </span>
+ <HealthChecksSummary checks={checks} issueCount={issueCount} isDraft={isDraft} />
+ </span>
+ </LemonButton>
</div>
{findings.length > 0 && (
// A note mounts, and so reports as shown, only when the reader opens the health tab. An issue
@@ -120,11 +149,29 @@ export function HealthChecksOverview({
All checks
</LemonButton>
{selection.kind === 'finding' ? (
- <HealthFindingDetail finding={selection.finding} onActedOn={onFindingActedOn} />
+ <HealthFindingDetail
+ finding={selection.finding}
+ check={checks.find(({ key }) => key === selection.finding.check) ?? null}
+ units={units}
+ exposures={exposures}
+ checkedAt={selection.finding.check ? checkedAt(selection.finding.check) : null}
+ onActedOn={onFindingActedOn}
+ />
) : selection.kind === 'check' ? (
- <HealthCheckDetail check={selection.check} />
+ <HealthCheckDetail
+ check={selection.check}
+ units={units}
+ exposures={exposures}
+ checkedAt={checkedAt(selection.check.key)}
+ />
) : (
- <HealthChecksSummaryDetail checks={checks} issueCount={issueCount} />
+ <HealthChecksSummaryDetail
+ checks={checks}
+ issueCount={issueCount}
+ units={units}
+ countedFrom={exposures?.date_range.date_from ?? null}
+ checkedAt={exposureCheckedAt}
+ />
)}
</section>
</div>
diff --git a/products/experiments/frontend/health/HealthChecksSummary.tsx b/products/experiments/frontend/health/HealthChecksSummary.tsx
index 10656bc82c75..060791b7409d 100644
--- a/products/experiments/frontend/health/HealthChecksSummary.tsx
+++ b/products/experiments/frontend/health/HealthChecksSummary.tsx
@@ -14,13 +14,14 @@ export function HealthChecksSummary({
const notCheckedCount = countHealthChecks(checks, 'not_checked')
return (
- <div className="flex flex-col gap-0.5" data-attr="experiment-health-summary">
+ // Spans only, because the summary sits inside the button that opens it.
+ <span className="flex flex-col gap-0.5" data-attr="experiment-health-summary">
<span className="font-semibold">{healthChecksVerdict(checks, issueCount)}</span>
- <p className="m-0 text-secondary">
+ <span className="text-secondary">
{notCheckedCount > 0 && <span>{pluralize(notCheckedCount, 'check')} couldn't run. </span>}
{isDraft && <span>More checks run after launch. </span>}
<span>We're still adding checks, so they don't cover every problem yet.</span>
- </p>
- </div>
+ </span>
+ </span>
)
}
diff --git a/products/experiments/frontend/health/HealthChecksSummaryDetail.tsx b/products/experiments/frontend/health/HealthChecksSummaryDetail.tsx
index bd2880d045a8..a4690c862fec 100644
--- a/products/experiments/frontend/health/HealthChecksSummaryDetail.tsx
+++ b/products/experiments/frontend/health/HealthChecksSummaryDetail.tsx
@@ -1,30 +1,55 @@
import { pluralize } from 'lib/utils/strings'
+import { type HealthReading, healthSummaryReadings } from './healthCheckValues'
import { type HealthPanelCheck, countHealthChecks, healthChecksVerdict } from './healthPanelChecks'
+import { HealthPanelSection } from './HealthPanelSection'
+import { HealthReadingList } from './HealthReadingList'
export function HealthChecksSummaryDetail({
checks,
issueCount,
+ units,
+ countedFrom,
+ checkedAt,
}: {
checks: HealthPanelCheck[]
issueCount: number
+ units: string
+ /** The start of the exposure count, when the page has an exposure answer. */
+ countedFrom: string | null
+ /** When the exposure checks ran. */
+ checkedAt: string | null
}): JSX.Element {
const passedCount = countHealthChecks(checks, 'passed')
const notApplicableCount = countHealthChecks(checks, 'not_applicable')
- const text = [
+ const counts = [
passedCount > 0 ? `${pluralize(passedCount, 'check')} found no issue.` : null,
notApplicableCount > 0
? `${pluralize(notApplicableCount, 'check')} ${notApplicableCount === 1 ? "doesn't" : "don't"} apply to this experiment.`
: null,
- 'Pick a check to see what it looks for.',
]
.filter(Boolean)
.join(' ')
+ const readings = healthSummaryReadings(checks, units)
+ const exposureTimes: HealthReading[] = readings.length
+ ? [
+ ...(countedFrom ? [{ label: 'Exposures counted from', time: countedFrom, asDate: true }] : []),
+ ...(checkedAt ? [{ label: 'Exposure checks ran', time: checkedAt }] : []),
+ ]
+ : []
return (
- <div className="flex flex-col gap-1" data-attr="experiment-health-summary-detail">
- <h3 className="m-0 text-base font-semibold">{healthChecksVerdict(checks, issueCount)}</h3>
- <p className="m-0 max-w-prose text-secondary">{text}</p>
+ <div className="flex flex-col gap-4" data-attr="experiment-health-summary-detail">
+ <div className="flex flex-col gap-1">
+ <h3 className="m-0 text-base font-semibold">{healthChecksVerdict(checks, issueCount)}</h3>
+ {counts && <p className="m-0 max-w-prose text-secondary">{counts}</p>}
+ </div>
+ {readings.length > 0 && (
+ <HealthPanelSection title="What the checks read">
+ <HealthReadingList readings={[...readings, ...exposureTimes]} />
+ </HealthPanelSection>
+ )}
+ <p className="m-0 max-w-prose text-secondary">Pick a check to see what it looks for and what it read.</p>
</div>
)
}
diff --git a/products/experiments/frontend/health/HealthExposureSplitTable.tsx b/products/experiments/frontend/health/HealthExposureSplitTable.tsx
new file mode 100644
index 000000000000..cc7dadffe639
--- /dev/null
+++ b/products/experiments/frontend/health/HealthExposureSplitTable.tsx
@@ -0,0 +1,72 @@
+import { useValues } from 'kea'
+
+import { LemonTable } from '@posthog/lemon-ui'
+
+import { LemonProgress } from 'lib/lemon-ui/LemonProgress'
+import { humanFriendlyNumber } from 'lib/utils/numbers'
+import { capitalizeFirstLetter } from 'lib/utils/strings'
+import { experimentLogic } from 'scenes/experiments/experimentLogic'
+import { VariantTag } from 'scenes/experiments/ExperimentView/VariantTag'
+import { getExperimentVariants, getVariantColor } from 'scenes/experiments/utils'
+
+import { type HealthSplitRow, formatHealthPercentage } from './healthCheckValues'
+
+export function HealthExposureSplitTable({ rows, units }: { rows: HealthSplitRow[]; units: string }): JSX.Element {
+ const { experiment } = useValues(experimentLogic)
+ const variants = getExperimentVariants(experiment)
+ const hasPlan = rows.some((row) => row.plannedPercentage !== null)
+
+ return (
+ <LemonTable
+ size="small"
+ className="max-w-xl"
+ dataSource={rows}
+ rowKey="variant"
+ data-attr="experiment-health-split"
+ columns={[
+ {
+ title: 'Variant',
+ key: 'variant',
+ render: (_, row) => <VariantTag variantKey={row.variant} />,
+ },
+ {
+ title: capitalizeFirstLetter(units),
+ key: 'exposed',
+ align: 'right',
+ render: (_, row) => <span className="tabular-nums">{humanFriendlyNumber(row.exposedCount)}</span>,
+ },
+ {
+ title: 'Share',
+ key: 'share',
+ render: (_, row) => (
+ <div className="flex items-center gap-2">
+ <div className="w-16 shrink-0">
+ <LemonProgress
+ percent={row.exposedPercentage}
+ strokeColor={getVariantColor(row.variant, variants)}
+ />
+ </div>
+ <span className="tabular-nums">{formatHealthPercentage(row.exposedPercentage)}</span>
+ </div>
+ ),
+ },
+ ...(hasPlan
+ ? [
+ {
+ title: 'Planned',
+ key: 'planned',
+ align: 'right' as const,
+ render: (_: unknown, row: HealthSplitRow) => (
+ <span className="tabular-nums">
+ {row.plannedPercentage === null
+ ? null
+ : formatHealthPercentage(row.plannedPercentage)}
+ </span>
+ ),
+ },
+ ]
+ : []),
+ ]}
+ />
+ )
+}
diff --git a/products/experiments/frontend/health/HealthFindingDetail.tsx b/products/experiments/frontend/health/HealthFindingDetail.tsx
index b9331f51f124..fd7073f6c653 100644
--- a/products/experiments/frontend/health/HealthFindingDetail.tsx
+++ b/products/experiments/frontend/health/HealthFindingDetail.tsx
@@ -1,13 +1,28 @@
-import { HealthFindingActionButton } from './HealthFindingActionButton'
+import type { ExperimentExposureQueryResponse } from '~/queries/schema/schema-general'
+
+import { HealthCheckReadSection } from './HealthCheckReadSection'
+import { HealthCheckRuleSection } from './HealthCheckRuleSection'
+import { HealthFindingFixes } from './HealthFindingFixes'
import { HealthFindingSeverityIcon } from './HealthFindingSeverityIcon'
+import type { HealthPanelCheck } from './healthPanelChecks'
import { HEALTH_FINDING_SEVERITY_LABELS, type HealthPanelFinding } from './healthPanelFindings'
+import { HealthPanelSection } from './HealthPanelSection'
import { useHealthFindingReporting } from './useHealthFindingReporting'
export function HealthFindingDetail({
finding,
+ check,
+ units,
+ exposures,
+ checkedAt,
onActedOn,
}: {
finding: HealthPanelFinding
+ /** The check that reported the finding. Missing on an exposure answer cached before findings named their check. */
+ check: HealthPanelCheck | null
+ units: string
+ exposures: ExperimentExposureQueryResponse | null
+ checkedAt: string | null
onActedOn?: () => void
}): JSX.Element {
const { reportActedOn } = useHealthFindingReporting({
@@ -16,35 +31,33 @@ export function HealthFindingDetail({
severity: finding.severity,
reason: finding.reason ?? undefined,
})
+ const status = [HEALTH_FINDING_SEVERITY_LABELS[finding.severity], check ? `Found by "${check.name}"` : null]
+ .filter(Boolean)
+ .join(' · ')
return (
- <div className="flex flex-col gap-3" data-attr="experiment-health-finding-detail">
- <div className="flex flex-wrap items-start justify-between gap-2">
- <div className="flex flex-col gap-1 min-w-0">
- <h3 className="m-0 flex items-center gap-2 text-base font-semibold">
- <HealthFindingSeverityIcon severity={finding.severity} />
- <span>{finding.title}</span>
- </h3>
- <span className="pl-6 text-xs text-secondary">
- {HEALTH_FINDING_SEVERITY_LABELS[finding.severity]}
- </span>
- </div>
- {finding.actions.length > 0 && (
- <div className="flex flex-wrap items-center gap-2">
- {finding.actions.map((actionKind) => (
- <HealthFindingActionButton
- key={actionKind}
- actionKind={actionKind}
- onClick={() => {
- reportActedOn(actionKind)
- onActedOn?.()
- }}
- />
- ))}
- </div>
- )}
+ <div className="flex flex-col gap-4" data-attr="experiment-health-finding-detail">
+ <div className="flex flex-col gap-1 min-w-0">
+ <h3 className="m-0 flex items-center gap-2 text-base font-semibold">
+ <HealthFindingSeverityIcon severity={finding.severity} />
+ <span>{finding.title}</span>
+ </h3>
+ <span className="pl-6 text-xs text-secondary">{status}</span>
</div>
- <p className="m-0 max-w-prose text-secondary whitespace-pre-line">{finding.detail}</p>
+ <HealthPanelSection title="What we found">
+ <p className="m-0 max-w-prose whitespace-pre-line">{finding.detail}</p>
+ </HealthPanelSection>
+ {check && <HealthCheckReadSection check={check} units={units} exposures={exposures} />}
+ {finding.actions.length > 0 && (
+ <HealthFindingFixes
+ actions={finding.actions}
+ onActedOn={(actionKind) => {
+ reportActedOn(actionKind)
+ onActedOn?.()
+ }}
+ />
+ )}
+ {check && <HealthCheckRuleSection check={check} units={units} checkedAt={checkedAt} />}
</div>
)
}
diff --git a/products/experiments/frontend/health/HealthFindingFixes.tsx b/products/experiments/frontend/health/HealthFindingFixes.tsx
new file mode 100644
index 000000000000..88a27e690b04
--- /dev/null
+++ b/products/experiments/frontend/health/HealthFindingFixes.tsx
@@ -0,0 +1,67 @@
+import type { ExperimentHealthFindingActionKind } from './experimentHealthFindingEvents'
+import { HealthFindingActionButton } from './HealthFindingActionButton'
+import { HealthPanelSection } from './HealthPanelSection'
+
+// What each action does and what it changes, so a reader can pick one before clicking.
+const FIXES: Record<ExperimentHealthFindingActionKind, { title: string; description: string }> = {
+ open_feature_flag: {
+ title: 'Open the feature flag',
+ description: 'Opens the feature flag in a new tab. A change there applies to every experiment on the flag.',
+ },
+ open_feature_flag_history: {
+ title: "See the flag's history",
+ description: 'Opens the changes to the feature flag in a new tab, with the time of each one. Nothing changes.',
+ },
+ adjust_distribution: {
+ title: 'Change the variant split',
+ description:
+ 'An even split keeps the variants the same size, and the rollout then decides how many users enter. A change after launch can move users who were already exposed to another variant.',
+ },
+ use_first_seen_variant: {
+ title: 'Count users in the first variant they saw',
+ description:
+ 'Sets the multiple variant handling to "Use first seen variant", so the results count each of these users in the variant they saw first. Your users see no change.',
+ },
+ edit_exposure_criteria: {
+ title: 'Change the exposure criteria',
+ description:
+ 'The exposure criteria decide which events and users count as exposed. The next calculation of the results uses the new criteria. Your users see no change.',
+ },
+ edit_release_conditions: {
+ title: 'Change the release conditions',
+ description:
+ 'The release conditions decide who gets a variant. A change after launch can let in other users or move users to another variant.',
+ },
+ add_primary_metric: {
+ title: 'Add a primary metric',
+ description: 'Adds a metric that the experiment compares the variants on. Your users see no change.',
+ },
+ add_secondary_metric: {
+ title: 'Add a secondary metric',
+ description: 'Adds a metric to watch next to the primary metrics. Your users see no change.',
+ },
+}
+
+export function HealthFindingFixes({
+ actions,
+ onActedOn,
+}: {
+ actions: ExperimentHealthFindingActionKind[]
+ onActedOn: (actionKind: ExperimentHealthFindingActionKind) => void
+}): JSX.Element {
+ return (
+ <HealthPanelSection title="Ways to fix">
+ <ul className="m-0 flex max-w-xl list-none flex-col divide-y rounded border p-0">
+ {actions.map((actionKind) => (
+ <li key={actionKind} className="flex flex-wrap items-start justify-between gap-2 px-3 py-2">
+ <div className="flex min-w-0 flex-1 basis-60 flex-col gap-0.5">
+ <span className="font-semibold">{FIXES[actionKind].title}</span>
+ <span className="text-secondary">{FIXES[actionKind].description}</span>
+ </div>
+ <HealthFindingActionButton actionKind={actionKind} onClick={() => onActedOn(actionKind)} />
+ </li>
+ ))}
+ </ul>
+ </HealthPanelSection>
+ )
+}
diff --git a/products/experiments/frontend/health/HealthPanelSection.tsx b/products/experiments/frontend/health/HealthPanelSection.tsx
new file mode 100644
index 000000000000..7e7341dfcfc5
--- /dev/null
+++ b/products/experiments/frontend/health/HealthPanelSection.tsx
@@ -0,0 +1,8 @@
+export function HealthPanelSection({ title, children }: { title: string; children: React.ReactNode }): JSX.Element {
+ return (
+ <section className="flex flex-col gap-1">
+ <h4 className="m-0 text-xs font-semibold text-secondary">{title}</h4>
+ {children}
+ </section>
+ )
+}
diff --git a/products/experiments/frontend/health/HealthReadingList.tsx b/products/experiments/frontend/health/HealthReadingList.tsx
new file mode 100644
index 000000000000..660da5aaaab2
--- /dev/null
+++ b/products/experiments/frontend/health/HealthReadingList.tsx
@@ -0,0 +1,28 @@
+import { Fragment } from 'react'
+
+import { TZLabel } from 'lib/components/TZLabel'
+
+import type { HealthReading } from './healthCheckValues'
+
+export function HealthReadingList({ readings }: { readings: HealthReading[] }): JSX.Element {
+ return (
+ <dl className="m-0 grid max-w-xl grid-cols-2 gap-x-4 gap-y-1">
+ {readings.map((reading) => (
+ <Fragment key={reading.label}>
+ <dt className="text-secondary">{reading.label}</dt>
+ <dd className="m-0 tabular-nums">
+ {'time' in reading ? (
+ reading.asDate ? (
+ <TZLabel time={reading.time} formatDate="MMMM D, YYYY" formatTime="h:mm A" />
+ ) : (
+ <TZLabel time={reading.time} />
+ )
+ ) : (
+ reading.text
+ )}
+ </dd>
+ </Fragment>
+ ))}
+ </dl>
+ )
+}
diff --git a/products/experiments/frontend/health/debug/healthDebugReport.test.ts b/products/experiments/frontend/health/debug/healthDebugReport.test.ts
index 7feef416cdc0..bf36578f0ad4 100644
--- a/products/experiments/frontend/health/debug/healthDebugReport.test.ts
+++ b/products/experiments/frontend/health/debug/healthDebugReport.test.ts
@@ -13,6 +13,7 @@ const PAUSED: ExperimentHealthFindingApi = {
evidence: {},
actions: ['open_feature_flag'],
diagnostic_ref: 'A5',
+ check: 'flag_state',
}
const input = (overrides: Partial<HealthDebugInput>): HealthDebugInput => ({
diff --git a/products/experiments/frontend/health/debug/healthDebugReport.ts b/products/experiments/frontend/health/debug/healthDebugReport.ts
index 9f1e9be35172..0b052affb0c3 100644
--- a/products/experiments/frontend/health/debug/healthDebugReport.ts
+++ b/products/experiments/frontend/health/debug/healthDebugReport.ts
@@ -9,6 +9,7 @@ import type {
HealthExposureInputsStatusEnumApi,
} from '../../generated/api.schemas'
import { FLAG_STATE_FINDING_CODES, experimentWarningFromHealth } from '../experimentHealthFindingEvents'
+import { exposureAnswerComputedAt } from '../exposureHealth'
import type { HealthPanelFinding } from '../healthPanelFindings'
export type HealthDebugDomain = 'experiment' | 'flag' | 'exposures' | 'metrics' | 'project' | 'raw'
@@ -321,13 +322,6 @@ function pageExposureResult(input: HealthDebugInput, code: HealthPanelFinding['c
return finding ? describeFinding(code, finding) : NO_FINDING
}
-// The page's answer comes from the cache, but its type does not declare the cache fields.
-function pageComputedAt(exposures: HealthDebugInput['exposures']): string | null {
- return exposures && 'last_refresh' in exposures && typeof exposures.last_refresh === 'string'
- ? exposures.last_refresh
- : null
-}
-
function exposureChecks(input: HealthDebugInput): HealthDebugCheck[] {
const exposureCheck = (
check: HealthPanelFinding['code'],
@@ -345,7 +339,7 @@ function exposureChecks(input: HealthDebugInput): HealthDebugCheck[] {
browser: page,
browserLabel: "Page's exposure answer",
computedAt: serverAnswered ? (input.serverExposures?.computed_at ?? null) : null,
- browserComputedAt: page !== null ? pageComputedAt(input.exposures) : null,
+ browserComputedAt: page !== null ? exposureAnswerComputedAt(input.exposures) : null,
difference: serverAnswered && page !== null && page !== server.result ? 'drift' : null,
note,
}
diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
index e0540f56550d..cd8ab93a08fa 100644
--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts
+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts
@@ -14,6 +14,7 @@ import { EXPERIMENT_VARIANT_MULTIPLE } from '../constants'
import { getExperimentStatus } from '../experimentStatus'
import type { ExperimentHealthApi, ExperimentHealthFindingApi } from '../generated/api.schemas'
import { getTotalExposures, hasSampleRatioMismatch } from './exposureHealth'
+import { healthCheckShowsDecidingValue } from './healthCheckValues'
import { type HealthPanelCheck, countHealthChecks } from './healthPanelChecks'
// pinned: `finding_code` property values. Insights group and filter on them, so a rename breaks those insights.
@@ -222,7 +223,26 @@ export function captureExperimentHealthCheckExpanded(
health_ui: healthUi,
check_key: check.key,
check_status: check.status,
- check_has_measurement: Boolean(check.measurement),
+ // pinned: property name. It means that the check shows the value that decided its status.
+ check_has_measurement: healthCheckShowsDecidingValue(check),
+ surface: 'experiment_page',
+ })
+}
+
+export function captureExperimentHealthSummaryOpened(
+ experiment: Experiment,
+ checks: HealthPanelCheck[],
+ findingCount: number,
+ healthUi: ExperimentHealthUi
+): void {
+ // pinned: analytics event name, so renaming it breaks insights
+ posthog.capture('experiment health summary opened', {
+ experiment_id: experiment.id,
+ experiment_status: getExperimentStatus(experiment),
+ health_ui: healthUi,
+ health_finding_count: findingCount,
+ checks_passed: countHealthChecks(checks, 'passed'),
+ checks_finding: countHealthChecks(checks, 'finding'),
surface: 'experiment_page',
})
}
diff --git a/products/experiments/frontend/health/exposureHealth.ts b/products/experiments/frontend/health/exposureHealth.ts
index 3b96aa23efee..1ca74edde26d 100644
--- a/products/experiments/frontend/health/exposureHealth.ts
+++ b/products/experiments/frontend/health/exposureHealth.ts
@@ -15,6 +15,13 @@ export function hasSampleRatioMismatch(exposures: ExperimentExposureQueryRespons
export const SAMPLE_RATIO_MATCH_DESCRIPTION =
'No sample ratio mismatch detected. The difference between actual and expected exposures is within normal random variation.'
+/** When the server computed the exposure answer. The page's answer comes from the cache, but its type doesn't declare the cache fields. */
+export function exposureAnswerComputedAt(exposures: ExperimentExposureQueryResponse | null | undefined): string | null {
+ return exposures && 'last_refresh' in exposures && typeof exposures.last_refresh === 'string'
+ ? exposures.last_refresh
+ : null
+}
+
/** The server's sample ratio finding in the exposure answer, or undefined when the server reports none. */
export function serverSampleRatioFinding(
exposures: ExperimentExposureQueryResponse | null | undefined
diff --git a/products/experiments/frontend/health/healthCheckValues.test.ts b/products/experiments/frontend/health/healthCheckValues.test.ts
new file mode 100644
index 000000000000..88020992df0f
--- /dev/null
+++ b/products/experiments/frontend/health/healthCheckValues.test.ts
@@ -0,0 +1,213 @@
+import { type ExperimentExposureQueryResponse, NodeKind } from '~/queries/schema/schema-general'
+
+import {
+ type HealthCheckRule,
+ type HealthCheckValues,
+ type HealthReading,
+ healthCheckReadings,
+ healthCheckShowsDecidingValue,
+ healthExposureSplit,
+ healthRuleReadings,
+ healthSummaryReadings,
+} from './healthCheckValues'
+import type { HealthPanelCheck } from './healthPanelChecks'
+
+const SRM_RULE: HealthCheckRule = {
+ value: 'p_value',
+ threshold: 0.001,
+ warns: 'below',
+ test: 'chi_squared',
+ minimum: { value: 'compared_count', threshold: 100 },
+}
+
+const MULTIPLE_RULE: HealthCheckRule = {
+ value: 'multiple_variant_percentage',
+ threshold: 5,
+ warns: 'above',
+ test: null,
+ minimum: { value: 'multiple_variant_count', threshold: 5 },
+}
+
+const check = (
+ key: HealthPanelCheck['key'],
+ status: HealthPanelCheck['status'],
+ values: HealthPanelCheck['values'],
+ rule: HealthCheckRule | null = null
+): HealthPanelCheck => ({ key, name: key, description: '', status, reason: null, values, rule }) as HealthPanelCheck
+
+const exposures = (
+ totalExposures: Record<string, number>,
+ expected?: Record<string, number>
+): ExperimentExposureQueryResponse => ({
+ kind: NodeKind.ExperimentExposureQuery,
+ timeseries: [],
+ total_exposures: totalExposures,
+ date_range: { date_from: '2026-10-19T00:00:00Z' },
+ sample_ratio_mismatch: expected ? { expected, p_value: 0.71 } : undefined,
+})
+
+describe('healthCheckValues', () => {
+ it.each<{ name: string; values: HealthCheckValues; expected: HealthReading[] }>([
+ {
+ name: 'the sample ratio check leaves its p-value to the rule',
+ values: { p_value: 0.71, compared_count: 9199, multiple_variant_count: 403 },
+ expected: [
+ { label: 'Users compared', text: '9,199' },
+ { label: 'In more than one variant', text: '403' },
+ ],
+ },
+ {
+ name: 'the share in more than one variant goes with its count',
+ values: {
+ exposed_count: 9602,
+ multiple_variant_count: 403,
+ multiple_variant_percentage: 4.197,
+ multiple_variant_handling: 'exclude',
+ },
+ expected: [
+ { label: 'Exposed users', text: '9,602' },
+ { label: 'In more than one variant', text: '403 · 4.2%' },
+ { label: 'Multiple variant handling', text: 'Exclude from analysis' },
+ ],
+ },
+ {
+ name: 'a metric count reads as n of m',
+ values: { metric_count: 3, completed_metric_count: 2, failed_metric_count: 1, calculated_at: null },
+ expected: [
+ { label: 'Metrics calculated', text: '2 of 3' },
+ { label: 'Metrics whose calculation failed', text: '1' },
+ ],
+ },
+ {
+ name: 'a deleted flag reads as deleted, whatever else it serves',
+ values: { flag_active: true, flag_deleted: true, flag_rolled_out: true, shipped_variant: 'test' },
+ expected: [{ label: 'Feature flag', text: 'Deleted' }],
+ },
+ {
+ name: 'a shipped variant names the variant',
+ values: { flag_active: true, flag_deleted: false, flag_rolled_out: true, shipped_variant: 'test' },
+ expected: [{ label: 'Feature flag', text: 'On, serves only "test"' }],
+ },
+ {
+ name: 'a setting that is off still shows',
+ values: { only_count_matured_users: false },
+ expected: [{ label: 'Require completed conversion or retention window', text: 'Off' }],
+ },
+ ])('healthCheckReadings: $name', ({ values, expected }) => {
+ expect(healthCheckReadings(values, 'users')).toEqual(expected)
+ })
+
+ it.each([
+ {
+ name: 'a passed test shows its result',
+ status: 'passed' as const,
+ expected: [
+ { label: 'Test', text: 'Chi-squared, exposed users against the planned split' },
+ { label: 'Result', text: 'p = 0.71' },
+ { label: 'Warns when', text: 'p is below 0.001' },
+ { label: 'Warns from', text: '100 users compared' },
+ ],
+ },
+ {
+ name: "a test that doesn't apply decided nothing, so it shows no result",
+ status: 'not_applicable' as const,
+ expected: [
+ { label: 'Test', text: 'Chi-squared, exposed users against the planned split' },
+ { label: 'Warns when', text: 'p is below 0.001' },
+ { label: 'Warns from', text: '100 users compared' },
+ ],
+ },
+ ])('healthRuleReadings: $name', ({ status, expected }) => {
+ expect(healthRuleReadings({ status, values: { p_value: 0.71 } }, SRM_RULE, 'users')).toEqual(expected)
+ })
+
+ it('healthRuleReadings: no exposure at all reads as none', () => {
+ const rule: HealthCheckRule = {
+ value: 'exposed_count',
+ threshold: 1,
+ warns: 'below',
+ test: null,
+ minimum: { value: 'since_launch_hours', threshold: 24 },
+ }
+
+ expect(healthRuleReadings({ status: 'passed', values: {} }, rule, 'groups')).toEqual([
+ { label: 'Warns when', text: 'No groups are exposed' },
+ { label: 'Warns from', text: '24 hours after launch' },
+ ])
+ })
+
+ it.each<{ name: string; values: HealthCheckValues; expected: boolean }>([
+ {
+ name: 'a share from enough users',
+ values: { multiple_variant_percentage: 60, multiple_variant_count: 30 },
+ expected: true,
+ },
+ {
+ name: 'a share from too few users',
+ values: { multiple_variant_percentage: 60, multiple_variant_count: 3 },
+ expected: false,
+ },
+ { name: 'no share', values: { multiple_variant_count: 30 }, expected: false },
+ ])('healthCheckShowsDecidingValue: $name', ({ values, expected }) => {
+ expect(healthCheckShowsDecidingValue({ values, rule: MULTIPLE_RULE })).toBe(expected)
+ })
+
+ it.each([
+ {
+ name: 'shares of the compared variants against the planned split',
+ answer: exposures({ control: 600, test: 400, $multiple: 50 }, { control: 500, test: 500 }),
+ expected: [
+ { variant: 'control', exposedCount: 600, exposedPercentage: 60, plannedPercentage: 50 },
+ { variant: 'test', exposedCount: 400, exposedPercentage: 40, plannedPercentage: 50 },
+ ],
+ },
+ {
+ name: 'no plan before the sample ratio test runs',
+ answer: exposures({ control: 30, test: 10, $multiple: 2 }),
+ expected: [
+ { variant: 'control', exposedCount: 30, exposedPercentage: 75, plannedPercentage: null },
+ { variant: 'test', exposedCount: 10, exposedPercentage: 25, plannedPercentage: null },
+ ],
+ },
+ {
+ name: 'the variants in the order of the experiment',
+ answer: {
+ ...exposures({ test: 10, control: 30 }),
+ timeseries: [
+ { variant: 'control', days: [], exposure_counts: [] },
+ { variant: 'test', days: [], exposure_counts: [] },
+ ],
+ },
+ expected: [
+ { variant: 'control', exposedCount: 30, exposedPercentage: 75, plannedPercentage: null },
+ { variant: 'test', exposedCount: 10, exposedPercentage: 25, plannedPercentage: null },
+ ],
+ },
+ { name: 'nothing without exposures', answer: exposures({ control: 0, test: 0 }), expected: null },
+ ])('healthExposureSplit: $name', ({ answer, expected }) => {
+ expect(healthExposureSplit(answer)).toEqual(expected)
+ })
+
+ it('healthSummaryReadings: picks each line from the check that read it', () => {
+ const checks = [
+ check('zero_exposures', 'passed', { exposed_count: 9602, since_launch_hours: 300 }),
+ check(
+ 'srm',
+ 'not_applicable',
+ { p_value: 0.0001, compared_count: 9199, multiple_variant_count: 403 },
+ SRM_RULE
+ ),
+ check('multiple_variants_share', 'passed', {
+ multiple_variant_count: 403,
+ multiple_variant_percentage: 4.2,
+ }),
+ check('metric_calculation_failed', 'passed', { metric_count: 3, completed_metric_count: 3 }),
+ ]
+
+ expect(healthSummaryReadings(checks, 'users')).toEqual([
+ { label: 'Exposed users', text: '9,602' },
+ { label: 'In more than one variant', text: '403 · 4.2%' },
+ { label: 'Metrics calculated', text: '3 of 3' },
+ ])
+ })
+})
diff --git a/products/experiments/frontend/health/healthCheckValues.ts b/products/experiments/frontend/health/healthCheckValues.ts
new file mode 100644
index 000000000000..1d0704dcb40e
--- /dev/null
+++ b/products/experiments/frontend/health/healthCheckValues.ts
@@ -0,0 +1,277 @@
+import { humanFriendlyNumber } from 'lib/utils/numbers'
+import { capitalizeFirstLetter } from 'lib/utils/strings'
+
+import type {
+ ExperimentExposureHealthCheckRule,
+ ExperimentExposureQueryResponse,
+} from '~/queries/schema/schema-general'
+
+import { EXPERIMENT_VARIANT_MULTIPLE } from '../constants'
+import type { ExperimentHealthCheckRuleApi, ExperimentHealthCheckResultApiValues } from '../generated/api.schemas'
+import type { HealthPanelCheck } from './healthPanelChecks'
+
+export type HealthCheckValues = ExperimentHealthCheckResultApiValues
+export type HealthCheckRule = ExperimentHealthCheckRuleApi | ExperimentExposureHealthCheckRule
+
+/**
+ * One line of what a check read: a label and either a text or a time that the panel shows in the reader's zone. A
+ * time that marks a start shows as a date, and the others show how long ago they were.
+ */
+export type HealthReading = { label: string } & ({ text: string } | { time: string; asDate?: boolean })
+
+export interface HealthSplitRow {
+ variant: string
+ exposedCount: number
+ exposedPercentage: number
+ plannedPercentage: number | null
+}
+
+const HANDLING_LABELS: Record<string, string> = {
+ exclude: 'Exclude from analysis',
+ first_seen: 'Use first seen variant',
+}
+
+const TEST_LABELS: Record<NonNullable<HealthCheckRule['test']>, string> = {
+ chi_squared: 'Chi-squared, exposed users against the planned split',
+}
+
+function numberValue(values: HealthCheckValues | undefined, key: string): number | null {
+ const value = values?.[key]
+ return typeof value === 'number' ? value : null
+}
+
+function formatCount(count: number): string {
+ return humanFriendlyNumber(count, 0)
+}
+
+export function formatHealthPercentage(percentage: number): string {
+ // One decimal would print a small share as 0%, which reads as none.
+ return percentage > 0 && percentage < 0.1 ? 'less than 0.1%' : `${Number(percentage.toFixed(1))}%`
+}
+
+function formatPValue(pValue: number): string {
+ return `p = ${Number(pValue.toPrecision(2))}`
+}
+
+function formatHours(hours: number): string {
+ return hours < 48 ? `${Math.floor(hours)} hours` : `${Math.floor(hours / 24)} days`
+}
+
+function reading(label: string, text: string | null): HealthReading | null {
+ return text === null ? null : { label, text }
+}
+
+function numberReading(
+ label: string,
+ values: HealthCheckValues | undefined,
+ key: string,
+ format: (value: number) => string
+): HealthReading | null {
+ const value = numberValue(values, key)
+ return reading(label, value === null ? null : format(value))
+}
+
+function exposedReading(values: HealthCheckValues | undefined, units: string): HealthReading | null {
+ return numberReading(`Exposed ${units}`, values, 'exposed_count', formatCount)
+}
+
+function multipleVariantReading(values: HealthCheckValues | undefined): HealthReading | null {
+ const count = numberValue(values, 'multiple_variant_count')
+ const percentage = numberValue(values, 'multiple_variant_percentage')
+ if (count === null) {
+ return null
+ }
+ return reading(
+ 'In more than one variant',
+ percentage === null ? formatCount(count) : `${formatCount(count)} · ${formatHealthPercentage(percentage)}`
+ )
+}
+
+function calculatedMetricsReading(values: HealthCheckValues | undefined): HealthReading | null {
+ const metricCount = numberValue(values, 'metric_count')
+ const completedCount = numberValue(values, 'completed_metric_count')
+ if (metricCount === null || completedCount === null) {
+ return null
+ }
+ return reading('Metrics calculated', `${formatCount(completedCount)} of ${formatCount(metricCount)}`)
+}
+
+function flagReading(values: HealthCheckValues | undefined): HealthReading | null {
+ if (typeof values?.flag_active !== 'boolean') {
+ return null
+ }
+ const shipped = values.shipped_variant
+ const text = values.flag_deleted
+ ? 'Deleted'
+ : !values.flag_active
+ ? 'Off'
+ : values.flag_rolled_out === false
+ ? 'On, rolled out to 0%'
+ : typeof shipped === 'string'
+ ? `On, serves only "${shipped}"`
+ : 'On'
+ return reading('Feature flag', text)
+}
+
+/** What a check read, in the order the panel lists it. The p-value shows with the rule that judges it instead. */
+export function healthCheckReadings(values: HealthCheckValues | undefined, units: string): HealthReading[] {
+ const handling = values?.multiple_variant_handling
+ const onlyMatured = values?.only_count_matured_users
+ const primaryMetrics = numberValue(values, 'primary_metric_count')
+ const waitingMetrics = numberValue(values, 'waiting_metric_count')
+ const calculatedAt = values?.calculated_at
+
+ return [
+ exposedReading(values, units),
+ numberReading(`${capitalizeFirstLetter(units)} compared`, values, 'compared_count', formatCount),
+ multipleVariantReading(values),
+ reading(
+ 'Multiple variant handling',
+ typeof handling === 'string' ? (HANDLING_LABELS[handling] ?? handling) : null
+ ),
+ numberReading('Since launch', values, 'since_launch_hours', formatHours),
+ numberReading(
+ `Exposed ${units} without the test account filters`,
+ values,
+ 'exposed_without_test_account_filter_count',
+ formatCount
+ ),
+ numberReading(
+ 'Kept by the test account filters',
+ values,
+ 'kept_by_test_account_filter_percentage',
+ formatHealthPercentage
+ ),
+ reading(
+ 'Require completed conversion or retention window',
+ typeof onlyMatured === 'boolean' ? (onlyMatured ? 'On' : 'Off') : null
+ ),
+ reading(
+ "Primary metrics that can't count users yet",
+ primaryMetrics === null || waitingMetrics === null
+ ? null
+ : `${formatCount(waitingMetrics)} of ${formatCount(primaryMetrics)}`
+ ),
+ calculatedMetricsReading(values),
+ numberReading('Metrics whose calculation failed', values, 'failed_metric_count', formatCount),
+ typeof calculatedAt === 'string' ? { label: 'Last calculated', time: calculatedAt } : null,
+ flagReading(values),
+ ].filter((line): line is HealthReading => line !== null)
+}
+
+function warnsWhen(rule: HealthCheckRule, units: string): string {
+ const threshold = rule.threshold
+ switch (rule.value) {
+ case 'p_value':
+ return `p is ${rule.warns} ${threshold}`
+ case 'multiple_variant_percentage':
+ return `${rule.warns === 'above' ? 'More' : 'Less'} than ${formatHealthPercentage(threshold)} of exposed ${units} are in more than one variant`
+ case 'kept_by_test_account_filter_percentage':
+ return `The test account filters keep ${rule.warns === 'above' ? 'more' : 'less'} than ${formatHealthPercentage(threshold)} of the exposed ${units}`
+ case 'exposed_count':
+ return rule.warns === 'below' && threshold === 1
+ ? `No ${units} are exposed`
+ : `${rule.warns === 'above' ? 'More' : 'Fewer'} than ${formatCount(threshold)} ${units} are exposed`
+ default:
+ return `${rule.value} is ${rule.warns} ${threshold}`
+ }
+}
+
+function minimumAmount(minimum: NonNullable<HealthCheckRule['minimum']>, units: string): string {
+ const amount = formatCount(minimum.threshold)
+ switch (minimum.value) {
+ case 'compared_count':
+ return `${amount} ${units} compared`
+ case 'multiple_variant_count':
+ return `${amount} ${units} in more than one variant`
+ case 'since_launch_hours':
+ return `${amount} hours after launch`
+ case 'exposed_without_test_account_filter_count':
+ return `${amount} ${units} exposed without the test account filters`
+ default:
+ return `${amount} (${minimum.value})`
+ }
+}
+
+/**
+ * How a check decides, as label and text lines. A statistical test also shows its result here, but only for a check
+ * that judged: a check that doesn't apply can still have read a p-value, which then decided nothing.
+ */
+export function healthRuleReadings(
+ check: Pick<HealthPanelCheck, 'status' | 'values'>,
+ rule: HealthCheckRule,
+ units: string
+): HealthReading[] {
+ const result =
+ check.status === 'passed' || check.status === 'finding' ? numberValue(check.values, rule.value) : null
+ const lines: (HealthReading | null)[] = [
+ rule.test ? { label: 'Test', text: TEST_LABELS[rule.test] } : null,
+ rule.test && result !== null ? { label: 'Result', text: formatPValue(result) } : null,
+ { label: 'Warns when', text: warnsWhen(rule, units) },
+ rule.minimum ? { label: 'Warns from', text: minimumAmount(rule.minimum, units) } : null,
+ ]
+ return lines.filter((line): line is HealthReading => line !== null)
+}
+
+/** Whether the check shows the value that decided its status: the rule's value, from enough data to judge it. */
+export function healthCheckShowsDecidingValue(check: Pick<HealthPanelCheck, 'values' | 'rule'>): boolean {
+ const rule = check.rule
+ if (!rule || numberValue(check.values, rule.value) === null) {
+ return false
+ }
+ const minimumValue = rule.minimum ? numberValue(check.values, rule.minimum.value) : null
+ return !rule.minimum || (minimumValue !== null && minimumValue >= rule.minimum.threshold)
+}
+
+/**
+ * The exposed users of each compared variant against the planned split, from the exposure answer. The planned shares
+ * come from the sample ratio test, so they are missing while too few users are exposed for it.
+ */
+export function healthExposureSplit(
+ exposures: ExperimentExposureQueryResponse | null | undefined
+): HealthSplitRow[] | null {
+ const totals = exposures?.total_exposures
+ if (!totals) {
+ return null
+ }
+ const expected = exposures.sample_ratio_mismatch?.expected
+ // The timeseries keeps the variant order of the experiment, which the totals don't.
+ const ordered = exposures.timeseries?.length
+ ? exposures.timeseries.map(({ variant }) => variant)
+ : Object.keys(totals)
+ const variants = ordered.filter((variant) =>
+ expected ? variant in expected : variant !== EXPERIMENT_VARIANT_MULTIPLE
+ )
+ const comparedCount = variants.reduce((sum, variant) => sum + (totals[variant] ?? 0), 0)
+ const plannedTotal = expected ? Object.values(expected).reduce((sum, count) => sum + count, 0) : 0
+ if (variants.length < 2 || comparedCount === 0) {
+ return null
+ }
+ return variants.map((variant) => ({
+ variant,
+ exposedCount: totals[variant] ?? 0,
+ exposedPercentage: ((totals[variant] ?? 0) / comparedCount) * 100,
+ plannedPercentage: expected && plannedTotal > 0 ? (expected[variant] / plannedTotal) * 100 : null,
+ }))
+}
+
+function findCheck(checks: HealthPanelCheck[], key: HealthPanelCheck['key']): HealthPanelCheck | undefined {
+ return checks.find((check) => check.key === key)
+}
+
+/** What the summary shows of the checks' values, from the checks that read each one. */
+export function healthSummaryReadings(checks: HealthPanelCheck[], units: string): HealthReading[] {
+ const srm = findCheck(checks, 'srm')
+ const pValue = numberValue(srm?.values, 'p_value')
+ const splitText =
+ srm && pValue !== null && (srm.status === 'passed' || srm.status === 'finding')
+ ? `${srm.status === 'passed' ? 'Matches' : "Doesn't match"}, ${formatPValue(pValue)}`
+ : null
+ return [
+ exposedReading(findCheck(checks, 'zero_exposures')?.values, units),
+ reading('Split against the plan', splitText),
+ multipleVariantReading(findCheck(checks, 'multiple_variants_share')?.values),
+ calculatedMetricsReading(findCheck(checks, 'metric_calculation_failed')?.values),
+ flagReading(findCheck(checks, 'flag_state')?.values),
+ ].filter((line): line is HealthReading => line !== null)
+}
diff --git a/products/experiments/frontend/health/healthPanelChecks.test.ts b/products/experiments/frontend/health/healthPanelChecks.test.ts
index 4e150d0b2c96..88b20f3cbef0 100644
--- a/products/experiments/frontend/health/healthPanelChecks.test.ts
+++ b/products/experiments/frontend/health/healthPanelChecks.test.ts
@@ -7,7 +7,6 @@ import {
type HealthPanelCheck,
exposureChecksState,
healthChipState,
- healthMeasurementText,
healthPanelChecks,
} from './healthPanelChecks'
import type { HealthPanelFinding } from './healthPanelFindings'
@@ -15,7 +14,7 @@ import type { HealthPanelFinding } from './healthPanelFindings'
const check = (
key: ExperimentHealthCheckResultApi['key'],
status: ExperimentHealthCheckResultApi['status']
-): ExperimentHealthCheckResultApi => ({ key, name: key, description: '', status, reason: null, measurement: null })
+): ExperimentHealthCheckResultApi => ({ key, name: key, description: '', status, reason: null, values: {}, rule: null })
const finding = (severity: HealthPanelFinding['severity']): HealthPanelFinding => ({
code: 'no_metric',
@@ -170,23 +169,4 @@ describe('healthPanelChecks', () => {
])('$name', ({ findings, checks, exposureChecks, expected }) => {
expect(healthChipState(findings, checks, exposureChecks)).toMatchObject(expected)
})
- it.each([
- {
- name: 'a share against its threshold',
- measurement: { value: 4.23, threshold: 5, unit: 'percentage', warns: 'above' },
- expected: 'Measured 4.2%. Warns above 5%.',
- },
- {
- name: 'a share too small for one decimal',
- measurement: { value: 0.04, threshold: 5, unit: 'percentage', warns: 'above' },
- expected: 'Measured less than 0.1%. Warns above 5%.',
- },
- {
- name: 'a small p-value',
- measurement: { value: 0.003456, threshold: 0.001, unit: 'p_value', warns: 'below' },
- expected: 'Measured p = 0.0035. Warns below 0.001.',
- },
- ] as const)('healthMeasurementText states $name', ({ measurement, expected }) => {
- expect(healthMeasurementText(measurement)).toEqual(expected)
- })
})
diff --git a/products/experiments/frontend/health/healthPanelChecks.ts b/products/experiments/frontend/health/healthPanelChecks.ts
index 0b8b5ba875c9..851da129bc18 100644
--- a/products/experiments/frontend/health/healthPanelChecks.ts
+++ b/products/experiments/frontend/health/healthPanelChecks.ts
@@ -1,15 +1,11 @@
import { pluralize } from 'lib/utils/strings'
-import type {
- ExperimentExposureHealthCheck,
- ExperimentExposureHealthMeasurement,
- ExperimentExposureQueryResponse,
-} from '~/queries/schema/schema-general'
+import type { ExperimentExposureHealthCheck, ExperimentExposureQueryResponse } from '~/queries/schema/schema-general'
import type { ExperimentHealthApi, ExperimentHealthCheckResultApi } from '../generated/api.schemas'
import { type HealthPanelFinding, healthIssues } from './healthPanelFindings'
-/** An exposure check from an answer cached before the server added a field has no description or measurement. */
+/** An exposure check from an answer cached before the server added a field has no description, values or rule. */
export type HealthPanelCheck = ExperimentHealthCheckResultApi | ExperimentExposureHealthCheck
/** A check in the list of checks. A check that reported a finding shows as that finding instead. */
@@ -121,20 +117,3 @@ export function healthChipState(
.join(' · ')
return { kind: 'checks', label: label || 'No checks apply yet', checks, findings }
}
-
-function formatMeasuredValue(value: number, unit: ExperimentExposureHealthMeasurement['unit']): string {
- if (unit === 'p_value') {
- return `p = ${Number(value.toPrecision(2))}`
- }
- // One decimal would print a small share as 0%, which reads as none.
- return value > 0 && value < 0.1 ? 'less than 0.1%' : `${Number(value.toFixed(1))}%`
-}
-
-/** The value that decided a passed check, against its threshold, for example "Measured 4.2%. Warns above 5%." */
-export function healthMeasurementText(measurement: ExperimentExposureHealthMeasurement): string {
- const threshold =
- measurement.unit === 'p_value'
- ? `${measurement.threshold}`
- : formatMeasuredValue(measurement.threshold, 'percentage')
- return `Measured ${formatMeasuredValue(measurement.value, measurement.unit)}. Warns ${measurement.warns} ${threshold}.`
-}
diff --git a/products/experiments/frontend/health/healthPanelFindings.test.ts b/products/experiments/frontend/health/healthPanelFindings.test.ts
index 612adb5ce281..2c7ef15f2387 100644
--- a/products/experiments/frontend/health/healthPanelFindings.test.ts
+++ b/products/experiments/frontend/health/healthPanelFindings.test.ts
@@ -21,6 +21,7 @@ const serverFinding = (
evidence: {},
actions: [],
diagnostic_ref: null,
+ check: 'no_metric',
})
const exposureFinding = (code: ExperimentExposureHealthFindingCode): ExperimentExposureHealthFinding => ({
diff --git a/products/experiments/frontend/health/healthPanelFindings.ts b/products/experiments/frontend/health/healthPanelFindings.ts
index 6b9a46bce08a..de27a38532eb 100644
--- a/products/experiments/frontend/health/healthPanelFindings.ts
+++ b/products/experiments/frontend/health/healthPanelFindings.ts
@@ -2,7 +2,11 @@ import { dayjs } from 'lib/dayjs'
import type { ExperimentExposureQueryResponse } from '~/queries/schema/schema-general'
-import type { ExperimentHealthApi, ExperimentHealthFindingApi } from '../generated/api.schemas'
+import type {
+ ExperimentHealthApi,
+ ExperimentHealthCheckKeyEnumApi,
+ ExperimentHealthFindingApi,
+} from '../generated/api.schemas'
import type { ExperimentHealthFindingActionKind, ExperimentHealthFindingCode } from './experimentHealthFindingEvents'
/** A finding of the health panel, from the experiment read or from the exposure answer. */
@@ -12,6 +16,8 @@ export type HealthPanelFinding = Pick<
> & {
code: ExperimentHealthFindingCode
actions: ExperimentHealthFindingActionKind[]
+ /** The check that reported it. Missing on an exposure answer cached before the server added the field. */
+ check?: ExperimentHealthCheckKeyEnumApi
}
const SEVERITY_ORDER: Record<HealthPanelFinding['severity'], number> = { critical: 0, warning: 1, info: 2 }
diff --git a/products/experiments/frontend/health/healthPanelSelection.test.ts b/products/experiments/frontend/health/healthPanelSelection.test.ts
new file mode 100644
index 000000000000..3b1c2cea17eb
--- /dev/null
+++ b/products/experiments/frontend/health/healthPanelSelection.test.ts
@@ -0,0 +1,26 @@
+import type { HealthPanelFinding } from './healthPanelFindings'
+import { resolveHealthPanelSelection } from './healthPanelSelection'
+
+const ISSUE: HealthPanelFinding = {
+ code: 'srm',
+ subcode: null,
+ reason: null,
+ severity: 'warning',
+ title: 'Users are not split as configured',
+ detail: '',
+ evidence: {},
+ actions: [],
+}
+
+describe('resolveHealthPanelSelection', () => {
+ it.each([
+ { name: 'opens the first issue without a pick', picked: null, expected: 'finding' },
+ {
+ name: 'keeps the summary that the reader picked over an issue',
+ picked: { kind: 'summary' },
+ expected: 'summary',
+ },
+ ] as const)('$name', ({ picked, expected }) => {
+ expect(resolveHealthPanelSelection(picked, [ISSUE], []).kind).toEqual(expected)
+ })
+})
diff --git a/products/experiments/frontend/health/healthPanelSelection.ts b/products/experiments/frontend/health/healthPanelSelection.ts
index a1ea6bfbfb43..ec2b6823f2a1 100644
--- a/products/experiments/frontend/health/healthPanelSelection.ts
+++ b/products/experiments/frontend/health/healthPanelSelection.ts
@@ -15,13 +15,16 @@ export function healthFindingKey(finding: HealthPanelFinding): string {
return `${finding.code}:${finding.subcode ?? ''}`
}
-// A refresh can remove the picked finding or check, so the pick holds only while it is still on the list.
+// A refresh can remove the picked finding or check, so the pick holds only while it is still on the list. The summary
+// is always there.
export function resolveHealthPanelSelection(
picked: HealthPanelSelection | null,
findings: HealthPanelFinding[],
checks: ListedHealthCheck[]
): ResolvedHealthPanelSelection {
switch (picked?.kind) {
+ case 'summary':
+ return picked
case 'finding': {
const finding = findings.find((candidate) => healthFindingKey(candidate) === picked.key)
if (finding) {
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index faf938b09c3e..1cbaa2dc99f0 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -43103,6 +43103,60 @@ export namespace Schemas {
EditReleaseConditions: 'edit_release_conditions',
} as const;
+ /**
+ * * `flag_state` - Flag State
+ * * `flag_shared` - Flag Shared
+ * * `flag_history` - Flag History
+ * * `split_changed_mid_run` - Split Changed Mid Run
+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
+ * * `flag_variants_mismatch` - Flag Variants Mismatch
+ * * `forced_variant_release_condition` - Forced Variant Release Condition
+ * * `new_flag_no_traffic` - New Flag No Traffic
+ * * `no_metric` - No Metric
+ * * `metric_event_not_ingested` - Metric Event Not Ingested
+ * * `metric_cannot_compile` - Metric Cannot Compile
+ * * `metric_calculation_failed` - Metric Calculation Failed
+ * * `metric_all_zero` - Metric All Zero
+ * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
+ * * `exposure_filters` - Exposure Filters
+ * * `early_significance` - Early Significance
+ * * `metrics_not_matured` - Metrics Not Matured
+ * * `zero_exposures` - Zero Exposures
+ * * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposures
+ * * `srm` - Srm
+ * * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
+ * * `multiple_variants_share` - Multiple Variants Share
+ */
+ export type ExperimentHealthCheckKeyEnum = typeof ExperimentHealthCheckKeyEnum[keyof typeof ExperimentHealthCheckKeyEnum];
+
+
+ export const ExperimentHealthCheckKeyEnum = {
+ FlagState: 'flag_state',
+ FlagShared: 'flag_shared',
+ FlagHistory: 'flag_history',
+ SplitChangedMidRun: 'split_changed_mid_run',
+ BucketingChangedMidRun: 'bucketing_changed_mid_run',
+ ReleaseConditionsChangedMidRun: 'release_conditions_changed_mid_run',
+ FlagVariantsMismatch: 'flag_variants_mismatch',
+ ForcedVariantReleaseCondition: 'forced_variant_release_condition',
+ NewFlagNoTraffic: 'new_flag_no_traffic',
+ NoMetric: 'no_metric',
+ MetricEventNotIngested: 'metric_event_not_ingested',
+ MetricCannotCompile: 'metric_cannot_compile',
+ MetricCalculationFailed: 'metric_calculation_failed',
+ MetricAllZero: 'metric_all_zero',
+ ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',
+ ExposureFilters: 'exposure_filters',
+ EarlySignificance: 'early_significance',
+ MetricsNotMatured: 'metrics_not_matured',
+ ZeroExposures: 'zero_exposures',
+ TestAccountFilterRemovesExposures: 'test_account_filter_removes_exposures',
+ Srm: 'srm',
+ BiasRiskMultipleExcluded: 'bias_risk_multiple_excluded',
+ MultipleVariantsShare: 'multiple_variants_share',
+ } as const;
+
/**
* The values behind the finding, such as the key of a shipped variant or the share of users exposed to multiple variants. The keys depend on the code.
*/
@@ -43166,62 +43220,34 @@ export namespace Schemas {
* @nullable
*/
diagnostic_ref: string | null;
+ /** The key of the check in `checks` that reported the finding. A check can report several codes, so the code doesn't name the check.
+ *
+ * * `flag_state` - Flag State
+ * * `flag_shared` - Flag Shared
+ * * `flag_history` - Flag History
+ * * `split_changed_mid_run` - Split Changed Mid Run
+ * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
+ * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
+ * * `flag_variants_mismatch` - Flag Variants Mismatch
+ * * `forced_variant_release_condition` - Forced Variant Release Condition
+ * * `new_flag_no_traffic` - New Flag No Traffic
+ * * `no_metric` - No Metric
+ * * `metric_event_not_ingested` - Metric Event Not Ingested
+ * * `metric_cannot_compile` - Metric Cannot Compile
+ * * `metric_calculation_failed` - Metric Calculation Failed
+ * * `metric_all_zero` - Metric All Zero
+ * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
+ * * `exposure_filters` - Exposure Filters
+ * * `early_significance` - Early Significance
+ * * `metrics_not_matured` - Metrics Not Matured
+ * * `zero_exposures` - Zero Exposures
+ * * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposures
+ * * `srm` - Srm
+ * * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
+ * * `multiple_variants_share` - Multiple Variants Share */
+ check: ExperimentHealthCheckKeyEnum;
}
- /**
- * * `flag_state` - Flag State
- * * `flag_shared` - Flag Shared
- * * `flag_history` - Flag History
- * * `split_changed_mid_run` - Split Changed Mid Run
- * * `bucketing_changed_mid_run` - Bucketing Changed Mid Run
- * * `release_conditions_changed_mid_run` - Release Conditions Changed Mid Run
- * * `flag_variants_mismatch` - Flag Variants Mismatch
- * * `forced_variant_release_condition` - Forced Variant Release Condition
- * * `new_flag_no_traffic` - New Flag No Traffic
- * * `no_metric` - No Metric
- * * `metric_event_not_ingested` - Metric Event Not Ingested
- * * `metric_cannot_compile` - Metric Cannot Compile
- * * `metric_calculation_failed` - Metric Calculation Failed
- * * `metric_all_zero` - Metric All Zero
- * * `exposure_criteria_cannot_match` - Exposure Criteria Cannot Match
- * * `exposure_filters` - Exposure Filters
- * * `early_significance` - Early Significance
- * * `metrics_not_matured` - Metrics Not Matured
- * * `zero_exposures` - Zero Exposures
- * * `test_account_filter_removes_exposures` - Test Account Filter Removes Exposures
- * * `srm` - Srm
- * * `bias_risk_multiple_excluded` - Bias Risk Multiple Excluded
- * * `multiple_variants_share` - Multiple Variants Share
- */
- export type ExperimentHealthCheckKeyEnum = typeof ExperimentHealthCheckKeyEnum[keyof typeof ExperimentHealthCheckKeyEnum];
-
-
- export const ExperimentHealthCheckKeyEnum = {
- FlagState: 'flag_state',
- FlagShared: 'flag_shared',
- FlagHistory: 'flag_history',
- SplitChangedMidRun: 'split_changed_mid_run',
- BucketingChangedMidRun: 'bucketing_changed_mid_run',
- ReleaseConditionsChangedMidRun: 'release_conditions_changed_mid_run',
- FlagVariantsMismatch: 'flag_variants_mismatch',
- ForcedVariantReleaseCondition: 'forced_variant_release_condition',
- NewFlagNoTraffic: 'new_flag_no_traffic',
- NoMetric: 'no_metric',
- MetricEventNotIngested: 'metric_event_not_ingested',
- MetricCannotCompile: 'metric_cannot_compile',
- MetricCalculationFailed: 'metric_calculation_failed',
- MetricAllZero: 'metric_all_zero',
- ExposureCriteriaCannotMatch: 'exposure_criteria_cannot_match',
- ExposureFilters: 'exposure_filters',
- EarlySignificance: 'early_significance',
- MetricsNotMatured: 'metrics_not_matured',
- ZeroExposures: 'zero_exposures',
- TestAccountFilterRemovesExposures: 'test_account_filter_removes_exposures',
- Srm: 'srm',
- BiasRiskMultipleExcluded: 'bias_risk_multiple_excluded',
- MultipleVariantsShare: 'multiple_variants_share',
- } as const;
-
/**
* * `passed` - Passed
* * `finding` - Finding
@@ -43239,46 +43265,147 @@ export namespace Schemas {
} as const;
/**
- * * `percentage` - Percentage of exposed units
- * * `p_value` - P-value of a statistical test
- */
- export type ExperimentHealthMeasurementUnitEnum = typeof ExperimentHealthMeasurementUnitEnum[keyof typeof ExperimentHealthMeasurementUnitEnum];
-
-
- export const ExperimentHealthMeasurementUnitEnum = {
- Percentage: 'percentage',
+ * * `exposed_count` - Users or groups exposed since launch, as the exposure criteria count them, including those exposed to multiple variants
+ * * `compared_count` - Exposed users or groups in the variants that the sample ratio test compares, without those exposed to multiple variants
+ * * `multiple_variant_count` - Users or groups exposed to more than one variant
+ * * `multiple_variant_percentage` - multiple_variant_count as a share of exposed_count
+ * * `multiple_variant_handling` - What the results do with users exposed to more than one variant: exclude or first_seen
+ * * `p_value` - P-value of the sample ratio test
+ * * `since_launch_hours` - Hours since launch
+ * * `exposed_without_test_account_filter_count` - Users or groups that the exposure criteria count without the test account filters
+ * * `kept_by_test_account_filter_percentage` - exposed_count as a share of exposed_without_test_account_filter_count
+ * * `only_count_matured_users` - Whether "Require completed conversion or retention window" is on
+ * * `primary_metric_count` - Primary metrics that the calculation computes
+ * * `waiting_metric_count` - Of the primary metrics, those whose conversion or retention window hasn't passed for any user yet
+ * * `metric_count` - Metrics that the calculation computes
+ * * `completed_metric_count` - Metrics whose result of the latest calculation the page shows
+ * * `failed_metric_count` - Metrics whose latest calculation failed
+ * * `calculated_at` - When the latest calculation of the metrics finished
+ * * `flag_active` - Whether the feature flag is enabled
+ * * `flag_deleted` - Whether the feature flag is deleted
+ * * `flag_rolled_out` - Whether a release condition of the feature flag rolls out to more than 0%
+ * * `shipped_variant` - The key of the variant that the feature flag serves to everyone, or null when it serves more than one
+ */
+ export type ExperimentHealthValueKeyEnum = typeof ExperimentHealthValueKeyEnum[keyof typeof ExperimentHealthValueKeyEnum];
+
+
+ export const ExperimentHealthValueKeyEnum = {
+ ExposedCount: 'exposed_count',
+ ComparedCount: 'compared_count',
+ MultipleVariantCount: 'multiple_variant_count',
+ MultipleVariantPercentage: 'multiple_variant_percentage',
+ MultipleVariantHandling: 'multiple_variant_handling',
PValue: 'p_value',
+ SinceLaunchHours: 'since_launch_hours',
+ ExposedWithoutTestAccountFilterCount: 'exposed_without_test_account_filter_count',
+ KeptByTestAccountFilterPercentage: 'kept_by_test_account_filter_percentage',
+ OnlyCountMaturedUsers: 'only_count_matured_users',
+ PrimaryMetricCount: 'primary_metric_count',
+ WaitingMetricCount: 'waiting_metric_count',
+ MetricCount: 'metric_count',
+ CompletedMetricCount: 'completed_metric_count',
+ FailedMetricCount: 'failed_metric_count',
+ CalculatedAt: 'calculated_at',
+ FlagActive: 'flag_active',
+ FlagDeleted: 'flag_deleted',
+ FlagRolledOut: 'flag_rolled_out',
+ ShippedVariant: 'shipped_variant',
} as const;
/**
* * `above` - Finding above the threshold
* * `below` - Finding below the threshold
*/
- export type ExperimentHealthMeasurementSideEnum = typeof ExperimentHealthMeasurementSideEnum[keyof typeof ExperimentHealthMeasurementSideEnum];
+ export type ExperimentHealthRuleSideEnum = typeof ExperimentHealthRuleSideEnum[keyof typeof ExperimentHealthRuleSideEnum];
- export const ExperimentHealthMeasurementSideEnum = {
+ export const ExperimentHealthRuleSideEnum = {
Above: 'above',
Below: 'below',
} as const;
- export interface ExperimentHealthMeasurement {
- /** The value that the check measured on the experiment. */
- value: number;
- /** The value at which the check reports a finding. */
- threshold: number;
- /** percentage: a share of the exposed users or groups, on the 0-100 scale. p_value: the p-value of a statistical test.
+ /**
+ * * `chi_squared` - Chi-squared goodness-of-fit test
+ */
+ export type ExperimentHealthRuleTestEnum = typeof ExperimentHealthRuleTestEnum[keyof typeof ExperimentHealthRuleTestEnum];
+
+
+ export const ExperimentHealthRuleTestEnum = {
+ ChiSquared: 'chi_squared',
+ } as const;
+
+ export interface ExperimentHealthRuleMinimum {
+ /** The key in `values` that must reach the threshold before the check judges.
*
- * * `percentage` - Percentage of exposed units
- * * `p_value` - P-value of a statistical test */
- unit: ExperimentHealthMeasurementUnitEnum;
- /** The side of the threshold that reports a finding: above or below.
+ * * `exposed_count` - Users or groups exposed since launch, as the exposure criteria count them, including those exposed to multiple variants
+ * * `compared_count` - Exposed users or groups in the variants that the sample ratio test compares, without those exposed to multiple variants
+ * * `multiple_variant_count` - Users or groups exposed to more than one variant
+ * * `multiple_variant_percentage` - multiple_variant_count as a share of exposed_count
+ * * `multiple_variant_handling` - What the results do with users exposed to more than one variant: exclude or first_seen
+ * * `p_value` - P-value of the sample ratio test
+ * * `since_launch_hours` - Hours since launch
+ * * `exposed_without_test_account_filter_count` - Users or groups that the exposure criteria count without the test account filters
+ * * `kept_by_test_account_filter_percentage` - exposed_count as a share of exposed_without_test_account_filter_count
+ * * `only_count_matured_users` - Whether "Require completed conversion or retention window" is on
+ * * `primary_metric_count` - Primary metrics that the calculation computes
+ * * `waiting_metric_count` - Of the primary metrics, those whose conversion or retention window hasn't passed for any user yet
+ * * `metric_count` - Metrics that the calculation computes
+ * * `completed_metric_count` - Metrics whose result of the latest calculation the page shows
+ * * `failed_metric_count` - Metrics whose latest calculation failed
+ * * `calculated_at` - When the latest calculation of the metrics finished
+ * * `flag_active` - Whether the feature flag is enabled
+ * * `flag_deleted` - Whether the feature flag is deleted
+ * * `flag_rolled_out` - Whether a release condition of the feature flag rolls out to more than 0%
+ * * `shipped_variant` - The key of the variant that the feature flag serves to everyone, or null when it serves more than one */
+ value: ExperimentHealthValueKeyEnum;
+ /** The value that the key must reach. Below it, the check reports no finding. */
+ threshold: number;
+ }
+
+ export interface ExperimentHealthCheckRule {
+ /** The key in `values` that the check compares with the threshold.
+ *
+ * * `exposed_count` - Users or groups exposed since launch, as the exposure criteria count them, including those exposed to multiple variants
+ * * `compared_count` - Exposed users or groups in the variants that the sample ratio test compares, without those exposed to multiple variants
+ * * `multiple_variant_count` - Users or groups exposed to more than one variant
+ * * `multiple_variant_percentage` - multiple_variant_count as a share of exposed_count
+ * * `multiple_variant_handling` - What the results do with users exposed to more than one variant: exclude or first_seen
+ * * `p_value` - P-value of the sample ratio test
+ * * `since_launch_hours` - Hours since launch
+ * * `exposed_without_test_account_filter_count` - Users or groups that the exposure criteria count without the test account filters
+ * * `kept_by_test_account_filter_percentage` - exposed_count as a share of exposed_without_test_account_filter_count
+ * * `only_count_matured_users` - Whether "Require completed conversion or retention window" is on
+ * * `primary_metric_count` - Primary metrics that the calculation computes
+ * * `waiting_metric_count` - Of the primary metrics, those whose conversion or retention window hasn't passed for any user yet
+ * * `metric_count` - Metrics that the calculation computes
+ * * `completed_metric_count` - Metrics whose result of the latest calculation the page shows
+ * * `failed_metric_count` - Metrics whose latest calculation failed
+ * * `calculated_at` - When the latest calculation of the metrics finished
+ * * `flag_active` - Whether the feature flag is enabled
+ * * `flag_deleted` - Whether the feature flag is deleted
+ * * `flag_rolled_out` - Whether a release condition of the feature flag rolls out to more than 0%
+ * * `shipped_variant` - The key of the variant that the feature flag serves to everyone, or null when it serves more than one */
+ value: ExperimentHealthValueKeyEnum;
+ /** The threshold, in the unit that the suffix of the key gives. */
+ threshold: number;
+ /** above: the check reports a finding when the value is above the threshold. below: when the value is below it. A value at the threshold passes.
*
* * `above` - Finding above the threshold
* * `below` - Finding below the threshold */
- warns: ExperimentHealthMeasurementSideEnum;
+ warns: ExperimentHealthRuleSideEnum;
+ /** The statistical test that computes the value, for example chi_squared. Null for a count or a share.
+ *
+ * * `chi_squared` - Chi-squared goodness-of-fit test */
+ test: ExperimentHealthRuleTestEnum | null;
+ /** What the check needs before it judges. Null when the check judges from the first value. */
+ minimum: ExperimentHealthRuleMinimum | null;
}
+ /**
+ * What the check read on the experiment, for every status, as numbers, strings, booleans or null. Empty when the check read nothing or shows nothing. The suffix of a key gives the unit: _count is a count, _percentage is on the 0-100 scale, _hours is hours and _at is an ISO 8601 time in UTC. A key without a suffix is a setting or a state. A key means the same thing in every check. `rule.value` names the key that decides.
+ */
+ export type ExperimentHealthCheckResultValues = {[key: string]: string | number | boolean | null};
+
export interface ExperimentHealthCheckResult {
/** Stable identifier of the check. A check can report several finding codes, so the key is not a finding code.
*
@@ -43322,8 +43449,10 @@ export namespace Schemas {
* @nullable
*/
reason: string | null;
- /** The value that decided a passed check, against its threshold. Null for the other statuses, and for a pass without a value. A finding carries its values in `evidence`. */
- measurement: ExperimentHealthMeasurement | null;
+ /** What the check read on the experiment, for every status, as numbers, strings, booleans or null. Empty when the check read nothing or shows nothing. The suffix of a key gives the unit: _count is a count, _percentage is on the 0-100 scale, _hours is hours and _at is an ISO 8601 time in UTC. A key without a suffix is a setting or a state. A key means the same thing in every check. `rule.value` names the key that decides. */
+ values: ExperimentHealthCheckResultValues;
+ /** How the check decides. The same for every experiment and every status. Null for a check without a numeric rule. */
+ rule: ExperimentHealthCheckRule | null;
}
export interface ExperimentHealth {
@@ -43579,6 +43708,11 @@ export namespace Schemas {
multiple_variant_handling?: MultipleVariantHandling | null;
}
+ /**
+ * What the check read on the experiment, for every status. The suffix of a key gives the unit: _count is a count, _percentage is on the 0-100 scale, _hours is hours. A key without a suffix is a setting. Missing on an answer cached before the field existed.
+ */
+ export type ExperimentExposureHealthCheckValues = {[key: string]: string | number | boolean | null} | null;
+
export type ExperimentExposureHealthCheckKey = typeof ExperimentExposureHealthCheckKey[keyof typeof ExperimentExposureHealthCheckKey];
@@ -43590,31 +43724,47 @@ export namespace Schemas {
TestAccountFilterRemovesExposures: 'test_account_filter_removes_exposures',
} as const;
- export type ExperimentExposureHealthMeasurementUnit = typeof ExperimentExposureHealthMeasurementUnit[keyof typeof ExperimentExposureHealthMeasurementUnit];
+ export type ExperimentExposureHealthValueKey = typeof ExperimentExposureHealthValueKey[keyof typeof ExperimentExposureHealthValueKey];
- export const ExperimentExposureHealthMeasurementUnit = {
- Percentage: 'percentage',
+ export const ExperimentExposureHealthValueKey = {
+ ExposedCount: 'exposed_count',
+ ComparedCount: 'compared_count',
+ MultipleVariantCount: 'multiple_variant_count',
+ MultipleVariantPercentage: 'multiple_variant_percentage',
+ MultipleVariantHandling: 'multiple_variant_handling',
PValue: 'p_value',
+ SinceLaunchHours: 'since_launch_hours',
+ ExposedWithoutTestAccountFilterCount: 'exposed_without_test_account_filter_count',
+ KeptByTestAccountFilterPercentage: 'kept_by_test_account_filter_percentage',
} as const;
- export type ExperimentExposureHealthMeasurementSide = typeof ExperimentExposureHealthMeasurementSide[keyof typeof ExperimentExposureHealthMeasurementSide];
+ export interface ExperimentExposureHealthRuleMinimum {
+ /** The value that the key must reach. Below it, the check reports no finding. */
+ threshold: number;
+ /** The key in `values` that must reach the threshold before the check judges. */
+ value: ExperimentExposureHealthValueKey;
+ }
+ export type ExperimentExposureHealthRuleSide = typeof ExperimentExposureHealthRuleSide[keyof typeof ExperimentExposureHealthRuleSide];
- export const ExperimentExposureHealthMeasurementSide = {
+
+ export const ExperimentExposureHealthRuleSide = {
Above: 'above',
Below: 'below',
} as const;
- export interface ExperimentExposureHealthMeasurement {
- /** The value at which the check reports a finding. */
+ export interface ExperimentExposureHealthCheckRule {
+ /** What the check needs before it judges. Null when the check judges from the first value. */
+ minimum: ExperimentExposureHealthRuleMinimum | null;
+ /** The statistical test that computes the value. Null for a count or a share. */
+ test: 'chi_squared' | null;
+ /** The threshold, in the unit that the suffix of the key gives. */
threshold: number;
- /** percentage: a share of the exposed users or groups, on the 0-100 scale. p_value: the p-value of a statistical test. */
- unit: ExperimentExposureHealthMeasurementUnit;
- /** The value that the check measured on the experiment. */
- value: number;
- /** The side of the threshold that reports a finding. */
- warns: ExperimentExposureHealthMeasurementSide;
+ /** The key in `values` that the check compares with the threshold. */
+ value: ExperimentExposureHealthValueKey;
+ /** above: the check reports a finding when the value is above the threshold. below: when the value is below it. A value at the threshold passes. */
+ warns: ExperimentExposureHealthRuleSide;
}
export type ExperimentExposureHealthCheckStatus = typeof ExperimentExposureHealthCheckStatus[keyof typeof ExperimentExposureHealthCheckStatus];
@@ -43632,14 +43782,16 @@ export namespace Schemas {
description?: string | null;
/** Stable identifier of the check. */
key: ExperimentExposureHealthCheckKey;
- /** The value that decided a passed check. Null for the other statuses and for a pass without a value. */
- measurement?: ExperimentExposureHealthMeasurement | null;
/** The problem the check looks for, as the health panel lists it. */
name: string;
/** Why the check did not look, for the statuses not_applicable and not_checked, for example 'Runs after launch'. Null for passed and finding. */
reason: string | null;
+ /** How the check decides, the same for every experiment. Null for a check without a numeric rule. Missing on an answer cached before the field existed. */
+ rule?: ExperimentExposureHealthCheckRule | null;
/** passed: no problem found. finding: a finding in `health_findings`. not_applicable: the check doesn't cover the experiment's state. not_checked: an input was missing or the check failed. */
status: ExperimentExposureHealthCheckStatus;
+ /** What the check read on the experiment, for every status. The suffix of a key gives the unit: _count is a count, _percentage is on the 0-100 scale, _hours is hours. A key without a suffix is a setting. Missing on an answer cached before the field existed. */
+ values?: ExperimentExposureHealthCheckValues;
}
/**
@@ -43680,6 +43832,8 @@ export namespace Schemas {
export interface ExperimentExposureHealthFinding {
/** The actions that fix the problem, in order of preference. */
actions: ExperimentExposureHealthFindingActionKind[];
+ /** The key of the check in `health_checks` that reported the finding. Missing on an answer cached before the field existed. */
+ check?: ExperimentExposureHealthCheckKey | null;
/** Stable identifier of the problem. Each code has one meaning across every surface that reports it. */
code: ExperimentExposureHealthFindingCode;
/** What is wrong, what it does to the experiment, and how to fix it. */