:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(experiments): make reason a field of a health finding (#125364)
Marcel Poelker1bcf906e7e3b
diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.jsonindex ec5bc3c9ecb6..5cc1dbd9422c 100644--- a/frontend/src/queries/schema.json+++ b/frontend/src/queries/schema.json@@ -26351,6 +26351,10 @@"description": "The values behind the finding, such as the p-value of the sample ratio test. The keys depend on the code.","type": "object"},+ "reason": {+ "description": "A stable qualifier below the subcode, for example where a sample ratio mismatch is located. Null when the finding names none.",+ "type": ["string", "null"]+ },"severity": {"$ref": "#/definitions/ExperimentExposureHealthFindingSeverity","description": "How much the problem affects the results: critical, warning, or info."@@ -26364,7 +26368,17 @@"type": "string"}},- "required": ["code", "subcode", "severity", "title", "detail", "evidence", "actions", "diagnostic_ref"],+ "required": [+ "code",+ "subcode",+ "reason",+ "severity",+ "title",+ "detail",+ "evidence",+ "actions",+ "diagnostic_ref"+ ],"type": "object"},"ExperimentExposureHealthFindingActionKind": {diff --git a/frontend/src/queries/schema/schema-general.ts b/frontend/src/queries/schema/schema-general.tsindex 80868767483a..71f54cf34754 100644--- a/frontend/src/queries/schema/schema-general.ts+++ b/frontend/src/queries/schema/schema-general.ts@@ -6560,6 +6560,8 @@ export interface ExperimentExposureHealthFinding {code: ExperimentExposureHealthFindingCode/** The case within the code, when a code covers several. Null when the code has one case. */subcode: string | null+ /** A stable qualifier below the subcode, for example where a sample ratio mismatch is located. Null when the finding names none. */+ reason: string | null/** How much the problem affects the results: critical, warning, or info. */severity: ExperimentExposureHealthFindingSeverity/** One-line summary of the problem. */diff --git a/frontend/src/scenes/experiments/ExperimentView/ExperimentWarningBanners.test.tsx b/frontend/src/scenes/experiments/ExperimentView/ExperimentWarningBanners.test.tsxindex 81a117ea21a0..6258902595fc 100644--- a/frontend/src/scenes/experiments/ExperimentView/ExperimentWarningBanners.test.tsx+++ b/frontend/src/scenes/experiments/ExperimentView/ExperimentWarningBanners.test.tsx@@ -60,6 +60,7 @@ describe('ExperimentWarningBanner', () => {{code: 'variant_shipped_while_running',subcode: 'running_but_single_variant_shipped',+ reason: null,severity: 'warning',title: '',detail: '',diff --git a/frontend/src/scenes/experiments/ExperimentView/Exposures.tsx b/frontend/src/scenes/experiments/ExperimentView/Exposures.tsxindex 403013eb2fa8..9ef683aa39b1 100644--- a/frontend/src/scenes/experiments/ExperimentView/Exposures.tsx+++ b/frontend/src/scenes/experiments/ExperimentView/Exposures.tsx@@ -18,7 +18,6 @@ import { teamLogic } from 'scenes/teamLogic'import { ExperimentExposureCriteria, ExperimentExposureQueryResponse } from '~/queries/schema/schema-general'import { EXPERIMENT_VARIANT_MULTIPLE } from 'products/experiments/frontend/constants'-import { healthFindingReason } from 'products/experiments/frontend/health/experimentHealthFindingEvents'import {getTotalExposures,hasSampleRatioMismatch,@@ -185,9 +184,9 @@ export function Exposures(): JSX.Element {? srmFinding? {code: 'srm',- variant: srmFinding.subcode ?? undefined,+ subcode: srmFinding.subcode ?? undefined,severity: srmFinding.severity,- reason: healthFindingReason(srmFinding.evidence),+ reason: srmFinding.reason ?? undefined,}: { code: 'srm' }: null,diff --git a/frontend/src/scenes/experiments/ExperimentView/healthFindingReporting.test.tsx b/frontend/src/scenes/experiments/ExperimentView/healthFindingReporting.test.tsxindex af5f76187945..e80e11f55a8f 100644--- a/frontend/src/scenes/experiments/ExperimentView/healthFindingReporting.test.tsx+++ b/frontend/src/scenes/experiments/ExperimentView/healthFindingReporting.test.tsx@@ -35,6 +35,7 @@ const NO_EXPOSURES = {const exposureFinding = (code: string, title: string, actions: string[]): Record<string, unknown> => ({code,subcode: null,+ reason: null,severity: 'warning',title,detail: title,@@ -108,6 +109,7 @@ describe('health finding reporting', () => {{code: 'flag_off_while_running',subcode: 'running_but_flag_disabled',+ reason: null,severity: 'warning',title: 'The experiment is paused',detail: 'The linked feature flag is disabled while the experiment has not been ended.',@@ -195,7 +197,7 @@ describe('health finding reporting', () => {const shown = captureSpy.mock.calls.filter(([event]) => event === 'experiment health finding shown')- .map(([, properties]) => [properties.finding_code, properties.finding_variant, properties.finding_severity])+ .map(([, properties]) => [properties.finding_code, properties.finding_subcode, properties.finding_severity])expect(shown).toEqual([['flag_off_while_running', 'running_but_flag_disabled', 'warning'],['srm', null, 'warning'],diff --git a/frontend/src/scenes/experiments/experimentLogic.test.ts b/frontend/src/scenes/experiments/experimentLogic.test.tsindex 0facbd4c3c92..2bbe72d77d26 100644--- a/frontend/src/scenes/experiments/experimentLogic.test.ts+++ b/frontend/src/scenes/experiments/experimentLogic.test.ts@@ -132,7 +132,7 @@ describe('experimentLogic', () => {sample_ratio_mismatch: { expected: { control: 475, test: 475 }, p_value: 0.0001 },bias_risk: { multiple_variant_percentage: 5 },health_findings: [- { code: 'srm', subcode: 'split_changed', evidence: { p_value: 0.0001, reason: 'library' } },+ { code: 'srm', subcode: 'split_changed', reason: 'library', evidence: { p_value: 0.0001 } },{ code: 'bias_risk_multiple_excluded' },],}@@ -217,7 +217,8 @@ describe('experimentLogic', () => {{code: 'zero_exposures',subcode: 'calls_not_counted',- evidence: { hours_since_launch: 48, reason: 'test_account_filter' },+ reason: 'test_account_filter',+ evidence: { hours_since_launch: 48 },},],},@@ -2376,6 +2377,7 @@ describe('experimentLogic', () => {): ExperimentHealthFindingApi => ({code,subcode,+ reason: null,severity: 'warning',title: '',detail: '',@@ -2647,7 +2649,8 @@ describe('experimentLogic', () => {health: {findings: [{- ...healthFinding('split_changed_mid_run', 'not_checked', { reason: 'exposure_filters' }),+ ...healthFinding('split_changed_mid_run', 'not_checked'),+ reason: 'exposure_filters',severity: 'info',},],@@ -2836,7 +2839,7 @@ describe('experimentLogic', () => {const draft = { ...experiment, id: 7, status: ExperimentStatus.Draft, start_date: undefined } as Experimentconst finding: ExperimentHealthFinding = {code: 'flag_live_before_launch',- variant: 'not_started_but_multiple_variants_rolled_out',+ subcode: 'not_started_but_multiple_variants_rolled_out',}logic.actions.loadExperimentSuccess(draft)@@ -2851,7 +2854,7 @@ describe('experimentLogic', () => {experiment_status: 'draft',experiment_days_since_start: null,finding_code: 'flag_live_before_launch',- finding_variant: 'not_started_but_multiple_variants_rolled_out',+ finding_subcode: 'not_started_but_multiple_variants_rolled_out',finding_severity: null,finding_reason: null,health_ui: 'warnings',@@ -2892,7 +2895,7 @@ describe('experimentLogic', () => {experiment_status: 'running',experiment_days_since_start: 3,finding_code: 'bias_risk_multiple_excluded',- finding_variant: null,+ finding_subcode: null,finding_severity: null,finding_reason: null,health_ui: 'panel',@@ -2919,7 +2922,7 @@ describe('experimentLogic', () => {experiment_status: 'running',experiment_days_since_start: 3,finding_code: 'zero_exposures',- finding_variant: null,+ finding_subcode: null,finding_severity: null,finding_reason: null,health_ui: 'warnings',diff --git a/frontend/src/scenes/experiments/experimentLogic.tsx b/frontend/src/scenes/experiments/experimentLogic.tsxindex db03f4db3d3b..e77aac5739de 100644--- a/frontend/src/scenes/experiments/experimentLogic.tsx+++ b/frontend/src/scenes/experiments/experimentLogic.tsx@@ -1549,7 +1549,7 @@ export const experimentLogic = kea<experimentLogicType>([reportHealthFindingShown: ({ finding }) => {// The page remounts its warnings within one load, for example on a tab change or after// a save, and each mount reports again.- const key = `${values.experimentLoadCount}:${finding.code}:${finding.variant ?? ''}`+ const key = `${values.experimentLoadCount}:${finding.code}:${finding.subcode ?? ''}`cache.shownHealthFindingKeys ??= new Set<string>()if (cache.shownHealthFindingKeys.has(key)) {returndiff --git a/posthog/schema.py b/posthog/schema.pyindex d9d777eadc0c..0bf1c8366694 100644--- a/posthog/schema.py+++ b/posthog/schema.py@@ -5913,6 +5913,13 @@ class ExperimentExposureHealthFinding(BaseModel):"The values behind the finding, such as the p-value of the sample ratio test. The keys depend on the code."),)+ reason: str | None = Field(+ ...,+ description=(+ "A stable qualifier below the subcode, for example where a sample ratio"+ " mismatch is located. Null when the finding names none."+ ),+ )severity: ExperimentExposureHealthFindingSeverity = Field(...,description=("How much the problem affects the results: critical, warning, or info."),diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.tsindex 62de6013b5ed..b40e405654a8 100644--- a/products/desktop/packages/api-client/src/generated.ts+++ b/products/desktop/packages/api-client/src/generated.ts@@ -10053,6 +10053,10 @@ export namespace Schemas {* The case within the code, when a code covers several, for example 'running_but_no_rollout' within 'flag_off_while_running'. Null when the code has one case.*/subcode: string | null;+ /**+ * A stable qualifier below the subcode: where a sample ratio mismatch is located ('library', 'page'), what leaves nobody exposed ('test_account_filter'), or why a check could not read what it needs ('no_history_access'). Null when the finding names none.+ */+ reason: string | null;/*** How much the problem affects the results: critical, warning, or info.*@@ -10576,6 +10580,10 @@ export namespace Schemas {* The values behind the finding, such as the p-value of the sample ratio test. The keys depend on the code.*/evidence: Record<string, string | number | null>;+ /**+ * A stable qualifier below the subcode, for example where a sample ratio mismatch is located. Null when the finding names none.+ */+ reason: string | null;/*** How much the problem affects the results: critical, warning, or info.*/diff --git a/products/experiments/backend/experiment_service.py b/products/experiments/backend/experiment_service.pyindex 396ae0a44e26..9f6f83b17d7a 100644--- a/products/experiments/backend/experiment_service.py+++ b/products/experiments/backend/experiment_service.py@@ -372,10 +372,7 @@ def _apply_holdout(filters: dict, holdout: ExperimentHoldout | None) -> dict:def _tracked_health_finding(finding: ExperimentHealthFinding) -> str:# pinned: `code:subcode:severity:reason`, as on the page's `experiment health checked`. Insights split each# string on `:`, so every field is present, empty when the finding has none.- reason = finding.evidence.get("reason")- return ":".join(- [finding.code.value, finding.subcode or "", finding.severity.value, reason if isinstance(reason, str) else ""]- )+ return ":".join([finding.code.value, finding.subcode or "", finding.severity.value, finding.reason or ""])class ExperimentVersionConflict(APIException):diff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.pyindex 07c44fc30b8f..a793696fc58f 100644--- a/products/experiments/backend/facade/contracts.py+++ b/products/experiments/backend/facade/contracts.py@@ -212,6 +212,8 @@ class ExperimentHealthFinding:code: ExperimentHealthFindingCodesubcode: str | None+ # pinned: the page sends it as `finding_reason` on the health finding events. Never customer text.+ reason: str | None = Noneseverity: ExperimentHealthFindingSeveritytitle: strdetail: strdiff --git a/products/experiments/backend/health/bucketing_changes.py b/products/experiments/backend/health/bucketing_changes.pyindex 70223c783cbc..9e38936980a1 100644--- a/products/experiments/backend/health/bucketing_changes.py+++ b/products/experiments/backend/health/bucketing_changes.py@@ -12,7 +12,7 @@ from products.feature_flags.backend.facade.edits import BucketingChange, Bucketiclass BucketingChangeShape(StrEnum):- # pinned: the finding's subcodes. The page sends them as `finding_variant` on the health finding events.+ # pinned: the finding's subcodes. The page sends them as `finding_subcode` on the health finding events.KEY_CHANGED = "key_changed"AGGREGATION_CHANGED = "aggregation_changed"BUCKETING_IDENTIFIER_CHANGED = "bucketing_identifier_changed"diff --git a/products/experiments/backend/health/checks/conditions_changed.py b/products/experiments/backend/health/checks/conditions_changed.pyindex ce4386dc2660..e6cab952bb71 100644--- a/products/experiments/backend/health/checks/conditions_changed.py+++ b/products/experiments/backend/health/checks/conditions_changed.py@@ -325,6 +325,7 @@ def _finding(sentences=sentences,kind_evidence={**_kind_evidence(effect), **(grade.evidence() if grade is not None else {})},diagnostic_ref=diagnostic_ref,+ reason=grade.finding_reason if grade is not None else None,)diff --git a/products/experiments/backend/health/checks/early_significance.py b/products/experiments/backend/health/checks/early_significance.pyindex 3585c2612d3c..0e34886b3b00 100644--- a/products/experiments/backend/health/checks/early_significance.py+++ b/products/experiments/backend/health/checks/early_significance.py@@ -27,7 +27,7 @@ from products.experiments.backend.logic.result_stats import MetricResultStats, Vfrom products.experiments.backend.running_time_calculator import calculate_detectable_effectfrom products.experiments.backend.running_time_target import RunningTimeTarget-# pinned: sent as `finding_variant` on the health finding events.+# pinned: sent as `finding_subcode` on the health finding events.BELOW_DETECTABLE_EFFECT = "below_detectable_effect"# The results page shows results from 50 exposures (`EXPERIMENT_MIN_EXPOSURES_FOR_RESULTS` indiff --git a/products/experiments/backend/health/checks/exposure_criteria.py b/products/experiments/backend/health/checks/exposure_criteria.pyindex 50e682937b9c..27bf725405b7 100644--- a/products/experiments/backend/health/checks/exposure_criteria.py+++ b/products/experiments/backend/health/checks/exposure_criteria.py@@ -32,7 +32,7 @@ from products.experiments.backend.health.skips import (not_checked,)-# pinned: the subcodes are sent as `finding_variant` on the health finding events, so insights group on them.+# pinned: the subcodes are sent as `finding_subcode` on the health finding events, so insights group on them.EXPOSURE_EVENT_NOT_INGESTED = "exposure_event_not_ingested"FILTER_ON_ANOTHER_FLAG = "filter_on_another_flag"FILTER_ON_PROPERTY_NOT_INGESTED = "filter_on_property_not_ingested"diff --git a/products/experiments/backend/health/checks/exposure_filters.py b/products/experiments/backend/health/checks/exposure_filters.pyindex 10f94cbaefa2..3a22e06646d6 100644--- a/products/experiments/backend/health/checks/exposure_filters.py+++ b/products/experiments/backend/health/checks/exposure_filters.py@@ -23,7 +23,7 @@ from products.experiments.backend.health.context import ()from products.experiments.backend.health.skips import NOT_ON_ARCHIVED, CheckOutcome, not_applicable, not_checked-# pinned: the subcodes are sent as `finding_variant` on the health finding events, so insights group on them.+# pinned: the subcodes are sent as `finding_subcode` on the health finding events, so insights group on them.COHORT_FORCES_VARIANT = "cohort_forces_variant"COHORT_IN_RELEASE_CONDITIONS = "cohort_in_release_conditions"diff --git a/products/experiments/backend/health/checks/flag_change_findings.py b/products/experiments/backend/health/checks/flag_change_findings.pyindex c37a2ec2a6a8..6aa606817725 100644--- a/products/experiments/backend/health/checks/flag_change_findings.py+++ b/products/experiments/backend/health/checks/flag_change_findings.py@@ -39,7 +39,7 @@ from products.experiments.backend.health.skips import ()# pinned: the subcode of a flag history that was not checked against the exposures. The page sends it as-# `finding_variant` on the health finding events.+# `finding_subcode` on the health finding events.NOT_CHECKED = "not_checked"EvidenceValue = str | int | float | bool | None@@ -120,7 +120,8 @@ def flag_history_not_readable(ctx: HealthContext) -> CheckOutcome:else "Review the flag history.")),- evidence={"reason": history.reason.value},+ reason=history.reason.value,+ evidence={},# The flag history is the page this caller can't open.actions=() if no_access else (ExperimentHealthFindingActionKind.OPEN_FEATURE_FLAG_HISTORY,),diagnostic_ref="E3",@@ -139,10 +140,12 @@ def change_finding(sentences: Sequence[str],kind_evidence: Mapping[str, EvidenceValue],diagnostic_ref: str,+ reason: str | None = None,) -> ExperimentHealthFinding:return ExperimentHealthFinding(code=code,subcode=subcode,+ reason=reason,severity=severity,title=title,detail=" ".join(sentences),@@ -201,6 +204,7 @@ def not_checked_finding(return ExperimentHealthFinding(code=code,subcode=NOT_CHECKED,+ reason=history.reason.value,severity=ExperimentHealthFindingSeverity.INFO,title=title,detail=" ".join(@@ -211,10 +215,7 @@ def not_checked_finding(not_checked_next_step(history.reason, change),]),- evidence={- "reason": history.reason.value,- **change_evidence(history, effect, kind_evidence=kind_evidence, exposed_before_change=None),- },+ evidence=change_evidence(history, effect, kind_evidence=kind_evidence, exposed_before_change=None),actions=(ExperimentHealthFindingActionKind.OPEN_FEATURE_FLAG_HISTORY,),diagnostic_ref=diagnostic_ref,)diff --git a/products/experiments/backend/health/checks/flag_state.py b/products/experiments/backend/health/checks/flag_state.pyindex 6a0e5d1142ff..4b22e22e7c1f 100644--- a/products/experiments/backend/health/checks/flag_state.py+++ b/products/experiments/backend/health/checks/flag_state.py@@ -40,14 +40,14 @@ from products.experiments.backend.health.split_changes import ()from products.feature_flags.backend.facade.edits import SplitChange-# pinned: the page's warning keys. The page sends them as `finding_variant` and+# pinned: the page's warning keys. The page sends them as `finding_subcode` and# `warning_key` on its events, and picks the banner text by them.RUNNING_BUT_FLAG_DISABLED = "running_but_flag_disabled"RUNNING_BUT_NO_ROLLOUT = "running_but_no_rollout"RUNNING_BUT_SINGLE_VARIANT_SHIPPED = "running_but_single_variant_shipped"ENDED_BUT_MULTIPLE_VARIANTS_ROLLED_OUT = "ended_but_multiple_variants_rolled_out"NOT_STARTED_BUT_MULTIPLE_VARIANTS_ROLLED_OUT = "not_started_but_multiple_variants_rolled_out"-# pinned: the page sends these as `finding_variant` on its events, as the keys above. No banner uses them.+# pinned: the page sends these as `finding_subcode` on its events, as the keys above. No banner uses them.RUNNING_BUT_FLAG_DELETED = "running_but_flag_deleted"ASSIGNED_OUTSIDE_FLAG_SERVICE = "assigned_outside_flag_service"PAUSED_FROM_EXPERIMENT = "paused_from_experiment"diff --git a/products/experiments/backend/health/checks/flag_variants.py b/products/experiments/backend/health/checks/flag_variants.pyindex b6f659b183dc..d1b92fd303e1 100644--- a/products/experiments/backend/health/checks/flag_variants.py+++ b/products/experiments/backend/health/checks/flag_variants.py@@ -20,7 +20,7 @@ from products.experiments.backend.health.skips import (not_applicable,)-# pinned: the subcodes are sent as `finding_variant` on the health finding events, so insights group on them.+# pinned: the subcodes are sent as `finding_subcode` on the health finding events, so insights group on them.FLAG_HAS_NO_VARIANTS = "flag_has_no_variants"NO_CONTROL_VARIANT = "no_control_variant"diff --git a/products/experiments/backend/health/checks/forced_variant.py b/products/experiments/backend/health/checks/forced_variant.pyindex dcf024f74044..89831703d782 100644--- a/products/experiments/backend/health/checks/forced_variant.py+++ b/products/experiments/backend/health/checks/forced_variant.py@@ -39,7 +39,7 @@ from products.experiments.backend.health.skips import CheckOutcome, CheckSkip, ffrom products.experiments.backend.health.split_changes import format_change_timefrom products.feature_flags.backend.facade.edits import ConditionsChange-# pinned: the subcodes are sent as `finding_variant` on the health finding events, so insights group on them.+# pinned: the subcodes are sent as `finding_subcode` on the health finding events, so insights group on them.# Every condition that includes users pins a variant, so no user is randomly assigned. One variant gets# everyone, or each variant gets a different group of users, so the results compare nothing valid.ALL_CONDITIONS_PINNED = "all_conditions_pinned"@@ -152,6 +152,7 @@ def _some_conditions_pinned(pins: _Pins, ctx: HealthContext, is_draft: bool) ->pinned_keys=pinned_keys,diagnostic_ref="A7",grade_evidence={**(grade.evidence() if grade is not None else {}), **set_after_launch.evidence},+ reason=grade.finding_reason if grade is not None else None,)@@ -321,10 +322,12 @@ def _finding(pinned_keys: Sequence[str],diagnostic_ref: str,grade_evidence: Mapping[str, str | int | float | None] | None = None,+ reason: str | None = None,) -> ExperimentHealthFinding:return ExperimentHealthFinding(code=ExperimentHealthFindingCode.FORCED_VARIANT_RELEASE_CONDITION,subcode=subcode,+ reason=reason,severity=severity,title=title,detail=detail,diff --git a/products/experiments/backend/health/checks/srm.py b/products/experiments/backend/health/checks/srm.pyindex f01b91f2289a..e7961845f239 100644--- a/products/experiments/backend/health/checks/srm.py+++ b/products/experiments/backend/health/checks/srm.py@@ -33,12 +33,12 @@ SEGMENT_LABEL_MAX_LENGTH = 60# The detail names the located segments furthest from the split, up to this many.NAMED_SEGMENTS = 2-# pinned: the subcode is sent as `finding_variant` on the health finding events, so insights group on it.+# pinned: the subcode is sent as `finding_subcode` on the health finding events, so insights group on it.SPLIT_CHANGED = "split_changed"class SrmReason(StrEnum):- # pinned: sent as `finding_reason` on the health finding events, so insights group on them.+ # pinned: the finding's `reason`.LIBRARY = ExposureSegmentDimension.LIBRARY.valueLIBRARY_VERSION = ExposureSegmentDimension.LIBRARY_VERSION.valuePAGE = ExposureSegmentDimension.PAGE.value@@ -137,14 +137,17 @@ def srm(ctx: HealthContext) -> CheckOutcome:"p_value": p_value,"p_value_after_split_change": p_value_after_change,}- cause: tuple[str, ...] = ()- if p_value_after_change is not None and p_value_after_change < SAMPLE_RATIO_MISMATCH_P_VALUE:- segment_cause = _segment_cause(exposures, unit)- cause = segment_cause.sentences- evidence.update(segment_cause.evidence)+ cause = (+ _segment_cause(exposures, unit)+ if p_value_after_change is not None and p_value_after_change < SAMPLE_RATIO_MISMATCH_P_VALUE+ else None+ )+ if cause is not None:+ evidence.update(cause.evidence)return ExperimentHealthFinding(code=ExperimentHealthFindingCode.SRM,subcode=SPLIT_CHANGED,+ reason=cause.reason if cause is not None else None,severity=ExperimentHealthFindingSeverity.WARNING,title=f"{unit.capitalize()}s are not split as configured after a split change",detail=" ".join(@@ -154,7 +157,7 @@ def srm(ctx: HealthContext) -> CheckOutcome:f"The variant split changed after launch while {unit}s were exposed, and some of them got their ""variants from another split, which can cause this.",*retest,- *cause,+ *(cause.sentences if cause is not None else ()),f'Follow the steps in "{split_change.title}" to fix the results.',]),@@ -222,6 +225,7 @@ def _mismatch(exposures: ExposureTotals, p_value: float) -> ExperimentHealthFindreturn ExperimentHealthFinding(code=ExperimentHealthFindingCode.SRM,subcode=None,+ reason=cause.reason,severity=ExperimentHealthFindingSeverity.WARNING,title=f"{unit.capitalize()}s are not split across variants as configured",detail=" ".join(@@ -255,10 +259,11 @@ class _Verdict:@frozenclass _SegmentCause:- """What the segments say about the mismatch: the sentences that follow it, and the reason and the segments as- evidence. Without a verdict the sentences are empty."""+ """What the segments say about the mismatch: the sentences that follow it, the finding's reason, and the+ segments as evidence. Without a verdict the sentences are empty and the reason is None."""sentences: tuple[str, ...]+ reason: str | Noneevidence: dict[str, str | int | float | None]@@ -270,9 +275,7 @@ def _segment_cause(exposures: ExposureTotals, unit: str) -> _SegmentCause:located = _located(verdicts)if located:evidence["located_dimensions"] = ",".join(segment.dimension.value for segment in located)- if reason is not None:- evidence["reason"] = reason.value- return _SegmentCause(sentences=tuple(sentences), evidence=evidence)+ return _SegmentCause(sentences=tuple(sentences), reason=None if reason is None else reason.value, evidence=evidence)def _segment_tests(exposures: ExposureTotals) -> list[_SegmentTest]:diff --git a/products/experiments/backend/health/checks/zero_exposures.py b/products/experiments/backend/health/checks/zero_exposures.pyindex 3e6fb00ab48e..d0e44ee6eaef 100644--- a/products/experiments/backend/health/checks/zero_exposures.py+++ b/products/experiments/backend/health/checks/zero_exposures.py@@ -9,12 +9,12 @@ from products.experiments.backend.health.context import HealthContextfrom products.experiments.backend.health.delays import LAST_CALLED_AT_LAG, ZERO_EXPOSURES_GRACE_HOURSfrom products.experiments.backend.health.skips import RUNS_AFTER_LAUNCH, CheckOutcome, not_applicable-# pinned: the subcodes are sent as `finding_variant` on the health finding events, so insights group on them.+# pinned: the finding's subcodes. The page sends them as `finding_subcode` on the health finding events.EXPERIMENT_PAUSED = "experiment_paused"FLAG_NOT_CALLED = "flag_not_called"CALLS_NOT_COUNTED = "calls_not_counted"CUSTOM_EXPOSURE_NOT_MATCHED = "custom_exposure_not_matched"-# pinned: sent as `finding_reason` on the health finding events.+# pinned: the finding's `reason`.TEST_ACCOUNT_FILTER = "test_account_filter"@@ -136,7 +136,6 @@ def zero_exposures(ctx: HealthContext) -> CheckOutcome:if exposed_without_filter is not None:evidence["exposed_without_test_account_filter"] = exposed_without_filterif exposed_without_filter:- evidence["reason"] = TEST_ACCOUNT_FILTERreturn _test_accounts_filtered_out(exposed_without_filter, flag_key, unit, ended=ctx.has_ended, evidence=evidence)@@ -204,6 +203,7 @@ def _test_accounts_filtered_out()return _finding(subcode=CALLS_NOT_COUNTED,+ reason=TEST_ACCOUNT_FILTER,severity=ExperimentHealthFindingSeverity.WARNING,title=title,detail=detail,@@ -239,6 +239,7 @@ def _cause_unknown(evidence: dict[str, int | str], unit: str, *, ended: bool) ->def _finding(*,subcode: str | None,+ reason: str | None = None,severity: ExperimentHealthFindingSeverity,title: str,detail: str,@@ -248,6 +249,7 @@ def _finding(return ExperimentHealthFinding(code=ExperimentHealthFindingCode.ZERO_EXPOSURES,subcode=subcode,+ reason=reason,severity=severity,title=title,detail=detail,diff --git a/products/experiments/backend/health/condition_changes.py b/products/experiments/backend/health/condition_changes.pyindex 472431226955..b5abae43dfb9 100644--- a/products/experiments/backend/health/condition_changes.py+++ b/products/experiments/backend/health/condition_changes.py@@ -35,7 +35,7 @@ from products.feature_flags.backend.facade.edits import (class ConditionsChangeShape(StrEnum):- # pinned: the finding's subcodes. The page sends them as `finding_variant` on the health finding events.+ # pinned: the finding's subcodes. The page sends them as `finding_subcode` on the health finding events.VARIANT_OVERRIDE_CHANGED = "variant_override_changed"CONDITIONS_CHANGED = "conditions_changed"REVERTED = "reverted"diff --git a/products/experiments/backend/health/context.py b/products/experiments/backend/health/context.pyindex 5d6ab4563158..a5ab3175d6a9 100644--- a/products/experiments/backend/health/context.py+++ b/products/experiments/backend/health/context.py@@ -117,7 +117,7 @@ class FlagHistory:class FlagHistoryNotCheckedReason(StrEnum):- # pinned: the finding's `evidence.reason`.+ # pinned: the finding's `reason`.NO_HISTORY_ACCESS = "no_history_access"NO_QUERY_ACCESS = "no_query_access"EXPOSURE_FILTERS = "exposure_filters"diff --git a/products/experiments/backend/health/pinned_share.py b/products/experiments/backend/health/pinned_share.pyindex 11eb85832d68..4632a4b16702 100644--- a/products/experiments/backend/health/pinned_share.py+++ b/products/experiments/backend/health/pinned_share.py@@ -31,7 +31,7 @@ _MIN_REMOTELY_EVALUATED_SHARE_TO_CLEAR = 0.5class PinnedShareUnknownReason(StrEnum):- # pinned: the finding's `evidence.reason`, which the health finding events track.+ # pinned: the finding's `reason`.NOT_MEASURED = "not_measured"TOO_FEW_PEOPLE = "too_few_people"# Most calls carry no version: local evaluation in a server SDK, a bootstrap, or the customer's own assignment.@@ -89,6 +89,11 @@ class PinnedShareGrade:share: float | Noneunknown_reason: PinnedShareUnknownReason | None+ @property+ def finding_reason(self) -> str | None:+ """The finding's `reason` when the share is unknown."""+ return None if self.unknown_reason is None else self.unknown_reason.value+def evidence(self) -> dict[str, str | int | float | None]:evidence: dict[str, str | int | float | None] = {}if self.measurement is not None:@@ -102,8 +107,6 @@ class PinnedShareGrade:)if self.share is not None:evidence["pinned_share"] = round(self.share, 4)- if self.unknown_reason is not None:- evidence["reason"] = self.unknown_reason.valuereturn evidencedef sentence(self, them: str) -> str:diff --git a/products/experiments/backend/health/split_changes.py b/products/experiments/backend/health/split_changes.pyindex ed0aa957e0e1..e7dcac47361e 100644--- a/products/experiments/backend/health/split_changes.py+++ b/products/experiments/backend/health/split_changes.py@@ -28,7 +28,7 @@ from products.feature_flags.backend.facade.edits import (class SplitChangeShape(StrEnum):- # pinned: the finding's subcodes. The page sends them as `finding_variant` on the health finding events.+ # pinned: the finding's subcodes. The page sends them as `finding_subcode` on the health finding events.VARIANT_ROLLED_OUT = "variant_rolled_out"VARIANT_REMOVED = "variant_removed"VARIANT_ADDED = "variant_added"diff --git a/products/experiments/backend/health/test/test_conditions_changed.py b/products/experiments/backend/health/test/test_conditions_changed.pyindex 67caa0bf5a28..824a52d16796 100644--- a/products/experiments/backend/health/test/test_conditions_changed.py+++ b/products/experiments/backend/health/test/test_conditions_changed.py@@ -157,6 +157,7 @@ class TestReleaseConditionsChangedMidRun(TestCase):f'The variant override "test" of Set 1 was removed on {FIRST_CHANGE}. {OVERRIDE_MOVED} PostHog '"can't measure how many users Set 1 matches. Reset the experiment, or move its start date to "f"{FIRST_CHANGE}.",+ "not_measured",),("override_removed_from_a_few_percent",@@ -171,6 +172,7 @@ class TestReleaseConditionsChangedMidRun(TestCase):f'The variant override "test" of Set 1 was removed on {FIRST_CHANGE}. {OVERRIDE_MOVED} Too few 'f"users called the flag since {TRAFFIC_SINCE} to measure how many users Set 1 matches. Reset the "f"experiment, or move its start date to {FIRST_CHANGE}.",+ "too_few_people",),("override_removed_without_a_measurement",@@ -179,6 +181,7 @@ class TestReleaseConditionsChangedMidRun(TestCase):f'The variant override "test" of Set 1 was removed on {FIRST_CHANGE}. {OVERRIDE_MOVED} PostHog '"can't measure how many users Set 1 matches. Reset the experiment, or move its start date to "f"{FIRST_CHANGE}.",+ "not_measured",),("override_changed_then_removed",@@ -247,6 +250,7 @@ class TestReleaseConditionsChangedMidRun(TestCase):"variant while it was in place and a random variant after, so those exposed while it was in place ""may have switched variants. Their exposures stay in the results. PostHog can't measure how many "f"users these conditions match. Reset the experiment, or move its start date to {FIRST_CHANGE}.",+ "not_measured",),("override_removed_with_its_condition",@@ -358,6 +362,7 @@ class TestReleaseConditionsChangedMidRun(TestCase):f"The release conditions changed on {FIRST_CHANGE}. This check can't tell whether anyone was exposed ""before the change, because this API key can't query events. Add the query:read scope to this API ""key, or review the change in the flag history.",+ "no_query_access",),])@@ -367,12 +372,14 @@ class TestReleaseConditionsChangedMidRun(TestCase):ctx: HealthContext,expected: tuple[str, ExperimentHealthFindingSeverity, str, str],expected_detail: str | None,+ expected_reason: str | None = None,) -> None:finding = release_conditions_changed_mid_run(ctx)assert isinstance(finding, ExperimentHealthFinding)self.assertEqual(finding.code, ExperimentHealthFindingCode.RELEASE_CONDITIONS_CHANGED_MID_RUN)self.assertEqual((finding.subcode, finding.severity, finding.title, finding.diagnostic_ref), expected)+ self.assertEqual(finding.reason, expected_reason)if expected_detail is not None:self.assertEqual(finding.detail, expected_detail)diff --git a/products/experiments/backend/health/test/test_forced_variant.py b/products/experiments/backend/health/test/test_forced_variant.pyindex 41e2be4f3f44..3185c8502024 100644--- a/products/experiments/backend/health/test/test_forced_variant.py+++ b/products/experiments/backend/health/test/test_forced_variant.py@@ -355,9 +355,7 @@ class TestForcedVariantReleaseCondition(TestCase):finding = forced_variant_release_condition(ctx)assert isinstance(finding, ExperimentHealthFinding)- self.assertEqual(- (finding.evidence.get("reason"), "can't measure" in finding.detail), (expected_reason, says_not_measurable)- )+ self.assertEqual((finding.reason, "can't measure" in finding.detail), (expected_reason, says_not_measurable))@parameterized.expand([diff --git a/products/experiments/backend/health/test/test_split_changed.py b/products/experiments/backend/health/test/test_split_changed.pyindex 6b742c11fc53..24e525d3ad8f 100644--- a/products/experiments/backend/health/test/test_split_changed.py+++ b/products/experiments/backend/health/test/test_split_changed.py@@ -303,6 +303,7 @@ class TestSplitChangedMidRun(TestCase):("unknown_exposure",_context(EVEN, UNEVEN, exposed_before_first=None),+ None,{"change_count": 1,"first_changed_at": "2026-09-12T13:04:00+00:00",@@ -317,6 +318,7 @@ class TestSplitChangedMidRun(TestCase):("few_exposed",_context(EVEN, UNEVEN, exposed_units=4),+ None,{"change_count": 1,"first_changed_at": "2026-09-12T13:04:00+00:00",@@ -331,8 +333,8 @@ class TestSplitChangedMidRun(TestCase):("not_checked",_context(EVEN, UNEVEN, not_checked=FlagHistoryNotCheckedReason.EXPOSURE_FILTERS),+ "exposure_filters",{- "reason": "exposure_filters","change_count": 1,"first_changed_at": "2026-09-12T13:04:00+00:00","last_changed_at": "2026-09-12T13:04:00+00:00",@@ -345,15 +347,16 @@ class TestSplitChangedMidRun(TestCase):("not_checked_without_history_access",_context(EVEN, not_checked=FlagHistoryNotCheckedReason.NO_HISTORY_ACCESS),- {"reason": "no_history_access"},+ "no_history_access",+ {},),])- def test_evidence(self, _name: str, ctx: HealthContext, expected: dict[str, Any]) -> None:+ def test_evidence(self, _name: str, ctx: HealthContext, reason: str | None, expected: dict[str, Any]) -> None:finding = _split_code_finding(ctx)assert isinstance(finding, ExperimentHealthFinding)- self.assertEqual(finding.evidence, expected)+ self.assertEqual((finding.reason, finding.evidence), (reason, expected))@parameterized.expand([diff --git a/products/experiments/backend/health/test/test_srm.py b/products/experiments/backend/health/test/test_srm.pyindex a715bd43b96e..c27fdbd5d60f 100644--- a/products/experiments/backend/health/test/test_srm.py+++ b/products/experiments/backend/health/test/test_srm.py@@ -167,7 +167,7 @@ class TestSrmSegments(TestCase):assert isinstance(outcome, ExperimentHealthFinding)self.assertEqual(- (outcome.subcode, outcome.title, outcome.detail, outcome.evidence.get("reason")),+ (outcome.subcode, outcome.title, outcome.detail, outcome.reason),(None, "Users are not split across variants as configured", detail, reason),)@@ -209,7 +209,6 @@ class TestSrmSegments(TestCase):evidence,{"p_value": 0.00001,- "reason": "library","located_dimensions": "library","library_1": "web","library_1_p_value": 1.0,diff --git a/products/experiments/backend/health/test/test_zero_exposures.py b/products/experiments/backend/health/test/test_zero_exposures.pyindex 0bfa6cfd55ae..0d18da1f063f 100644--- a/products/experiments/backend/health/test/test_zero_exposures.py+++ b/products/experiments/backend/health/test/test_zero_exposures.py@@ -214,7 +214,8 @@ class TestZeroExposures(TestCase):"running",RUNNING_CONTEXT,1200,- {"exposed_without_test_account_filter": 1200, "reason": "test_account_filter"},+ "test_account_filter",+ {"exposed_without_test_account_filter": 1200},"No users exposed: the test account filter removes all of them","The exposure criteria filter out test accounts, and the project's test account filters match all 1,200 "'users that got a variant of the feature flag "checkout-test" during the experiment, so nobody is '@@ -225,7 +226,8 @@ class TestZeroExposures(TestCase):"ended_one_user",replace(RUNNING_CONTEXT, **ENDED),1,- {"exposed_without_test_account_filter": 1, "reason": "test_account_filter"},+ "test_account_filter",+ {"exposed_without_test_account_filter": 1},"No users exposed: the test account filter removed all of them","No users were exposed while the experiment ran, because the exposure criteria filter out test ""accounts, and the project's test account filters matched the only user that got a variant of the "@@ -236,6 +238,7 @@ class TestZeroExposures(TestCase):"nobody_without_the_filter",RUNNING_CONTEXT,0,+ None,{"exposed_without_test_account_filter": 0},"No users exposed, although the feature flag is called",'PostHog has recorded calls of the feature flag "checkout-test" during the experiment, but none of them '@@ -246,7 +249,14 @@ class TestZeroExposures(TestCase):])def test_counts_the_users_that_the_test_account_filter_removes(- self, _name: str, base: HealthContext, exposed: int, evidence: dict[str, int | str], title: str, detail: str+ self,+ _name: str,+ base: HealthContext,+ exposed: int,+ reason: str | None,+ evidence: dict[str, int | str],+ title: str,+ detail: str,) -> None:ctx = replace(base,@@ -260,8 +270,8 @@ class TestZeroExposures(TestCase):assert isinstance(finding, ExperimentHealthFinding)self.assertEqual(- (finding.subcode, finding.title, finding.detail, finding.evidence),- (CALLS_NOT_COUNTED, title, detail, {"hours_since_launch": 48, **evidence}),+ (finding.subcode, finding.reason, finding.title, finding.detail, finding.evidence),+ (CALLS_NOT_COUNTED, reason, title, detail, {"hours_since_launch": 48, **evidence}),)diff --git a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.pyindex e71ad1b3d6db..64192fa1a539 100644--- a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py+++ b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py@@ -433,6 +433,7 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):ExperimentExposureHealthFinding(code=ExperimentExposureHealthFindingCode(finding.code.value),subcode=finding.subcode,+ reason=finding.reason,severity=ExperimentExposureHealthFindingSeverity(finding.severity.value),title=finding.title,detail=finding.detail,@@ -586,7 +587,7 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):def get_cache_payload(self) -> dict:payload = super().get_cache_payload()- payload["experiment_exposures_response_version"] = 2+ payload["experiment_exposures_response_version"] = 3# The sample ratio finding can name a split change from the flag's history, so a reader who can't read the# history, a userless run included, must not get a cached answer of one who can.if not self._can_read_flag_history():diff --git a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.pyindex a9acde80de81..99983c4618ac 100644--- a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py+++ b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py@@ -1641,8 +1641,8 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):True,'The mismatch comes from users whose first exposure was sent by version "4.2.1" of the "posthog-node" '"library.",+ "library_version",{- "reason": "library_version","located_dimensions": "library,library_version","library_1": "web","library_1_exposed_control": 100,@@ -1692,13 +1692,14 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):"reader_who_does_not_see_them",False,"This may indicate issues with randomization or data collection.",+ None,{},),])@time_machine.travel("2024-01-07T12:00:00Z", tick=False)def test_srm_splits_the_exposures_by_the_library_and_page_of_the_first_exposure(- self, _name: str, findings_shown: bool, detail: str, evidence: dict[str, str | int]+ self, _name: str, findings_shown: bool, detail: str, reason: str | None, evidence: dict[str, str | int]) -> None:def exposure(variant: str, timestamp: str, library: str, version: str, **place: str | None) -> dict[str, Any]:return {@@ -1761,7 +1762,9 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):).calculate()[finding] = response.health_findings or []- self.assertEqual((finding.code, finding.subcode), (ExperimentExposureHealthFindingCode.SRM, None))+ self.assertEqual(+ (finding.code, finding.subcode, finding.reason), (ExperimentExposureHealthFindingCode.SRM, None, reason)+ )self.assertIn(detail, finding.detail)self.assertEqual({key: value for key, value in finding.evidence.items() if not key.endswith("p_value")}, evidencediff --git a/products/experiments/backend/presentation/serializers.py b/products/experiments/backend/presentation/serializers.pyindex 03e9d871cb11..a8b04af886b4 100644--- a/products/experiments/backend/presentation/serializers.py+++ b/products/experiments/backend/presentation/serializers.py@@ -220,6 +220,14 @@ class ExperimentHealthFindingSerializer(serializers.Serializer):"'running_but_no_rollout' within 'flag_off_while_running'. Null when the code has one case."),)+ reason = serializers.CharField(+ allow_null=True,+ help_text=(+ "A stable qualifier below the subcode: where a sample ratio mismatch is located ('library', 'page'), "+ "what leaves nobody exposed ('test_account_filter'), or why a check could not read what it needs "+ "('no_history_access'). Null when the finding names none."+ ),+ )severity = serializers.ChoiceField(choices=ExperimentHealthFindingSeverity.choices,help_text="How much the problem affects the results: critical, warning, or info.",diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.pyindex 7314ca7c1534..051e41a1939b 100644--- a/products/experiments/backend/presentation/views.py+++ b/products/experiments/backend/presentation/views.py@@ -967,7 +967,7 @@ class EnterpriseExperimentsViewSet(for exposure criteria that name an event without property filters. When the split changedafter launch and the criteria have filters or an action, or an API key lacks `query:read`,it returns an info finding with the subcode `not_checked` and the cause in- `evidence.reason`. A caller who can't view the linked flag's history (viewer access to the+ `reason`. A caller who can't view the linked flag's history (viewer access to theflag and, with an API key, `activity_log:read`) gets that finding for every runningexperiment, because the changes are not read.@@ -1032,7 +1032,7 @@ class EnterpriseExperimentsViewSet(the flag's calls that name the matched condition, not the experiment's exposure criteria:info up to 5%, warning above, and no finding under 1% when at least half of the callsthat returned a variant name a condition. The finding is info when the share is unknown,- and `evidence.reason` says why: `not_measured` (for example an API key without+ and `reason` says why: `not_measured` (for example an API key without`query:read`, or a caller denied a flag call property), `too_few_people` (under 100), or`evaluated_outside_flag_service` (most calls come from local evaluation, a bootstrap orthe caller's own assignment). A pin to a variant with 0% in the split adds an info,diff --git a/products/experiments/backend/test/test_presentation_api.py b/products/experiments/backend/test/test_presentation_api.pyindex 6f3db84e3c94..974f8b4db197 100644--- a/products/experiments/backend/test/test_presentation_api.py+++ b/products/experiments/backend/test/test_presentation_api.py@@ -918,7 +918,7 @@ class TestExperimentCRUD(_HoistFlagConfigClientMixin, APILicensedTest):self.assertEqual(response.status_code, status.HTTP_200_OK, response.content)split_findings = [- (finding["subcode"], finding["evidence"].get("reason"))+ (finding["subcode"], finding["reason"])for finding in response.json()["findings"]if finding["code"] == "split_changed_mid_run"]diff --git a/products/experiments/frontend/generated/api.schemas.ts b/products/experiments/frontend/generated/api.schemas.tsindex 012dcb5eba19..bdd51818d7ea 100644--- a/products/experiments/frontend/generated/api.schemas.ts+++ b/products/experiments/frontend/generated/api.schemas.ts@@ -2421,6 +2421,11 @@ export interface ExperimentHealthFindingApi {* @nullable*/subcode: string | null+ /**+ * A stable qualifier below the subcode: where a sample ratio mismatch is located ('library', 'page'), what leaves nobody exposed ('test_account_filter'), or why a check could not read what it needs ('no_history_access'). Null when the finding names none.+ * @nullable+ */+ reason: string | null/** How much the problem affects the results: critical, warning, or info.** * `critical` - Critical severitydiff --git a/products/experiments/frontend/generated/api.ts b/products/experiments/frontend/generated/api.tsindex 265ea3364f07..994be9c77257 100644--- a/products/experiments/frontend/generated/api.ts+++ b/products/experiments/frontend/generated/api.ts@@ -712,7 +712,7 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =* for exposure criteria that name an event without property filters. When the split changed* after launch and the criteria have filters or an action, or an API key lacks `query:read`,* it returns an info finding with the subcode `not_checked` and the cause in- * `evidence.reason`. A caller who can't view the linked flag's history (viewer access to the+ * `reason`. A caller who can't view the linked flag's history (viewer access to the* flag and, with an API key, `activity_log:read`) gets that finding for every running* experiment, because the changes are not read.*@@ -777,7 +777,7 @@ export const getExperimentsHealthRetrieveUrl = (projectId: string, id: number) =* the flag's calls that name the matched condition, not the experiment's exposure criteria:* info up to 5%, warning above, and no finding under 1% when at least half of the calls* that returned a variant name a condition. The finding is info when the share is unknown,- * and `evidence.reason` says why: `not_measured` (for example an API key without+ * and `reason` says why: `not_measured` (for example an API key without* `query:read`, or a caller denied a flag call property), `too_few_people` (under 100), or* `evaluated_outside_flag_service` (most calls come from local evaluation, a bootstrap or* the caller's own assignment). A pin to a variant with 0% in the split adds an info,diff --git a/products/experiments/frontend/health/HealthChecksOverview.stories.tsx b/products/experiments/frontend/health/HealthChecksOverview.stories.tsxindex aa2332d69341..7feb4a9f65e2 100644--- a/products/experiments/frontend/health/HealthChecksOverview.stories.tsx+++ b/products/experiments/frontend/health/HealthChecksOverview.stories.tsx@@ -52,6 +52,7 @@ export const Draft: Story = {{code: 'new_flag_no_traffic',subcode: null,+ reason: null,severity: 'info',title: 'The feature flag has no traffic yet',detail: "PostHog hasn't recorded any calls to the linked feature flag yet. The experiment only counts users after your code evaluates the flag, so launch after the flag starts getting traffic. New calls can take up to an hour to show up here.",@@ -83,6 +84,7 @@ export const DraftWithEventsNotReceived: Story = {{code: 'metric_event_not_ingested',subcode: null,+ reason: null,severity: 'info',title: "A metric uses an event this project hasn't received",detail: 'The primary metric "Revenue" uses the event "purchase_completed", which this project hasn\'t received yet. Until it arrives, the metric can\'t compare the variants. Launch the experiment after your code starts sending it.',@@ -96,6 +98,7 @@ export const DraftWithEventsNotReceived: Story = {{code: 'exposure_criteria_cannot_match',subcode: 'exposure_event_not_ingested',+ reason: null,severity: 'info',title: "This project hasn't received the exposure event",detail: 'The experiment counts users as exposed when they send the event "checkout_viewed", which this project hasn\'t received yet. Until it arrives, nobody is counted as exposed. Launch the experiment after your code starts sending it.',@@ -119,6 +122,7 @@ export const RunningWithSignificanceOnASmallSample: Story = {{code: 'early_significance',subcode: 'below_detectable_effect',+ reason: null,severity: 'info',title: 'Significant result on a small sample',detail: 'The variant "test" shows a +20% difference on the metric "Checkout", which is significant. With 2,000 users so far, the experiment can reliably detect differences of about 38% or more, so this difference may shrink or go away as more users arrive. The running time estimate needs about 3,201 users to detect a 30% difference. Wait for that sample before you decide.',@@ -167,6 +171,7 @@ export const RunningWithAnIssue: Story = {{code: 'srm',subcode: null,+ reason: null,severity: 'warning',title: 'Users are not split across variants as configured',detail: "The distribution of users across variants doesn't match your configured rollout percentages (p < 0.001). This may indicate issues with randomization or data collection.",diff --git a/products/experiments/frontend/health/HealthFindingRow.tsx b/products/experiments/frontend/health/HealthFindingRow.tsxindex 440d0449a8bf..d3907268d1b3 100644--- a/products/experiments/frontend/health/HealthFindingRow.tsx+++ b/products/experiments/frontend/health/HealthFindingRow.tsx@@ -5,7 +5,6 @@ import { LemonButton, Tooltip } from '@posthog/lemon-ui'import { cn } from 'lib/utils/css-classes'-import { healthFindingReason } from './experimentHealthFindingEvents'import { HealthFindingActionButton } from './HealthFindingActionButton'import type { HealthPanelFinding } from './healthPanelFindings'import { useHealthFindingReporting } from './useHealthFindingReporting'@@ -28,9 +27,9 @@ export function HealthFindingRow({const detailId = useId()const { reportOpened, reportActedOn } = useHealthFindingReporting({code: finding.code,- variant: finding.subcode ?? undefined,+ subcode: finding.subcode ?? undefined,severity: finding.severity,- reason: healthFindingReason(finding.evidence),+ reason: finding.reason ?? undefined,})return (diff --git a/products/experiments/frontend/health/debug/healthDebugReport.test.ts b/products/experiments/frontend/health/debug/healthDebugReport.test.tsindex c2a5f908466c..7feef416cdc0 100644--- a/products/experiments/frontend/health/debug/healthDebugReport.test.ts+++ b/products/experiments/frontend/health/debug/healthDebugReport.test.ts@@ -6,6 +6,7 @@ import { type HealthDebugInput, buildHealthDebugChecks } from './healthDebugRepoconst PAUSED: ExperimentHealthFindingApi = {code: 'flag_off_while_running',subcode: 'running_but_flag_disabled',+ reason: null,severity: 'warning',title: 'The experiment is paused',detail: '',diff --git a/products/experiments/frontend/health/debug/healthDebugReport.ts b/products/experiments/frontend/health/debug/healthDebugReport.tsindex b9587ba017a8..6f3d6b56e691 100644--- a/products/experiments/frontend/health/debug/healthDebugReport.ts+++ b/products/experiments/frontend/health/debug/healthDebugReport.ts@@ -177,8 +177,8 @@ function splitChangedNote(finding: ExperimentHealthFindingApi): string {const { evidence } = findingif (finding.subcode === 'not_checked') {return evidence.change_count == null- ? `Not checked: ${evidence.reason}.`- : `Not checked: ${evidence.reason}. ${evidence.change_count} changes since launch, from ${evidence.from} to ${evidence.to}.`+ ? `Not checked: ${finding.reason}.`+ : `Not checked: ${finding.reason}. ${evidence.change_count} changes since launch, from ${evidence.from} to ${evidence.to}.`}return `${evidence.change_count} counted changes, from ${evidence.from} to ${evidence.to}. ${exposedBeforeFirst(finding)}`}@@ -187,7 +187,7 @@ function bucketingChangedNote(finding: ExperimentHealthFindingApi): string {const { evidence } = findingconst changed = `${evidence.change_count} saves changed ${evidence.changed_settings}.`return finding.subcode === 'not_checked'- ? `Not checked: ${evidence.reason}. ${changed}`+ ? `Not checked: ${finding.reason}. ${changed}`: `${changed} ${exposedBeforeFirst(finding)}`}@@ -195,13 +195,13 @@ function conditionsChangedNote(finding: ExperimentHealthFindingApi): string {const { evidence } = findingconst changed = `${evidence.change_count} changes, from ${evidence.conditions_before} to ${evidence.conditions_after} conditions.`if (finding.subcode === 'not_checked') {- return `Not checked: ${evidence.reason}. ${changed}`+ return `Not checked: ${finding.reason}. ${changed}`}const pinnedShare = share(evidence.pinned_share)const removedOverride = pinnedShare? `Removed override share ${pinnedShare}.`- : typeof evidence.reason === 'string'- ? `Removed override share unknown: ${evidence.reason}.`+ : finding.reason+ ? `Removed override share unknown: ${finding.reason}.`: nullreturn [changed, exposedBeforeFirst(finding), removedOverride].filter(Boolean).join(' ')}@@ -220,8 +220,8 @@ function forcedVariantNote(finding: ExperimentHealthFindingApi): string {if (remoteShare) {parts.push(`${remoteShare} of calls evaluated by the flag service.`)}- if (typeof evidence.reason === 'string') {- parts.push(`Share unknown: ${evidence.reason}.`)+ if (finding.reason) {+ parts.push(`Share unknown: ${finding.reason}.`)}return parts.join(' ')}diff --git a/products/experiments/frontend/health/experimentHealthFindingEvents.ts b/products/experiments/frontend/health/experimentHealthFindingEvents.tsindex b407247ee906..129ef2dcad42 100644--- a/products/experiments/frontend/health/experimentHealthFindingEvents.ts+++ b/products/experiments/frontend/health/experimentHealthFindingEvents.ts@@ -39,14 +39,11 @@ export type ExperimentHealthFindingCode =export interface ExperimentHealthFinding {code: ExperimentHealthFindingCode- /** The sub-case of the code, for example the warning key. It must not hold customer text. */- variant?: string+ /** The server finding's subcode, or the warning key of a separate warning. It must not hold customer text. */+ subcode?: string/** Set for a server finding. The separate warnings have none. */severity?: ExperimentHealthFindingApi['severity']- /**- * Why a check was not checked, or the cause the server's finding names in its `reason` evidence. Both are- * pinned server values, never customer text.- */+ /** The server finding's reason: a pinned qualifier below the subcode, never customer text. */reason?: string}@@ -91,7 +88,7 @@ export function experimentHealthUi(experiment: Experiment, showsServerHealth: bo}export function healthFindingForExperimentWarning(warningKey: ExperimentWarningKey): ExperimentHealthFinding {- return { code: EXPERIMENT_WARNING_FINDING_CODES[warningKey], variant: warningKey }+ return { code: EXPERIMENT_WARNING_FINDING_CODES[warningKey], subcode: warningKey }}function isExperimentWarningKey(subcode: string | null): subcode is ExperimentWarningKey {@@ -126,7 +123,7 @@ function healthFindingEventProperties(experiment_status: getExperimentStatus(experiment),experiment_days_since_start: experiment.start_date ? dayjs().diff(experiment.start_date, 'day') : null,finding_code: finding.code,- finding_variant: finding.variant ?? null,+ finding_subcode: finding.subcode ?? null,finding_severity: finding.severity ?? null,finding_reason: finding.reason ?? null,health_ui: healthUi,@@ -136,14 +133,10 @@ function healthFindingEventProperties(}}-export function healthFindingReason(evidence: ExperimentHealthFindingApi['evidence'] | undefined): string | undefined {- return typeof evidence?.reason === 'string' ? evidence.reason : undefined-}--function trackedHealthFinding({ code, subcode, severity, evidence }: ExperimentHealthFindingApi): string {+function trackedHealthFinding({ code, subcode, severity, reason }: ExperimentHealthFindingApi): string {// pinned: `code:subcode:severity:reason`, as on the server's `experiment launched`. Insights split each string// on `:`, so every field is present, empty when the finding has none.- return [code, subcode ?? '', severity, healthFindingReason(evidence) ?? ''].join(':')+ return [code, subcode ?? '', severity, reason ?? ''].join(':')}export function captureExperimentHealthFindingShown(@@ -251,8 +244,7 @@ export function captureExperimentHealthChecked(* `has_srm`, `has_bias_risk` and a zero `exposures_total` are the page's own rules. The `_server` values are* the server's checks on the same answer, null when the answer holds no server findings (a cached answer* from before the server ran them). `srm_subcode` and `zero_exposures_subcode` are the `subcode` of those findings, and- * `srm_reason` and `zero_exposures_reason` the `reason` in their evidence, each null without the finding or when it- * names none.+ * `srm_reason` and `zero_exposures_reason` their `reason`, each null without the finding or when it names none.*/export function exposureHealthEventProperties(exposures: ExperimentExposureQueryResponse | null | undefined,@@ -302,12 +294,12 @@ export function exposureHealthEventProperties(// pinned: analytics property, so renaming it breaks insights. Its values are the server's pinned subcodes.srm_subcode: srmFinding?.subcode ?? null,// pinned: analytics property, so renaming it breaks insights. Its values are the server's pinned reasons.- srm_reason: healthFindingReason(srmFinding?.evidence) ?? null,+ srm_reason: srmFinding?.reason ?? null,zero_exposures_server: serverCodes ? serverCodes.has('zero_exposures') : null,// pinned: analytics property, so renaming it breaks insights. Its values are the server's pinned subcodes.zero_exposures_subcode: zeroExposuresFinding?.subcode ?? null,// pinned: analytics property, so renaming it breaks insights. Its values are the server's pinned reasons.- zero_exposures_reason: healthFindingReason(zeroExposuresFinding?.evidence) ?? null,+ zero_exposures_reason: zeroExposuresFinding?.reason ?? null,bias_risk_server: serverCodes ? serverCodes.has('bias_risk_multiple_excluded') : null,}}diff --git a/products/experiments/frontend/health/exposureHealth.test.ts b/products/experiments/frontend/health/exposureHealth.test.tsindex cfc795558c47..3c1c49fb7b87 100644--- a/products/experiments/frontend/health/exposureHealth.test.ts+++ b/products/experiments/frontend/health/exposureHealth.test.ts@@ -7,6 +7,7 @@ const srmFinding = (): NonNullable<ExperimentExposureQueryResponse['health_findings']>[number] => ({code: 'srm',subcode,+ reason: null,severity: 'warning',title: 'Users are not split as configured',detail: `Server detail for ${subcode ?? 'a mismatch'}`,diff --git a/products/experiments/frontend/health/healthPanelChecks.test.ts b/products/experiments/frontend/health/healthPanelChecks.test.tsindex 0c356ae1764f..2a9b2b46df5d 100644--- a/products/experiments/frontend/health/healthPanelChecks.test.ts+++ b/products/experiments/frontend/health/healthPanelChecks.test.ts@@ -19,6 +19,7 @@ const check = (const finding = (severity: HealthPanelFinding['severity']): HealthPanelFinding => ({code: 'no_metric',subcode: null,+ reason: null,severity,title: 'No metrics defined',detail: '',diff --git a/products/experiments/frontend/health/healthPanelFindings.test.ts b/products/experiments/frontend/health/healthPanelFindings.test.tsindex 349b3def0fb9..612adb5ce281 100644--- a/products/experiments/frontend/health/healthPanelFindings.test.ts+++ b/products/experiments/frontend/health/healthPanelFindings.test.ts@@ -14,6 +14,7 @@ const serverFinding = (): ExperimentHealthFindingApi => ({code,subcode: null,+ reason: null,severity,title: code,detail: code,@@ -25,6 +26,7 @@ const serverFinding = (const exposureFinding = (code: ExperimentExposureHealthFindingCode): ExperimentExposureHealthFinding => ({code,subcode: null,+ reason: null,severity: 'warning',title: code,detail: code,diff --git a/products/experiments/frontend/health/healthPanelFindings.ts b/products/experiments/frontend/health/healthPanelFindings.tsindex 5574d10015bf..c40b4eae2820 100644--- a/products/experiments/frontend/health/healthPanelFindings.ts+++ b/products/experiments/frontend/health/healthPanelFindings.ts@@ -8,7 +8,7 @@ import type { ExperimentHealthFindingActionKind, ExperimentHealthFindingCode } f/** A finding of the health panel, from the experiment read or from the exposure answer. */export type HealthPanelFinding = Pick<ExperimentHealthFindingApi,- 'subcode' | 'severity' | 'title' | 'detail' | 'evidence'+ 'subcode' | 'reason' | 'severity' | 'title' | 'detail' | 'evidence'> & {code: ExperimentHealthFindingCodeactions: ExperimentHealthFindingActionKind[]diff --git a/products/experiments/frontend/health/useHealthFindingReporting.ts b/products/experiments/frontend/health/useHealthFindingReporting.tsindex e5d2d328096a..7987bb9342ee 100644--- a/products/experiments/frontend/health/useHealthFindingReporting.ts+++ b/products/experiments/frontend/health/useHealthFindingReporting.ts@@ -27,32 +27,32 @@ export function useHealthFindingReporting(const { reportHealthFindingShown, reportHealthFindingOpened, reportHealthFindingActedOn } =useActions(experimentLogic)const code = finding?.code- const variant = finding?.variant+ const subcode = finding?.subcodeconst severity = finding?.severityconst reason = finding?.reasonuseEffect(() => {if (code && isShown) {- reportHealthFindingShown({ code, variant, severity, reason })+ reportHealthFindingShown({ code, subcode, severity, reason })}- }, [code, variant, severity, reason, isShown, experimentLoadCount, reportHealthFindingShown])+ }, [code, subcode, severity, reason, isShown, experimentLoadCount, reportHealthFindingShown])const reportOpened = useCallback((openKind: ExperimentHealthFindingOpenKind): void => {if (code) {- reportHealthFindingOpened({ code, variant, severity, reason }, openKind)+ reportHealthFindingOpened({ code, subcode, severity, reason }, openKind)}},- [code, variant, severity, reason, reportHealthFindingOpened]+ [code, subcode, severity, reason, reportHealthFindingOpened])const reportActedOn = useCallback((actionKind: ExperimentHealthFindingActionKind): void => {if (code) {- reportHealthFindingActedOn({ code, variant, severity, reason }, actionKind)+ reportHealthFindingActedOn({ code, subcode, severity, reason }, actionKind)}},- [code, variant, severity, reason, reportHealthFindingActedOn]+ [code, subcode, severity, reason, reportHealthFindingActedOn])return { reportOpened, reportActedOn }diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 1ce2168a847c..2558557e5c02 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -47845,6 +47845,11 @@ export namespace Schemas {* @nullable*/subcode: string | null;+ /**+ * A stable qualifier below the subcode: where a sample ratio mismatch is located ('library', 'page'), what leaves nobody exposed ('test_account_filter'), or why a check could not read what it needs ('no_history_access'). Null when the finding names none.+ * @nullable+ */+ reason: string | null;/** How much the problem affects the results: critical, warning, or info.** * `critical` - Critical severity@@ -48296,6 +48301,8 @@ export namespace Schemas {diagnostic_ref: string | null;/** The values behind the finding, such as the p-value of the sample ratio test. The keys depend on the code. */evidence: ExperimentExposureHealthFindingEvidence;+ /** A stable qualifier below the subcode, for example where a sample ratio mismatch is located. Null when the finding names none. */+ reason: string | null;/** How much the problem affects the results: critical, warning, or info. */severity: ExperimentExposureHealthFindingSeverity;/** The case within the code, when a code covers several. Null when the code has one case. */