:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(desktop): drop legacy experiment result fallbacks (#120551)
Anders8144eb493ebc
diff --git a/products/desktop/packages/api-client/src/evidence-previews.test.ts b/products/desktop/packages/api-client/src/evidence-previews.test.tsindex aafe1b127af8..b53a64e6eb23 100644--- a/products/desktop/packages/api-client/src/evidence-previews.test.ts+++ b/products/desktop/packages/api-client/src/evidence-previews.test.ts@@ -85,7 +85,6 @@ describe("evidence preview shaping", () => {significant: true,},],- significance_code: "significant",is_cached: true,last_refresh: new Date().toISOString(),} as unknown as Schemas.ExperimentQueryResponse;@@ -254,29 +253,84 @@ describe("evidence preview shaping", () => {expect(results.secondaryMetrics[0].state).toBe("ready");});- it("shows outcomes while marking low-exposure results as insufficient", () => {- const insufficientResponse = {+ it.each([+ {+ method: "frequentist",+ statistics: { confidence_interval: null, p_value: null },+ },+ {+ method: "bayesian",+ statistics: { credible_interval: null, chance_to_win: null },+ },+ ])(+ "shows outcomes while marking low-exposure $method results as insufficient",+ ({ method, statistics }) => {+ const insufficientResponse = {+ ...readyMetricResponse,+ variant_results: [+ {+ key: "test",+ method,+ number_of_samples: 40,+ sum: 6,+ sum_squares: 6,+ significant: null,+ validation_failures: ["not-enough-exposures"],+ ...statistics,+ },+ ],+ } as unknown as Schemas.ExperimentQueryResponse;+ const results = shapeExperimentResults(+ experimentWithMetrics(),+ exposureResponse,+ [{ response: insufficientResponse }],+ [{ response: insufficientResponse }],+ );++ expect(results.state).toBe("insufficient_data");+ expect(results.primaryMetrics[0]).toMatchObject({+ state: "insufficient_data",+ variants: [+ expect.any(Object),+ expect.objectContaining({+ outcome: "6 · 15.0%",+ pValue: null,+ chanceToWin: null,+ significance: "insufficient_data",+ }),+ ],+ });+ },+ );++ it("shows a Bayesian result's chance to win instead of a p-value", () => {+ const bayesianResponse = {...readyMetricResponse,- significance_code: "not_enough_exposure",- significant: false,+ variant_results: [+ {+ key: "test",+ method: "bayesian",+ number_of_samples: 1000,+ sum: 125,+ sum_squares: 125,+ credible_interval: [0.05, 0.45],+ chance_to_win: 0.97,+ significant: true,+ },+ ],} as unknown as Schemas.ExperimentQueryResponse;const results = shapeExperimentResults(experimentWithMetrics(),exposureResponse,- [{ response: insufficientResponse }],- [{ response: insufficientResponse }],+ [{ response: bayesianResponse }],+ [],);- expect(results.state).toBe("insufficient_data");- expect(results.primaryMetrics[0]).toMatchObject({- state: "insufficient_data",- variants: [- expect.any(Object),- expect.objectContaining({- outcome: "125 · 12.5%",- significance: "insufficient_data",- }),- ],+ expect(results.primaryMetrics[0].variants[1]).toMatchObject({+ interval: "+5.00% to +45.0%",+ pValue: null,+ chanceToWin: "97.0%",+ significance: "significant",});});diff --git a/products/desktop/packages/api-client/src/evidence-previews.ts b/products/desktop/packages/api-client/src/evidence-previews.tsindex a1ed832c408a..4de972a870d8 100644--- a/products/desktop/packages/api-client/src/evidence-previews.ts+++ b/products/desktop/packages/api-client/src/evidence-previews.ts@@ -658,10 +658,8 @@ function outcomeValue(}function hasInsufficientData(- response: Schemas.ExperimentQueryResponse,stats: Schemas.ExperimentStatsBaseValidated[],): boolean {- if (response.significance_code === "not_enough_exposure") return true;return stats.some((entry) =>(entry.validation_failures ?? []).some((failure) =>@@ -736,7 +734,7 @@ function shapeMetricResult(const baselineOutcome = outcomeValue(baseline, definition.valueType);const allStats = [baseline, ...variants];- const insufficient = hasInsufficientData(response, allStats);+ const insufficient = hasInsufficientData(allStats);const upliftedRows: Array<{row: ExperimentVariantResultPresentation;value: number;@@ -746,22 +744,19 @@ function shapeMetricResult(const isControl = index === 0;const statsRecord = stats as unknown as QueryRecord;const interval = validInterval(- statsRecord.confidence_interval ??- statsRecord.credible_interval ??- response.credible_intervals?.[stats.key],- );- const pValue = cellNumber(- statsRecord.p_value ?? (variants.length === 1 ? response.p_value : null),- );- const chanceToWin = cellNumber(- statsRecord.chance_to_win ?? response.probability?.[stats.key],+ statsRecord.confidence_interval ?? statsRecord.credible_interval,);+ // The API sends an unset statistic as null, and cellNumber reads null as 0.+ const pValue =+ typeof statsRecord.p_value === "number" ? statsRecord.p_value : null;+ const chanceToWin =+ typeof statsRecord.chance_to_win === "number"+ ? statsRecord.chance_to_win+ : null;const significantValue =typeof statsRecord.significant === "boolean"? statsRecord.significant- : variants.length === 1 && typeof response.significant === "boolean"- ? response.significant- : null;+ : null;const uplift =!isControl &&outcome.comparable !== null &&diff --git a/products/desktop/packages/api-client/src/posthog-client.test.ts b/products/desktop/packages/api-client/src/posthog-client.test.tsindex 7223b6779b8a..34fd544dff33 100644--- a/products/desktop/packages/api-client/src/posthog-client.test.ts+++ b/products/desktop/packages/api-client/src/posthog-client.test.ts@@ -513,7 +513,6 @@ describe("PostHogAPIClient", () => {confidence_interval: [0.01, 0.39],},],- significance_code: "significant",is_cached: true,last_refresh: new Date().toISOString(),};