Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(desktop): drop legacy experiment result fallbacks (#120551)

Anders8144eb493ebc
diff --git a/products/desktop/packages/api-client/src/evidence-previews.test.ts b/products/desktop/packages/api-client/src/evidence-previews.test.ts
index aafe1b127af8..b53a64e6eb23 100644
--- a/products/desktop/packages/api-client/src/evidence-previews.test.ts
+++ b/products/desktop/packages/api-client/src/evidence-previews.test.ts
@@ -85,7 +85,6 @@ describe("evidence preview shaping", () => {
significant: true,
},
],
- significance_code: "significant",
is_cached: true,
last_refresh: new Date().toISOString(),
} as unknown as Schemas.ExperimentQueryResponse;
@@ -254,29 +253,84 @@ describe("evidence preview shaping", () => {
expect(results.secondaryMetrics[0].state).toBe("ready");
});
- it("shows outcomes while marking low-exposure results as insufficient", () => {
- const insufficientResponse = {
+ it.each([
+ {
+ method: "frequentist",
+ statistics: { confidence_interval: null, p_value: null },
+ },
+ {
+ method: "bayesian",
+ statistics: { credible_interval: null, chance_to_win: null },
+ },
+ ])(
+ "shows outcomes while marking low-exposure $method results as insufficient",
+ ({ method, statistics }) => {
+ const insufficientResponse = {
+ ...readyMetricResponse,
+ variant_results: [
+ {
+ key: "test",
+ method,
+ number_of_samples: 40,
+ sum: 6,
+ sum_squares: 6,
+ significant: null,
+ validation_failures: ["not-enough-exposures"],
+ ...statistics,
+ },
+ ],
+ } as unknown as Schemas.ExperimentQueryResponse;
+ const results = shapeExperimentResults(
+ experimentWithMetrics(),
+ exposureResponse,
+ [{ response: insufficientResponse }],
+ [{ response: insufficientResponse }],
+ );
+
+ expect(results.state).toBe("insufficient_data");
+ expect(results.primaryMetrics[0]).toMatchObject({
+ state: "insufficient_data",
+ variants: [
+ expect.any(Object),
+ expect.objectContaining({
+ outcome: "6 · 15.0%",
+ pValue: null,
+ chanceToWin: null,
+ significance: "insufficient_data",
+ }),
+ ],
+ });
+ },
+ );
+
+ it("shows a Bayesian result's chance to win instead of a p-value", () => {
+ const bayesianResponse = {
...readyMetricResponse,
- significance_code: "not_enough_exposure",
- significant: false,
+ variant_results: [
+ {
+ key: "test",
+ method: "bayesian",
+ number_of_samples: 1000,
+ sum: 125,
+ sum_squares: 125,
+ credible_interval: [0.05, 0.45],
+ chance_to_win: 0.97,
+ significant: true,
+ },
+ ],
} as unknown as Schemas.ExperimentQueryResponse;
const results = shapeExperimentResults(
experimentWithMetrics(),
exposureResponse,
- [{ response: insufficientResponse }],
- [{ response: insufficientResponse }],
+ [{ response: bayesianResponse }],
+ [],
);
- expect(results.state).toBe("insufficient_data");
- expect(results.primaryMetrics[0]).toMatchObject({
- state: "insufficient_data",
- variants: [
- expect.any(Object),
- expect.objectContaining({
- outcome: "125 · 12.5%",
- significance: "insufficient_data",
- }),
- ],
+ expect(results.primaryMetrics[0].variants[1]).toMatchObject({
+ interval: "+5.00% to +45.0%",
+ pValue: null,
+ chanceToWin: "97.0%",
+ significance: "significant",
});
});
diff --git a/products/desktop/packages/api-client/src/evidence-previews.ts b/products/desktop/packages/api-client/src/evidence-previews.ts
index a1ed832c408a..4de972a870d8 100644
--- a/products/desktop/packages/api-client/src/evidence-previews.ts
+++ b/products/desktop/packages/api-client/src/evidence-previews.ts
@@ -658,10 +658,8 @@ function outcomeValue(
}
function hasInsufficientData(
- response: Schemas.ExperimentQueryResponse,
stats: Schemas.ExperimentStatsBaseValidated[],
): boolean {
- if (response.significance_code === "not_enough_exposure") return true;
return stats.some((entry) =>
(entry.validation_failures ?? []).some(
(failure) =>
@@ -736,7 +734,7 @@ function shapeMetricResult(
const baselineOutcome = outcomeValue(baseline, definition.valueType);
const allStats = [baseline, ...variants];
- const insufficient = hasInsufficientData(response, allStats);
+ const insufficient = hasInsufficientData(allStats);
const upliftedRows: Array<{
row: ExperimentVariantResultPresentation;
value: number;
@@ -746,22 +744,19 @@ function shapeMetricResult(
const isControl = index === 0;
const statsRecord = stats as unknown as QueryRecord;
const interval = validInterval(
- statsRecord.confidence_interval ??
- statsRecord.credible_interval ??
- response.credible_intervals?.[stats.key],
- );
- const pValue = cellNumber(
- statsRecord.p_value ?? (variants.length === 1 ? response.p_value : null),
- );
- const chanceToWin = cellNumber(
- statsRecord.chance_to_win ?? response.probability?.[stats.key],
+ statsRecord.confidence_interval ?? statsRecord.credible_interval,
);
+ // The API sends an unset statistic as null, and cellNumber reads null as 0.
+ const pValue =
+ typeof statsRecord.p_value === "number" ? statsRecord.p_value : null;
+ const chanceToWin =
+ typeof statsRecord.chance_to_win === "number"
+ ? statsRecord.chance_to_win
+ : null;
const significantValue =
typeof statsRecord.significant === "boolean"
? statsRecord.significant
- : variants.length === 1 && typeof response.significant === "boolean"
- ? response.significant
- : null;
+ : null;
const uplift =
!isControl &&
outcome.comparable !== null &&
diff --git a/products/desktop/packages/api-client/src/posthog-client.test.ts b/products/desktop/packages/api-client/src/posthog-client.test.ts
index 7223b6779b8a..34fd544dff33 100644
--- a/products/desktop/packages/api-client/src/posthog-client.test.ts
+++ b/products/desktop/packages/api-client/src/posthog-client.test.ts
@@ -513,7 +513,6 @@ describe("PostHogAPIClient", () => {
confidence_interval: [0.01, 0.39],
},
],
- significance_code: "significant",
is_cached: true,
last_refresh: new Date().toISOString(),
};