:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(slack-app): show primary metric results on experiment unfurls (#124692)
Shy Alterca73c7cd5f9d
diff --git a/products/experiments/backend/facade/api.py b/products/experiments/backend/facade/api.pyindex 4fa956d4ce6b..f08b46aa1a0f 100644--- a/products/experiments/backend/facade/api.py+++ b/products/experiments/backend/facade/api.py@@ -32,8 +32,14 @@ from products.experiments.backend.hogql_queries.exposure_query_logic import (normalize_to_exposure_criteria,resolve_default_exposure_event,)+from products.experiments.backend.metric_calculation.config import (+ build_primary_calculation_configs,+ calculation_metric_name,+)+from products.experiments.backend.metric_calculation.results import MetricResultStorefrom products.experiments.backend.models.experiment import Experiment as ExperimentModelfrom products.experiments.backend.running_time_target import load_running_time_target+from products.experiments.stats.shared.enums import DifferenceTypefrom .contracts import (CreateExperimentInput,@@ -47,6 +53,8 @@ from .contracts import (ExperimentValidationError,RunningExperiment,RunningTimeMode,+ StoredArmResult,+ StoredPrimaryResult,)logger = structlog.get_logger(__name__)@@ -174,6 +182,39 @@ def get_experiment_health_inputs(return health_inputs_payload(inputs)+def get_stored_primary_result(*, team_id: int, experiment_id: int) -> StoredPrimaryResult | None:+ """The current stored result of the experiment's primary metric. It reads Postgres only, not ClickHouse.++ None for an experiment that reports absolute differences: the calculation key leaves out the difference type,+ so the stored interval can be in either unit."""+ experiment = ExperimentModel.objects.filter(team_id=team_id, pk=experiment_id, deleted=False).first()+ config = next(iter(build_primary_calculation_configs(experiment)), None) if experiment else None+ if experiment is None or config is None or config.settings.stats.difference_type == DifferenceType.ABSOLUTE:+ return None+ stats = MetricResultStore(experiment_id=experiment_id).current_result(config)+ if stats is None:+ return None+ base = stats.baseline.mean+ definition = config.definition+ return StoredPrimaryResult(+ metric_name=calculation_metric_name(experiment, config),+ metric_type=definition.get("metric_type"),+ proportion=definition.get("metric_type") == "funnel" or definition.get("threshold") is not None,+ goal=definition.get("goal"),+ baseline=StoredArmResult(key=stats.baseline.key, number_of_samples=stats.baseline.number_of_samples, mean=base),+ variants=tuple(+ StoredArmResult(+ key=variant.key,+ number_of_samples=variant.number_of_samples,+ mean=variant.mean,+ change=variant.interval.midpoint if variant.interval and base and base > 0 else None,+ significant=variant.significant,+ )+ for variant in stats.variants+ ),+ )++def get_running_time_target(*, team_id: int, experiment_id: int, mode: RunningTimeMode | None = None) -> ExperimentRunningTimeTarget:diff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.pyindex 3410e66c454e..1c533f34818a 100644--- a/products/experiments/backend/facade/contracts.py+++ b/products/experiments/backend/facade/contracts.py@@ -20,6 +20,27 @@ if TYPE_CHECKING:MAX_METRICS_TO_SUMMARIZE = 50+@frozen+class StoredArmResult:+ key: str+ number_of_samples: int+ mean: float | None+ change: float | None = None+ significant: bool | None = None+++@frozen+class StoredPrimaryResult:+ """The current stored result of an experiment's primary metric."""++ metric_name: str+ metric_type: str | None+ proportion: bool+ goal: str | None+ baseline: StoredArmResult+ variants: tuple[StoredArmResult, ...]++@frozenclass ExperimentSummaryData:"""Result of fetching experiment data for the AI results summary."""diff --git a/products/experiments/backend/test/test_facade.py b/products/experiments/backend/test/test_facade.pyindex 2ceb5e8c064a..d692105c169e 100644--- a/products/experiments/backend/test/test_facade.py+++ b/products/experiments/backend/test/test_facade.py@@ -3,6 +3,7 @@import uuidfrom datetime import UTC, datetime, timedelta+import pytestimport time_machinefrom posthog.test.base import (APIBaseTest,@@ -27,8 +28,10 @@ from products.experiments.backend.facade.contracts import (CreateExperimentInput,EndedExperiment,ExperimentValidationError,+ StoredPrimaryResult,)-from products.experiments.backend.models.experiment import Experiment, ExperimentHoldout+from products.experiments.backend.metric_calculation.config import build_primary_calculation_configs+from products.experiments.backend.models.experiment import Experiment, ExperimentHoldout, ExperimentMetricResultfrom products.feature_flags.backend.facade.enums import FlagEvaluationsModefrom products.feature_flags.backend.models.feature_flag import FeatureFlagfrom products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig@@ -339,3 +342,66 @@ class TestCountEnrolledPeople(ClickhouseTestMixin, BaseTest):flush_persons_and_events()assert api.count_enrolled_people(team=self.team, user=self.user, flag_keys=["checkout"]) == {"checkout": 2}+++_CHECKOUT = {"kind": "EventsNode", "event": "checkout"}+++class TestGetStoredPrimaryResult(BaseTest):+ def _stored(self, metric: dict, stats_config: dict, baseline_sum: int) -> StoredPrimaryResult | None:+ flag = FeatureFlag.objects.create(team=self.team, created_by=self.user, key="stored-result")+ experiment = Experiment.objects.create(+ team=self.team,+ feature_flag=flag,+ name="stored result",+ start_date=datetime(2026, 8, 1, tzinfo=UTC),+ stats_config=stats_config,+ metrics=[{"uuid": "m1", "kind": "ExperimentMetric", "name": "Checkout", "goal": "decrease", **metric}],+ )+ ExperimentMetricResult.objects.create(+ experiment=experiment,+ metric_uuid="m1",+ fingerprint=build_primary_calculation_configs(experiment)[0].calculation_key(),+ query_from=datetime(2026, 8, 1, tzinfo=UTC),+ query_to=datetime(2026, 8, 5, tzinfo=UTC),+ status=ExperimentMetricResult.Status.COMPLETED,+ completed_at=datetime(2026, 8, 5, tzinfo=UTC),+ result={+ "baseline": {"key": "control", "number_of_samples": 100, "sum": baseline_sum, "sum_squares": 20},+ "variant_results": [+ {+ "key": "test",+ "method": "frequentist",+ "number_of_samples": 50,+ "sum": 15,+ "sum_squares": 15,+ "confidence_interval": [0.1, 0.4],+ "significant": True,+ }+ ],+ },+ )+ return api.get_stored_primary_result(team_id=self.team.pk, experiment_id=experiment.pk)++ @parameterized.expand(+ [+ ("funnel", {"metric_type": "funnel", "series": [_CHECKOUT]}, 20, 0.25, True),+ ("thresholded_mean", {"metric_type": "mean", "threshold": 5, "source": _CHECKOUT}, 20, 0.25, True),+ ("negative_baseline_mean", {"metric_type": "mean", "source": _CHECKOUT}, -20, None, False),+ ]+ )+ def test_returns_the_current_stored_result_of_the_primary_metric(+ self, _name: str, metric: dict, baseline_sum: int, change: float | None, proportion: bool+ ) -> None:+ result = self._stored(metric, {"method": "frequentist"}, baseline_sum)++ assert result is not None+ assert (result.metric_name, result.proportion, result.goal) == ("Checkout", proportion, "decrease")+ assert result.baseline.mean == pytest.approx(baseline_sum / 100)+ assert (result.variants[0].mean, result.variants[0].significant) == (0.3, True)+ assert result.variants[0].change == (None if change is None else pytest.approx(change))++ def test_returns_nothing_for_absolute_differences(self) -> None:+ stats_config = {"method": "frequentist", "frequentist": {"difference_type": "ABSOLUTE"}}++ assert self._stored({"metric_type": "funnel", "series": [_CHECKOUT]}, stats_config, 20) is Nonediff --git a/products/slack_app/backend/slack_link_unfurl.py b/products/slack_app/backend/slack_link_unfurl.pyindex 3d954ee1f818..58f4c04f7a52 100644--- a/products/slack_app/backend/slack_link_unfurl.py+++ b/products/slack_app/backend/slack_link_unfurl.py@@ -45,6 +45,8 @@ from products.conversations.backend.models.ticket import Ticketfrom products.dashboards.backend.models.dashboard import Dashboardfrom products.dashboards.backend.models.dashboard_tile import DashboardTilefrom products.error_tracking.backend.facade import api as error_tracking_facade+from products.experiments.backend.facade import api as experiments_facade+from products.experiments.backend.facade.contracts import StoredArmResult, StoredPrimaryResultfrom products.experiments.backend.models.experiment import Experimentfrom products.exports.backend.facade import api as exports_facadefrom products.feature_flags.backend.facade.config import ConfigFormatError@@ -1006,20 +1008,89 @@ def _experiment_card(ctx: _UnfurlContext, ref: str, url: str) -> _UnfurlCard | _if experiment.start_date:days = ((experiment.end_date or timezone.now()) - experiment.start_date).days + 1details.append(f"Ran {_count(days, 'day')}" if experiment.end_date else f"Day {days}")- metric = _primary_metric(experiment.metrics)+ stats = _stored_primary_result(experiment)+ metric = stats.metric_name if stats else _primary_metric(experiment.metrics)if metric:details.append(f"Metric: {metric}")+ variants = _flag_variants(experiment.feature_flag)+ if stats:+ details.append(f"Split {'/'.join(str(variant['rollout_percentage']) for variant in variants)}")return _UnfurlCard(label="Experiment",title=experiment.name,status=_humanize(experiment.status_label),details=tuple(details),- chart=_chart_rows(_variant_rows(_flag_variants(experiment.feature_flag))),- body=experiment.description,+ chart=_experiment_chart(stats) if stats else _chart_rows(_variant_rows(variants)),+ body="\n\n".join(+ part for part in (_experiment_verdict(stats) if stats else None, experiment.description) if part+ ),timestamp=experiment.updated_at,)+def _stored_primary_result(experiment: Experiment) -> StoredPrimaryResult | None:+ try:+ stats = experiments_facade.get_stored_primary_result(team_id=experiment.team_id, experiment_id=experiment.pk)+ except Exception:+ logger.exception("slack_app_unfurl_experiment_results_failed", experiment_id=experiment.pk)+ return None+ if (+ stats is None+ or stats.metric_type not in ("funnel", "mean")+ or stats.baseline.mean is None+ or not stats.variants+ ):+ return None+ return stats+++def _variant_delta(stats: StoredPrimaryResult, variant: StoredArmResult) -> float | None:+ if variant.change is not None:+ return variant.change+ if variant.mean is None or stats.baseline.mean is None or stats.baseline.mean <= 0:+ return None+ return variant.mean / stats.baseline.mean - 1+++def _experiment_verdict(stats: StoredPrimaryResult) -> str:+ control = stats.baseline.key+ if all(variant.change is None for variant in stats.variants):+ exposed = stats.baseline.number_of_samples + sum(variant.number_of_samples for variant in stats.variants)+ return f"⏳ *Too early to call.* {exposed:,} exposed so far."+ direction = -1 if stats.goal == "decrease" else 1+ ranked = sorted(+ ((variant, delta) for variant in stats.variants if (delta := _variant_delta(stats, variant)) is not None),+ key=lambda pair: pair[1] * direction,+ reverse=True,+ )+ significant = [(variant, delta) for variant, delta in ranked if variant.significant]+ if significant and significant[0][1] * direction > 0:+ return f"🏆 *Winner so far: {significant[0][0].key}.* {significant[0][1]:+.1%} vs {control}."+ if significant:+ return f"🔻 *Losing: {significant[-1][0].key}.* {significant[-1][1]:+.1%} vs {control}."+ if ranked and ranked[0][1] * direction > 0:+ return f"⚖️ *No clear winner yet.* {ranked[0][0].key} leads at {ranked[0][1]:+.1%} vs {control}, but it is not significant."+ return f"⚖️ *No clear winner yet.* No variant beats {control}."+++def _experiment_chart(stats: StoredPrimaryResult) -> str | None:+ arms = [stats.baseline, *stats.variants]+ means = [arm.mean for arm in arms if arm.mean is not None]+ show_bars = all(mean >= 0 for mean in means)+ top = max(means, default=0.0)+ rows = []+ for arm in arms:+ if arm.mean is None:+ value = "—"+ else:+ value = f"{arm.mean:.1%}" if stats.proportion else f"{arm.mean:,.2f}"+ if arm is not stats.baseline and (delta := _variant_delta(stats, arm)) is not None:+ value += f" {delta:+.1%}{' ✓' if arm.significant else ''}"+ bar = _bar((arm.mean or 0.0) / top if top else 0.0) if show_bars else ""+ rows.append(_ChartRow(label=_chart_label(arm.key), bar=bar, value=value))+ return _chart_rows(rows)++def _survey_status(survey: Survey) -> str:if survey.archived:return "Archived"diff --git a/products/slack_app/backend/tests/test_link_unfurl.py b/products/slack_app/backend/tests/test_link_unfurl.pyindex 782222ddba15..f5df34b7bb5c 100644--- a/products/slack_app/backend/tests/test_link_unfurl.py+++ b/products/slack_app/backend/tests/test_link_unfurl.py@@ -36,6 +36,7 @@ from products.conversations.backend.models.ticket import Ticketfrom products.dashboards.backend.models.dashboard import Dashboardfrom products.dashboards.backend.models.dashboard_tile import DashboardTilefrom products.error_tracking.backend.facade import testing as error_tracking_testing+from products.experiments.backend.facade.contracts import StoredArmResult, StoredPrimaryResultfrom products.experiments.backend.models.experiment import Experimentfrom products.feature_flags.backend.models.feature_flag import FeatureFlagfrom products.notebooks.backend.models import Notebook@@ -48,6 +49,7 @@ from products.slack_app.backend.services.slack_scopes import REQUIRED_SLACK_SCOPfrom products.slack_app.backend.slack_link_unfurl import (_card_blocks,_date_range,+ _experiment_chart,_flag_chart,_insight_resource_label,_question_list,@@ -284,6 +286,42 @@ class TestCardText:questions = [{"question": f"Question {index}"} for index in range(7)]assert _question_list(questions) == "1. Question 0\n2. Question 1\n3. Question 2\n+ 4 more"+ @parameterized.expand(+ [+ (+ "variant_without_data",+ "funnel",+ 0.2,+ (0.25, None),+ ["control ████████████████░░░░ 20.0%", "empty ░░░░░░░░░░░░░░░░░░░░ —"],+ ),+ ("negative_means", "mean", -10.0, (-5.0, -20.0), ["control -10.00", "test -5.00"]),+ ]+ )+ def test_experiment_chart_shows_measured_values_only(+ self,+ _name: str,+ metric_type: str,+ control: float,+ means: tuple[float | None, float | None],+ expected: list[str],+ ) -> None:+ stats = StoredPrimaryResult(+ metric_name="Metric",+ metric_type=metric_type,+ proportion=metric_type == "funnel",+ goal=None,+ baseline=StoredArmResult(key="control", number_of_samples=100, mean=control),+ variants=(+ StoredArmResult(key="test", number_of_samples=100, mean=means[0]),+ StoredArmResult(key="empty", number_of_samples=0, mean=means[1]),+ ),+ )++ lines = (_experiment_chart(stats) or "").splitlines()++ assert all(line in lines for line in expected)+@pytest.mark.django_dbclass TestHandlePosthogLinkUnfurl(APIBaseTest):@@ -1162,6 +1200,65 @@ class TestHandlePosthogLinkUnfurl(APIBaseTest):text = self._unfurl_text(mock_client.chat_unfurl.call_args.kwargs["unfurls"][url])assert "*Paused*" in text+ @parameterized.expand(+ [+ ("winner", 0.15, True, ["*Winner so far: test.* +15.0% vs control.", "20.0%", "23.0% +15.0% ✓"]),+ ("too_early", None, None, ["*Too early to call.* 10,400 exposed so far.", "23.0% +15.0%"]),+ ]+ )+ @patch("products.slack_app.backend.api.resolve_slack_user")+ @patch("products.slack_app.backend.slack_link_unfurl.SlackIntegration")+ def test_experiment_card_shows_the_stored_primary_result(+ self,+ _name: str,+ change: float | None,+ significant: bool | None,+ expected: list[str],+ mock_slack_integration_class: MagicMock,+ mock_resolve: MagicMock,+ ) -> None:+ flag = FeatureFlag.objects.create(+ team=self.team,+ key="checkout-copy",+ created_by=self.user,+ filters={+ "groups": [{"properties": [], "rollout_percentage": 100}],+ "multivariate": {+ "variants": [+ {"key": "control", "rollout_percentage": 50},+ {"key": "test", "rollout_percentage": 50},+ ]+ },+ },+ )+ experiment = Experiment.objects.create(team=self.team, name="Checkout copy", feature_flag=flag)+ stored = StoredPrimaryResult(+ metric_name="Completed checkout",+ metric_type="funnel",+ proportion=True,+ goal=None,+ baseline=StoredArmResult(key="control", number_of_samples=5200, mean=0.2),+ variants=(+ StoredArmResult(key="test", number_of_samples=5200, mean=0.23, change=change, significant=significant),+ ),+ )+ mock_resolve.return_value = MagicMock(user=self.user)+ mock_client = MagicMock()+ mock_slack_integration_class.return_value.client = mock_client+ url = f"http://testserver/project/{self.team.pk}/experiments/{experiment.pk}"++ with patch(+ "products.slack_app.backend.slack_link_unfurl.experiments_facade.get_stored_primary_result",+ return_value=stored,+ ):+ self._unfurl_links([url])++ text = self._unfurl_text(mock_client.chat_unfurl.call_args.kwargs["unfurls"][url])+ assert "Metric: Completed checkout" in text+ assert "Split 50/50" in text+ for fragment in expected:+ assert fragment in text+@staticmethoddef _assert_slack_accepts(unfurl: dict) -> None:for block in unfurl["blocks"]: