Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(slack-app): show primary metric results on experiment unfurls (#124692)

Shy Alterca73c7cd5f9d
diff --git a/products/experiments/backend/facade/api.py b/products/experiments/backend/facade/api.py
index 4fa956d4ce6b..f08b46aa1a0f 100644
--- a/products/experiments/backend/facade/api.py
+++ b/products/experiments/backend/facade/api.py
@@ -32,8 +32,14 @@ from products.experiments.backend.hogql_queries.exposure_query_logic import (
normalize_to_exposure_criteria,
resolve_default_exposure_event,
)
+from products.experiments.backend.metric_calculation.config import (
+ build_primary_calculation_configs,
+ calculation_metric_name,
+)
+from products.experiments.backend.metric_calculation.results import MetricResultStore
from products.experiments.backend.models.experiment import Experiment as ExperimentModel
from products.experiments.backend.running_time_target import load_running_time_target
+from products.experiments.stats.shared.enums import DifferenceType
from .contracts import (
CreateExperimentInput,
@@ -47,6 +53,8 @@ from .contracts import (
ExperimentValidationError,
RunningExperiment,
RunningTimeMode,
+ StoredArmResult,
+ StoredPrimaryResult,
)
logger = structlog.get_logger(__name__)
@@ -174,6 +182,39 @@ def get_experiment_health_inputs(
return health_inputs_payload(inputs)
+def get_stored_primary_result(*, team_id: int, experiment_id: int) -> StoredPrimaryResult | None:
+ """The current stored result of the experiment's primary metric. It reads Postgres only, not ClickHouse.
+
+ None for an experiment that reports absolute differences: the calculation key leaves out the difference type,
+ so the stored interval can be in either unit."""
+ experiment = ExperimentModel.objects.filter(team_id=team_id, pk=experiment_id, deleted=False).first()
+ config = next(iter(build_primary_calculation_configs(experiment)), None) if experiment else None
+ if experiment is None or config is None or config.settings.stats.difference_type == DifferenceType.ABSOLUTE:
+ return None
+ stats = MetricResultStore(experiment_id=experiment_id).current_result(config)
+ if stats is None:
+ return None
+ base = stats.baseline.mean
+ definition = config.definition
+ return StoredPrimaryResult(
+ metric_name=calculation_metric_name(experiment, config),
+ metric_type=definition.get("metric_type"),
+ proportion=definition.get("metric_type") == "funnel" or definition.get("threshold") is not None,
+ goal=definition.get("goal"),
+ baseline=StoredArmResult(key=stats.baseline.key, number_of_samples=stats.baseline.number_of_samples, mean=base),
+ variants=tuple(
+ StoredArmResult(
+ key=variant.key,
+ number_of_samples=variant.number_of_samples,
+ mean=variant.mean,
+ change=variant.interval.midpoint if variant.interval and base and base > 0 else None,
+ significant=variant.significant,
+ )
+ for variant in stats.variants
+ ),
+ )
+
+
def get_running_time_target(
*, team_id: int, experiment_id: int, mode: RunningTimeMode | None = None
) -> ExperimentRunningTimeTarget:
diff --git a/products/experiments/backend/facade/contracts.py b/products/experiments/backend/facade/contracts.py
index 3410e66c454e..1c533f34818a 100644
--- a/products/experiments/backend/facade/contracts.py
+++ b/products/experiments/backend/facade/contracts.py
@@ -20,6 +20,27 @@ if TYPE_CHECKING:
MAX_METRICS_TO_SUMMARIZE = 50
+@frozen
+class StoredArmResult:
+ key: str
+ number_of_samples: int
+ mean: float | None
+ change: float | None = None
+ significant: bool | None = None
+
+
+@frozen
+class StoredPrimaryResult:
+ """The current stored result of an experiment's primary metric."""
+
+ metric_name: str
+ metric_type: str | None
+ proportion: bool
+ goal: str | None
+ baseline: StoredArmResult
+ variants: tuple[StoredArmResult, ...]
+
+
@frozen
class ExperimentSummaryData:
"""Result of fetching experiment data for the AI results summary."""
diff --git a/products/experiments/backend/test/test_facade.py b/products/experiments/backend/test/test_facade.py
index 2ceb5e8c064a..d692105c169e 100644
--- a/products/experiments/backend/test/test_facade.py
+++ b/products/experiments/backend/test/test_facade.py
@@ -3,6 +3,7 @@
import uuid
from datetime import UTC, datetime, timedelta
+import pytest
import time_machine
from posthog.test.base import (
APIBaseTest,
@@ -27,8 +28,10 @@ from products.experiments.backend.facade.contracts import (
CreateExperimentInput,
EndedExperiment,
ExperimentValidationError,
+ StoredPrimaryResult,
)
-from products.experiments.backend.models.experiment import Experiment, ExperimentHoldout
+from products.experiments.backend.metric_calculation.config import build_primary_calculation_configs
+from products.experiments.backend.models.experiment import Experiment, ExperimentHoldout, ExperimentMetricResult
from products.feature_flags.backend.facade.enums import FlagEvaluationsMode
from products.feature_flags.backend.models.feature_flag import FeatureFlag
from products.feature_flags.backend.models.organization_feature_flags_config import OrganizationFeatureFlagsConfig
@@ -339,3 +342,66 @@ class TestCountEnrolledPeople(ClickhouseTestMixin, BaseTest):
flush_persons_and_events()
assert api.count_enrolled_people(team=self.team, user=self.user, flag_keys=["checkout"]) == {"checkout": 2}
+
+
+_CHECKOUT = {"kind": "EventsNode", "event": "checkout"}
+
+
+class TestGetStoredPrimaryResult(BaseTest):
+ def _stored(self, metric: dict, stats_config: dict, baseline_sum: int) -> StoredPrimaryResult | None:
+ flag = FeatureFlag.objects.create(team=self.team, created_by=self.user, key="stored-result")
+ experiment = Experiment.objects.create(
+ team=self.team,
+ feature_flag=flag,
+ name="stored result",
+ start_date=datetime(2026, 8, 1, tzinfo=UTC),
+ stats_config=stats_config,
+ metrics=[{"uuid": "m1", "kind": "ExperimentMetric", "name": "Checkout", "goal": "decrease", **metric}],
+ )
+ ExperimentMetricResult.objects.create(
+ experiment=experiment,
+ metric_uuid="m1",
+ fingerprint=build_primary_calculation_configs(experiment)[0].calculation_key(),
+ query_from=datetime(2026, 8, 1, tzinfo=UTC),
+ query_to=datetime(2026, 8, 5, tzinfo=UTC),
+ status=ExperimentMetricResult.Status.COMPLETED,
+ completed_at=datetime(2026, 8, 5, tzinfo=UTC),
+ result={
+ "baseline": {"key": "control", "number_of_samples": 100, "sum": baseline_sum, "sum_squares": 20},
+ "variant_results": [
+ {
+ "key": "test",
+ "method": "frequentist",
+ "number_of_samples": 50,
+ "sum": 15,
+ "sum_squares": 15,
+ "confidence_interval": [0.1, 0.4],
+ "significant": True,
+ }
+ ],
+ },
+ )
+ return api.get_stored_primary_result(team_id=self.team.pk, experiment_id=experiment.pk)
+
+ @parameterized.expand(
+ [
+ ("funnel", {"metric_type": "funnel", "series": [_CHECKOUT]}, 20, 0.25, True),
+ ("thresholded_mean", {"metric_type": "mean", "threshold": 5, "source": _CHECKOUT}, 20, 0.25, True),
+ ("negative_baseline_mean", {"metric_type": "mean", "source": _CHECKOUT}, -20, None, False),
+ ]
+ )
+ def test_returns_the_current_stored_result_of_the_primary_metric(
+ self, _name: str, metric: dict, baseline_sum: int, change: float | None, proportion: bool
+ ) -> None:
+ result = self._stored(metric, {"method": "frequentist"}, baseline_sum)
+
+ assert result is not None
+ assert (result.metric_name, result.proportion, result.goal) == ("Checkout", proportion, "decrease")
+ assert result.baseline.mean == pytest.approx(baseline_sum / 100)
+ assert (result.variants[0].mean, result.variants[0].significant) == (0.3, True)
+ assert result.variants[0].change == (None if change is None else pytest.approx(change))
+
+ def test_returns_nothing_for_absolute_differences(self) -> None:
+ stats_config = {"method": "frequentist", "frequentist": {"difference_type": "ABSOLUTE"}}
+
+ assert self._stored({"metric_type": "funnel", "series": [_CHECKOUT]}, stats_config, 20) is None
diff --git a/products/slack_app/backend/slack_link_unfurl.py b/products/slack_app/backend/slack_link_unfurl.py
index 3d954ee1f818..58f4c04f7a52 100644
--- a/products/slack_app/backend/slack_link_unfurl.py
+++ b/products/slack_app/backend/slack_link_unfurl.py
@@ -45,6 +45,8 @@ from products.conversations.backend.models.ticket import Ticket
from products.dashboards.backend.models.dashboard import Dashboard
from products.dashboards.backend.models.dashboard_tile import DashboardTile
from products.error_tracking.backend.facade import api as error_tracking_facade
+from products.experiments.backend.facade import api as experiments_facade
+from products.experiments.backend.facade.contracts import StoredArmResult, StoredPrimaryResult
from products.experiments.backend.models.experiment import Experiment
from products.exports.backend.facade import api as exports_facade
from products.feature_flags.backend.facade.config import ConfigFormatError
@@ -1006,20 +1008,89 @@ def _experiment_card(ctx: _UnfurlContext, ref: str, url: str) -> _UnfurlCard | _
if experiment.start_date:
days = ((experiment.end_date or timezone.now()) - experiment.start_date).days + 1
details.append(f"Ran {_count(days, 'day')}" if experiment.end_date else f"Day {days}")
- metric = _primary_metric(experiment.metrics)
+ stats = _stored_primary_result(experiment)
+ metric = stats.metric_name if stats else _primary_metric(experiment.metrics)
if metric:
details.append(f"Metric: {metric}")
+ variants = _flag_variants(experiment.feature_flag)
+ if stats:
+ details.append(f"Split {'/'.join(str(variant['rollout_percentage']) for variant in variants)}")
return _UnfurlCard(
label="Experiment",
title=experiment.name,
status=_humanize(experiment.status_label),
details=tuple(details),
- chart=_chart_rows(_variant_rows(_flag_variants(experiment.feature_flag))),
- body=experiment.description,
+ chart=_experiment_chart(stats) if stats else _chart_rows(_variant_rows(variants)),
+ body="\n\n".join(
+ part for part in (_experiment_verdict(stats) if stats else None, experiment.description) if part
+ ),
timestamp=experiment.updated_at,
)
+def _stored_primary_result(experiment: Experiment) -> StoredPrimaryResult | None:
+ try:
+ stats = experiments_facade.get_stored_primary_result(team_id=experiment.team_id, experiment_id=experiment.pk)
+ except Exception:
+ logger.exception("slack_app_unfurl_experiment_results_failed", experiment_id=experiment.pk)
+ return None
+ if (
+ stats is None
+ or stats.metric_type not in ("funnel", "mean")
+ or stats.baseline.mean is None
+ or not stats.variants
+ ):
+ return None
+ return stats
+
+
+def _variant_delta(stats: StoredPrimaryResult, variant: StoredArmResult) -> float | None:
+ if variant.change is not None:
+ return variant.change
+ if variant.mean is None or stats.baseline.mean is None or stats.baseline.mean <= 0:
+ return None
+ return variant.mean / stats.baseline.mean - 1
+
+
+def _experiment_verdict(stats: StoredPrimaryResult) -> str:
+ control = stats.baseline.key
+ if all(variant.change is None for variant in stats.variants):
+ exposed = stats.baseline.number_of_samples + sum(variant.number_of_samples for variant in stats.variants)
+ return f"⏳ *Too early to call.* {exposed:,} exposed so far."
+ direction = -1 if stats.goal == "decrease" else 1
+ ranked = sorted(
+ ((variant, delta) for variant in stats.variants if (delta := _variant_delta(stats, variant)) is not None),
+ key=lambda pair: pair[1] * direction,
+ reverse=True,
+ )
+ significant = [(variant, delta) for variant, delta in ranked if variant.significant]
+ if significant and significant[0][1] * direction > 0:
+ return f"🏆 *Winner so far: {significant[0][0].key}.* {significant[0][1]:+.1%} vs {control}."
+ if significant:
+ return f"🔻 *Losing: {significant[-1][0].key}.* {significant[-1][1]:+.1%} vs {control}."
+ if ranked and ranked[0][1] * direction > 0:
+ return f"⚖️ *No clear winner yet.* {ranked[0][0].key} leads at {ranked[0][1]:+.1%} vs {control}, but it is not significant."
+ return f"⚖️ *No clear winner yet.* No variant beats {control}."
+
+
+def _experiment_chart(stats: StoredPrimaryResult) -> str | None:
+ arms = [stats.baseline, *stats.variants]
+ means = [arm.mean for arm in arms if arm.mean is not None]
+ show_bars = all(mean >= 0 for mean in means)
+ top = max(means, default=0.0)
+ rows = []
+ for arm in arms:
+ if arm.mean is None:
+ value = "—"
+ else:
+ value = f"{arm.mean:.1%}" if stats.proportion else f"{arm.mean:,.2f}"
+ if arm is not stats.baseline and (delta := _variant_delta(stats, arm)) is not None:
+ value += f" {delta:+.1%}{' ✓' if arm.significant else ''}"
+ bar = _bar((arm.mean or 0.0) / top if top else 0.0) if show_bars else ""
+ rows.append(_ChartRow(label=_chart_label(arm.key), bar=bar, value=value))
+ return _chart_rows(rows)
+
+
def _survey_status(survey: Survey) -> str:
if survey.archived:
return "Archived"
diff --git a/products/slack_app/backend/tests/test_link_unfurl.py b/products/slack_app/backend/tests/test_link_unfurl.py
index 782222ddba15..f5df34b7bb5c 100644
--- a/products/slack_app/backend/tests/test_link_unfurl.py
+++ b/products/slack_app/backend/tests/test_link_unfurl.py
@@ -36,6 +36,7 @@ from products.conversations.backend.models.ticket import Ticket
from products.dashboards.backend.models.dashboard import Dashboard
from products.dashboards.backend.models.dashboard_tile import DashboardTile
from products.error_tracking.backend.facade import testing as error_tracking_testing
+from products.experiments.backend.facade.contracts import StoredArmResult, StoredPrimaryResult
from products.experiments.backend.models.experiment import Experiment
from products.feature_flags.backend.models.feature_flag import FeatureFlag
from products.notebooks.backend.models import Notebook
@@ -48,6 +49,7 @@ from products.slack_app.backend.services.slack_scopes import REQUIRED_SLACK_SCOP
from products.slack_app.backend.slack_link_unfurl import (
_card_blocks,
_date_range,
+ _experiment_chart,
_flag_chart,
_insight_resource_label,
_question_list,
@@ -284,6 +286,42 @@ class TestCardText:
questions = [{"question": f"Question {index}"} for index in range(7)]
assert _question_list(questions) == "1. Question 0\n2. Question 1\n3. Question 2\n+ 4 more"
+ @parameterized.expand(
+ [
+ (
+ "variant_without_data",
+ "funnel",
+ 0.2,
+ (0.25, None),
+ ["control ████████████████░░░░ 20.0%", "empty ░░░░░░░░░░░░░░░░░░░░ —"],
+ ),
+ ("negative_means", "mean", -10.0, (-5.0, -20.0), ["control -10.00", "test -5.00"]),
+ ]
+ )
+ def test_experiment_chart_shows_measured_values_only(
+ self,
+ _name: str,
+ metric_type: str,
+ control: float,
+ means: tuple[float | None, float | None],
+ expected: list[str],
+ ) -> None:
+ stats = StoredPrimaryResult(
+ metric_name="Metric",
+ metric_type=metric_type,
+ proportion=metric_type == "funnel",
+ goal=None,
+ baseline=StoredArmResult(key="control", number_of_samples=100, mean=control),
+ variants=(
+ StoredArmResult(key="test", number_of_samples=100, mean=means[0]),
+ StoredArmResult(key="empty", number_of_samples=0, mean=means[1]),
+ ),
+ )
+
+ lines = (_experiment_chart(stats) or "").splitlines()
+
+ assert all(line in lines for line in expected)
+
@pytest.mark.django_db
class TestHandlePosthogLinkUnfurl(APIBaseTest):
@@ -1162,6 +1200,65 @@ class TestHandlePosthogLinkUnfurl(APIBaseTest):
text = self._unfurl_text(mock_client.chat_unfurl.call_args.kwargs["unfurls"][url])
assert "*Paused*" in text
+ @parameterized.expand(
+ [
+ ("winner", 0.15, True, ["*Winner so far: test.* +15.0% vs control.", "20.0%", "23.0% +15.0% ✓"]),
+ ("too_early", None, None, ["*Too early to call.* 10,400 exposed so far.", "23.0% +15.0%"]),
+ ]
+ )
+ @patch("products.slack_app.backend.api.resolve_slack_user")
+ @patch("products.slack_app.backend.slack_link_unfurl.SlackIntegration")
+ def test_experiment_card_shows_the_stored_primary_result(
+ self,
+ _name: str,
+ change: float | None,
+ significant: bool | None,
+ expected: list[str],
+ mock_slack_integration_class: MagicMock,
+ mock_resolve: MagicMock,
+ ) -> None:
+ flag = FeatureFlag.objects.create(
+ team=self.team,
+ key="checkout-copy",
+ created_by=self.user,
+ filters={
+ "groups": [{"properties": [], "rollout_percentage": 100}],
+ "multivariate": {
+ "variants": [
+ {"key": "control", "rollout_percentage": 50},
+ {"key": "test", "rollout_percentage": 50},
+ ]
+ },
+ },
+ )
+ experiment = Experiment.objects.create(team=self.team, name="Checkout copy", feature_flag=flag)
+ stored = StoredPrimaryResult(
+ metric_name="Completed checkout",
+ metric_type="funnel",
+ proportion=True,
+ goal=None,
+ baseline=StoredArmResult(key="control", number_of_samples=5200, mean=0.2),
+ variants=(
+ StoredArmResult(key="test", number_of_samples=5200, mean=0.23, change=change, significant=significant),
+ ),
+ )
+ mock_resolve.return_value = MagicMock(user=self.user)
+ mock_client = MagicMock()
+ mock_slack_integration_class.return_value.client = mock_client
+ url = f"http://testserver/project/{self.team.pk}/experiments/{experiment.pk}"
+
+ with patch(
+ "products.slack_app.backend.slack_link_unfurl.experiments_facade.get_stored_primary_result",
+ return_value=stored,
+ ):
+ self._unfurl_links([url])
+
+ text = self._unfurl_text(mock_client.chat_unfurl.call_args.kwargs["unfurls"][url])
+ assert "Metric: Completed checkout" in text
+ assert "Split 50/50" in text
+ for fragment in expected:
+ assert fragment in text
+
@staticmethod
def _assert_slack_accepts(unfurl: dict) -> None:
for block in unfurl["blocks"]: