Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

refactor(experiments): add facade seed helpers for outside tests (#125602)

Andersadf88b4a6849
diff --git a/products/experiments/backend/facade/testing.py b/products/experiments/backend/facade/testing.py
new file mode 100644
index 000000000000..29c97dc93d1c
--- /dev/null
+++ b/products/experiments/backend/facade/testing.py
@@ -0,0 +1,136 @@
+"""Test-support facade for experiments.
+
+Test suites outside this product plant experiments, holdouts, saved metrics and the team
+experiments config through here instead of importing the models. Ids go in and ids come out.
+
+The helpers write rows directly. `facade.api.create_experiment` is the production door, but it
+creates the feature flag from a key and validates the metrics, and the outside suites bring a
+flag they made themselves.
+"""
+
+from datetime import datetime
+from typing import Any
+
+from asgiref.sync import sync_to_async
+
+from products.experiments.backend.models.experiment import (
+ Experiment,
+ ExperimentHoldout,
+ ExperimentSavedMetric,
+ ExperimentToSavedMetric,
+)
+from products.experiments.backend.models.team_experiments_config import TeamExperimentsConfig
+
+
+def create_experiment(
+ *,
+ team_id: int,
+ feature_flag_id: int,
+ name: str = "Test experiment",
+ description: str | None = None,
+ created_by_id: int | None = None,
+ start_date: datetime | None = None,
+ end_date: datetime | None = None,
+ holdout_id: int | None = None,
+ parameters: dict[str, Any] | None = None,
+ filters: dict[str, Any] | None = None,
+ metrics: list[dict[str, Any]] | None = None,
+ metrics_secondary: list[dict[str, Any]] | None = None,
+ exposure_criteria: dict[str, Any] | None = None,
+ excluded_variants: list[str] | None = None,
+ archived: bool = False,
+ deleted: bool = False,
+ updated_at: datetime | None = None,
+) -> int:
+ """Create an experiment on an existing feature flag, and return its id.
+
+ The save derives the status from the dates: no `start_date` is a draft, a `start_date` is
+ running, and both dates are stopped. JSON fields left as None take the model defaults.
+ """
+ json_fields = {
+ "parameters": parameters,
+ "filters": filters,
+ "metrics": metrics,
+ "metrics_secondary": metrics_secondary,
+ "exposure_criteria": exposure_criteria,
+ }
+ experiment = Experiment.objects.create(
+ team_id=team_id,
+ feature_flag_id=feature_flag_id,
+ name=name,
+ description=description,
+ created_by_id=created_by_id,
+ start_date=start_date,
+ end_date=end_date,
+ holdout_id=holdout_id,
+ excluded_variants=excluded_variants,
+ archived=archived,
+ deleted=deleted,
+ **{field: value for field, value in json_fields.items() if value is not None},
+ )
+ if updated_at is not None:
+ # updated_at is auto_now, so the insert overwrites it and only an update can backdate it.
+ Experiment.objects.filter(team_id=team_id, id=experiment.id).update(updated_at=updated_at)
+ return experiment.id
+
+
+acreate_experiment = sync_to_async(create_experiment)
+
+
+def update_experiment(*, team_id: int, experiment_id: int, **fields: Any) -> None:
+ """Set fields on an experiment and save it, so the save signals fire as they do for a production
+ write. `deleted=True` is a soft delete."""
+ experiment = Experiment.objects.get(team_id=team_id, id=experiment_id)
+ for field, value in fields.items():
+ # setattr accepts any name, so a misspelled field would save nothing and pass silently.
+ Experiment._meta.get_field(field)
+ setattr(experiment, field, value)
+ experiment.save()
+
+
+def delete_experiment(*, team_id: int, experiment_id: int) -> None:
+ """Remove the experiment row, which fires the delete signals."""
+ Experiment.objects.get(team_id=team_id, id=experiment_id).delete()
+
+
+def create_holdout(
+ *,
+ team_id: int,
+ name: str,
+ filters: list[dict[str, Any]] | None = None,
+ created_by_id: int | None = None,
+) -> int:
+ holdout = ExperimentHoldout.objects.create(
+ team_id=team_id,
+ name=name,
+ created_by_id=created_by_id,
+ **({} if filters is None else {"filters": filters}),
+ )
+ return holdout.id
+
+
+def create_saved_metric(*, team_id: int, name: str, query: dict[str, Any], created_by_id: int | None = None) -> int:
+ saved_metric = ExperimentSavedMetric.objects.create(
+ team_id=team_id, name=name, query=query, created_by_id=created_by_id
+ )
+ return saved_metric.id
+
+
+def link_saved_metric(*, experiment_id: int, saved_metric_id: int, metadata: dict[str, Any] | None = None) -> int:
+ """Attach a saved metric to an experiment, and return the link id.
+
+ The metric is primary unless `metadata` holds `"type": "secondary"`.
+ """
+ link = ExperimentToSavedMetric.objects.create(
+ experiment_id=experiment_id, saved_metric_id=saved_metric_id, metadata=metadata or {}
+ )
+ return link.id
+
+
+def set_team_experiments_config(*, team_id: int, **fields: Any) -> None:
+ """Write fields on the team's experiments config.
+
+ A missing row is created with the fields in one insert, so its activity log records them as
+ changes from the defaults, the same as a row created in the Django admin.
+ """
+ TeamExperimentsConfig.objects.update_or_create(team_id=team_id, defaults=fields)
diff --git a/products/experiments/backend/test/test_facade_testing.py b/products/experiments/backend/test/test_facade_testing.py
new file mode 100644
index 000000000000..fa0f05e53a6d
--- /dev/null
+++ b/products/experiments/backend/test/test_facade_testing.py
@@ -0,0 +1,69 @@
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from posthog.test.base import BaseTest
+
+from django.core.exceptions import FieldDoesNotExist
+
+from posthog.models.team import Team
+
+from products.experiments.backend.facade import testing
+from products.experiments.backend.models.experiment import Experiment, ExperimentToSavedMetric
+from products.experiments.backend.models.team_experiments_config import TeamExperimentsConfig
+from products.feature_flags.backend.models.feature_flag import FeatureFlag
+
+
+class TestFacadeTesting(BaseTest):
+ def test_seed_helpers_write_team_scoped_rows_and_relations(self) -> None:
+ flag = FeatureFlag.objects.create(team=self.team, key="seeded-flag", created_by=self.user)
+ holdout_id = testing.create_holdout(
+ team_id=self.team.id, name="Holdout", filters=[{"properties": [], "rollout_percentage": 10}]
+ )
+ start_date = datetime(2026, 1, 1, tzinfo=UTC)
+ experiment_id = testing.create_experiment(
+ team_id=self.team.id,
+ feature_flag_id=flag.id,
+ created_by_id=self.user.id,
+ start_date=start_date,
+ end_date=start_date + timedelta(days=1),
+ holdout_id=holdout_id,
+ metrics=[{"kind": "ExperimentMetric", "metric_type": "mean"}],
+ updated_at=start_date,
+ )
+ saved_metric_id = testing.create_saved_metric(
+ team_id=self.team.id, name="Shared metric", query={"kind": "ExperimentMetric", "metric_type": "funnel"}
+ )
+ testing.link_saved_metric(
+ experiment_id=experiment_id, saved_metric_id=saved_metric_id, metadata={"type": "secondary"}
+ )
+ testing.set_team_experiments_config(team_id=self.team.id, default_cuped_enabled=True)
+ testing.set_team_experiments_config(team_id=self.team.id, experiment_precomputation_enabled=True)
+
+ experiment = Experiment.objects.get(id=experiment_id)
+ assert experiment.team_id == self.team.id
+ assert experiment.feature_flag_id == flag.id
+ assert experiment.created_by_id == self.user.id
+ assert experiment.holdout is not None and experiment.holdout.team_id == self.team.id
+ assert experiment.status == Experiment.Status.STOPPED
+ assert experiment.metrics == [{"kind": "ExperimentMetric", "metric_type": "mean"}]
+ assert experiment.exposure_criteria == {}
+ assert experiment.updated_at == start_date
+ link = ExperimentToSavedMetric.objects.get(experiment_id=experiment_id)
+ assert (link.saved_metric_id, link.metadata) == (saved_metric_id, {"type": "secondary"})
+ assert link.saved_metric.team_id == self.team.id
+ config = TeamExperimentsConfig.objects.get(team_id=self.team.id)
+ assert (config.default_cuped_enabled, config.experiment_precomputation_enabled) == (True, True)
+
+ other_team = Team.objects.create(organization=self.organization, name="Other team")
+ with pytest.raises(Experiment.DoesNotExist):
+ testing.update_experiment(team_id=other_team.id, experiment_id=experiment_id, deleted=True)
+ with pytest.raises(Experiment.DoesNotExist):
+ testing.delete_experiment(team_id=other_team.id, experiment_id=experiment_id)
+ with pytest.raises(FieldDoesNotExist):
+ testing.update_experiment(team_id=self.team.id, experiment_id=experiment_id, delted=True)
+
+ testing.update_experiment(team_id=self.team.id, experiment_id=experiment_id, deleted=True, holdout_id=None)
+ experiment.refresh_from_db()
+ assert (experiment.deleted, experiment.holdout_id) == (True, None)
+ testing.delete_experiment(team_id=self.team.id, experiment_id=experiment_id)
+ assert not Experiment.objects.filter(id=experiment_id).exists()