:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(experiments): stop PostHog AI creating empty web-type experiments (#119532)
ruby childs1217391d9a21
diff --git a/products/experiments/backend/max_tools.py b/products/experiments/backend/max_tools.pyindex a257988a328c..e5b9da1ca79c 100644--- a/products/experiments/backend/max_tools.py+++ b/products/experiments/backend/max_tools.py@@ -34,9 +34,14 @@ CREATE_EXPERIMENT_TOOL_DESCRIPTION = dedent("""using the `create_feature_flag` tool with 2 to 20 variants (conventionally control and test).The baseline defaults to the variant named "control" when present, else the first variant.- # Experiment Types- - **product**: For backend/API changes, server-side experiments- - **web**: For frontend UI changes, client-side experiments+ # Scope+ This tool creates code-based experiments (type "product"): the user implements the variants+ in their own code via the feature flag. UX/UI and frontend experiments are fully supported+ this way, including when an agent writes the variant code in the user's repo.+ The one thing this tool cannot create is a no-code web experiment — those store DOM+ transforms this tool cannot author, and are created with the PostHog toolbar on the user's+ own site. Only when the user explicitly wants to author changes visually without touching+ code should you point them to the toolbar instead of using this tool.# Workflow1. Create a multivariate feature flag with `create_feature_flag` (variants: control + test)@@ -86,14 +91,6 @@ class CreateExperimentToolArgs(BaseModel):Control shows existing 3-step checkout, test shows new 1-page checkout."""").strip(),)- type: Literal["product", "web"] = Field(- default="product",- description=dedent("""- The experiment type:- - "product": For backend/API changes, server-side experiments (default)- - "web": For frontend UI changes, client-side experiments- """).strip(),- )class CreateExperimentTool(MaxTool):@@ -109,7 +106,6 @@ class CreateExperimentTool(MaxTool):name: str,feature_flag_key: str,description: str | None = None,- type: Literal["product", "web"] = "product",) -> tuple[str, dict[str, Any] | None]:if not name or not name.strip():return "Experiment name cannot be empty", {"error": "invalid_name"}@@ -153,7 +149,9 @@ class CreateExperimentTool(MaxTool):name=name,feature_flag_key=feature_flag_key,description=description or "",- type=type,+ # Always "product": this tool cannot author the DOM transforms a no-code+ # web experiment needs, so a `type="web"` row it creates is an empty shell.+ type="product",event_source=EventSource.POSTHOG_AI,analytics_properties=analytics_properties,)@@ -169,7 +167,7 @@ class CreateExperimentTool(MaxTool):"experiment_id": experiment.id,"experiment_name": experiment.name,"feature_flag_key": feature_flag_key,- "type": type,+ "type": experiment.type,"url": experiment_url,},)diff --git a/products/experiments/backend/test/test_max_tools.py b/products/experiments/backend/test/test_max_tools.pyindex d40f18eeec77..c45b26e3c22e 100644--- a/products/experiments/backend/test/test_max_tools.py+++ b/products/experiments/backend/test/test_max_tools.py@@ -14,7 +14,7 @@ from posthog.event_usage import EventSourcefrom posthog.models.team.extensions import get_or_create_team_extensionfrom products.experiments.backend.experiment_summary_data_service import ExperimentSummaryData-from products.experiments.backend.max_tools import CreateExperimentTool, ExperimentSummaryTool+from products.experiments.backend.max_tools import CreateExperimentTool, CreateExperimentToolArgs, ExperimentSummaryToolfrom products.experiments.backend.models.experiment import Experimentfrom products.experiments.backend.models.team_experiments_config import TeamExperimentsConfigfrom products.feature_flags.backend.models.feature_flag import FeatureFlag@@ -94,22 +94,25 @@ class TestCreateExperimentTool(APIBaseTest):experiment = await Experiment.objects.aget(name="Checkout Experiment", team=self.team)assert experiment.description == "Testing new checkout flow to improve conversion rates"- async def test_create_experiment_web_type(self):+ async def test_create_experiment_cannot_create_web_type(self):+ # No-code web experiments carry DOM transforms this tool cannot author, so the+ # tool must not expose `type` and must always create product experiments.await self._create_multivariate_flag(key="homepage-redesign", name="Homepage Redesign Flag")tool = self._create_tool()+ assert "type" not in CreateExperimentToolArgs.model_fields+result, artifact = await tool._arun_impl(name="Homepage Redesign",feature_flag_key="homepage-redesign",- type="web",)assert "Successfully created" in resultassert artifact is not None- assert artifact["type"] == "web"+ assert artifact["type"] == "product"experiment = await Experiment.objects.aget(name="Homepage Redesign", team=self.team)- assert experiment.type == "web"+ assert experiment.type == "product"async def test_create_experiment_duplicate_name(self):flag = await self._create_multivariate_flag(key="existing-flag")