Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(experiments): stop PostHog AI creating empty web-type experiments (#119532)

ruby childs1217391d9a21
diff --git a/products/experiments/backend/max_tools.py b/products/experiments/backend/max_tools.py
index a257988a328c..e5b9da1ca79c 100644
--- a/products/experiments/backend/max_tools.py
+++ b/products/experiments/backend/max_tools.py
@@ -34,9 +34,14 @@ CREATE_EXPERIMENT_TOOL_DESCRIPTION = dedent("""
using the `create_feature_flag` tool with 2 to 20 variants (conventionally control and test).
The baseline defaults to the variant named "control" when present, else the first variant.
- # Experiment Types
- - **product**: For backend/API changes, server-side experiments
- - **web**: For frontend UI changes, client-side experiments
+ # Scope
+ This tool creates code-based experiments (type "product"): the user implements the variants
+ in their own code via the feature flag. UX/UI and frontend experiments are fully supported
+ this way, including when an agent writes the variant code in the user's repo.
+ The one thing this tool cannot create is a no-code web experiment — those store DOM
+ transforms this tool cannot author, and are created with the PostHog toolbar on the user's
+ own site. Only when the user explicitly wants to author changes visually without touching
+ code should you point them to the toolbar instead of using this tool.
# Workflow
1. Create a multivariate feature flag with `create_feature_flag` (variants: control + test)
@@ -86,14 +91,6 @@ class CreateExperimentToolArgs(BaseModel):
Control shows existing 3-step checkout, test shows new 1-page checkout."
""").strip(),
)
- type: Literal["product", "web"] = Field(
- default="product",
- description=dedent("""
- The experiment type:
- - "product": For backend/API changes, server-side experiments (default)
- - "web": For frontend UI changes, client-side experiments
- """).strip(),
- )
class CreateExperimentTool(MaxTool):
@@ -109,7 +106,6 @@ class CreateExperimentTool(MaxTool):
name: str,
feature_flag_key: str,
description: str | None = None,
- type: Literal["product", "web"] = "product",
) -> tuple[str, dict[str, Any] | None]:
if not name or not name.strip():
return "Experiment name cannot be empty", {"error": "invalid_name"}
@@ -153,7 +149,9 @@ class CreateExperimentTool(MaxTool):
name=name,
feature_flag_key=feature_flag_key,
description=description or "",
- type=type,
+ # Always "product": this tool cannot author the DOM transforms a no-code
+ # web experiment needs, so a `type="web"` row it creates is an empty shell.
+ type="product",
event_source=EventSource.POSTHOG_AI,
analytics_properties=analytics_properties,
)
@@ -169,7 +167,7 @@ class CreateExperimentTool(MaxTool):
"experiment_id": experiment.id,
"experiment_name": experiment.name,
"feature_flag_key": feature_flag_key,
- "type": type,
+ "type": experiment.type,
"url": experiment_url,
},
)
diff --git a/products/experiments/backend/test/test_max_tools.py b/products/experiments/backend/test/test_max_tools.py
index d40f18eeec77..c45b26e3c22e 100644
--- a/products/experiments/backend/test/test_max_tools.py
+++ b/products/experiments/backend/test/test_max_tools.py
@@ -14,7 +14,7 @@ from posthog.event_usage import EventSource
from posthog.models.team.extensions import get_or_create_team_extension
from products.experiments.backend.experiment_summary_data_service import ExperimentSummaryData
-from products.experiments.backend.max_tools import CreateExperimentTool, ExperimentSummaryTool
+from products.experiments.backend.max_tools import CreateExperimentTool, CreateExperimentToolArgs, ExperimentSummaryTool
from products.experiments.backend.models.experiment import Experiment
from products.experiments.backend.models.team_experiments_config import TeamExperimentsConfig
from products.feature_flags.backend.models.feature_flag import FeatureFlag
@@ -94,22 +94,25 @@ class TestCreateExperimentTool(APIBaseTest):
experiment = await Experiment.objects.aget(name="Checkout Experiment", team=self.team)
assert experiment.description == "Testing new checkout flow to improve conversion rates"
- async def test_create_experiment_web_type(self):
+ async def test_create_experiment_cannot_create_web_type(self):
+ # No-code web experiments carry DOM transforms this tool cannot author, so the
+ # tool must not expose `type` and must always create product experiments.
await self._create_multivariate_flag(key="homepage-redesign", name="Homepage Redesign Flag")
tool = self._create_tool()
+ assert "type" not in CreateExperimentToolArgs.model_fields
+
result, artifact = await tool._arun_impl(
name="Homepage Redesign",
feature_flag_key="homepage-redesign",
- type="web",
)
assert "Successfully created" in result
assert artifact is not None
- assert artifact["type"] == "web"
+ assert artifact["type"] == "product"
experiment = await Experiment.objects.aget(name="Homepage Redesign", team=self.team)
- assert experiment.type == "web"
+ assert experiment.type == "product"
async def test_create_experiment_duplicate_name(self):
flag = await self._create_multivariate_flag(key="existing-flag")