Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(experiments): reject default confidence levels outside 0 to 1 (#119369)

Juraj Majerikf21474bcbf2d
diff --git a/posthog/api/team/integration_config.py b/posthog/api/team/integration_config.py
index 3a468a366660..31bba87ff5f1 100644
--- a/posthog/api/team/integration_config.py
+++ b/posthog/api/team/integration_config.py
@@ -1,5 +1,6 @@
"""Serializers and handlers for product-specific project settings."""
+from decimal import Decimal
from typing import Any, cast
from django.core.exceptions import ValidationError as DjangoValidationError
@@ -280,6 +281,17 @@ def handle_experiments_config(request: request.Request, team: Team) -> response.
f"{MIN_RECALCULATION_GAP_HOURS} hours apart. Null means the default time (02:00 UTC)."
),
)
+ # The frequentist engine derives alpha as 1 - confidence level, so anything outside (0, 1)
+ # is meaningless. The field stores two decimal places, so these bounds are that open range.
+ default_experiment_confidence_level = serializers.DecimalField(
+ max_digits=3,
+ decimal_places=2,
+ min_value=Decimal("0.01"),
+ max_value=Decimal("0.99"),
+ required=False,
+ allow_null=True,
+ help_text="Default confidence level for new experiments, above 0 and below 1, for example 0.95.",
+ )
class Meta:
model = TeamExperimentsConfig
diff --git a/posthog/api/test/test_project.py b/posthog/api/test/test_project.py
index e59b3094daca..3d6913ec5aaa 100644
--- a/posthog/api/test/test_project.py
+++ b/posthog/api/test/test_project.py
@@ -1545,6 +1545,27 @@ class TestProjectAPI(team_api_test_factory()): # type: ignore
)
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.json())
+ @parameterized.expand(
+ [
+ ("zero", 0, status.HTTP_400_BAD_REQUEST),
+ ("one", 1, status.HTTP_400_BAD_REQUEST),
+ ("above_one", 1.5, status.HTTP_400_BAD_REQUEST),
+ ("negative", -0.5, status.HTTP_400_BAD_REQUEST),
+ ("ninety_percent", 0.9, status.HTTP_200_OK),
+ ]
+ )
+ def test_experiments_config_confidence_level_must_be_between_zero_and_one(self, _name, level, expected_status):
+ response = self.client.patch(
+ f"/api/projects/{self.project.id}/experiments_config/",
+ {"default_experiment_confidence_level": level},
+ format="json",
+ )
+ self.assertEqual(response.status_code, expected_status, response.json())
+ if expected_status == status.HTTP_400_BAD_REQUEST:
+ self.assertEqual(response.json()["attr"], "default_experiment_confidence_level")
+ else:
+ self.assertEqual(response.json()["default_experiment_confidence_level"], "0.90")
+
def test_tags_round_trip_and_land_in_the_project_team_namespace(self):
# `tags` is not a Project column, so it must be pulled out before the serializer's
# passthrough loop setattr()s everything left in validated_data onto the model.