:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(aio): allow per-project dataset limit overrides via flag payload (#119658)
Radu Raicea6ef5f3594490
diff --git a/products/ai_observability/backend/api/datasets.py b/products/ai_observability/backend/api/datasets.pyindex ca76ab4c2beb..debe34f372b3 100644--- a/products/ai_observability/backend/api/datasets.py+++ b/products/ai_observability/backend/api/datasets.py@@ -32,6 +32,7 @@ from posthog.permissions import (APIScopePermission,PostHogFeatureFlagPermission,TeamMemberAccessPermission,+ get_authenticator_client,)from posthog.rate_limit import PersonalApiKeyOrUserRateThrottle@@ -69,6 +70,7 @@ from products.ai_observability.backend.dataset_service import (archive_dataset_item,create_dataset,create_dataset_item,+ max_datasets_for_team,restore_dataset,restore_dataset_item,update_dataset,@@ -738,11 +740,17 @@ class DatasetViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, GenericVserializer = DatasetCreateSerializer(data=request.data)serializer.is_valid(raise_exception=True)data = serializer.validated_data+ max_datasets = max_datasets_for_team(+ team=self.team,+ user=cast(User, request.user),+ caller_properties=get_authenticator_client(request.successful_authenticator),+ )try:with transaction.atomic():dataset = create_dataset(team=self.team,created_by=cast(User, request.user),+ max_datasets=max_datasets,name=data["name"],description=data["description"],metadata=data["metadata"],diff --git a/products/ai_observability/backend/api/test/test_datasets.py b/products/ai_observability/backend/api/test/test_datasets.pyindex 717e71bbc5f8..7cc8f90cc928 100644--- a/products/ai_observability/backend/api/test/test_datasets.py+++ b/products/ai_observability/backend/api/test/test_datasets.py@@ -1,7 +1,9 @@+import jsonfrom datetime import timedelta+from functools import partialfrom posthog.test.base import APIBaseTest-from unittest.mock import AsyncMock, MagicMock, patch+from unittest.mock import DEFAULT, AsyncMock, MagicMock, patchfrom django.core.cache import cachefrom django.db import IntegrityError, connection, transaction@@ -9,6 +11,7 @@ from django.db.models.deletion import RestrictedErrorfrom django.utils.timezone import nowfrom parameterized import parameterized+from posthoganalytics.client import Clientfrom rest_framework import statusfrom posthog.models.activity_logging.activity_log import ActivityLog@@ -806,6 +809,80 @@ class TestDatasetsApi(APIBaseTest):)self.assertEqual(Dataset.objects.for_team(self.team.id).count(), 1)+ @parameterized.expand(+ [+ ("session_person", None, "session", "posthog"),+ ("api_key_organization", 0, "personal_api_key", "personal_api_key"),+ ("session_project", 1, "session", "posthog"),+ ]+ )+ def test_dataset_limit_uses_the_team_override_evaluated_for_the_requesting_user(+ self, _name: str, group_index: int | None, credential_type: str, client_id: str+ ) -> None:+ if credential_type == "personal_api_key":+ key = self.create_personal_api_key_with_scopes(["dataset:write"])+ self.client.logout()+ self.client.credentials(HTTP_AUTHORIZATION=f"Bearer {key}")++ caller_properties = {"credential_type": credential_type, "client_id": client_id}+ payload = {"team_configs": {str(self.team.id): {"max_datasets": 2}}}+ filters: dict[str, object] = {+ "groups": [+ {+ "properties": [+ {+ "key": key,+ "value": [value],+ "operator": "exact",+ "type": "person" if group_index is None else "group",+ }+ for key, value in caller_properties.items()+ ],+ "rollout_percentage": 100,+ }+ ],+ "payloads": {"true": json.dumps(payload)},+ }+ if group_index is not None:+ filters["aggregation_group_type_index"] = group_index+ flag_client = Client("test-project-token", send=False, sync_mode=True)+ self.addCleanup(flag_client.shutdown)+ flag_client.group_type_mapping = {"0": "organization", "1": "project"}+ flag_client.feature_flags = [{"key": "llm-analytics-datasets", "active": True, "filters": filters}]+ transaction_depth = len(connection.atomic_blocks)+ payload_transaction_depths: list[int] = []++ def record_payload_transaction_depth(*args: object, **kwargs: object) -> object:+ payload_transaction_depths.append(len(connection.atomic_blocks))+ return DEFAULT++ with (+ patch("products.ai_observability.backend.dataset_service.MAX_DATASETS_PER_TEAM", 1),+ patch(+ "products.ai_observability.backend.dataset_service.posthoganalytics.get_feature_flag_payload",+ side_effect=record_payload_transaction_depth,+ wraps=partial(flag_client.get_feature_flag_payload, only_evaluate_locally=True),+ ) as get_payload,+ ):+ self._create_dataset("First dataset")+ self._create_dataset("Second dataset")+ response = self.client.post(self.datasets_url, {"name": "Over the limit"}, format="json")++ self.assertEqual(response.status_code, status.HTTP_409_CONFLICT)+ self.assertEqual((response.data["current_count"], response.data["limit"]), (2, 2))+ self.assertEqual(payload_transaction_depths, [transaction_depth] * 3)+ get_payload.assert_called_with(+ "llm-analytics-datasets",+ str(self.user.distinct_id),+ groups={"organization": str(self.organization.id), "project": str(self.team.id)},+ group_properties={+ "organization": {**caller_properties, "id": str(self.organization.id)},+ "project": {**caller_properties, "id": str(self.team.id)},+ },+ person_properties=caller_properties,+ send_feature_flag_events=False,+ )+def test_item_limit_counts_archived_items_without_creating_a_revision(self) -> None:dataset = self._create_dataset()item = self._create_item(dataset["id"])diff --git a/products/ai_observability/backend/dataset_service.py b/products/ai_observability/backend/dataset_service.pyindex 752f664dacff..ee8b99b1e96d 100644--- a/products/ai_observability/backend/dataset_service.py+++ b/products/ai_observability/backend/dataset_service.py@@ -9,6 +9,9 @@ from uuid import UUIDfrom django.db import IntegrityError, connection, transactionfrom django.db.models import Max+import posthoganalytics++from posthog.exceptions_capture import capture_exceptionfrom posthog.models import Team, Userfrom products.ai_observability.backend.dataset_limits import MAX_DATASET_ITEM_PAYLOAD_BYTES, MAX_ITEMS_PER_DATASET@@ -25,6 +28,13 @@ MAX_SOURCE_ID_LENGTH = 255MAX_DATASETS_PER_TEAM = 20MAX_VERSIONS_PER_ITEM = 40+# The flag that gates the dataset API. Its payload also carries per-project overrides of+# `MAX_DATASETS_PER_TEAM`, so a project that asks for more datasets gets them with no deploy:+# {"team_configs": {"<team_id>": {"max_datasets": 50}}}+DATASETS_FEATURE_FLAG = "llm-analytics-datasets"+PAYLOAD_TEAM_CONFIGS_KEY = "team_configs"+PAYLOAD_MAX_DATASETS_KEY = "max_datasets"+class DatasetValidationError(Exception):def __init__(self, field: str, detail: str) -> None:@@ -391,6 +401,43 @@ def _create_item_version(return version+def max_datasets_for_team(*, team: Team, user: User | None, caller_properties: dict[str, str] | None = None) -> int:+ """The project's dataset limit: its override in the flag payload, or `MAX_DATASETS_PER_TEAM`.++ The payload is evaluated for the requesting user with the groups and caller properties+ that `PostHogFeatureFlagPermission` uses, so it follows the same rollout that gates the API.+ A missing, malformed, or unreadable override resolves to the default.+ """+ if user is None:+ return MAX_DATASETS_PER_TEAM+ organization_id = str(team.organization_id)+ project_id = str(team.id)+ caller = caller_properties or {}+ try:+ payload = posthoganalytics.get_feature_flag_payload(+ DATASETS_FEATURE_FLAG,+ str(user.distinct_id),+ groups={"organization": organization_id, "project": project_id},+ group_properties={+ "organization": {**caller, "id": organization_id},+ "project": {**caller, "id": project_id},+ },+ person_properties=caller or None,+ send_feature_flag_events=False,+ )+ if isinstance(payload, str):+ payload = json.loads(payload)+ except Exception as error:+ capture_exception(error)+ return MAX_DATASETS_PER_TEAM+ team_configs = payload.get(PAYLOAD_TEAM_CONFIGS_KEY) if isinstance(payload, dict) else None+ team_config = team_configs.get(project_id) if isinstance(team_configs, dict) else None+ limit = team_config.get(PAYLOAD_MAX_DATASETS_KEY) if isinstance(team_config, dict) else None+ if isinstance(limit, bool) or not isinstance(limit, int) or limit < 1:+ return MAX_DATASETS_PER_TEAM+ return limit++@transaction.atomicdef create_dataset(*,@@ -399,19 +446,20 @@ def create_dataset(name: str,description: str = "",metadata: dict[str, JSONValue] | None = None,+ max_datasets: int = MAX_DATASETS_PER_TEAM,) -> Dataset:normalized_metadata = metadata if metadata is not None else {}normalized_name = _validate_dataset_fields(name=name, description=description, metadata=normalized_metadata)with connection.cursor() as cursor:cursor.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", [f"aio-dataset-allocation:{team.id}"])current_count = Dataset.objects.for_team(team.id, canonical=True).count()- if current_count >= MAX_DATASETS_PER_TEAM:+ if current_count >= max_datasets:raise DatasetLimitExceeded(resource="datasets",current_count=current_count,- limit=MAX_DATASETS_PER_TEAM,+ limit=max_datasets,detail=(- f"No more datasets can be added to this project. The limit is {MAX_DATASETS_PER_TEAM}. "+ f"No more datasets can be added to this project. The limit is {max_datasets}. ""Contact support if you need more."),)diff --git a/products/ai_observability/backend/test/test_dataset_service.py b/products/ai_observability/backend/test/test_dataset_service.pynew file mode 100644index 000000000000..d9ae7f4f4ad4--- /dev/null+++ b/products/ai_observability/backend/test/test_dataset_service.py@@ -0,0 +1,56 @@+import json+import uuid++from unittest.mock import patch++from django.test import SimpleTestCase++from parameterized import parameterized++from posthog.models import Team, User++from products.ai_observability.backend.dataset_service import MAX_DATASETS_PER_TEAM, max_datasets_for_team++TEAM_ID = 42+GET_PAYLOAD = "products.ai_observability.backend.dataset_service.posthoganalytics.get_feature_flag_payload"+++class TestMaxDatasetsForTeam(SimpleTestCase):+ def setUp(self) -> None:+ self.team = Team(id=TEAM_ID, organization_id=uuid.uuid4())+ self.user = User(distinct_id="user-distinct-id")++ @parameterized.expand(+ [+ ("override", {"team_configs": {str(TEAM_ID): {"max_datasets": 50}}}, 50),+ ("override_as_json_string", json.dumps({"team_configs": {str(TEAM_ID): {"max_datasets": 50}}}), 50),+ ("override_below_default", {"team_configs": {str(TEAM_ID): {"max_datasets": 5}}}, 5),+ ("no_payload", None, MAX_DATASETS_PER_TEAM),+ ("malformed_json", "{not json", MAX_DATASETS_PER_TEAM),+ ("payload_not_an_object", ["unexpected"], MAX_DATASETS_PER_TEAM),+ ("other_team_only", {"team_configs": {"999999": {"max_datasets": 50}}}, MAX_DATASETS_PER_TEAM),+ ("team_configs_not_an_object", {"team_configs": [50]}, MAX_DATASETS_PER_TEAM),+ ("team_config_not_an_object", {"team_configs": {str(TEAM_ID): 50}}, MAX_DATASETS_PER_TEAM),+ ("zero", {"team_configs": {str(TEAM_ID): {"max_datasets": 0}}}, MAX_DATASETS_PER_TEAM),+ ("boolean", {"team_configs": {str(TEAM_ID): {"max_datasets": True}}}, MAX_DATASETS_PER_TEAM),+ ("string", {"team_configs": {str(TEAM_ID): {"max_datasets": "50"}}}, MAX_DATASETS_PER_TEAM),+ ("float", {"team_configs": {str(TEAM_ID): {"max_datasets": 50.5}}}, MAX_DATASETS_PER_TEAM),+ ]+ )+ def test_resolves_the_limit_from_the_payload(self, _name: str, payload: object, expected: int) -> None:+ with patch(GET_PAYLOAD, return_value=payload):+ self.assertEqual(max_datasets_for_team(team=self.team, user=self.user), expected)++ def test_read_error_resolves_to_the_default(self) -> None:+ error = RuntimeError("flag read failed")+ with (+ patch(GET_PAYLOAD, side_effect=error),+ patch("products.ai_observability.backend.dataset_service.capture_exception") as capture,+ ):+ self.assertEqual(max_datasets_for_team(team=self.team, user=self.user), MAX_DATASETS_PER_TEAM)+ capture.assert_called_once_with(error)++ def test_no_user_resolves_to_the_default_without_a_flag_read(self) -> None:+ with patch(GET_PAYLOAD) as get_payload:+ self.assertEqual(max_datasets_for_team(team=self.team, user=None), MAX_DATASETS_PER_TEAM)+ get_payload.assert_not_called()