Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(aio): allow per-project dataset limit overrides via flag payload (#119658)

Radu Raicea6ef5f3594490
diff --git a/products/ai_observability/backend/api/datasets.py b/products/ai_observability/backend/api/datasets.py
index ca76ab4c2beb..debe34f372b3 100644
--- a/products/ai_observability/backend/api/datasets.py
+++ b/products/ai_observability/backend/api/datasets.py
@@ -32,6 +32,7 @@ from posthog.permissions import (
APIScopePermission,
PostHogFeatureFlagPermission,
TeamMemberAccessPermission,
+ get_authenticator_client,
)
from posthog.rate_limit import PersonalApiKeyOrUserRateThrottle
@@ -69,6 +70,7 @@ from products.ai_observability.backend.dataset_service import (
archive_dataset_item,
create_dataset,
create_dataset_item,
+ max_datasets_for_team,
restore_dataset,
restore_dataset_item,
update_dataset,
@@ -738,11 +740,17 @@ class DatasetViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, GenericV
serializer = DatasetCreateSerializer(data=request.data)
serializer.is_valid(raise_exception=True)
data = serializer.validated_data
+ max_datasets = max_datasets_for_team(
+ team=self.team,
+ user=cast(User, request.user),
+ caller_properties=get_authenticator_client(request.successful_authenticator),
+ )
try:
with transaction.atomic():
dataset = create_dataset(
team=self.team,
created_by=cast(User, request.user),
+ max_datasets=max_datasets,
name=data["name"],
description=data["description"],
metadata=data["metadata"],
diff --git a/products/ai_observability/backend/api/test/test_datasets.py b/products/ai_observability/backend/api/test/test_datasets.py
index 717e71bbc5f8..7cc8f90cc928 100644
--- a/products/ai_observability/backend/api/test/test_datasets.py
+++ b/products/ai_observability/backend/api/test/test_datasets.py
@@ -1,7 +1,9 @@
+import json
from datetime import timedelta
+from functools import partial
from posthog.test.base import APIBaseTest
-from unittest.mock import AsyncMock, MagicMock, patch
+from unittest.mock import DEFAULT, AsyncMock, MagicMock, patch
from django.core.cache import cache
from django.db import IntegrityError, connection, transaction
@@ -9,6 +11,7 @@ from django.db.models.deletion import RestrictedError
from django.utils.timezone import now
from parameterized import parameterized
+from posthoganalytics.client import Client
from rest_framework import status
from posthog.models.activity_logging.activity_log import ActivityLog
@@ -806,6 +809,80 @@ class TestDatasetsApi(APIBaseTest):
)
self.assertEqual(Dataset.objects.for_team(self.team.id).count(), 1)
+ @parameterized.expand(
+ [
+ ("session_person", None, "session", "posthog"),
+ ("api_key_organization", 0, "personal_api_key", "personal_api_key"),
+ ("session_project", 1, "session", "posthog"),
+ ]
+ )
+ def test_dataset_limit_uses_the_team_override_evaluated_for_the_requesting_user(
+ self, _name: str, group_index: int | None, credential_type: str, client_id: str
+ ) -> None:
+ if credential_type == "personal_api_key":
+ key = self.create_personal_api_key_with_scopes(["dataset:write"])
+ self.client.logout()
+ self.client.credentials(HTTP_AUTHORIZATION=f"Bearer {key}")
+
+ caller_properties = {"credential_type": credential_type, "client_id": client_id}
+ payload = {"team_configs": {str(self.team.id): {"max_datasets": 2}}}
+ filters: dict[str, object] = {
+ "groups": [
+ {
+ "properties": [
+ {
+ "key": key,
+ "value": [value],
+ "operator": "exact",
+ "type": "person" if group_index is None else "group",
+ }
+ for key, value in caller_properties.items()
+ ],
+ "rollout_percentage": 100,
+ }
+ ],
+ "payloads": {"true": json.dumps(payload)},
+ }
+ if group_index is not None:
+ filters["aggregation_group_type_index"] = group_index
+ flag_client = Client("test-project-token", send=False, sync_mode=True)
+ self.addCleanup(flag_client.shutdown)
+ flag_client.group_type_mapping = {"0": "organization", "1": "project"}
+ flag_client.feature_flags = [{"key": "llm-analytics-datasets", "active": True, "filters": filters}]
+ transaction_depth = len(connection.atomic_blocks)
+ payload_transaction_depths: list[int] = []
+
+ def record_payload_transaction_depth(*args: object, **kwargs: object) -> object:
+ payload_transaction_depths.append(len(connection.atomic_blocks))
+ return DEFAULT
+
+ with (
+ patch("products.ai_observability.backend.dataset_service.MAX_DATASETS_PER_TEAM", 1),
+ patch(
+ "products.ai_observability.backend.dataset_service.posthoganalytics.get_feature_flag_payload",
+ side_effect=record_payload_transaction_depth,
+ wraps=partial(flag_client.get_feature_flag_payload, only_evaluate_locally=True),
+ ) as get_payload,
+ ):
+ self._create_dataset("First dataset")
+ self._create_dataset("Second dataset")
+ response = self.client.post(self.datasets_url, {"name": "Over the limit"}, format="json")
+
+ self.assertEqual(response.status_code, status.HTTP_409_CONFLICT)
+ self.assertEqual((response.data["current_count"], response.data["limit"]), (2, 2))
+ self.assertEqual(payload_transaction_depths, [transaction_depth] * 3)
+ get_payload.assert_called_with(
+ "llm-analytics-datasets",
+ str(self.user.distinct_id),
+ groups={"organization": str(self.organization.id), "project": str(self.team.id)},
+ group_properties={
+ "organization": {**caller_properties, "id": str(self.organization.id)},
+ "project": {**caller_properties, "id": str(self.team.id)},
+ },
+ person_properties=caller_properties,
+ send_feature_flag_events=False,
+ )
+
def test_item_limit_counts_archived_items_without_creating_a_revision(self) -> None:
dataset = self._create_dataset()
item = self._create_item(dataset["id"])
diff --git a/products/ai_observability/backend/dataset_service.py b/products/ai_observability/backend/dataset_service.py
index 752f664dacff..ee8b99b1e96d 100644
--- a/products/ai_observability/backend/dataset_service.py
+++ b/products/ai_observability/backend/dataset_service.py
@@ -9,6 +9,9 @@ from uuid import UUID
from django.db import IntegrityError, connection, transaction
from django.db.models import Max
+import posthoganalytics
+
+from posthog.exceptions_capture import capture_exception
from posthog.models import Team, User
from products.ai_observability.backend.dataset_limits import MAX_DATASET_ITEM_PAYLOAD_BYTES, MAX_ITEMS_PER_DATASET
@@ -25,6 +28,13 @@ MAX_SOURCE_ID_LENGTH = 255
MAX_DATASETS_PER_TEAM = 20
MAX_VERSIONS_PER_ITEM = 40
+# The flag that gates the dataset API. Its payload also carries per-project overrides of
+# `MAX_DATASETS_PER_TEAM`, so a project that asks for more datasets gets them with no deploy:
+# {"team_configs": {"<team_id>": {"max_datasets": 50}}}
+DATASETS_FEATURE_FLAG = "llm-analytics-datasets"
+PAYLOAD_TEAM_CONFIGS_KEY = "team_configs"
+PAYLOAD_MAX_DATASETS_KEY = "max_datasets"
+
class DatasetValidationError(Exception):
def __init__(self, field: str, detail: str) -> None:
@@ -391,6 +401,43 @@ def _create_item_version(
return version
+def max_datasets_for_team(*, team: Team, user: User | None, caller_properties: dict[str, str] | None = None) -> int:
+ """The project's dataset limit: its override in the flag payload, or `MAX_DATASETS_PER_TEAM`.
+
+ The payload is evaluated for the requesting user with the groups and caller properties
+ that `PostHogFeatureFlagPermission` uses, so it follows the same rollout that gates the API.
+ A missing, malformed, or unreadable override resolves to the default.
+ """
+ if user is None:
+ return MAX_DATASETS_PER_TEAM
+ organization_id = str(team.organization_id)
+ project_id = str(team.id)
+ caller = caller_properties or {}
+ try:
+ payload = posthoganalytics.get_feature_flag_payload(
+ DATASETS_FEATURE_FLAG,
+ str(user.distinct_id),
+ groups={"organization": organization_id, "project": project_id},
+ group_properties={
+ "organization": {**caller, "id": organization_id},
+ "project": {**caller, "id": project_id},
+ },
+ person_properties=caller or None,
+ send_feature_flag_events=False,
+ )
+ if isinstance(payload, str):
+ payload = json.loads(payload)
+ except Exception as error:
+ capture_exception(error)
+ return MAX_DATASETS_PER_TEAM
+ team_configs = payload.get(PAYLOAD_TEAM_CONFIGS_KEY) if isinstance(payload, dict) else None
+ team_config = team_configs.get(project_id) if isinstance(team_configs, dict) else None
+ limit = team_config.get(PAYLOAD_MAX_DATASETS_KEY) if isinstance(team_config, dict) else None
+ if isinstance(limit, bool) or not isinstance(limit, int) or limit < 1:
+ return MAX_DATASETS_PER_TEAM
+ return limit
+
+
@transaction.atomic
def create_dataset(
*,
@@ -399,19 +446,20 @@ def create_dataset(
name: str,
description: str = "",
metadata: dict[str, JSONValue] | None = None,
+ max_datasets: int = MAX_DATASETS_PER_TEAM,
) -> Dataset:
normalized_metadata = metadata if metadata is not None else {}
normalized_name = _validate_dataset_fields(name=name, description=description, metadata=normalized_metadata)
with connection.cursor() as cursor:
cursor.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", [f"aio-dataset-allocation:{team.id}"])
current_count = Dataset.objects.for_team(team.id, canonical=True).count()
- if current_count >= MAX_DATASETS_PER_TEAM:
+ if current_count >= max_datasets:
raise DatasetLimitExceeded(
resource="datasets",
current_count=current_count,
- limit=MAX_DATASETS_PER_TEAM,
+ limit=max_datasets,
detail=(
- f"No more datasets can be added to this project. The limit is {MAX_DATASETS_PER_TEAM}. "
+ f"No more datasets can be added to this project. The limit is {max_datasets}. "
"Contact support if you need more."
),
)
diff --git a/products/ai_observability/backend/test/test_dataset_service.py b/products/ai_observability/backend/test/test_dataset_service.py
new file mode 100644
index 000000000000..d9ae7f4f4ad4
--- /dev/null
+++ b/products/ai_observability/backend/test/test_dataset_service.py
@@ -0,0 +1,56 @@
+import json
+import uuid
+
+from unittest.mock import patch
+
+from django.test import SimpleTestCase
+
+from parameterized import parameterized
+
+from posthog.models import Team, User
+
+from products.ai_observability.backend.dataset_service import MAX_DATASETS_PER_TEAM, max_datasets_for_team
+
+TEAM_ID = 42
+GET_PAYLOAD = "products.ai_observability.backend.dataset_service.posthoganalytics.get_feature_flag_payload"
+
+
+class TestMaxDatasetsForTeam(SimpleTestCase):
+ def setUp(self) -> None:
+ self.team = Team(id=TEAM_ID, organization_id=uuid.uuid4())
+ self.user = User(distinct_id="user-distinct-id")
+
+ @parameterized.expand(
+ [
+ ("override", {"team_configs": {str(TEAM_ID): {"max_datasets": 50}}}, 50),
+ ("override_as_json_string", json.dumps({"team_configs": {str(TEAM_ID): {"max_datasets": 50}}}), 50),
+ ("override_below_default", {"team_configs": {str(TEAM_ID): {"max_datasets": 5}}}, 5),
+ ("no_payload", None, MAX_DATASETS_PER_TEAM),
+ ("malformed_json", "{not json", MAX_DATASETS_PER_TEAM),
+ ("payload_not_an_object", ["unexpected"], MAX_DATASETS_PER_TEAM),
+ ("other_team_only", {"team_configs": {"999999": {"max_datasets": 50}}}, MAX_DATASETS_PER_TEAM),
+ ("team_configs_not_an_object", {"team_configs": [50]}, MAX_DATASETS_PER_TEAM),
+ ("team_config_not_an_object", {"team_configs": {str(TEAM_ID): 50}}, MAX_DATASETS_PER_TEAM),
+ ("zero", {"team_configs": {str(TEAM_ID): {"max_datasets": 0}}}, MAX_DATASETS_PER_TEAM),
+ ("boolean", {"team_configs": {str(TEAM_ID): {"max_datasets": True}}}, MAX_DATASETS_PER_TEAM),
+ ("string", {"team_configs": {str(TEAM_ID): {"max_datasets": "50"}}}, MAX_DATASETS_PER_TEAM),
+ ("float", {"team_configs": {str(TEAM_ID): {"max_datasets": 50.5}}}, MAX_DATASETS_PER_TEAM),
+ ]
+ )
+ def test_resolves_the_limit_from_the_payload(self, _name: str, payload: object, expected: int) -> None:
+ with patch(GET_PAYLOAD, return_value=payload):
+ self.assertEqual(max_datasets_for_team(team=self.team, user=self.user), expected)
+
+ def test_read_error_resolves_to_the_default(self) -> None:
+ error = RuntimeError("flag read failed")
+ with (
+ patch(GET_PAYLOAD, side_effect=error),
+ patch("products.ai_observability.backend.dataset_service.capture_exception") as capture,
+ ):
+ self.assertEqual(max_datasets_for_team(team=self.team, user=self.user), MAX_DATASETS_PER_TEAM)
+ capture.assert_called_once_with(error)
+
+ def test_no_user_resolves_to_the_default_without_a_flag_read(self) -> None:
+ with patch(GET_PAYLOAD) as get_payload:
+ self.assertEqual(max_datasets_for_team(team=self.team, user=None), MAX_DATASETS_PER_TEAM)
+ get_payload.assert_not_called()