:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(aio): stop new tagger creation and point users to categorical evals (#120133)
Bernat Torresf0b6523da944
diff --git a/frontend/snapshots.yml b/frontend/snapshots.ymlindex a22793a9f27d..4f03b9530842 100644--- a/frontend/snapshots.yml+++ b/frontend/snapshots.yml@@ -12734,6 +12734,18 @@ snapshots:hash: v1.k794b7964.ef34c395a9c714e408b11f16c269b26faf1702c47d28b1a4a75a6dfc53843c86.bGIL3-MblhBwE8TbLeV2b950OxPKA6zuY845SbU3Eykscenes-app-llm-observability-numericevaluationconfig--system-one-narrow--light:hash: v1.k794b7964.0582a7952e17e17dcbcf88734e3593fda4adc9e4ef4d18dc33e3264f1f54db30.L3V9MhG1KqtPXLmo0YNTOyblMDqwyN5ByhYuKSUz3-g+ scenes-app-llm-observability-tagger-creation-retired-modal--default--dark:+ hash: v1.k794b7964.a50e0bc709ba60817178e68de22478ed2246b4b0fe368b2bc0980d6685db75f1.TrIdwsauxW1KI2-THnOP2aeIOaxBp2FYLo_S6yaiJ78+ scenes-app-llm-observability-tagger-creation-retired-modal--default--light:+ hash: v1.k794b7964.3d11df71f93f30412159fd9f1dd312ff71a6d717840fd489c1b074d9ab08c32d.9YSVVJe1RobDnHYUhTN4A7IgSv-h-Rtq5CHh9DVzKKU+ scenes-app-llm-observability-tagger-creation-retired-modal--turn-back-on--dark:+ hash: v1.k794b7964.d0d918ec528620c42a9db7a571b17df37f1766388fdb22a16f145f69064848da.6-c8xtoOfnqBHlZIjtTHrVeUuLcJ8HMxe1N75_0X1Zc+ scenes-app-llm-observability-tagger-creation-retired-modal--turn-back-on--light:+ hash: v1.k794b7964.dd4d8aef334145fa385feb77f79a384b7ffbe718f93d6f347b2f65a97d2d11aa.pSRDtGAxPvPj7aSzrg0OHapd58TfcJksaMcu2rVLUbA+ scenes-app-llm-observability-taggers-deprecation-banner--default--dark:+ hash: v1.k794b7964.4e5e8fcbc5e9ff76ccda6b84a6bfb7a99d28334c6b2afc4ee6bb08e221fde8e6.Euz0mSfdHk2e8kSuHfbkcOEuljQWO8FJAMneFQE5EHk+ scenes-app-llm-observability-taggers-deprecation-banner--default--light:+ hash: v1.k794b7964.4afb0ca40061311438558a25b30e8afe725a6645742a036410f3803dcb7ce3b1.I-UIHserKVYTZtQOJ1HTlRwwhbn-uFPSzYzjgdbUGK0scenes-app-logs-logsimpactcounts--no-coverage--dark:hash: v1.k794b7964.fcfa5be29c5721d0db9fa21ecab20f16c04a4074c9af7fb3a58311900e168a48.2eslY8aigbAuKJ2Bsn8AjSOah5Q77FoWONVdG2fb6C4scenes-app-logs-logsimpactcounts--no-coverage--light:diff --git a/frontend/src/lib/agentScopes.generated.ts b/frontend/src/lib/agentScopes.generated.tsindex 8f804a4b58b5..21e23a3b06a0 100644--- a/frontend/src/lib/agentScopes.generated.ts+++ b/frontend/src/lib/agentScopes.generated.ts@@ -132,7 +132,6 @@ export const AGENT_USE_CASE_SCOPES = ['survey:read','survey:write','tagger:read',- 'tagger:write','ticket:read','ticket:write','task:read',diff --git a/products/ai_observability/backend/api/taggers.py b/products/ai_observability/backend/api/taggers.pyindex f670c3663fbd..54c5f6be99e6 100644--- a/products/ai_observability/backend/api/taggers.py+++ b/products/ai_observability/backend/api/taggers.py@@ -10,7 +10,7 @@ import django_filtersfrom django_filters.rest_framework import DjangoFilterBackendfrom drf_spectacular.utils import OpenApiResponse, PolymorphicProxySerializer, extend_schema, extend_schema_fieldfrom pydantic import ValidationError-from rest_framework import serializers, status, viewsets+from rest_framework import mixins, serializers, status, viewsetsfrom rest_framework.decorators import actionfrom rest_framework.permissions import IsAuthenticatedfrom rest_framework.request import Request@@ -189,6 +189,14 @@ class TaggerBaseWriteSerializer(serializers.ModelSerializer):]def validate(self, data: dict[str, Any]) -> dict[str, Any]:+ if self.instance is not None and not self.instance.enabled and data.get("enabled") is True:+ raise serializers.ValidationError(+ {+ "enabled": "Taggers are deprecated, so a switched-off tagger can't be turned back on. "+ "Create an evaluation with a categorical output instead."+ }+ )+tagger_type = data.get("tagger_type", self.instance.tagger_type if self.instance else TaggerType.LLM)tagger_config = data.get("tagger_config")@@ -256,24 +264,6 @@ class TaggerBaseWriteSerializer(serializers.ModelSerializer):model_config.save()return model_config- def create(self, validated_data: dict[str, Any]) -> Tagger:- request = self.context["request"]- team = self.context["get_team"]()- validated_data["team"] = team- validated_data["created_by"] = request.user-- model_config_data = validated_data.pop("model_configuration", None)-- # Wrap in a transaction so the model_configuration row and the tagger row- # land together — otherwise a Hog bytecode compile error on the tagger save- # would leave an orphaned LLMModelConfiguration row behind.- with transaction.atomic():- if model_config_data:- validated_data["model_configuration"] = self._create_or_update_model_configuration(- model_config_data, team.id- )- return super().create(validated_data)-def update(self, instance: Tagger, validated_data: dict[str, Any]) -> Tagger:model_config_data = validated_data.pop("model_configuration", None)@@ -288,13 +278,6 @@ class TaggerBaseWriteSerializer(serializers.ModelSerializer):return super().update(instance, validated_data)-class TaggerCreateSerializer(TaggerBaseWriteSerializer):- def to_internal_value(self, data: Any) -> dict[str, Any]:- if isinstance(data, dict) and "deleted" in data:- raise serializers.ValidationError({"deleted": "This field cannot be set when creating a tagger."})- return super().to_internal_value(data)--class TaggerUpdateSerializer(TaggerBaseWriteSerializer):class Meta(TaggerBaseWriteSerializer.Meta):fields = [*TaggerBaseWriteSerializer.Meta.fields, "deleted"]@@ -407,7 +390,16 @@ class TestHogTaggerResponseSerializer(serializers.Serializer):)-class TaggerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, ForbidDestroyModel, viewsets.ModelViewSet):+# Taggers are deprecated in favor of categorical evaluations, so the viewset has no create action.+class TaggerViewSet(+ TeamAndOrgViewSetMixin,+ AccessControlViewSetMixin,+ ForbidDestroyModel,+ mixins.ListModelMixin,+ mixins.RetrieveModelMixin,+ mixins.UpdateModelMixin,+ viewsets.GenericViewSet,+):scope_object = "tagger"permission_classes = [IsAuthenticated, AccessControlPermission, PostHogFeatureFlagPermission]posthog_feature_flag = "llm-analytics-tags"@@ -417,8 +409,6 @@ class TaggerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, ForbidDesfilterset_class = TaggerFilterdef get_serializer_class(self) -> type[BaseSerializer[Any]]:- if self.action == "create":- return TaggerCreateSerializerif self.action in ("update", "partial_update"):return TaggerUpdateSerializerreturn super().get_serializer_class()@@ -434,30 +424,6 @@ class TaggerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, ForbidDesreturn queryset- def perform_create(self, serializer: BaseSerializer[Any]) -> None:- instance = serializer.save()-- conditions = instance.conditions or []- tagger_config = instance.tagger_config or {}- tag_count = len(tagger_config.get("tags", []))-- report_user_action(- self.request.user,- "llma tagger created",- {- "tagger_id": str(instance.id),- "tagger_name": instance.name,- "tag_count": tag_count,- "has_description": bool(instance.description),- "enabled": instance.enabled,- "condition_count": len(conditions),- "has_rollout_percentage": any(c.get("rollout_percentage", 100) < 100 for c in conditions),- "prompt_length": len(tagger_config.get("prompt", "")),- },- team=self.team,- request=self.request,- )-def perform_update(self, serializer: BaseSerializer[Any]) -> None:instance_before = cast(Tagger, serializer.instance)is_deletion = serializer.validated_data.get("deleted") is True and not instance_before.deleted@@ -511,18 +477,6 @@ class TaggerViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, ForbidDesdef retrieve(self, request: Request, *args: Any, **kwargs: Any) -> Response:return super().retrieve(request, *args, **kwargs)- @extend_schema(request=TaggerCreateSerializer, responses={201: OpenApiResponse(response=TaggerSerializer)})- @llma_track_latency("llma_taggers_create")- @monitor(feature=Feature.QUERY, endpoint="llma_taggers_create", method="POST")- def create(self, request: Request, *args: Any, **kwargs: Any) -> Response:- serializer = self.get_serializer(data=request.data)- serializer.is_valid(raise_exception=True)- self.perform_create(serializer)- instance = cast(Tagger, serializer.instance)- response_data = self._serialize_saved_tagger(instance)- headers = self.get_success_headers(response_data)- return Response(response_data, status=status.HTTP_201_CREATED, headers=headers)-@extend_schema(request=TaggerUpdateSerializer, responses={200: OpenApiResponse(response=TaggerSerializer)})@llma_track_latency("llma_taggers_update")@monitor(feature=Feature.QUERY, endpoint="llma_taggers_update", method="PUT")diff --git a/products/ai_observability/backend/api/test/test_taggers.py b/products/ai_observability/backend/api/test/test_taggers.pyindex 50239894310d..e2ccf6a60401 100644--- a/products/ai_observability/backend/api/test/test_taggers.py+++ b/products/ai_observability/backend/api/test/test_taggers.py@@ -63,6 +63,11 @@ class TestTaggersApi(APIBaseTest):feature_flag_patch.start()self.addCleanup(feature_flag_patch.stop)+ def _create_tagger(self) -> Tagger:+ return Tagger.objects.create(+ name="Feature Tagger", tagger_config=_make_tagger_config(), team=self.team, created_by=self.user+ )+def test_feature_flag_gates_the_api_server_side(self):with patch("posthog.permissions.posthog_feature_flag_enabled", return_value=False):response = self.client.get(f"/api/environments/{self.team.id}/taggers/")@@ -74,58 +79,21 @@ class TestTaggersApi(APIBaseTest):response = self.client.get(f"/api/environments/{self.team.id}/taggers/")assert response.status_code == status.HTTP_401_UNAUTHORIZED- def test_can_create_tagger(self):+ def test_create_is_not_allowed(self):response = self.client.post(f"/api/environments/{self.team.id}/taggers/",- {- "name": "Feature Tagger",- "description": "Tags product features",- "enabled": True,- "tagger_config": _make_tagger_config(),- "conditions": [{"id": "cond-1", "rollout_percentage": 50, "properties": []}],- },- format="json",- )- assert response.status_code == status.HTTP_201_CREATED- assert Tagger.objects.count() == 1-- tagger = Tagger.objects.first()- assert tagger is not None- assert tagger.name == "Feature Tagger"- assert tagger.description == "Tags product features"- assert tagger.enabled is True- assert tagger.tagger_config["prompt"] == "Which product features were discussed?"- assert len(tagger.tagger_config["tags"]) == 2- assert len(tagger.conditions) == 1- assert tagger.conditions[0]["id"] == "cond-1"- assert tagger.team == self.team- assert tagger.created_by == self.user- assert tagger.deleted is False-- def test_create_cannot_soft_delete_tagger(self):- response = self.client.post(- f"/api/environments/{self.team.id}/taggers/",- {- "name": "Feature Tagger",- "tagger_config": _make_tagger_config(),- "deleted": True,- },+ {"name": "Feature Tagger", "tagger_config": _make_tagger_config()},format="json",)- assert response.status_code == status.HTTP_400_BAD_REQUEST- assert response.data == self.validation_error_response(- "This field cannot be set when creating a tagger.",- attr="deleted",- )+ assert response.status_code == status.HTTP_405_METHOD_NOT_ALLOWEDassert Tagger.objects.count() == 0- def test_create_rejects_read_only_provider_key_name(self):- response = self.client.post(- f"/api/environments/{self.team.id}/taggers/",+ def test_update_rejects_read_only_provider_key_name(self):+ tagger = self._create_tagger()+ response = self.client.patch(+ f"/api/environments/{self.team.id}/taggers/{tagger.id}/",{- "name": "Feature Tagger",- "tagger_config": _make_tagger_config(),"model_configuration": {"provider": LLMProvider.OPENAI,"model": "gpt-4o",@@ -140,7 +108,6 @@ class TestTaggersApi(APIBaseTest):"This field is read-only.",attr="model_configuration__provider_key_name",)- assert Tagger.objects.count() == 0def test_can_retrieve_list_of_taggers(self):Tagger.objects.create(@@ -243,6 +210,22 @@ class TestTaggersApi(APIBaseTest):assert tagger.enabled is Falseassert tagger.tagger_config["prompt"] == "Updated prompt"+ @parameterized.expand(+ [("switched_off", False, status.HTTP_400_BAD_REQUEST), ("switched_on", True, status.HTTP_200_OK)]+ )+ def test_enabling_a_tagger(self, _name: str, currently_enabled: bool, expected_status: int):+ tagger = self._create_tagger()+ tagger.enabled = currently_enabled+ tagger.save()++ response = self.client.patch(+ f"/api/environments/{self.team.id}/taggers/{tagger.id}/", {"enabled": True}, format="json"+ )++ assert response.status_code == expected_status, response.json()+ tagger.refresh_from_db()+ assert tagger.enabled is currently_enabled+def test_delete_method_returns_405(self):tagger = Tagger.objects.create(name="Test Tagger",@@ -347,26 +330,6 @@ class TestTaggersApi(APIBaseTest):assert response.status_code == status.HTTP_200_OKassert len(response.data["results"]) == 0- def test_validation_requires_name(self):- response = self.client.post(- f"/api/environments/{self.team.id}/taggers/",- {- "tagger_config": _make_tagger_config(),- },- format="json",- )- assert response.status_code == status.HTTP_400_BAD_REQUEST-- def test_validation_requires_tagger_config(self):- response = self.client.post(- f"/api/environments/{self.team.id}/taggers/",- {- "name": "Test",- },- format="json",- )- assert response.status_code == status.HTTP_400_BAD_REQUEST-@parameterized.expand([("empty_tags", {"tags": []}),@@ -384,30 +347,25 @@ class TestTaggersApi(APIBaseTest):])def test_validation_rejects_invalid_tagger_config(self, _name: str, config_overrides: dict):- response = self.client.post(- f"/api/environments/{self.team.id}/taggers/",- {- "name": "Test",- "tagger_config": _make_tagger_config(**config_overrides),- },+ tagger = self._create_tagger()+ response = self.client.patch(+ f"/api/environments/{self.team.id}/taggers/{tagger.id}/",+ {"tagger_config": _make_tagger_config(**config_overrides)},format="json",)assert response.status_code == status.HTTP_400_BAD_REQUEST@parameterized.expand([("fractional", 33.3), ("slider_minimum", 0.1), ("whole", 50)])def test_can_save_fractional_rollout_percentage(self, _name: str, rollout_percentage: float):- response = self.client.post(- f"/api/environments/{self.team.id}/taggers/",- {- "name": "Feature Tagger",- "tagger_config": _make_tagger_config(),- "conditions": [{"id": "cond-1", "rollout_percentage": rollout_percentage, "properties": []}],- },+ tagger = self._create_tagger()+ response = self.client.patch(+ f"/api/environments/{self.team.id}/taggers/{tagger.id}/",+ {"conditions": [{"id": "cond-1", "rollout_percentage": rollout_percentage, "properties": []}]},format="json",)- assert response.status_code == status.HTTP_201_CREATED, response.json()- tagger = Tagger.objects.get(id=response.data["id"])+ assert response.status_code == status.HTTP_200_OK, response.json()+ tagger.refresh_from_db()assert tagger.conditions[0]["rollout_percentage"] == rollout_percentagedef test_patch_tagger_type_without_fresh_config_is_rejected(self):@@ -465,20 +423,12 @@ class TestTaggersAccessControl(APIBaseTest):organization_member=membership,)- def test_no_access_user_cannot_list_or_create_taggers(self):+ def test_no_access_user_cannot_list_taggers(self):self.client.force_login(self.no_access_user)list_response = self.client.get(f"/api/environments/{self.team.id}/taggers/")assert list_response.status_code == status.HTTP_403_FORBIDDEN- create_response = self.client.post(- f"/api/environments/{self.team.id}/taggers/",- {"name": "Blocked Tagger", "tagger_config": _make_tagger_config()},- format="json",- )- assert create_response.status_code == status.HTTP_403_FORBIDDEN- assert Tagger.objects.count() == 0-def test_viewer_user_can_read_but_not_write_taggers(self):tagger = Tagger.objects.create(name="Existing Tagger",@@ -495,14 +445,6 @@ class TestTaggersAccessControl(APIBaseTest):retrieve_response = self.client.get(f"/api/environments/{self.team.id}/taggers/{tagger.id}/")assert retrieve_response.status_code == status.HTTP_200_OK- create_response = self.client.post(- f"/api/environments/{self.team.id}/taggers/",- {"name": "New Tagger", "tagger_config": _make_tagger_config()},- format="json",- )- assert create_response.status_code == status.HTTP_403_FORBIDDEN- assert Tagger.objects.count() == 1-edit_response = self.client.patch(f"/api/environments/{self.team.id}/taggers/{tagger.id}/",{"name": "Renamed by viewer"},@@ -512,17 +454,12 @@ class TestTaggersAccessControl(APIBaseTest):tagger.refresh_from_db()assert tagger.name == "Existing Tagger"- def test_editor_user_can_create_and_edit_taggers(self):+ def test_editor_user_can_edit_taggers(self):+ tagger_id = Tagger.objects.create(+ name="Editor Tagger", tagger_config=_make_tagger_config(), team=self.team, created_by=self.user+ ).idself.client.force_login(self.editor_user)- create_response = self.client.post(- f"/api/environments/{self.team.id}/taggers/",- {"name": "Editor Tagger", "tagger_config": _make_tagger_config()},- format="json",- )- assert create_response.status_code == status.HTTP_201_CREATED- tagger_id = create_response.data["id"]-edit_response = self.client.patch(f"/api/environments/{self.team.id}/taggers/{tagger_id}/",{"name": "Renamed by editor"},diff --git a/products/ai_observability/frontend/generated/api.schemas.ts b/products/ai_observability/frontend/generated/api.schemas.tsindex 4f1ad20157a7..e5c598199f66 100644--- a/products/ai_observability/frontend/generated/api.schemas.ts+++ b/products/ai_observability/frontend/generated/api.schemas.ts@@ -4609,19 +4609,6 @@ export interface TaggerModelConfigurationWriteApi {provider_key_id?: string | null}-export interface TaggerCreateApi {- /** @maxLength 400 */- name: string- description?: string- enabled?: boolean- tagger_type?: TaggerTypeEnumApi- /** Tagger configuration. For tagger_type 'llm': {prompt, tags, min_tags?, max_tags?}. For tagger_type 'hog': {source, tags?}. */- tagger_config: TaggerConfigApi- /** Conditions that scope when the tagger runs */- conditions?: TaggerConditionApi[]- model_configuration?: TaggerModelConfigurationWriteApi | null-}-export interface TaggerUpdateApi {/** @maxLength 400 */name: stringdiff --git a/products/ai_observability/frontend/generated/api.ts b/products/ai_observability/frontend/generated/api.tsindex 3d85f12610c8..2c9637cbd771 100644--- a/products/ai_observability/frontend/generated/api.ts+++ b/products/ai_observability/frontend/generated/api.ts@@ -133,7 +133,6 @@ import type {SummarizeRequestApi,SummarizeResponseApi,TaggerApi,- TaggerCreateApi,TaggerUpdateApi,TaggersListParams,TestHogRequestApi,@@ -2949,23 +2948,6 @@ export const taggersList = async (})}-export const getTaggersCreateUrl = (projectId: string) => {- return `/api/projects/${projectId}/taggers/`-}--export const taggersCreate = async (- projectId: string,- taggerCreateApi: TaggerCreateApi,- options?: RequestInit-): Promise<TaggerApi> => {- return apiMutator<TaggerApi>(getTaggersCreateUrl(projectId), {- ...options,- method: 'POST',- headers: { 'Content-Type': 'application/json', ...options?.headers },- body: JSON.stringify(taggerCreateApi),- })-}-export const getTaggersRetrieveUrl = (projectId: string, id: string) => {return `/api/projects/${projectId}/taggers/${id}/`}diff --git a/products/ai_observability/frontend/generated/api.zod.ts b/products/ai_observability/frontend/generated/api.zod.tsindex 4ca36fa22143..4f7d9c32beff 100644--- a/products/ai_observability/frontend/generated/api.zod.ts+++ b/products/ai_observability/frontend/generated/api.zod.ts@@ -3016,145 +3016,6 @@ export const LlmPromptsNameLabelsUpdateBody = /* @__PURE__ */ zod.object({),})-export const taggersCreateBodyNameMax = 400--export const taggersCreateBodyTaggerTypeDefault = `llm`-export const taggersCreateBodyTaggerConfigOneOneTagsItemNameMax = 100--export const taggersCreateBodyTaggerConfigOneOneTagsItemDescriptionDefault = ``-export const taggersCreateBodyTaggerConfigOneOneTagsItemDescriptionMax = 500--export const taggersCreateBodyTaggerConfigOneOneMinTagsDefault = 0-export const taggersCreateBodyTaggerConfigOneOneMinTagsMin = 0--export const taggersCreateBodyTaggerConfigOneTwoTagsItemNameMax = 100--export const taggersCreateBodyTaggerConfigOneTwoTagsItemDescriptionDefault = ``-export const taggersCreateBodyTaggerConfigOneTwoTagsItemDescriptionMax = 500--export const taggersCreateBodyConditionsItemIdMax = 100--export const taggersCreateBodyConditionsItemRolloutPercentageDefault = 100-export const taggersCreateBodyConditionsItemRolloutPercentageMin = 0-export const taggersCreateBodyConditionsItemRolloutPercentageMax = 100--export const taggersCreateBodyModelConfigurationOneModelMax = 100--export const TaggersCreateBody = /* @__PURE__ */ zod.object({- name: zod.string().max(taggersCreateBodyNameMax),- description: zod.string().optional(),- enabled: zod.boolean().optional(),- tagger_type: zod- .enum(['llm', 'hog'])- .describe('\* `llm` - LLM\n\* `hog` - Hog')- .default(taggersCreateBodyTaggerTypeDefault),- tagger_config: zod- .union([- zod.object({- prompt: zod.string().min(1).describe('Prompt instructing the LLM how to tag generations'),- tags: zod- .array(- zod.object({- name: zod- .string()- .max(taggersCreateBodyTaggerConfigOneOneTagsItemNameMax)- .describe('Tag identifier'),- description: zod- .string()- .max(taggersCreateBodyTaggerConfigOneOneTagsItemDescriptionMax)- .default(taggersCreateBodyTaggerConfigOneOneTagsItemDescriptionDefault)- .describe('Description to help the LLM classify'),- })- )- .describe('Available tags the LLM can assign'),- min_tags: zod- .number()- .min(taggersCreateBodyTaggerConfigOneOneMinTagsMin)- .default(taggersCreateBodyTaggerConfigOneOneMinTagsDefault)- .describe('Minimum number of tags to apply'),- max_tags: zod.number().min(1).nullish().describe('Maximum number of tags to apply (null = no limit)'),- }),- zod.object({- source: zod.string().min(1).describe('Hog source code to classify a generation into tags.'),- tags: zod- .array(- zod.object({- name: zod- .string()- .max(taggersCreateBodyTaggerConfigOneTwoTagsItemNameMax)- .describe('Tag identifier'),- description: zod- .string()- .max(taggersCreateBodyTaggerConfigOneTwoTagsItemDescriptionMax)- .default(taggersCreateBodyTaggerConfigOneTwoTagsItemDescriptionDefault)- .describe('Description to help the LLM classify'),- })- )- .optional()- .describe('Optional tag whitelist. Leave empty to allow any tag returned by the Hog code.'),- }),- ])- .describe(- "Tagger configuration. For tagger_type 'llm': {prompt, tags, min_tags?, max_tags?}. For tagger_type 'hog': {source, tags?}."- ),- conditions: zod- .array(- zod.object({- id: zod- .string()- .max(taggersCreateBodyConditionsItemIdMax)- .describe('Stable identifier for this condition'),- rollout_percentage: zod- .number()- .min(taggersCreateBodyConditionsItemRolloutPercentageMin)- .max(taggersCreateBodyConditionsItemRolloutPercentageMax)- .default(taggersCreateBodyConditionsItemRolloutPercentageDefault)- .describe('Percentage of matching events to apply this condition to'),- properties: zod- .array(zod.record(zod.string(), zod.unknown()))- .optional()- .describe('Property filters that scope when this condition fires'),- })- )- .optional()- .describe('Conditions that scope when the tagger runs'),- model_configuration: zod- .union([- zod.object({- provider: zod- .enum([- 'openai',- 'anthropic',- 'gemini',- 'openrouter',- 'fireworks',- 'azure_openai',- 'together_ai',- 'minimax',- 'openai_compatible',- ])- .describe(- '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `openai_compatible` - OpenAI-compatible'- )- .describe(- 'LLM provider to use for this tagger.\n\n\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `openai_compatible` - OpenAI-compatible'- ),- model: zod- .string()- .max(taggersCreateBodyModelConfigurationOneModelMax)- .describe('Provider model identifier to use for this tagger.'),- provider_key_id: zod- .uuid()- .nullish()- .describe(- 'Existing LLM provider key UUID for the current project. Do not invent this value; use a real provider key ID returned by PostHog, or omit\/null when no provider key should be pinned.'- ),- }),- zod.null(),- ])- .optional(),-})-export const taggersUpdateBodyNameMax = 400export const taggersUpdateBodyTaggerTypeDefault = `llm`diff --git a/products/ai_observability/frontend/tags/AIObservabilityTag.tsx b/products/ai_observability/frontend/tags/AIObservabilityTag.tsxindex cfa55f8de7bb..98fc7da8c396 100644--- a/products/ai_observability/frontend/tags/AIObservabilityTag.tsx+++ b/products/ai_observability/frontend/tags/AIObservabilityTag.tsx@@ -1,5 +1,7 @@import { BindLogic, useActions, useValues } from 'kea'import { Field, Form } from 'kea-forms'+import { router } from 'kea-router'+import { useState } from 'react'import { IconArrowLeft, IconCopy, IconPlus, IconTrash, IconWarning } from '@posthog/icons'import {@@ -43,6 +45,8 @@ import {} from '../settings/providerKeyStateUtils'import { HOG_TAGGER_EXAMPLES } from './hogTaggerExamples'import { HogTestResult, TagRun, llmTaggerLogic } from './llmTaggerLogic'+import { TaggerCreationRetiredModal } from './TaggerCreationRetiredModal'+import { TaggersDeprecationBanner } from './TaggersDeprecationBanner'import { Tagger, TaggerConditionSet } from './types'const DEFAULT_HOG_SOURCE = `// Return a list of tag names that apply to this generation@@ -375,8 +379,9 @@ function HogTaggerTestSection({ id }: { id: string }): JSX.Element {function AIObservabilityTaggerForm({ id }: { id: string }): JSX.Element {const logic = llmTaggerLogic({ id })- const { taggerForm, taggerFormChanged, isTaggerFormSubmitting } = useValues(logic)+ const { tagger, taggerForm, taggerFormChanged, isTaggerFormSubmitting } = useValues(logic)const { setTaggerFormValues, submitTaggerForm, deleteTagger } = useActions(logic)+ const [isEnableRetiredModalOpen, setIsEnableRetiredModalOpen] = useState(false)return (<BindLogic logic={llmTaggerLogic} props={{ id }}>@@ -404,9 +409,18 @@ function AIObservabilityTaggerForm({ id }: { id: string }): JSX.Element {<div className="flex items-center gap-2"><LemonSwitchchecked={taggerForm.enabled}- onChange={(checked) => setTaggerFormValues({ enabled: checked })}+ onChange={(checked) =>+ checked && !tagger?.enabled+ ? setIsEnableRetiredModalOpen(true)+ : setTaggerFormValues({ enabled: checked })+ }label="Enabled"/>+ <TaggerCreationRetiredModal+ action="enable"+ isOpen={isEnableRetiredModalOpen}+ onClose={() => setIsEnableRetiredModalOpen(false)}+ /></div></div>@@ -581,29 +595,22 @@ function AIObservabilityTaggerForm({ id }: { id: string }): JSX.Element {type="primary"onClick={submitTaggerForm}loading={isTaggerFormSubmitting}- disabledReason={!taggerFormChanged && id !== 'new' ? 'No changes to save' : undefined}+ disabledReason={!taggerFormChanged ? 'No changes to save' : undefined}>- {id === 'new' ? 'Create tagger' : 'Save changes'}+ Save changes</LemonButton></AccessControlAction><LemonButton type="secondary" to={urls.aiObservabilityTags()}>Cancel</LemonButton>- {id !== 'new' && (- <AccessControlAction- resourceType={AccessControlResourceType.Tagger}- minAccessLevel={AccessControlLevel.Editor}- >- <LemonButton- type="secondary"- status="danger"- className="ml-auto"- onClick={deleteTagger}- >- Delete- </LemonButton>- </AccessControlAction>- )}+ <AccessControlAction+ resourceType={AccessControlResourceType.Tagger}+ minAccessLevel={AccessControlLevel.Editor}+ >+ <LemonButton type="secondary" status="danger" className="ml-auto" onClick={deleteTagger}>+ Delete+ </LemonButton>+ </AccessControlAction></div></div></Form>@@ -741,6 +748,14 @@ export function AIObservabilityTag({ id }: { id?: string }): JSX.Element {const { setActiveTab } = useActions(llmTaggerLogic({ id: taggerId }))const providerKeyIssue = tagger?.enabled ? getTaggerProviderKeyIssue(tagger, providerKeys) : null+ if (isNew) {+ return (+ <SceneContent>+ <TaggerCreationRetiredModal isOpen onClose={() => router.actions.push(urls.aiObservabilityTags())} />+ </SceneContent>+ )+ }+if (taggerLoading) {return (<SceneContent>@@ -751,12 +766,13 @@ export function AIObservabilityTag({ id }: { id?: string }): JSX.Element {return (<SceneContent>+ <TaggersDeprecationBanner /><div className="space-y-6">{/* Header */}<div className="flex justify-between items-start pb-4 border-b"><div className="space-y-2">- <h1 className="text-2xl font-semibold">{isNew ? 'New tagger' : tagger?.name || 'Tagger'}</h1>- {!isNew && tagger && (+ <h1 className="text-2xl font-semibold">{tagger?.name || 'Tagger'}</h1>+ {tagger && (<div className="flex items-center gap-2"><LemonTag type={tagger.enabled ? 'success' : 'default'}>{tagger.enabled ? 'Enabled' : 'Disabled'}@@ -792,24 +808,20 @@ export function AIObservabilityTag({ id }: { id?: string }): JSX.Element {)}<LemonTabs- activeKey={isNew ? 'configuration' : activeTab}+ activeKey={activeTab}onChange={(key) => setActiveTab(key as 'runs' | 'configuration')}data-attr="llma-tagger-tabs"tabs={[- ...(!isNew- ? [- {- key: 'runs',- label: 'Runs',- content: (- <div className="max-w-6xl">- <TagRunsChart id={taggerId} />- <TagRunsTable id={taggerId} />- </div>- ),- },- ]- : []),+ {+ key: 'runs',+ label: 'Runs',+ content: (+ <div className="max-w-6xl">+ <TagRunsChart id={taggerId} />+ <TagRunsTable id={taggerId} />+ </div>+ ),+ },{key: 'configuration',label: 'Configuration',diff --git a/products/ai_observability/frontend/tags/AIObservabilityTagsScene.tsx b/products/ai_observability/frontend/tags/AIObservabilityTagsScene.tsxindex 6c851886dd83..5c6131a1fe7a 100644--- a/products/ai_observability/frontend/tags/AIObservabilityTagsScene.tsx+++ b/products/ai_observability/frontend/tags/AIObservabilityTagsScene.tsx@@ -1,5 +1,6 @@import { useActions, useValues } from 'kea'import { combineUrl, router } from 'kea-router'+import { useState } from 'react'import { IconPencil, IconPlus, IconSearch, IconTrash, IconWarning } from '@posthog/icons'import {@@ -20,7 +21,6 @@ import { DateFilter } from 'lib/components/DateFilter/DateFilter'import { FEATURE_FLAGS } from 'lib/constants'import { useFeatureFlag } from 'lib/hooks/useFeatureFlag'import { LemonTableColumns } from 'lib/lemon-ui/LemonTable'-import { LinkPrimitive } from 'lib/lemon-ui/Link'import { featureFlagLogic } from 'lib/logic/featureFlagLogic'import { deleteWithUndo } from 'lib/utils/deleteWithUndo'import { SceneExport } from 'scenes/sceneTypes'@@ -36,6 +36,8 @@ import { AccessControlLevel, AccessControlResourceType } from '~/types'import { LLMProviderKey, llmProviderKeysLogic } from '../settings/llmProviderKeysLogic'import { getUnhealthyProviderKey, providerKeyStateIssueDescription } from '../settings/providerKeyStateUtils'import { llmTaggersLogic } from './llmTaggersLogic'+import { TaggerCreationRetiredModal } from './TaggerCreationRetiredModal'+import { TaggersDeprecationBanner } from './TaggersDeprecationBanner'import { Tagger } from './types'export const scene: SceneExport = {@@ -76,9 +78,7 @@ function TaggerMetrics(): JSX.Element {</div>) : (<div className="flex-[2] bg-bg-light border rounded p-8 flex items-center justify-center">- <div className="text-muted text-center">- No enabled taggers to display. Create and enable taggers to see metrics.- </div>+ <div className="text-muted text-center">No enabled taggers to display.</div></div>)}@@ -118,6 +118,7 @@ function AIObservabilityTagsContent(): JSX.Element {const { searchParams } = useValues(router)const taggerUrl = (id: string): string => combineUrl(urls.aiObservabilityTag(id), searchParams).urlconst settingsUrl = urls.settings('project-ai-observability', 'ai-observability-byok')+ const [isEnableRetiredModalOpen, setIsEnableRetiredModalOpen] = useState(false)const columns: LemonTableColumns<Tagger> = [{@@ -166,7 +167,9 @@ function AIObservabilityTagsContent(): JSX.Element {><LemonSwitchchecked={tagger.enabled}- onChange={() => toggleTaggerEnabled(tagger.id)}+ onChange={() =>+ tagger.enabled ? toggleTaggerEnabled(tagger.id) : setIsEnableRetiredModalOpen(true)+ }size="small"data-attr="toggle-tagger-enabled"/>@@ -314,12 +317,17 @@ function AIObservabilityTagsContent(): JSX.Element {nouns={['tagger', 'taggers']}emptyState={<div className="text-center p-8 text-muted">No taggers found.</div>}/>+ <TaggerCreationRetiredModal+ action="enable"+ isOpen={isEnableRetiredModalOpen}+ onClose={() => setIsEnableRetiredModalOpen(false)}+ /></div>)}export function AIObservabilityTagsScene(): JSX.Element {- const { searchParams } = useValues(router)+ const [isCreationRetiredModalOpen, setIsCreationRetiredModalOpen] = useState(false)const { featureFlags, receivedFeatureFlags } = useValues(featureFlagLogic)const todayRailNav = useFeatureFlag('TODAY_RAIL_NAV')@@ -353,9 +361,7 @@ export function AIObservabilityTagsScene(): JSX.Element {{todayRailNav ? (<Buttonvariant="primary"- render={- <LinkPrimitive to={combineUrl(urls.aiObservabilityTag('new'), searchParams).url} />- }+ onClick={() => setIsCreationRetiredModalOpen(true)}data-attr="create-tagger-button"><IconPlus />@@ -365,7 +371,7 @@ export function AIObservabilityTagsScene(): JSX.Element {<LemonButtontype="primary"icon={<IconPlus />}- to={combineUrl(urls.aiObservabilityTag('new'), searchParams).url}+ onClick={() => setIsCreationRetiredModalOpen(true)}data-attr="create-tagger-button">Create tagger@@ -374,7 +380,12 @@ export function AIObservabilityTagsScene(): JSX.Element {</AccessControlAction>}/>+ <TaggersDeprecationBanner /><AIObservabilityTagsContent />+ <TaggerCreationRetiredModal+ isOpen={isCreationRetiredModalOpen}+ onClose={() => setIsCreationRetiredModalOpen(false)}+ /></SceneContent>)}diff --git a/products/ai_observability/frontend/tags/TaggerCreationRetiredModal.stories.tsx b/products/ai_observability/frontend/tags/TaggerCreationRetiredModal.stories.tsxnew file mode 100644index 000000000000..5a3d7f3c1814--- /dev/null+++ b/products/ai_observability/frontend/tags/TaggerCreationRetiredModal.stories.tsx@@ -0,0 +1,22 @@+import type { Meta, StoryObj } from '@storybook/react'++import { TaggerCreationRetiredModal } from './TaggerCreationRetiredModal'++const meta: Meta<typeof TaggerCreationRetiredModal> = {+ title: 'Scenes-App/LLM observability/Tagger creation retired modal',+ component: TaggerCreationRetiredModal,+ parameters: {+ viewMode: 'story',+ },+}+export default meta++type Story = StoryObj<typeof TaggerCreationRetiredModal>++export const Default: Story = {+ args: { isOpen: true, onClose: () => {}, inline: true },+}++export const TurnBackOn: Story = {+ args: { isOpen: true, onClose: () => {}, action: 'enable', inline: true },+}diff --git a/products/ai_observability/frontend/tags/TaggerCreationRetiredModal.tsx b/products/ai_observability/frontend/tags/TaggerCreationRetiredModal.tsxnew file mode 100644index 000000000000..fcff869caadc--- /dev/null+++ b/products/ai_observability/frontend/tags/TaggerCreationRetiredModal.tsx@@ -0,0 +1,50 @@+import * as transformerPng from '@posthog/brand/hoggies/png/transformer'+import { LemonButton, LemonModal } from '@posthog/lemon-ui'++import { pngHoggie } from 'lib/brand/hoggies'+import { urls } from 'scenes/urls'++const HedgehogTransformer = pngHoggie(transformerPng)++export type TaggerRetiredAction = 'create' | 'enable'++const MESSAGES: Record<TaggerRetiredAction, string> = {+ create: "New taggers can't be created. To tag generations, create an evaluation with a categorical output.",+ enable: "Switched-off taggers can't be turned back on. To keep tagging generations, create an evaluation with a categorical output.",+}++export function TaggerCreationRetiredModal({+ isOpen,+ onClose,+ action = 'create',+ inline = false,+}: {+ isOpen: boolean+ onClose: () => void+ action?: TaggerRetiredAction+ // Renders inline rather than in a portal, so Storybook can snapshot it.+ inline?: boolean+}): JSX.Element {+ return (+ <LemonModal+ isOpen={isOpen}+ onClose={onClose}+ inline={inline}+ title="Taggers are deprecated in favor of evaluations"+ width={520}+ data-attr="tagger-creation-retired-modal"+ >+ <div className="flex flex-col items-center gap-4 text-center pb-2">+ <HedgehogTransformer className="w-28 h-28" />+ <p>{MESSAGES[action]}</p>+ <LemonButton+ type="primary"+ to={urls.aiObservabilityEvaluationTemplates()}+ data-attr="tagger-creation-retired-create-evaluation"+ >+ Create categorical evaluation+ </LemonButton>+ </div>+ </LemonModal>+ )+}diff --git a/products/ai_observability/frontend/tags/TaggersDeprecationBanner.stories.tsx b/products/ai_observability/frontend/tags/TaggersDeprecationBanner.stories.tsxnew file mode 100644index 000000000000..07cf8d48fdd4--- /dev/null+++ b/products/ai_observability/frontend/tags/TaggersDeprecationBanner.stories.tsx@@ -0,0 +1,16 @@+import type { Meta, StoryObj } from '@storybook/react'++import { TaggersDeprecationBanner } from './TaggersDeprecationBanner'++const meta: Meta<typeof TaggersDeprecationBanner> = {+ title: 'Scenes-App/LLM observability/Taggers deprecation banner',+ component: TaggersDeprecationBanner,+ parameters: {+ viewMode: 'story',+ },+}+export default meta++type Story = StoryObj<typeof TaggersDeprecationBanner>++export const Default: Story = {}diff --git a/products/ai_observability/frontend/tags/TaggersDeprecationBanner.tsx b/products/ai_observability/frontend/tags/TaggersDeprecationBanner.tsxnew file mode 100644index 000000000000..9dcac370c29e--- /dev/null+++ b/products/ai_observability/frontend/tags/TaggersDeprecationBanner.tsx@@ -0,0 +1,25 @@+import * as townCrierPng from '@posthog/brand/hoggies/png/town-crier'+import { LemonBanner } from '@posthog/lemon-ui'++import { pngHoggie } from 'lib/brand/hoggies'+import { urls } from 'scenes/urls'++const HedgehogTownCrier = pngHoggie(townCrierPng)++export function TaggersDeprecationBanner(): JSX.Element {+ return (+ <LemonBanner+ type="warning"+ // The banner is a size container, so the hog only takes text width when there is room for it.+ icon={<HedgehogTownCrier className="hidden @lg:block w-12 h-12 shrink-0" />}+ action={{+ children: 'Create categorical evaluation',+ to: urls.aiObservabilityEvaluationTemplates(),+ 'data-attr': 'taggers-deprecation-banner-create-evaluation',+ }}+ >+ <p className="font-semibold">Taggers are deprecated in favor of evaluations</p>+ <p>To tag new generations, create an evaluation with a categorical output.</p>+ </LemonBanner>+ )+}diff --git a/products/ai_observability/frontend/tags/llmTaggerLogic.ts b/products/ai_observability/frontend/tags/llmTaggerLogic.tsindex 3f6cc7ad854c..41461ffc3a18 100644--- a/products/ai_observability/frontend/tags/llmTaggerLogic.ts+++ b/products/ai_observability/frontend/tags/llmTaggerLogic.ts@@ -429,16 +429,10 @@ export const llmTaggerLogic = kea<llmTaggerLogicType>([},}- if (props.id === 'new') {- // nosemgrep: prefer-codegen-api -- Legacy raw API call with a hand-written URL and an unchecked response type. Use taggersCreate() from 'products/ai_observability/frontend/generated/api' instead.- await api.create(`api/projects/${teamLogic.values.currentTeamId}/taggers/`, payload)- lemonToast.success('Tagger created')- } else {- // nosemgrep: prefer-codegen-api -- Legacy raw API call with a hand-written URL and an unchecked response type. Use taggersPartialUpdate() from 'products/ai_observability/frontend/generated/api' instead.- await api.update(`api/projects/${teamLogic.values.currentTeamId}/taggers/${props.id}/`, payload)- lemonToast.success('Tagger updated')- }- // Reload list before navigating so the new/updated tagger is visible+ // nosemgrep: prefer-codegen-api -- Legacy raw API call with a hand-written URL and an unchecked response type. Use taggersPartialUpdate() from 'products/ai_observability/frontend/generated/api' instead.+ await api.update(`api/projects/${teamLogic.values.currentTeamId}/taggers/${props.id}/`, payload)+ lemonToast.success('Tagger updated')+ // Reload list before navigating so the updated tagger is visiblellmTaggersLogic.findMounted()?.actions.loadTaggers()router.actions.push(urls.aiObservabilityTags())},diff --git a/products/ai_observability/mcp/tools.yaml b/products/ai_observability/mcp/tools.yamlindex 6e0126fc9a54..714c91bf3588 100644--- a/products/ai_observability/mcp/tools.yaml+++ b/products/ai_observability/mcp/tools.yaml@@ -571,13 +571,14 @@ tools:boolean, a finite number, or category keys). For 'hog' and 'llm_judge', choose output_type='boolean','numeric', or 'categorical'; numeric output_config accepts min, max, step, allows_na, and passing_rule;categorical output_config accepts options, selection_mode, allows_na, and passing_rule. 'sentiment':- output_type='sentiment', omit model_configuration, optional evaluation_config.source='user_messages'.- `target` accepts 'generation' (default), 'trace', or 'session'. A session target runs once per- `$ai_session_id` and defaults to the 'inactivity' settle strategy. Pass `enabled` as a boolean (true to- start scoring new $ai_generation events immediately, false to create it paused). To organize the evaluation,- pass a `directory_id` returned by llma-evaluation-directory-list. Omit `directory_id` or pass null to keep- it at the top level. Results are '$ai_evaluation' events. An evaluation only scores data that arrives after- it is enabled, so once it is enabled, offer the user a backfill over past data with+ output_type='sentiment', omit model_configuration, optional evaluation_config.source='user_messages'. To tag+ or classify generations into labels (what taggers used to do), use output_type='categorical' with+ selection_mode='multiple'. `target` accepts 'generation' (default), 'trace', or 'session'. A session target+ runs once per `$ai_session_id` and defaults to the 'inactivity' settle strategy. Pass `enabled` as a boolean+ (true to start scoring new $ai_generation events immediately, false to create it paused). To organize the+ evaluation, pass a `directory_id` returned by llma-evaluation-directory-list. Omit `directory_id` or pass+ null to keep it at the top level. Results are '$ai_evaluation' events. An evaluation only scores data that+ arrives after it is enabled, so once it is enabled, offer the user a backfill over past data withllma-evaluation-backfill-estimate.llma-evaluation-delete:operation: evaluations_destroy@@ -1708,25 +1709,6 @@ tools:bullets, and interesting notes. Results are cached. Use mode "minimal" (default) for 3-5 points or"detailed" for 5-10 points. Rate-limited; requires AI data processing approval for the organization.requires_ai_consent: true- llma-tagger-create:- operation: taggers_create- enabled: true- scopes:- - tagger:write- annotations:- readOnly: false- destructive: false- idempotent: false- enrich_url: '{id}'- title: Create tagger- description: >- Create an AI observability tagger that automatically adds custom tags to $ai_generation events. For- tagger_type "llm", provide tagger_config.prompt plus tagger_config.tags (each with name and optional- description); optionally set min_tags and max_tags. For tagger_type "hog", provide tagger_config.source Hog- code that returns a tag name string, a list of tag name strings, or null; optionally provide- tagger_config.tags as a whitelist. Use enabled=false while drafting, then update the tagger from the UI.- feature_flag: llm-analytics-tags- url_prefix: /ai-evals/taggersllma-tagger-list:operation: taggers_listenabled: true@@ -1742,7 +1724,8 @@ tools:description: >List AI observability taggers for the current project. Supports filtering by enabled status, id__in, search,and ordering. Returns each tagger's name, description, enabled state, type, configuration, conditions, model- configuration, creator, and timestamps.+ configuration, creator, and timestamps. Taggers are deprecated and new ones can't be created. To tag or+ classify generations, create a categorical evaluation with llma-evaluation-create instead.feature_flag: llm-analytics-tagsurl_prefix: /ai-evals/taggersllma-tagger-test-hog:diff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.jsonindex 42728376d1fc..061a0321ad67 100644--- a/services/mcp/schema/generated-tool-definitions.json+++ b/services/mcp/schema/generated-tool-definitions.json@@ -7978,7 +7978,7 @@}},"llma-evaluation-create": {- "description": "Create an evaluation. 'llm_judge': set evaluation_config.prompt and model_configuration with provider and model; provider_key_id may be null when no key is pinned. 'hog': evaluation_config.source (Hog returning a boolean, a finite number, or category keys). For 'hog' and 'llm_judge', choose output_type='boolean', 'numeric', or 'categorical'; numeric output_config accepts min, max, step, allows_na, and passing_rule; categorical output_config accepts options, selection_mode, allows_na, and passing_rule. 'sentiment': output_type='sentiment', omit model_configuration, optional evaluation_config.source='user_messages'. `target` accepts 'generation' (default), 'trace', or 'session'. A session target runs once per `$ai_session_id` and defaults to the 'inactivity' settle strategy. Pass `enabled` as a boolean (true to start scoring new $ai_generation events immediately, false to create it paused). To organize the evaluation, pass a `directory_id` returned by llma-evaluation-directory-list. Omit `directory_id` or pass null to keep it at the top level. Results are '$ai_evaluation' events. An evaluation only scores data that arrives after it is enabled, so once it is enabled, offer the user a backfill over past data with llma-evaluation-backfill-estimate.",+ "description": "Create an evaluation. 'llm_judge': set evaluation_config.prompt and model_configuration with provider and model; provider_key_id may be null when no key is pinned. 'hog': evaluation_config.source (Hog returning a boolean, a finite number, or category keys). For 'hog' and 'llm_judge', choose output_type='boolean', 'numeric', or 'categorical'; numeric output_config accepts min, max, step, allows_na, and passing_rule; categorical output_config accepts options, selection_mode, allows_na, and passing_rule. 'sentiment': output_type='sentiment', omit model_configuration, optional evaluation_config.source='user_messages'. To tag or classify generations into labels (what taggers used to do), use output_type='categorical' with selection_mode='multiple'. `target` accepts 'generation' (default), 'trace', or 'session'. A session target runs once per `$ai_session_id` and defaults to the 'inactivity' settle strategy. Pass `enabled` as a boolean (true to start scoring new $ai_generation events immediately, false to create it paused). To organize the evaluation, pass a `directory_id` returned by llma-evaluation-directory-list. Omit `directory_id` or pass null to keep it at the top level. Results are '$ai_evaluation' events. An evaluation only scores data that arrives after it is enabled, so once it is enabled, offer the user a backfill over past data with llma-evaluation-backfill-estimate.","category": "AI observability","feature": "llm_analytics","summary": "Create evaluation",@@ -8861,23 +8861,8 @@},"requires_ai_consent": true},- "llma-tagger-create": {- "description": "Create an AI observability tagger that automatically adds custom tags to $ai_generation events. For tagger_type \"llm\", provide tagger_config.prompt plus tagger_config.tags (each with name and optional description); optionally set min_tags and max_tags. For tagger_type \"hog\", provide tagger_config.source Hog code that returns a tag name string, a list of tag name strings, or null; optionally provide tagger_config.tags as a whitelist. Use enabled=false while drafting, then update the tagger from the UI.",- "category": "AI observability",- "feature": "llm_analytics",- "summary": "Create tagger",- "title": "Create tagger",- "required_scopes": ["tagger:write"],- "annotations": {- "destructiveHint": false,- "idempotentHint": false,- "openWorldHint": true,- "readOnlyHint": false- },- "feature_flag": "llm-analytics-tags"- },"llma-tagger-list": {- "description": "List AI observability taggers for the current project. Supports filtering by enabled status, id__in, search, and ordering. Returns each tagger's name, description, enabled state, type, configuration, conditions, model configuration, creator, and timestamps.",+ "description": "List AI observability taggers for the current project. Supports filtering by enabled status, id__in, search, and ordering. Returns each tagger's name, description, enabled state, type, configuration, conditions, model configuration, creator, and timestamps. Taggers are deprecated and new ones can't be created. To tag or classify generations, create a categorical evaluation with llma-evaluation-create instead.","category": "AI observability","feature": "llm_analytics","summary": "List taggers",diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.jsonindex de836de2ca12..b0ffda933f63 100644--- a/services/mcp/schema/tool-definitions-all.json+++ b/services/mcp/schema/tool-definitions-all.json@@ -8204,7 +8204,7 @@}},"llma-evaluation-create": {- "description": "Create an evaluation. 'llm_judge': set evaluation_config.prompt and model_configuration with provider and model; provider_key_id may be null when no key is pinned. 'hog': evaluation_config.source (Hog returning a boolean, a finite number, or category keys). For 'hog' and 'llm_judge', choose output_type='boolean', 'numeric', or 'categorical'; numeric output_config accepts min, max, step, allows_na, and passing_rule; categorical output_config accepts options, selection_mode, allows_na, and passing_rule. 'sentiment': output_type='sentiment', omit model_configuration, optional evaluation_config.source='user_messages'. `target` accepts 'generation' (default), 'trace', or 'session'. A session target runs once per `$ai_session_id` and defaults to the 'inactivity' settle strategy. Pass `enabled` as a boolean (true to start scoring new $ai_generation events immediately, false to create it paused). To organize the evaluation, pass a `directory_id` returned by llma-evaluation-directory-list. Omit `directory_id` or pass null to keep it at the top level. Results are '$ai_evaluation' events. An evaluation only scores data that arrives after it is enabled, so once it is enabled, offer the user a backfill over past data with llma-evaluation-backfill-estimate.",+ "description": "Create an evaluation. 'llm_judge': set evaluation_config.prompt and model_configuration with provider and model; provider_key_id may be null when no key is pinned. 'hog': evaluation_config.source (Hog returning a boolean, a finite number, or category keys). For 'hog' and 'llm_judge', choose output_type='boolean', 'numeric', or 'categorical'; numeric output_config accepts min, max, step, allows_na, and passing_rule; categorical output_config accepts options, selection_mode, allows_na, and passing_rule. 'sentiment': output_type='sentiment', omit model_configuration, optional evaluation_config.source='user_messages'. To tag or classify generations into labels (what taggers used to do), use output_type='categorical' with selection_mode='multiple'. `target` accepts 'generation' (default), 'trace', or 'session'. A session target runs once per `$ai_session_id` and defaults to the 'inactivity' settle strategy. Pass `enabled` as a boolean (true to start scoring new $ai_generation events immediately, false to create it paused). To organize the evaluation, pass a `directory_id` returned by llma-evaluation-directory-list. Omit `directory_id` or pass null to keep it at the top level. Results are '$ai_evaluation' events. An evaluation only scores data that arrives after it is enabled, so once it is enabled, offer the user a backfill over past data with llma-evaluation-backfill-estimate.","category": "AI observability","feature": "llm_analytics","summary": "Create evaluation",@@ -9255,23 +9255,8 @@},"requires_ai_consent": true},- "llma-tagger-create": {- "description": "Create an AI observability tagger that automatically adds custom tags to $ai_generation events. For tagger_type \"llm\", provide tagger_config.prompt plus tagger_config.tags (each with name and optional description); optionally set min_tags and max_tags. For tagger_type \"hog\", provide tagger_config.source Hog code that returns a tag name string, a list of tag name strings, or null; optionally provide tagger_config.tags as a whitelist. Use enabled=false while drafting, then update the tagger from the UI.",- "category": "AI observability",- "feature": "llm_analytics",- "summary": "Create tagger",- "title": "Create tagger",- "required_scopes": ["tagger:write"],- "annotations": {- "destructiveHint": false,- "idempotentHint": false,- "openWorldHint": true,- "readOnlyHint": false- },- "feature_flag": "llm-analytics-tags"- },"llma-tagger-list": {- "description": "List AI observability taggers for the current project. Supports filtering by enabled status, id__in, search, and ordering. Returns each tagger's name, description, enabled state, type, configuration, conditions, model configuration, creator, and timestamps.",+ "description": "List AI observability taggers for the current project. Supports filtering by enabled status, id__in, search, and ordering. Returns each tagger's name, description, enabled state, type, configuration, conditions, model configuration, creator, and timestamps. Taggers are deprecated and new ones can't be created. To tag or classify generations, create a categorical evaluation with llma-evaluation-create instead.","category": "AI observability","feature": "llm_analytics","summary": "List taggers",diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 59c4647a25e4..a887a162cc06 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -110723,19 +110723,6 @@ export namespace Schemas {force_refresh?: boolean;}- export interface TaggerCreate {- /** @maxLength 400 */- name: string;- description?: string;- enabled?: boolean;- tagger_type?: TaggerTypeEnum;- /** Tagger configuration. For tagger_type 'llm': {prompt, tags, min_tags?, max_tags?}. For tagger_type 'hog': {source, tags?}. */- tagger_config: TaggerConfig;- /** Conditions that scope when the tagger runs */- conditions?: TaggerCondition[];- model_configuration?: TaggerModelConfigurationWrite | null;- }-export interface TaggerUpdate {/** @maxLength 400 */name: string;diff --git a/services/mcp/src/generated/ai_observability/api.ts b/services/mcp/src/generated/ai_observability/api.tsindex 0687c98c1f37..edd7a9e48735 100644--- a/services/mcp/src/generated/ai_observability/api.ts+++ b/services/mcp/src/generated/ai_observability/api.ts@@ -3,7 +3,7 @@* MCP service uses these Zod schemas for generated tool handlers.* To regenerate: hogli build:openapi*- * PostHog API - MCP 98 enabled ops+ * PostHog API - MCP 97 enabled ops* OpenAPI spec version: 1.0.0*/import * as zod from 'zod'@@ -3733,153 +3733,6 @@ export const TaggersListQueryParams = () => zod.object({search: zod.string().optional().describe('Search in name or description'),})-export const TaggersCreateParams = () => zod.object({- project_id: zod- .string()- .describe(- "Project ID of the project you're trying to access. To find the ID of the project, make a call to \/api\/projects\/."- ),-})--export const taggersCreateBodyNameMax = 400--export const taggersCreateBodyTaggerTypeDefault = `llm`-export const taggersCreateBodyTaggerConfigOneOneTagsItemNameMax = 100--export const taggersCreateBodyTaggerConfigOneOneTagsItemDescriptionDefault = ``-export const taggersCreateBodyTaggerConfigOneOneTagsItemDescriptionMax = 500--export const taggersCreateBodyTaggerConfigOneOneMinTagsDefault = 0-export const taggersCreateBodyTaggerConfigOneOneMinTagsMin = 0--export const taggersCreateBodyTaggerConfigOneTwoTagsItemNameMax = 100--export const taggersCreateBodyTaggerConfigOneTwoTagsItemDescriptionDefault = ``-export const taggersCreateBodyTaggerConfigOneTwoTagsItemDescriptionMax = 500--export const taggersCreateBodyConditionsItemIdMax = 100--export const taggersCreateBodyConditionsItemRolloutPercentageDefault = 100-export const taggersCreateBodyConditionsItemRolloutPercentageMin = 0-export const taggersCreateBodyConditionsItemRolloutPercentageMax = 100--export const taggersCreateBodyModelConfigurationOneModelMax = 100--export const TaggersCreateBody = () => zod.object({- name: zod.string().max(taggersCreateBodyNameMax),- description: zod.string().optional(),- enabled: zod.boolean().optional(),- tagger_type: zod- .enum(['llm', 'hog'])- .describe('\* `llm` - LLM\n\* `hog` - Hog')- .default(taggersCreateBodyTaggerTypeDefault),- tagger_config: zod- .union([- zod.object({- prompt: zod.string().min(1).describe('Prompt instructing the LLM how to tag generations'),- tags: zod- .array(- zod.object({- name: zod- .string()- .max(taggersCreateBodyTaggerConfigOneOneTagsItemNameMax)- .describe('Tag identifier'),- description: zod- .string()- .max(taggersCreateBodyTaggerConfigOneOneTagsItemDescriptionMax)- .default(taggersCreateBodyTaggerConfigOneOneTagsItemDescriptionDefault)- .describe('Description to help the LLM classify'),- })- )- .describe('Available tags the LLM can assign'),- min_tags: zod- .number()- .min(taggersCreateBodyTaggerConfigOneOneMinTagsMin)- .default(taggersCreateBodyTaggerConfigOneOneMinTagsDefault)- .describe('Minimum number of tags to apply'),- max_tags: zod.number().min(1).nullish().describe('Maximum number of tags to apply (null = no limit)'),- }),- zod.object({- source: zod.string().min(1).describe('Hog source code to classify a generation into tags.'),- tags: zod- .array(- zod.object({- name: zod- .string()- .max(taggersCreateBodyTaggerConfigOneTwoTagsItemNameMax)- .describe('Tag identifier'),- description: zod- .string()- .max(taggersCreateBodyTaggerConfigOneTwoTagsItemDescriptionMax)- .default(taggersCreateBodyTaggerConfigOneTwoTagsItemDescriptionDefault)- .describe('Description to help the LLM classify'),- })- )- .optional()- .describe('Optional tag whitelist. Leave empty to allow any tag returned by the Hog code.'),- }),- ])- .describe(- "Tagger configuration. For tagger_type 'llm': {prompt, tags, min_tags?, max_tags?}. For tagger_type 'hog': {source, tags?}."- ),- conditions: zod- .array(- zod.object({- id: zod- .string()- .max(taggersCreateBodyConditionsItemIdMax)- .describe('Stable identifier for this condition'),- rollout_percentage: zod- .number()- .min(taggersCreateBodyConditionsItemRolloutPercentageMin)- .max(taggersCreateBodyConditionsItemRolloutPercentageMax)- .default(taggersCreateBodyConditionsItemRolloutPercentageDefault)- .describe('Percentage of matching events to apply this condition to'),- properties: zod- .array(zod.record(zod.string(), zod.unknown()))- .optional()- .describe('Property filters that scope when this condition fires'),- })- )- .optional()- .describe('Conditions that scope when the tagger runs'),- model_configuration: zod- .union([- zod.object({- provider: zod- .enum([- 'openai',- 'anthropic',- 'gemini',- 'openrouter',- 'fireworks',- 'azure_openai',- 'together_ai',- 'minimax',- 'openai_compatible',- ])- .describe(- '\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `openai_compatible` - OpenAI-compatible'- )- .describe(- 'LLM provider to use for this tagger.\n\n\* `openai` - Openai\n\* `anthropic` - Anthropic\n\* `gemini` - Gemini\n\* `openrouter` - Openrouter\n\* `fireworks` - Fireworks\n\* `azure_openai` - Azure OpenAI\n\* `together_ai` - Together AI\n\* `minimax` - MiniMax\n\* `openai_compatible` - OpenAI-compatible'- ),- model: zod- .string()- .max(taggersCreateBodyModelConfigurationOneModelMax)- .describe('Provider model identifier to use for this tagger.'),- provider_key_id: zod- .string()- .nullish()- .describe(- 'Existing LLM provider key UUID for the current project. Do not invent this value; use a real provider key ID returned by PostHog, or omit\/null when no provider key should be pinned.'- ),- }),- zod.null(),- ])- .optional(),-})-/*** Test Hog tagger code against sample events without saving.*/diff --git a/services/mcp/src/tools/exec.ts b/services/mcp/src/tools/exec.tsindex ad998d3ae861..0118a76b4b78 100644--- a/services/mcp/src/tools/exec.ts+++ b/services/mcp/src/tools/exec.ts@@ -567,6 +567,8 @@ const DEPRECATED_TOOL_REDIRECTS: Record<string, (allTools: Tool<ZodObjectAny>[])? `${base} For governed business metrics, search \`system.information_schema.metrics\` instead of \`system.insights\`.`: base},+ 'llma-tagger-create': () =>+ 'Tool "llma-tagger-create" was removed because taggers are deprecated. To tag or classify generations, use "llma-evaluation-create" with output_type="categorical", output_config.selection_mode="multiple", and one option per tag in output_config.options. Each option is { key, label }: the key is lowercase snake_case or kebab-case (for example "billing-issue"), and the label keeps the tag name. For an LLM tagger, set evaluation_type="llm_judge" with model_configuration (provider and model), and put the tagger prompt plus any tag descriptions in evaluation_config.prompt. For a Hog tagger, set evaluation_type="hog" with Hog source in evaluation_config.source that returns option keys. Run "info llma-evaluation-create" for the full schema.','event-definitions-list': () =>'Tool "event-definitions-list" was removed. Use "read-data-schema" with input { "query": { "kind": "events" } } to list event definitions.','properties-list': () =>diff --git a/services/mcp/src/tools/generated/ai_observability.ts b/services/mcp/src/tools/generated/ai_observability.tsindex 51396f1d96a6..5ec3ee898295 100644--- a/services/mcp/src/tools/generated/ai_observability.ts+++ b/services/mcp/src/tools/generated/ai_observability.ts@@ -2921,47 +2921,6 @@ const llmaSummarizationCreate = (): ToolBase<},})-const LlmaTaggerCreateSchema = () => {- const TaggersCreateBody = orvalSchemas.TaggersCreateBody()- return TaggersCreateBody-}--const llmaTaggerCreate = (): ToolBase<ReturnType<typeof LlmaTaggerCreateSchema>, WithPostHogUrl<Schemas.Tagger>> => ({- name: 'llma-tagger-create',- schema: LlmaTaggerCreateSchema(),- handler: async (context: Context, params: z.infer<ReturnType<typeof LlmaTaggerCreateSchema>>) => {- const projectId = await context.stateManager.getProjectId()- const body: Record<string, unknown> = {}- if (params.name !== undefined) {- body['name'] = params.name- }- if (params.description !== undefined) {- body['description'] = params.description- }- if (params.enabled !== undefined) {- body['enabled'] = params.enabled- }- if (params.tagger_type !== undefined) {- body['tagger_type'] = params.tagger_type- }- if (params.tagger_config !== undefined) {- body['tagger_config'] = params.tagger_config- }- if (params.conditions !== undefined) {- body['conditions'] = params.conditions- }- if (params.model_configuration !== undefined) {- body['model_configuration'] = params.model_configuration- }- const result = await context.api.request<Schemas.Tagger>({- method: 'POST',- path: `/api/projects/${encodeURIComponent(String(projectId))}/taggers/`,- body,- })- return await withPostHogUrl(context, result, `/ai-evals/taggers/${result.id}`)- },-})-const LlmaTaggerListSchema = () => {const TaggersListQueryParams = orvalSchemas.TaggersListQueryParams()return TaggersListQueryParams@@ -3598,7 +3557,6 @@ export const GENERATED_TOOLS: Record<string, () => ToolBase<ZodObjectAny>> = {'llma-score-definition-version-get': llmaScoreDefinitionVersionGet,'llma-score-definition-version-list': llmaScoreDefinitionVersionList,'llma-summarization-create': llmaSummarizationCreate,- 'llma-tagger-create': llmaTaggerCreate,'llma-tagger-list': llmaTaggerList,'llma-tagger-test-hog': llmaTaggerTestHog,'llma-trace-review-create': llmaTraceReviewCreate,diff --git a/services/mcp/tests/unit/exec.test.ts b/services/mcp/tests/unit/exec.test.tsindex 72699d04f889..90f6046dd6d8 100644--- a/services/mcp/tests/unit/exec.test.ts+++ b/services/mcp/tests/unit/exec.test.ts@@ -1969,6 +1969,7 @@ describe('exec tool', () => {['query-run', 'execute-sql'],['self-driving-inbox-get', 'inbox-reports-list'],['experiment-get-all', 'experiment-list'],+ ['llma-tagger-create', 'llma-evaluation-create'],])('throws redirect when calling deprecated %s', async (deprecated, replacement) => {const exec = createExec()await expect(exec.handler(mockContext, { command: `call ${deprecated} {}` })).rejects.toThrow(