:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(aio): remove the system one evaluations feature flag (#124222)
Bernat Torres4bd5145d06e4
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.mdindex 680193dd40eb..1fb1a2fb9ad7 100644--- a/docs/internal/ai-observability-judge-inputs.md+++ b/docs/internal/ai-observability-judge-inputs.md@@ -72,11 +72,8 @@ Changing the judge method marks the evaluation as edited, including when the selThe selection is stored as `evaluation_config.judge_method` (`llm` or `decision`). Evaluations without this field keep automatic routing until a method or model is explicitly selected.Automatic OpenRouter routing selects decisions only for models without text output. Models supporting both methods keep chat routing unless decisions are explicitly selected.Prompt-only `evaluation_config` PATCH requests preserve the saved method; an explicit `judge_method: null` restores automatic routing.-The detail editor PATCHes only changed fields on existing evaluations, so renaming or disabling a saved decision evaluation still works after its feature flag is turned off. Re-enabling it or changing its model or output configuration still validates decision access.+The detail editor PATCHes only changed fields on existing evaluations. Re-enabling an evaluation or changing its model or output configuration still validates decision access.Explicit method selection requires a selected model, including for legacy evaluations that use the project's default connection.-The `llm-analytics-system-one-evaluations` project-group feature flag controls access in the browser and background workers.-Both use the project's UUID as its group key; the numeric project ID is a group property.-Deploy the ingestion and evaluation worker changes before enabling the flag.Projects use their saved provider keys or configure a System One-compatible deployment and its authentication.Evaluation connections never fall back to an instance credential or gateway configuration.The configured endpoint authorizes the supplied credential, including connections to PostHog's regional AI gateway endpoints.diff --git a/frontend/snapshots.yml b/frontend/snapshots.ymlindex a38c92f7f341..2d2585c07dcb 100644--- a/frontend/snapshots.yml+++ b/frontend/snapshots.yml@@ -9364,10 +9364,6 @@ snapshots:hash: v1.k794b7964.81ce7999f24877f6e4ac2d9ab89fb4e584e5bc0749e594742c15377c8da8a1a7.BlSbtKwSVk0n9eZKEpRMG7hADbYEDAXAedKVuCaBN_8scenes-app-ai-observability-system-one-connection--provider-settings--light:hash: v1.k794b7964.05b7de1c84c25e03badf722fa567182ff449e74a6299d6a543c554ecd20e3184.eccRHOq726ohqws6q_XlZPELbvyUZPfAZk1cwlog_QY- scenes-app-ai-observability-system-one-connection--provider-settings-flag-off--dark:- hash: v1.k794b7964.81ce7999f24877f6e4ac2d9ab89fb4e584e5bc0749e594742c15377c8da8a1a7.DaP59ecAHMR8eaDYuWmJyKEowz34lJ3qbuKNJ4DsN8U- scenes-app-ai-observability-system-one-connection--provider-settings-flag-off--light:- hash: v1.k794b7964.05b7de1c84c25e03badf722fa567182ff449e74a6299d6a543c554ecd20e3184.3SudNZ7cfTDqUdodzpr3A8w1rZBmSjpUHNJ-qyTQQC4scenes-app-ai-observability-trace--full--dark:hash: v1.k794b7964.4d3ad4ebbe259bff3cd17afce36a4f9e9e69b5def84e53c89a543b970751e695.a2alK28PTn9_H4Th1Gy1knhB3uSsWmZ77ZmrV0jqQsoscenes-app-ai-observability-trace--full--light:@@ -13236,10 +13232,6 @@ snapshots:hash: v1.k794b7964.58dc8d73bf85d6c8bcd7a10b08c1e38efefd840fbe912cbbdff34bc4596041d8.5_RSGKERITVltMJ5ap2pmDAF2IC77kXkHPRfyXJi2oYscenes-app-llm-observability-evaluationmodelpicker--decision-narrow--light:hash: v1.k794b7964.28a30a7cb478ba7b585a0736e53e74735c643e3b61b0b84059494f6040493c29.haO943nTQB3hFa_f-TgDhACaCqM3_bOKVEdmp0SXTzg- scenes-app-llm-observability-evaluationmodelpicker--flag-off--dark:- hash: v1.k794b7964.277344d00b8edc7551d8850029611c83e1740ac1ea3adb9e679ceffd2037064b.0gLUQZyT3_xppzOwry0tkGL0HlIjU-jdxq6V8VIAGs4- scenes-app-llm-observability-evaluationmodelpicker--flag-off--light:- hash: v1.k794b7964.060fc008f850bbab373cb0c43a70e91454426b70a2d259ab371e1c2cfa7d1834.YTIqtwuIVdRppnvMFv142GGjOIDBIehxdylH9qJJQLYscenes-app-llm-observability-evaluationmodelpicker--llm--dark:hash: v1.k794b7964.43b905fdb235f4c15e7b906f14467c4ee9e8df010a3f26e476a2b3e6f59a4323.MZUQFFzWautdc3zzDJVWS6YbBa4qxX69-2HNwn-4wXQscenes-app-llm-observability-evaluationmodelpicker--llm--light:diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsxindex 12d5639b9e1a..4977094a2398 100644--- a/frontend/src/lib/constants.tsx+++ b/frontend/src/lib/constants.tsx@@ -372,7 +372,6 @@ export const FEATURE_FLAGS = {LLM_ANALYTICS_COMMUNITY_SKILLS: 'llm-analytics-community-skills', // owner: #team-ai-observabilityLLM_ANALYTICS_DATASETS: 'llm-analytics-datasets', // owner: #team-ai-observability #team-posthog-aiLLM_ANALYTICS_EVAL_SETTLING_STRATEGY: 'llm-analytics-eval-settling-strategy', // owner: #team-ai-observability- LLM_ANALYTICS_SYSTEM_ONE_EVALUATIONS: 'llm-analytics-system-one-evaluations', // owner: #team-ai-observabilityLLM_ANALYTICS_TAGS: 'llm-analytics-tags', // owner: #team-ai-observabilityLLM_ANALYTICS_TRACE_NAVIGATION: 'llm-analytics-trace-navigation', // owner: #team-ai-observabilityLLM_OBSERVABILITY_TRACE_SEARCH: 'llm-observability-trace-search', // owner: #team-ai-observabilitydiff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.pyindex 742f4e4b2193..0825d945d7b0 100644--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py@@ -57,7 +57,6 @@ from products.ai_observability.backend.llm import DEFAULT_MODEL_BY_PROVIDER, Clifrom products.ai_observability.backend.llm.decisions import (DecisionClient,DecisionEndpointBlockedError,- decision_evaluations_enabled,is_decision_model,)from products.ai_observability.backend.llm.errors import (@@ -81,7 +80,7 @@ from products.ai_observability.backend.llm.errors import ()from products.ai_observability.backend.llm.providers.openai import OPENAI_DECISIONS_BASE_URLfrom products.ai_observability.backend.llm.providers.openai_decisions import OpenAIDecisionAdapter-from products.ai_observability.backend.llm.providers.openrouter import OPENROUTER_DECISIONS_BASE_URL, decision_model_ids+from products.ai_observability.backend.llm.providers.openrouter import OPENROUTER_DECISIONS_BASE_URLfrom products.ai_observability.backend.llm.types import CompletionResponsefrom products.ai_observability.backend.models.evaluation_configs import (BooleanOutputConfig,@@ -655,19 +654,14 @@ def call_llm_judge(probability: float | None = Nonedecision_result = Nonetry:- openrouter_enabled = provider == "openrouter" and decision_evaluations_enabled(- team_id, base_url=OPENROUTER_DECISIONS_BASE_URL- )judge_method = evaluation["evaluation_config"].get("judge_method")# Explicit selection takes precedence for models that support both methods.# Models gaining decisions must not move existing chat evaluations onto a different API.uses_decisions = (judge_method == "decision"if judge_method is not None- else is_decision_model(provider, model, openrouter_enabled=openrouter_enabled, decision_only=True)+ else is_decision_model(provider, model, decision_only=True))- if judge_method is None and provider == "openrouter" and not openrouter_enabled:- uses_decisions = model in (decision_model_ids(refresh=False, decision_only=True) or ())if uses_decisions:if output_type not in ("boolean", "categorical", "numeric"):return build_skipped_evaluation_result(@@ -685,15 +679,6 @@ def call_llm_judge(if provider_keyelse "")- if (provider == "openrouter" and not openrouter_enabled) or (- provider in ("system_one", "openai") and not decision_evaluations_enabled(team_id, base_url=base_url)- ):- return build_skipped_evaluation_result(- output_type=output_type,- allows_na=allows_na,- reasoning="Decision model evaluations are not available for this project.",- skip_reason="system_one_unavailable",- )prompt = evaluation["evaluation_config"]["prompt"]categorical_config = (CategoricalOutputConfig.model_validate(output_config) if output_type == "categorical" else None@@ -960,18 +945,6 @@ def call_llm_judge(non_retryable=True,)except UnsupportedModelError as e:- # A failed chat call can populate a cold catalogue; a disabled flag must not disable the evaluation.- if (- provider == "openrouter"- and not openrouter_enabled- and model in (decision_model_ids(refresh=False, decision_only=True) or ())- ):- return build_skipped_evaluation_result(- output_type=output_type,- allows_na=allows_na,- reasoning="Decision model evaluations are not available for this project.",- skip_reason="system_one_unavailable",- )increment_user_errors("model_not_supported", provider=provider)return terminal_user_error_result(spec=require_user_error_spec("model_not_supported", is_byok=is_byok),diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.pyindex c76e819fdca8..a1210eb83260 100644--- a/posthog/temporal/ai_observability/test_run_evaluation.py+++ b/posthog/temporal/ai_observability/test_run_evaluation.py@@ -9,7 +9,6 @@ from typing import Any, Literal, castimport pytestfrom unittest.mock import AsyncMock, MagicMock, patch-from django.core.cache import cachefrom django.test import override_settingsimport httpx@@ -48,11 +47,6 @@ from products.ai_observability.backend.llm.errors import (StructuredOutputParseError,UnsupportedModelError,)-from products.ai_observability.backend.llm.providers.openai import OpenAIAdapter-from products.ai_observability.backend.llm.providers.openrouter import (- MODEL_MODALITIES_CACHE_KEY,- MODEL_MODALITIES_LAST_GOOD_CACHE_KEY,-)from products.ai_observability.backend.models.evaluation_config import EvaluationConfigfrom products.ai_observability.backend.models.evaluation_directories import EvaluationDirectoryfrom products.ai_observability.backend.models.evaluations import Evaluation@@ -312,18 +306,14 @@ def _mock_config_with_active_key(provider: str = "openai") -> MagicMock:return MagicMock(active_provider_key=key)[email protected]("flag", [True, False, None])@pytest.mark.parametrize("judge_method", [None, "llm"])-def test_openrouter_catalogue_outage_only_affects_projects_with_decisions_enabled(- flag: bool | None, judge_method: str | None-) -> None:+def test_openrouter_catalogue_outage_only_affects_automatic_routing(judge_method: str | None) -> None:key = MagicMock(provider="openrouter", encrypted_config={"api_key": "example-token"})with (patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,patch("products.ai_observability.backend.llm.providers.openrouter._model_output_modalities", return_value=None) as catalogue,- patch("posthog.temporal.ai_observability.evaluation_llm_judge.decision_evaluations_enabled", return_value=flag),patch("posthog.temporal.ai_observability.evaluation_llm_judge.Client.complete") as complete,patch("posthog.temporal.ai_observability.evaluation_llm_judge.DecisionClient.evaluate") as decide,):@@ -333,7 +323,7 @@ def test_openrouter_catalogue_outage_only_affects_projects_with_decisions_enablecomplete.return_value = MagicMock(parsed=BooleanEvalResult(verdict=True, reasoning="Polite"), usage=None)with (pytest.raises(TransientJudgeError, match="OpenRouter model capabilities")- if flag and judge_method is None+ if judge_method is Noneelse nullcontext()):result = call_llm_judge(@@ -348,22 +338,18 @@ def test_openrouter_catalogue_outage_only_affects_projects_with_decisions_enableuser_prompt="Hello!",allows_na=False,)- if flag and judge_method is None:+ if judge_method is None:complete.assert_not_called()else:assert result["verdict"] is True- if judge_method is None:- catalogue.assert_called_once_with(refresh=False)- else:- catalogue.assert_not_called()+ catalogue.assert_not_called()decide.assert_not_called()@pytest.mark.parametrize("provider,model", [("openrouter", "example/dual-model"), ("openai", "gpt-6-luna")])[email protected]("flag", [True, False, None])@pytest.mark.parametrize("judge_method", [None, "llm"])def test_legacy_and_explicit_llm_methods_use_chat_for_a_dual_capability_model(- provider: str, model: str, flag: bool | None, judge_method: str | None+ provider: str, model: str, judge_method: str | None) -> None:key = MagicMock(provider=provider, encrypted_config={"api_key": "example-token"})with (@@ -372,7 +358,6 @@ def test_legacy_and_explicit_llm_methods_use_chat_for_a_dual_capability_model("products.ai_observability.backend.llm.providers.openrouter._model_output_modalities",return_value={"example/dual-model": ["text", "decisions"]},),- patch("posthog.temporal.ai_observability.evaluation_llm_judge.decision_evaluations_enabled", return_value=flag),patch("posthog.temporal.ai_observability.evaluation_llm_judge.Client.complete") as complete,patch("posthog.temporal.ai_observability.evaluation_llm_judge.DecisionClient.evaluate") as decide,patch("posthog.temporal.ai_observability.evaluation_llm_judge.OpenAIDecisionAdapter.evaluate") as native_decide,@@ -475,7 +460,6 @@ def test_legacy_and_explicit_llm_methods_use_chat_for_a_dual_capability_model(def test_openai_native_decisions_use_shared_result_semantics(applicable: bool, output_type: str, output_config: dict, answers: list[dict], expected: dict) -> None:- team = Team(id=1, organization_id=uuid.uuid4(), uuid=uuid.uuid4())evaluation = {"id": "example-evaluation","name": "Response quality",@@ -500,11 +484,8 @@ def test_openai_native_decisions_use_shared_result_semantics(with (patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,- patch("products.ai_observability.backend.llm.decisions.Team.objects.only") as teams,- patch("products.ai_observability.backend.llm.decisions.get_feature_flag_or_none", return_value=True),patch("httpx.AsyncHTTPTransport.handle_async_request", return_value=response) as transport,):- teams.return_value.get.return_value = teamspec.return_value.resolve.return_value = MagicMock(provider="openai",model="gpt-6-luna",@@ -538,72 +519,6 @@ def test_openai_native_decisions_use_shared_result_semantics(assert "probability" not in result[email protected]("flag", [False, None])-def test_openai_decisions_disabled_flag_skips_without_invalidating_key(flag: bool | None) -> None:- with (- patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,- patch("posthog.temporal.ai_observability.evaluation_llm_judge.decision_evaluations_enabled", return_value=flag),- patch("httpx.AsyncHTTPTransport.handle_async_request") as transport,- ):- spec.return_value.resolve.return_value = MagicMock(- provider="openai",- model="gpt-6-luna",- is_byok=True,- provider_key=MagicMock(encrypted_config={"api_key": "example-token"}),- )- result = call_llm_judge(- evaluation={"team_id": 1, "evaluation_config": {"prompt": "Check the reply.", "judge_method": "decision"}},- system_prompt="",- user_prompt="Example reply.",- allows_na=False,- )- assert result["skip_reason"] == "system_one_unavailable"- assert "terminal_user_error" not in result- assert "provider_key_state" not in result- transport.assert_not_called()--[email protected]("flag", [False, None])[email protected]("cached", [True, False])-def test_openrouter_decision_model_with_disabled_flag_skips_without_disabling(flag: bool | None, cached: bool) -> None:- cache.delete_many([MODEL_MODALITIES_CACHE_KEY, MODEL_MODALITIES_LAST_GOOD_CACHE_KEY])- model = "example/decision"- if cached:- cache.set(MODEL_MODALITIES_LAST_GOOD_CACHE_KEY, {model: ["decisions"]}, timeout=None)- key = MagicMock(provider="openrouter", encrypted_config={"api_key": "example-token"})- try:- with (- patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,- patch(- "posthog.temporal.ai_observability.evaluation_llm_judge.decision_evaluations_enabled", return_value=flag- ),- patch.object(OpenAIAdapter, "complete", side_effect=ModelNotFoundError(model)) as complete,- patch("posthog.temporal.ai_observability.evaluation_llm_judge.DecisionClient.evaluate") as decide,- patch("products.ai_observability.backend.llm.providers.openrouter.httpx.get") as catalogue,- ):- spec.return_value.resolve.return_value = MagicMock(- provider="openrouter", model=model, provider_key=key, is_byok=True- )- catalogue.return_value.json.return_value = {- "data": [{"id": model, "architecture": {"output_modalities": ["decisions"]}}]- }- result = call_llm_judge(- evaluation={"team_id": 1, "evaluation_config": {"prompt": "Is the response polite?"}},- system_prompt="",- user_prompt="Hello!",- allows_na=False,- )- assert result["skipped"] is True- assert result["skip_reason"] == "system_one_unavailable"- assert "terminal_user_error" not in result- assert "provider_key_state" not in result- assert complete.call_count == (0 if cached else 1)- assert catalogue.call_count == (0 if cached else 1)- decide.assert_not_called()- finally:- cache.delete_many([MODEL_MODALITIES_CACHE_KEY, MODEL_MODALITIES_LAST_GOOD_CACHE_KEY])--@pytest.mark.parametrize("provider,connection_config,base_url,model,usage",[@@ -698,11 +613,8 @@ def test_system_one_judge_emits_boolean_probability_without_reasoning(),patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,- patch("products.ai_observability.backend.llm.decisions.Team.objects.only") as teams,- patch("products.ai_observability.backend.llm.decisions.get_feature_flag_or_none", return_value=True),patch("httpx.AsyncHTTPTransport.handle_async_request", return_value=response) as request,):- teams.return_value.get.return_value = Team(id=1, organization_id=uuid.uuid4(), uuid=uuid.uuid4())spec.return_value.resolve.return_value = resolvedresult = call_llm_judge(evaluation=evaluation,@@ -795,7 +707,6 @@ def test_system_one_categorical_results_use_category_keys_without_boolean_probab),patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,- patch("posthog.temporal.ai_observability.evaluation_llm_judge.decision_evaluations_enabled", return_value=True),patch("httpx.AsyncHTTPTransport.handle_async_request") as request,):spec.return_value.resolve.return_value = MagicMock(@@ -878,7 +789,6 @@ def test_decision_numeric_scores_use_configured_bounds(),patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,- patch("posthog.temporal.ai_observability.evaluation_llm_judge.decision_evaluations_enabled", return_value=True),patch("httpx.AsyncHTTPTransport.handle_async_request",return_value=httpx.Response(@@ -924,7 +834,6 @@ def test_system_one_numeric_requires_a_score_range(output_config: dict[str, floawith (patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,- patch("posthog.temporal.ai_observability.evaluation_llm_judge.decision_evaluations_enabled", return_value=True),patch("httpx.AsyncHTTPTransport.handle_async_request") as request,):spec.return_value.resolve.return_value = MagicMock(@@ -946,42 +855,6 @@ def test_system_one_numeric_requires_a_score_range(output_config: dict[str, floarequest.assert_not_called()- "provider,base_url,flag",- [- ("system_one", "https://decisions.example.com/v1", False),- ("system_one", "https://ai-gateway.us.posthog.com/v1", False),- ],-)-def test_system_one_restricted_connection_does_not_send_evaluation_data(- provider: str, base_url: str, flag: bool-) -> None:- with (- override_settings(POSTHOG_INTERNAL_ORG_IDS=[]),- patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,- patch("products.ai_observability.backend.llm.decisions.Team.objects.only") as teams,- patch("products.ai_observability.backend.llm.decisions.get_feature_flag_or_none", return_value=flag),- patch("httpx.AsyncHTTPTransport.handle_async_request") as request,- ):- teams.return_value.get.return_value = Team(id=1, organization_id=uuid.uuid4(), uuid=uuid.uuid4())- spec.return_value.resolve.return_value = MagicMock(- provider=provider,- model="custom-model",- provider_key=MagicMock(- provider=provider, encrypted_config={"base_url": base_url, "api_key": "example-token"}- ),- is_byok=True,- )- result = call_llm_judge(- evaluation={"team_id": 1, "evaluation_config": {"prompt": "Is this a greeting?"}},- system_prompt="",- user_prompt="Hello!",- allows_na=False,- )- assert result["skipped"] is True- request.assert_not_called()--@pytest.mark.parametrize("provider,status,encoding,expected_skip_reason",[@@ -1015,7 +888,6 @@ def test_provider_rejections_distinguish_blocked_endpoints_from_bad_inputs(),patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,- patch("posthog.temporal.ai_observability.evaluation_llm_judge.decision_evaluations_enabled", return_value=True),patch("httpx.AsyncHTTPTransport.handle_async_request", return_value=response) as request,):spec.return_value.resolve.return_value = MagicMock(@@ -1192,7 +1064,6 @@ def test_custom_provider_rate_limit_retries_without_disabling_the_evaluation(with (patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,- patch("posthog.temporal.ai_observability.evaluation_llm_judge.decision_evaluations_enabled", return_value=True),patch("httpx.AsyncHTTPTransport.handle_async_request",side_effect=[diff --git a/products/ai_observability/backend/api/evaluations.py b/products/ai_observability/backend/api/evaluations.pyindex df4377acccf5..ca1b961c85b1 100644--- a/products/ai_observability/backend/api/evaluations.py+++ b/products/ai_observability/backend/api/evaluations.py@@ -59,10 +59,10 @@ from ..facade import (from ..facade.contracts import Contractfrom ..hog import compile_ai_observability_hogfrom ..llm import DEFAULT_MODEL_BY_PROVIDER-from ..llm.decisions import decision_evaluations_enabled, is_decision_model+from ..llm.decisions import is_decision_modelfrom ..llm.errors import ProviderConnectionError-from ..llm.providers.openai import OPENAI_DECISIONS_BASE_URL, OpenAIConfig-from ..llm.providers.openrouter import OPENROUTER_BASE_URL, is_non_chat_model+from ..llm.providers.openai import OpenAIConfig+from ..llm.providers.openrouter import is_non_chat_modelfrom ..models.evaluation_config import EvaluationConfigfrom ..models.evaluation_configs import (EVALUATION_TEST_LOOKBACK_DAYS,@@ -572,12 +572,9 @@ class EvaluationSerializer(UserAccessControlSerializerMixin, serializers.ModelSemodel_provider,model_configuration.get("model"),decision_only=judge_method is None,- openrouter_enabled=model_provider == "openrouter"- and decision_evaluations_enabled(self.context["get_team"]().id, base_url=OPENROUTER_BASE_URL),openai_enabled=judge_method == "decision"and model_provider == LLMProvider.OPENAI- and model_configuration.get("model") in OpenAIConfig.DECISION_MODELS- and decision_evaluations_enabled(self.context["get_team"]().id, base_url=OPENAI_DECISIONS_BASE_URL),+ and model_configuration.get("model") in OpenAIConfig.DECISION_MODELS,)except ProviderConnectionError as e:raise serializers.ValidationError({"model_configuration": str(e)}) from e@@ -738,13 +735,11 @@ class EvaluationSerializer(UserAccessControlSerializerMixin, serializers.ModelSeif uses_decision_model:returnif model and is_non_chat_model(model):- guidance = (- "Choose a chat model or a supported decision model."- if decision_evaluations_enabled(self.context["get_team"]().id, base_url=OPENROUTER_BASE_URL)- else "Choose a chat model."- )raise serializers.ValidationError(- {"model_configuration": f"'{model}' is not supported as an evaluation judge. {guidance}"}+ {+ "model_configuration": f"'{model}' is not supported as an evaluation judge. "+ "Choose a chat model or a supported decision model."+ })def _validate_can_run(self, data: dict) -> None:diff --git a/products/ai_observability/backend/api/provider_keys.py b/products/ai_observability/backend/api/provider_keys.pyindex da445f700a5d..a0d88dc9599b 100644--- a/products/ai_observability/backend/api/provider_keys.py+++ b/products/ai_observability/backend/api/provider_keys.py@@ -26,7 +26,7 @@ from posthog.security.url_validation import strip_userinfofrom products.access_control.backend.presentation.access_control import AccessControlViewSetMixinfrom ..llm.client import Client-from ..llm.decisions import DecisionClient, decision_evaluations_enabled+from ..llm.decisions import DecisionClientfrom ..llm.providers.anthropic import (DISALLOWED_FOUNDRY_ENDPOINT_MESSAGE,error_field_for_validation_message as anthropic_error_field,@@ -82,10 +82,6 @@ def _reload_model_config_dependents_on_commit(team_id: int, model_config_ids: lidef validate_provider_key(provider: str, api_key: str, *, team_id: int, **kwargs: str) -> tuple[str, str | None]:"""Validate an API key for any supported provider using the unified client."""- if provider == LLMProvider.SYSTEM_ONE and not decision_evaluations_enabled(- team_id, base_url=kwargs.get("base_url", "")- ):- raise exceptions.PermissionDenied("System One evaluations are not available for this project.")try:return Client.validate_key(provider, api_key, **kwargs)except Exception:diff --git a/products/ai_observability/backend/api/proxy.py b/products/ai_observability/backend/api/proxy.pyindex fc1de5ce3dad..fd2d2678fc1f 100644--- a/products/ai_observability/backend/api/proxy.py+++ b/products/ai_observability/backend/api/proxy.py@@ -57,18 +57,13 @@ from products.ai_observability.backend.llm import (OutputSchema,get_playground_models,)-from products.ai_observability.backend.llm.decisions import decision_evaluations_enabledfrom products.ai_observability.backend.llm.errors import (ProviderConfigurationError,ProviderHostUnresolvedError,UnsupportedProviderError,)-from products.ai_observability.backend.llm.providers.openai import OPENAI_DECISIONS_BASE_URL, OpenAIConfig-from products.ai_observability.backend.llm.providers.openrouter import (- OPENROUTER_BASE_URL,- decision_model_ids,- non_chat_model_ids,-)+from products.ai_observability.backend.llm.providers.openai import OpenAIConfig+from products.ai_observability.backend.llm.providers.openrouter import decision_model_ids, non_chat_model_idsfrom products.ai_observability.backend.models.provider_keys import (LLMProvider,LLMProviderKey,@@ -556,15 +551,11 @@ class LLMProxyViewSet(viewsets.ViewSet):provider_display = PROVIDER_DISPLAY_NAMES.get(provider_key.provider, provider_key.provider.title())decision_models: frozenset[str] = frozenset()non_chat_models: frozenset[str] = frozenset()- if provider_key.provider == LLMProvider.OPENROUTER and decision_evaluations_enabled(- provider_key.team_id, base_url=OPENROUTER_BASE_URL- ):+ if provider_key.provider == LLMProvider.OPENROUTER:decision_models = decision_model_ids() or frozenset()non_chat_models = non_chat_model_ids() or frozenset()models = list(dict.fromkeys([*models, *sorted(decision_models)]))- elif provider_key.provider == LLMProvider.OPENAI and decision_evaluations_enabled(- provider_key.team_id, base_url=OPENAI_DECISIONS_BASE_URL- ):+ elif provider_key.provider == LLMProvider.OPENAI:decision_models = OpenAIConfig.DECISION_MODELSmodels = list(dict.fromkeys([*models, *sorted(decision_models)]))return Response(diff --git a/products/ai_observability/backend/api/test/test_evaluations.py b/products/ai_observability/backend/api/test/test_evaluations.pyindex 8f0504a2cb45..5fcc123c9561 100644--- a/products/ai_observability/backend/api/test/test_evaluations.py+++ b/products/ai_observability/backend/api/test/test_evaluations.py@@ -161,7 +161,6 @@ class TestModelConfigurationSerializer(SimpleTestCase):context={"get_team": lambda: Mock(id=1)},)with (- patch("products.ai_observability.backend.api.evaluations.decision_evaluations_enabled", return_value=True),patch("products.ai_observability.backend.llm.providers.openrouter._model_output_modalities",return_value={"example/dual-model": ["text", "decisions"]},@@ -217,10 +216,9 @@ class TestModelConfigurationSerializer(SimpleTestCase):("unsupported_decision", "decision", "openai", ["text"], None),("custom_chat", "llm", "system_one", ["decisions"], None),("decision_only_chat", "llm", "openrouter", ["decisions"], None),- ("openai_decision", "decision", "openai", ["text"], True, "gpt-6-luna", True),- ("openai_legacy", None, "openai", ["text"], False, "gpt-6-luna", True),- ("openai_chat", "llm", "openai", ["text"], False, "gpt-6-luna", True),- ("openai_disabled", "decision", "openai", ["text"], None, "gpt-6-luna", False),+ ("openai_decision", "decision", "openai", ["text"], True, "gpt-6-luna"),+ ("openai_legacy", None, "openai", ["text"], False, "gpt-6-luna"),+ ("openai_chat", "llm", "openai", ["text"], False, "gpt-6-luna"),])def test_judge_method_matches_model_capabilities(@@ -231,7 +229,6 @@ class TestModelConfigurationSerializer(SimpleTestCase):modalities: list[str],expected: bool | None,model: str = "example/model",- flag: bool = True,) -> None:config = {"prompt": "Check the response."}if method:@@ -245,7 +242,6 @@ class TestModelConfigurationSerializer(SimpleTestCase):)serializer = EvaluationSerializer(instance=evaluation, partial=True, context={"get_team": lambda: Mock(id=1)})with (- patch("products.ai_observability.backend.api.evaluations.decision_evaluations_enabled", return_value=flag),patch("products.ai_observability.backend.llm.providers.openrouter._model_output_modalities",return_value={"example/model": modalities},@@ -262,36 +258,6 @@ class TestModelConfigurationSerializer(SimpleTestCase):validated = serializer.validate(data)self.assertEqual(validated["evaluation_config"], config)- @parameterized.expand(- [- ({"name": "Renamed decision evaluation"}, True),- ({"enabled": False}, True),- ({"enabled": True}, False),- ({"output_config": {"allows_na": True}}, False),- ]- )- def test_native_decision_flag_off_allows_rename_and_disable(self, data: dict[str, object], valid: bool) -> None:- evaluation = Evaluation(- evaluation_type="llm_judge",- evaluation_config={"prompt": "Check the response.", "judge_method": "decision"},- enabled=data.get("enabled") is not True,- output_type="boolean",- output_config={},- model_configuration=LLMModelConfiguration(provider="openai", model="gpt-6-luna"),- )- serializer = EvaluationSerializer(- instance=evaluation, data=data, partial=True, context={"get_team": lambda: Mock(id=1)}- )- with patch(- "products.ai_observability.backend.api.evaluations.decision_evaluations_enabled", return_value=False- ):- self.assertEqual(serializer.is_valid(), valid, serializer.errors)- if valid:- for field, value in data.items():- self.assertEqual(serializer.validated_data[field], value)- else:- self.assertIn("model_configuration", serializer.errors)-def test_judge_method_change_validates_the_stored_model(self) -> None:evaluation = Evaluation(evaluation_type="llm_judge",@@ -320,13 +286,12 @@ class TestModelConfigurationSerializer(SimpleTestCase):"selection_mode": mode,"options": [{"key": f"option_{i}", "label": f"Option {i}"} for i in range(count)],}- with patch("products.ai_observability.backend.api.evaluations.decision_evaluations_enabled", return_value=True):- if valid:- assert serializer.validate({"output_config": config})["output_config"]["selection_mode"] == mode- else:- with self.assertRaises(ValidationError) as error:- serializer.validate({"output_config": config})- assert "at least two options" in str(error.exception)+ if valid:+ assert serializer.validate({"output_config": config})["output_config"]["selection_mode"] == mode+ else:+ with self.assertRaises(ValidationError) as error:+ serializer.validate({"output_config": config})+ assert "at least two options" in str(error.exception)@parameterized.expand([("boolean", {}), ("categorical", {}), ("numeric", {"min": 0, "max": 10})])def test_system_one_supports_evaluation_output_types(@@ -378,7 +343,6 @@ class TestModelConfigurationSerializer(SimpleTestCase):"products.ai_observability.backend.llm.providers.openrouter._model_output_modalities",return_value={"typesafe/jev-1.13": ["decisions"]},),- patch("products.ai_observability.backend.api.evaluations.decision_evaluations_enabled", return_value=True),):self.assertEqual(serializer.is_valid(), valid, serializer.errors)if valid:@@ -388,17 +352,15 @@ class TestModelConfigurationSerializer(SimpleTestCase):@parameterized.expand([- (True, {"name": "Updated judge"}, True),- (True, {"enabled": False}, True),- (True, {"output_config": {"max": None}}, False),- (False, {"output_config": {"max": None}}, True),- (True, {"model_configuration": {"provider": "openrouter", "model": "openai/gpt-4o"}}, False),- (False, {"model_configuration": {"provider": "openrouter", "model": "openai/gpt-4o"}}, True),- (True, {"enabled": True}, False),+ ({"name": "Updated judge"}, True),+ ({"enabled": False}, True),+ ({"output_config": {"max": None}}, False),+ ({"model_configuration": {"provider": "openrouter", "model": "openai/gpt-4o"}}, False),+ ({"enabled": True}, False),])def test_openrouter_catalogue_outage_only_blocks_model_validation(- self, flag: bool, data: dict[str, object], valid: bool+ self, data: dict[str, object], valid: bool) -> None:evaluation = Evaluation(evaluation_type="llm_judge",@@ -415,7 +377,6 @@ class TestModelConfigurationSerializer(SimpleTestCase):patch("products.ai_observability.backend.llm.providers.openrouter._model_output_modalities", return_value=None) as catalogue,- patch("products.ai_observability.backend.api.evaluations.decision_evaluations_enabled", return_value=flag),):self.assertEqual(serializer.is_valid(), valid, serializer.errors)if valid:@@ -1146,14 +1107,13 @@ class TestEvaluationConfigsApi(APIBaseTest):@parameterized.expand([- ("decision_model_enabled", "typesafe/jev-1.13", True, 201),- ("decision_model_disabled", "typesafe/jev-1.13", False, 400),- ("decision_model_alias", "~typesafe/jev-latest", True, 201),- ("chat_model", "openai/gpt-4o", False, 201),- ("other_non_chat_model", "example/embedding", True, 400),+ ("decision_model", "typesafe/jev-1.13", 201),+ ("decision_model_alias", "~typesafe/jev-latest", 201),+ ("chat_model", "openai/gpt-4o", 201),+ ("other_non_chat_model", "example/embedding", 400),])- def test_llm_judge_creation_checks_openrouter_model_capabilities(self, _name, model, flag, expected_status):+ def test_llm_judge_creation_checks_openrouter_model_capabilities(self, _name, model, expected_status):with (patch("products.ai_observability.backend.llm.providers.openrouter._model_output_modalities",@@ -1163,7 +1123,6 @@ class TestEvaluationConfigsApi(APIBaseTest):"example/embedding": ["embeddings"],},),- patch("products.ai_observability.backend.api.evaluations.decision_evaluations_enabled", return_value=flag),):response = self.client.post(f"/api/environments/{self.team.id}/evaluations/",@@ -1181,10 +1140,7 @@ class TestEvaluationConfigsApi(APIBaseTest):self.assertEqual(response.status_code, expected_status, response.json())if expected_status == 400:self.assertEqual(response.data["attr"], "model_configuration")- if flag:- self.assertIn("Choose a chat model or a supported decision model.", response.data["detail"])- else:- self.assertIn("Choose a chat model.", response.data["detail"])+ self.assertIn("Choose a chat model or a supported decision model.", response.data["detail"])@parameterized.expand([("omitted", False), ("null", True)])def test_llm_judge_creation_requires_model_configuration(self, _name, include_null_configuration):@@ -2469,8 +2425,7 @@ class TestReEnableValidatesRootCauseResolved(APIBaseTest):self.assertTrue(eval_obj.enabled)self.assertIsNone(eval_obj.status_reason)- @parameterized.expand([(False, 400), (True, 200)])- def test_re_enable_openrouter_decision_model_depends_on_rollout(self, flag, expected_status):+ def test_re_enable_openrouter_decision_model(self):key = LLMProviderKey.objects.create(team=self.team,provider="openrouter",@@ -2488,7 +2443,6 @@ class TestReEnableValidatesRootCauseResolved(APIBaseTest):"products.ai_observability.backend.llm.providers.openrouter._model_output_modalities",return_value={"typesafe/jev-1.13": ["decisions"]},),- patch("products.ai_observability.backend.api.evaluations.decision_evaluations_enabled", return_value=flag),):response = self.client.patch(f"/api/environments/{self.team.id}/evaluations/{eval_obj.id}/",@@ -2496,9 +2450,9 @@ class TestReEnableValidatesRootCauseResolved(APIBaseTest):format="json",)- self.assertEqual(response.status_code, expected_status, response.json())+ self.assertEqual(response.status_code, 200, response.json())eval_obj.refresh_from_db()- self.assertEqual(eval_obj.enabled, flag)+ self.assertTrue(eval_obj.enabled)def test_allows_re_enable_when_model_not_found_with_new_model(self):key = LLMProviderKey.objects.create(diff --git a/products/ai_observability/backend/api/test/test_provider_keys.py b/products/ai_observability/backend/api/test/test_provider_keys.pyindex 86c4834c317b..300f4f7d16e6 100644--- a/products/ai_observability/backend/api/test/test_provider_keys.py+++ b/products/ai_observability/backend/api/test/test_provider_keys.py@@ -207,50 +207,6 @@ class TestLLMProviderKeyViewSet(APIBaseTest):self.organization_membership.level = OrganizationMembership.Level.ADMINself.organization_membership.save()- @parameterized.expand([("create",), ("update",), ("validate",), ("prevalidate",)])- def test_system_one_connection_requires_enabled_feature(self, operation: str) -> None:- gateway_url = "https://ai-gateway.us.posthog.com/v1"- key = LLMProviderKey.objects.create(- team=self.team,- provider="system_one",- name="Example connection",- encrypted_config={"api_key": "example-token", "base_url": gateway_url},- created_by=self.user,- )- base_url = f"/api/environments/{self.team.id}/llm_analytics/"- with (- self.settings(POSTHOG_INTERNAL_ORG_IDS=[]),- patch(- "products.ai_observability.backend.llm.decisions.get_feature_flag_or_none",- return_value=False,- ),- patch("httpx.AsyncHTTPTransport.handle_async_request") as request,- ):- if operation == "create":- response = self.client.post(- f"{base_url}provider_keys/",- {- "provider": "system_one",- "name": "Example connection",- "api_key": "example-token",- "base_url": gateway_url,- "system_one_model": "custom-model",- },- )- elif operation == "update":- response = self.client.patch(f"{base_url}provider_keys/{key.id}/", {"api_key": "new-example-token"})- elif operation == "validate":- response = self.client.post(f"{base_url}provider_keys/{key.id}/validate/")- else:- response = self.client.post(- f"{base_url}provider_key_validations/",- {"provider": "system_one", "api_key": "example-token", "base_url": gateway_url},- )- self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN, response.data)- request.assert_not_called()- key.refresh_from_db()- self.assertEqual(key.encrypted_config["api_key"], "example-token")-def test_unauthenticated_user_cannot_access_provider_keys(self):self.client.logout()response = self.client.get(f"/api/environments/{self.team.id}/llm_analytics/provider_keys/")@@ -296,8 +252,7 @@ class TestLLMProviderKeyViewSet(APIBaseTest):@patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")})@patch("httpx.AsyncHTTPTransport.handle_async_request")- @patch("products.ai_observability.backend.llm.decisions.get_feature_flag_or_none", return_value=True)- def test_custom_system_one_connection_round_trip(self, _flag: Mock, request: Mock, _dns: Mock) -> None:+ def test_custom_system_one_connection_round_trip(self, request: Mock, _dns: Mock) -> None:body = json.dumps({"model": "custom-model",diff --git a/products/ai_observability/backend/api/test/test_proxy.py b/products/ai_observability/backend/api/test/test_proxy.pyindex 6549a769e914..efb3fe6ebb66 100644--- a/products/ai_observability/backend/api/test/test_proxy.py+++ b/products/ai_observability/backend/api/test/test_proxy.py@@ -365,8 +365,7 @@ class TestPlaygroundModelEnforcement(APIBaseTest):returned_ids = {m["id"] for m in response.json()}assert returned_ids == PLAYGROUND_MODEL_IDS- @parameterized.expand([(True,), (False,)])- def test_openai_decision_models_use_the_existing_key(self, flag: bool) -> None:+ def test_openai_decision_models_use_the_existing_key(self) -> None:key = LLMProviderKey.objects.create(team=self.team,provider="openai",@@ -375,23 +374,17 @@ class TestPlaygroundModelEnforcement(APIBaseTest):encrypted_config={"api_key": "example-token"},created_by=self.user,)- with (- patch("products.ai_observability.backend.api.proxy.Client.list_models", return_value=["gpt-5-mini"]),- patch("products.ai_observability.backend.api.proxy.decision_evaluations_enabled", return_value=flag),- ):+ with patch("products.ai_observability.backend.api.proxy.Client.list_models", return_value=["gpt-5-mini"]):response = self.client.get("/api/llm_proxy/models/", {"provider_key_id": str(key.id)})assert response.status_code == 200models = {model["id"]: model for model in response.json()}assert models["gpt-5-mini"]["supports_decisions"] is Falseassert models["gpt-5-mini"]["supports_chat"] is True- assert ("gpt-6-luna" in models) is flag- if flag:- assert models["gpt-6-luna"]["supports_decisions"] is True- assert models["gpt-6-luna"]["supports_chat"] is True- assert models["gpt-6-luna"]["provider"] == "OpenAI"-- @parameterized.expand([(True,), (False,)])- def test_openrouter_decision_models_use_the_existing_key(self, flag: bool) -> None:+ assert models["gpt-6-luna"]["supports_decisions"] is True+ assert models["gpt-6-luna"]["supports_chat"] is True+ assert models["gpt-6-luna"]["provider"] == "OpenAI"++ def test_openrouter_decision_models_use_the_existing_key(self) -> None:key = LLMProviderKey.objects.create(team=self.team,provider="openrouter",@@ -404,7 +397,6 @@ class TestPlaygroundModelEnforcement(APIBaseTest):"products.ai_observability.backend.api.proxy.Client.list_models",return_value=["typesafe/jev-router", "example/dual-model"],),- patch("products.ai_observability.backend.api.proxy.decision_evaluations_enabled", return_value=flag),patch("products.ai_observability.backend.api.proxy.decision_model_ids",return_value=frozenset({"typesafe/jev-1.13", "example/dual-model"}),@@ -412,7 +404,7 @@ class TestPlaygroundModelEnforcement(APIBaseTest):patch("products.ai_observability.backend.api.proxy.non_chat_model_ids",return_value=frozenset({"typesafe/jev-1.13"}),- ) as non_chat_models,+ ),):response = self.client.get("/api/llm_proxy/models/", {"provider_key_id": str(key.id)})@@ -421,11 +413,7 @@ class TestPlaygroundModelEnforcement(APIBaseTest):assert models["typesafe/jev-router"]["supports_decisions"] is Falseassert models["typesafe/jev-router"]["supports_chat"] is Trueassert models["example/dual-model"]["supports_chat"] is True- assert models["example/dual-model"]["supports_decisions"] is flag- assert ("typesafe/jev-1.13" in models) is flag- if flag:- assert models["typesafe/jev-1.13"]["provider"] == "OpenRouter"- assert models["typesafe/jev-1.13"]["supports_decisions"] is True- assert models["typesafe/jev-1.13"]["supports_chat"] is False- else:- non_chat_models.assert_not_called()+ assert models["example/dual-model"]["supports_decisions"] is True+ assert models["typesafe/jev-1.13"]["provider"] == "OpenRouter"+ assert models["typesafe/jev-1.13"]["supports_decisions"] is True+ assert models["typesafe/jev-1.13"]["supports_chat"] is Falsediff --git a/products/ai_observability/backend/llm/decisions.py b/products/ai_observability/backend/llm/decisions.pyindex 8dea28310178..aecc2da91d53 100644--- a/products/ai_observability/backend/llm/decisions.py+++ b/products/ai_observability/backend/llm/decisions.py@@ -13,8 +13,6 @@ from posthog.llm.system_one import (build_system_one_body,parse_system_one_response,)-from posthog.models import Team-from posthog.ph_client import get_feature_flag_or_nonefrom posthog.security.pinned_requests import SSRFBlockedErrorfrom posthog.security.url_validation import UNRESOLVED_HOST_REASON, has_authority_bypass_chars, validate_url_and_pin_ips@@ -43,7 +41,6 @@ def is_decision_model(provider: str | None,model: str | None,*,- openrouter_enabled: bool,decision_only: bool = False,openai_enabled: bool = False,) -> bool:@@ -51,8 +48,7 @@ def is_decision_model(return Trueif provider == "openai":return openai_enabled and model in OpenAIConfig.DECISION_MODELS- # Disabled projects keep the chat path independent of catalogue availability.- if provider != "openrouter" or not model or not openrouter_enabled:+ if provider != "openrouter" or not model:return Falsemodels = decision_model_ids(decision_only=decision_only)if models is None:@@ -60,25 +56,6 @@ def is_decision_model(return model in models-def decision_evaluations_enabled(team_id: int, *, base_url: str) -> bool:- try:- host = (urlsplit(base_url).hostname or "").encode("idna").decode("ascii").lower().rstrip(".")- except (ValueError, UnicodeError):- return False- if not host:- return False- team = Team.objects.only("uuid", "organization_id").get(id=team_id)- return (- get_feature_flag_or_none(- "llm-analytics-system-one-evaluations",- str(team.uuid),- groups={"organization": str(team.organization_id), "project": str(team.uuid)},- send_feature_flag_events=False,- )- is True- )--class DecisionRequestRejectedError(ProviderRequestRejectedError):passdiff --git a/products/ai_observability/backend/llm/test/test_decisions.py b/products/ai_observability/backend/llm/test/test_decisions.pyindex d707df0a2ed0..4ce2ea0164a0 100644--- a/products/ai_observability/backend/llm/test/test_decisions.py+++ b/products/ai_observability/backend/llm/test/test_decisions.py@@ -3,7 +3,6 @@ from collections.abc import Iteratorfrom contextlib import nullcontextfrom ipaddress import ip_addressfrom typing import Literal-from uuid import uuid4import pytestfrom unittest.mock import patch@@ -13,7 +12,6 @@ from django.test import override_settingsimport httpxfrom posthog.llm.system_one import NoulAnswer, NoulQuestion, ScoreAnswer, ScoreQuestion-from posthog.models import Teamfrom products.ai_observability.backend.llm.client import Clientfrom products.ai_observability.backend.llm.decisions import (@@ -21,7 +19,6 @@ from products.ai_observability.backend.llm.decisions import (DecisionEndpointBlockedError,DecisionRateLimitError,DecisionRequestRejectedError,- decision_evaluations_enabled,)from products.ai_observability.backend.llm.errors import (AuthenticationError,@@ -49,38 +46,6 @@ def public_endpoint_dns() -> Iterator[None]:yield- "base_url,flag,enabled",- [- ("https://api.typesafe.ai/v1", True, True),- ("", True, False),- ("https://decisions.example.com/v1", True, True),- ("https://decisions.example.com/v1", False, False),- ("https://decisions.example.com/v1", None, False),- ("https://ai-gateway.us.posthog.com/v1", True, True),- ("https://ai-gateway.eu.posthog.com/v1", True, True),- ("https://AI-GATEWAY.US.POSTHOG.COM.:443/v1", True, True),- ("https://ai-gateway.us.posthog.com/v1", True, True),- ("https://ai-gateway.us.posthog.com/v1", False, False),- ("https://ai-gateway.us.posthog.com/v1", None, False),- ],-)-def test_system_one_connections_require_flag_and_supported_endpoint(- base_url: str, flag: bool | None, enabled: bool-) -> None:- team = Team(id=1, organization_id=uuid4(), uuid=uuid4())- with (- override_settings(POSTHOG_INTERNAL_ORG_IDS=[]),- patch("products.ai_observability.backend.llm.decisions.Team.objects.only") as teams,- patch(- "products.ai_observability.backend.llm.decisions.get_feature_flag_or_none",- side_effect=lambda *args, groups, **kwargs: flag if groups["project"] == str(team.uuid) else False,- ),- ):- teams.return_value.get.return_value = team- assert decision_evaluations_enabled(team.id, base_url=base_url) is enabled--@pytest.mark.parametrize("status, expected_state", [(200, "ok"), (401, "invalid"), (402, "error"), (403, "invalid"), (500, "error")])diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.stories.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.stories.tsxindex e5b6b372b3e6..e84ceed4588f 100644--- a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.stories.tsx+++ b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.stories.tsx@@ -1,8 +1,6 @@import type { Meta, StoryObj } from '@storybook/react'import { BindLogic } from 'kea'-import { FEATURE_FLAGS } from 'lib/constants'-import { useStorybookMocks } from '~/mocks/browser'import { llmEvaluationLogic } from '../llmEvaluationLogic'@@ -14,7 +12,6 @@ const meta: Meta<{ method?: JudgeMethod; narrow?: boolean; empty?: boolean; naticomponent: EvaluationModelPicker,parameters: {layout: 'padded',- featureFlags: [FEATURE_FLAGS.LLM_ANALYTICS_SYSTEM_ONE_EVALUATIONS],},render: ({ method = 'llm', narrow = false, empty = false, nativeOpenAI = false }) => {useStorybookMocks({@@ -107,6 +104,5 @@ export const LLM: Story = { args: { method: 'llm' } }export const Decision: Story = { args: { method: 'decision' } }export const DecisionNarrow: Story = { args: { method: 'decision', narrow: true } }export const NoDecisionModels: Story = { args: { method: 'decision', empty: true } }-export const FlagOff: Story = { parameters: { featureFlags: [] } }export const NativeOpenAI: Story = { args: { method: 'decision', nativeOpenAI: true } }export const NativeOpenAINarrow: Story = { args: { method: 'decision', nativeOpenAI: true, narrow: true } }diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsxindex 6b16f8d7d15f..07b4b2de9cd1 100644--- a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx+++ b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx@@ -2,9 +2,7 @@ import { useActions, useValues } from 'kea'import { LemonInput, LemonSegmentedButton } from '@posthog/lemon-ui'-import { FEATURE_FLAGS } from 'lib/constants'import { LemonField } from 'lib/lemon-ui/LemonField'-import { featureFlagLogic } from 'lib/logic/featureFlagLogic'import { ByokModelPickerNotice } from '../../ByokModelPickerNotice'import { getModelPickerFooterLink, ModelPicker } from '../../ModelPicker'@@ -25,7 +23,6 @@ export function EvaluationModelPicker(): JSX.Element {judgeModelGroups: groups,} = useValues(llmEvaluationLogic)const { selectModelFromPicker, setJudgeMethod, patchOutputConfig } = useActions(llmEvaluationLogic)- const { featureFlags } = useValues(featureFlagLogic)// Evals always run on the team's own provider key, so only BYOK models are offered.const selectedModelName = byokModels.find(@@ -65,9 +62,6 @@ export function EvaluationModelPicker(): JSX.Element {Decision model</span>),- disabledReason: !featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_SYSTEM_ONE_EVALUATIONS]- ? 'Decision model evaluations are not enabled for this project.'- : undefined,},]}/>diff --git a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsxindex b8e98ba85ec0..8afe8e5f46f4 100644--- a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx+++ b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx@@ -15,10 +15,9 @@ import {} from '@posthog/lemon-ui'import { RestrictionScope, useRestrictedArea } from 'lib/components/RestrictedArea'-import { FEATURE_FLAGS, TeamMembershipLevel } from 'lib/constants'+import { TeamMembershipLevel } from 'lib/constants'import { IconKey } from 'lib/lemon-ui/icons'import { LemonTableColumns } from 'lib/lemon-ui/LemonTable'-import { featureFlagLogic } from 'lib/logic/featureFlagLogic'import { LLMProviderIcon, LLM_PROVIDER_SELECT_OPTIONS } from '../LLMProviderIcon'import {@@ -240,7 +239,6 @@ function KeyValidationStatus({}function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }): JSX.Element {- const { featureFlags } = useValues(featureFlagLogic)const { systemOneBaseUrl, systemOneModel } = useValues(llmProviderKeysLogic)const { newKeyModalOpen, providerKeysLoading, preValidationResult, preValidationResultLoading, evaluationConfig } =useValues(llmProviderKeysLogic)@@ -435,11 +433,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }<LemonSelectvalue={provider}onChange={handleProviderChange}- options={LLM_PROVIDER_SELECT_OPTIONS.filter(- ({ value }) =>- value !== 'system_one' ||- featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_SYSTEM_ONE_EVALUATIONS] === true- )}+ options={LLM_PROVIDER_SELECT_OPTIONS}className="mt-1"fullWidth/>diff --git a/products/ai_observability/frontend/settings/SystemOneConnectionFields.stories.tsx b/products/ai_observability/frontend/settings/SystemOneConnectionFields.stories.tsxindex 04831a61e3cf..ba7483ff4310 100644--- a/products/ai_observability/frontend/settings/SystemOneConnectionFields.stories.tsx+++ b/products/ai_observability/frontend/settings/SystemOneConnectionFields.stories.tsx@@ -1,7 +1,5 @@import type { Meta, StoryObj } from '@storybook/react'-import { FEATURE_FLAGS } from 'lib/constants'-import { useStorybookMocks } from '~/mocks/browser'import { LLMProviderKeysSettings } from './LLMProviderKeysSettings'@@ -27,8 +25,4 @@ export default metatype Story = StoryObj<typeof meta>export const Default: Story = {}-export const ProviderSettings: Story = {- parameters: { featureFlags: [FEATURE_FLAGS.LLM_ANALYTICS_SYSTEM_ONE_EVALUATIONS] },- render: () => <LLMProviderKeysSettings />,-}-export const ProviderSettingsFlagOff: Story = { render: () => <LLMProviderKeysSettings /> }+export const ProviderSettings: Story = { render: () => <LLMProviderKeysSettings /> }