Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(aio): remove the system one evaluations feature flag (#124222)

Bernat Torres4bd5145d06e4
diff --git a/docs/internal/ai-observability-judge-inputs.md b/docs/internal/ai-observability-judge-inputs.md
index 680193dd40eb..1fb1a2fb9ad7 100644
--- a/docs/internal/ai-observability-judge-inputs.md
+++ b/docs/internal/ai-observability-judge-inputs.md
@@ -72,11 +72,8 @@ Changing the judge method marks the evaluation as edited, including when the sel
The selection is stored as `evaluation_config.judge_method` (`llm` or `decision`). Evaluations without this field keep automatic routing until a method or model is explicitly selected.
Automatic OpenRouter routing selects decisions only for models without text output. Models supporting both methods keep chat routing unless decisions are explicitly selected.
Prompt-only `evaluation_config` PATCH requests preserve the saved method; an explicit `judge_method: null` restores automatic routing.
-The detail editor PATCHes only changed fields on existing evaluations, so renaming or disabling a saved decision evaluation still works after its feature flag is turned off. Re-enabling it or changing its model or output configuration still validates decision access.
+The detail editor PATCHes only changed fields on existing evaluations. Re-enabling an evaluation or changing its model or output configuration still validates decision access.
Explicit method selection requires a selected model, including for legacy evaluations that use the project's default connection.
-The `llm-analytics-system-one-evaluations` project-group feature flag controls access in the browser and background workers.
-Both use the project's UUID as its group key; the numeric project ID is a group property.
-Deploy the ingestion and evaluation worker changes before enabling the flag.
Projects use their saved provider keys or configure a System One-compatible deployment and its authentication.
Evaluation connections never fall back to an instance credential or gateway configuration.
The configured endpoint authorizes the supplied credential, including connections to PostHog's regional AI gateway endpoints.
diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml
index a38c92f7f341..2d2585c07dcb 100644
--- a/frontend/snapshots.yml
+++ b/frontend/snapshots.yml
@@ -9364,10 +9364,6 @@ snapshots:
hash: v1.k794b7964.81ce7999f24877f6e4ac2d9ab89fb4e584e5bc0749e594742c15377c8da8a1a7.BlSbtKwSVk0n9eZKEpRMG7hADbYEDAXAedKVuCaBN_8
scenes-app-ai-observability-system-one-connection--provider-settings--light:
hash: v1.k794b7964.05b7de1c84c25e03badf722fa567182ff449e74a6299d6a543c554ecd20e3184.eccRHOq726ohqws6q_XlZPELbvyUZPfAZk1cwlog_QY
- scenes-app-ai-observability-system-one-connection--provider-settings-flag-off--dark:
- hash: v1.k794b7964.81ce7999f24877f6e4ac2d9ab89fb4e584e5bc0749e594742c15377c8da8a1a7.DaP59ecAHMR8eaDYuWmJyKEowz34lJ3qbuKNJ4DsN8U
- scenes-app-ai-observability-system-one-connection--provider-settings-flag-off--light:
- hash: v1.k794b7964.05b7de1c84c25e03badf722fa567182ff449e74a6299d6a543c554ecd20e3184.3SudNZ7cfTDqUdodzpr3A8w1rZBmSjpUHNJ-qyTQQC4
scenes-app-ai-observability-trace--full--dark:
hash: v1.k794b7964.4d3ad4ebbe259bff3cd17afce36a4f9e9e69b5def84e53c89a543b970751e695.a2alK28PTn9_H4Th1Gy1knhB3uSsWmZ77ZmrV0jqQso
scenes-app-ai-observability-trace--full--light:
@@ -13236,10 +13232,6 @@ snapshots:
hash: v1.k794b7964.58dc8d73bf85d6c8bcd7a10b08c1e38efefd840fbe912cbbdff34bc4596041d8.5_RSGKERITVltMJ5ap2pmDAF2IC77kXkHPRfyXJi2oY
scenes-app-llm-observability-evaluationmodelpicker--decision-narrow--light:
hash: v1.k794b7964.28a30a7cb478ba7b585a0736e53e74735c643e3b61b0b84059494f6040493c29.haO943nTQB3hFa_f-TgDhACaCqM3_bOKVEdmp0SXTzg
- scenes-app-llm-observability-evaluationmodelpicker--flag-off--dark:
- hash: v1.k794b7964.277344d00b8edc7551d8850029611c83e1740ac1ea3adb9e679ceffd2037064b.0gLUQZyT3_xppzOwry0tkGL0HlIjU-jdxq6V8VIAGs4
- scenes-app-llm-observability-evaluationmodelpicker--flag-off--light:
- hash: v1.k794b7964.060fc008f850bbab373cb0c43a70e91454426b70a2d259ab371e1c2cfa7d1834.YTIqtwuIVdRppnvMFv142GGjOIDBIehxdylH9qJJQLY
scenes-app-llm-observability-evaluationmodelpicker--llm--dark:
hash: v1.k794b7964.43b905fdb235f4c15e7b906f14467c4ee9e8df010a3f26e476a2b3e6f59a4323.MZUQFFzWautdc3zzDJVWS6YbBa4qxX69-2HNwn-4wXQ
scenes-app-llm-observability-evaluationmodelpicker--llm--light:
diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx
index 12d5639b9e1a..4977094a2398 100644
--- a/frontend/src/lib/constants.tsx
+++ b/frontend/src/lib/constants.tsx
@@ -372,7 +372,6 @@ export const FEATURE_FLAGS = {
LLM_ANALYTICS_COMMUNITY_SKILLS: 'llm-analytics-community-skills', // owner: #team-ai-observability
LLM_ANALYTICS_DATASETS: 'llm-analytics-datasets', // owner: #team-ai-observability #team-posthog-ai
LLM_ANALYTICS_EVAL_SETTLING_STRATEGY: 'llm-analytics-eval-settling-strategy', // owner: #team-ai-observability
- LLM_ANALYTICS_SYSTEM_ONE_EVALUATIONS: 'llm-analytics-system-one-evaluations', // owner: #team-ai-observability
LLM_ANALYTICS_TAGS: 'llm-analytics-tags', // owner: #team-ai-observability
LLM_ANALYTICS_TRACE_NAVIGATION: 'llm-analytics-trace-navigation', // owner: #team-ai-observability
LLM_OBSERVABILITY_TRACE_SEARCH: 'llm-observability-trace-search', // owner: #team-ai-observability
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index 742f4e4b2193..0825d945d7b0 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -57,7 +57,6 @@ from products.ai_observability.backend.llm import DEFAULT_MODEL_BY_PROVIDER, Cli
from products.ai_observability.backend.llm.decisions import (
DecisionClient,
DecisionEndpointBlockedError,
- decision_evaluations_enabled,
is_decision_model,
)
from products.ai_observability.backend.llm.errors import (
@@ -81,7 +80,7 @@ from products.ai_observability.backend.llm.errors import (
)
from products.ai_observability.backend.llm.providers.openai import OPENAI_DECISIONS_BASE_URL
from products.ai_observability.backend.llm.providers.openai_decisions import OpenAIDecisionAdapter
-from products.ai_observability.backend.llm.providers.openrouter import OPENROUTER_DECISIONS_BASE_URL, decision_model_ids
+from products.ai_observability.backend.llm.providers.openrouter import OPENROUTER_DECISIONS_BASE_URL
from products.ai_observability.backend.llm.types import CompletionResponse
from products.ai_observability.backend.models.evaluation_configs import (
BooleanOutputConfig,
@@ -655,19 +654,14 @@ def call_llm_judge(
probability: float | None = None
decision_result = None
try:
- openrouter_enabled = provider == "openrouter" and decision_evaluations_enabled(
- team_id, base_url=OPENROUTER_DECISIONS_BASE_URL
- )
judge_method = evaluation["evaluation_config"].get("judge_method")
# Explicit selection takes precedence for models that support both methods.
# Models gaining decisions must not move existing chat evaluations onto a different API.
uses_decisions = (
judge_method == "decision"
if judge_method is not None
- else is_decision_model(provider, model, openrouter_enabled=openrouter_enabled, decision_only=True)
+ else is_decision_model(provider, model, decision_only=True)
)
- if judge_method is None and provider == "openrouter" and not openrouter_enabled:
- uses_decisions = model in (decision_model_ids(refresh=False, decision_only=True) or ())
if uses_decisions:
if output_type not in ("boolean", "categorical", "numeric"):
return build_skipped_evaluation_result(
@@ -685,15 +679,6 @@ def call_llm_judge(
if provider_key
else ""
)
- if (provider == "openrouter" and not openrouter_enabled) or (
- provider in ("system_one", "openai") and not decision_evaluations_enabled(team_id, base_url=base_url)
- ):
- return build_skipped_evaluation_result(
- output_type=output_type,
- allows_na=allows_na,
- reasoning="Decision model evaluations are not available for this project.",
- skip_reason="system_one_unavailable",
- )
prompt = evaluation["evaluation_config"]["prompt"]
categorical_config = (
CategoricalOutputConfig.model_validate(output_config) if output_type == "categorical" else None
@@ -960,18 +945,6 @@ def call_llm_judge(
non_retryable=True,
)
except UnsupportedModelError as e:
- # A failed chat call can populate a cold catalogue; a disabled flag must not disable the evaluation.
- if (
- provider == "openrouter"
- and not openrouter_enabled
- and model in (decision_model_ids(refresh=False, decision_only=True) or ())
- ):
- return build_skipped_evaluation_result(
- output_type=output_type,
- allows_na=allows_na,
- reasoning="Decision model evaluations are not available for this project.",
- skip_reason="system_one_unavailable",
- )
increment_user_errors("model_not_supported", provider=provider)
return terminal_user_error_result(
spec=require_user_error_spec("model_not_supported", is_byok=is_byok),
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index c76e819fdca8..a1210eb83260 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -9,7 +9,6 @@ from typing import Any, Literal, cast
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
-from django.core.cache import cache
from django.test import override_settings
import httpx
@@ -48,11 +47,6 @@ from products.ai_observability.backend.llm.errors import (
StructuredOutputParseError,
UnsupportedModelError,
)
-from products.ai_observability.backend.llm.providers.openai import OpenAIAdapter
-from products.ai_observability.backend.llm.providers.openrouter import (
- MODEL_MODALITIES_CACHE_KEY,
- MODEL_MODALITIES_LAST_GOOD_CACHE_KEY,
-)
from products.ai_observability.backend.models.evaluation_config import EvaluationConfig
from products.ai_observability.backend.models.evaluation_directories import EvaluationDirectory
from products.ai_observability.backend.models.evaluations import Evaluation
@@ -312,18 +306,14 @@ def _mock_config_with_active_key(provider: str = "openai") -> MagicMock:
return MagicMock(active_provider_key=key)
[email protected]("flag", [True, False, None])
@pytest.mark.parametrize("judge_method", [None, "llm"])
-def test_openrouter_catalogue_outage_only_affects_projects_with_decisions_enabled(
- flag: bool | None, judge_method: str | None
-) -> None:
+def test_openrouter_catalogue_outage_only_affects_automatic_routing(judge_method: str | None) -> None:
key = MagicMock(provider="openrouter", encrypted_config={"api_key": "example-token"})
with (
patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
patch(
"products.ai_observability.backend.llm.providers.openrouter._model_output_modalities", return_value=None
) as catalogue,
- patch("posthog.temporal.ai_observability.evaluation_llm_judge.decision_evaluations_enabled", return_value=flag),
patch("posthog.temporal.ai_observability.evaluation_llm_judge.Client.complete") as complete,
patch("posthog.temporal.ai_observability.evaluation_llm_judge.DecisionClient.evaluate") as decide,
):
@@ -333,7 +323,7 @@ def test_openrouter_catalogue_outage_only_affects_projects_with_decisions_enable
complete.return_value = MagicMock(parsed=BooleanEvalResult(verdict=True, reasoning="Polite"), usage=None)
with (
pytest.raises(TransientJudgeError, match="OpenRouter model capabilities")
- if flag and judge_method is None
+ if judge_method is None
else nullcontext()
):
result = call_llm_judge(
@@ -348,22 +338,18 @@ def test_openrouter_catalogue_outage_only_affects_projects_with_decisions_enable
user_prompt="Hello!",
allows_na=False,
)
- if flag and judge_method is None:
+ if judge_method is None:
complete.assert_not_called()
else:
assert result["verdict"] is True
- if judge_method is None:
- catalogue.assert_called_once_with(refresh=False)
- else:
- catalogue.assert_not_called()
+ catalogue.assert_not_called()
decide.assert_not_called()
@pytest.mark.parametrize("provider,model", [("openrouter", "example/dual-model"), ("openai", "gpt-6-luna")])
[email protected]("flag", [True, False, None])
@pytest.mark.parametrize("judge_method", [None, "llm"])
def test_legacy_and_explicit_llm_methods_use_chat_for_a_dual_capability_model(
- provider: str, model: str, flag: bool | None, judge_method: str | None
+ provider: str, model: str, judge_method: str | None
) -> None:
key = MagicMock(provider=provider, encrypted_config={"api_key": "example-token"})
with (
@@ -372,7 +358,6 @@ def test_legacy_and_explicit_llm_methods_use_chat_for_a_dual_capability_model(
"products.ai_observability.backend.llm.providers.openrouter._model_output_modalities",
return_value={"example/dual-model": ["text", "decisions"]},
),
- patch("posthog.temporal.ai_observability.evaluation_llm_judge.decision_evaluations_enabled", return_value=flag),
patch("posthog.temporal.ai_observability.evaluation_llm_judge.Client.complete") as complete,
patch("posthog.temporal.ai_observability.evaluation_llm_judge.DecisionClient.evaluate") as decide,
patch("posthog.temporal.ai_observability.evaluation_llm_judge.OpenAIDecisionAdapter.evaluate") as native_decide,
@@ -475,7 +460,6 @@ def test_legacy_and_explicit_llm_methods_use_chat_for_a_dual_capability_model(
def test_openai_native_decisions_use_shared_result_semantics(
applicable: bool, output_type: str, output_config: dict, answers: list[dict], expected: dict
) -> None:
- team = Team(id=1, organization_id=uuid.uuid4(), uuid=uuid.uuid4())
evaluation = {
"id": "example-evaluation",
"name": "Response quality",
@@ -500,11 +484,8 @@ def test_openai_native_decisions_use_shared_result_semantics(
with (
patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),
patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
- patch("products.ai_observability.backend.llm.decisions.Team.objects.only") as teams,
- patch("products.ai_observability.backend.llm.decisions.get_feature_flag_or_none", return_value=True),
patch("httpx.AsyncHTTPTransport.handle_async_request", return_value=response) as transport,
):
- teams.return_value.get.return_value = team
spec.return_value.resolve.return_value = MagicMock(
provider="openai",
model="gpt-6-luna",
@@ -538,72 +519,6 @@ def test_openai_native_decisions_use_shared_result_semantics(
assert "probability" not in result
[email protected]("flag", [False, None])
-def test_openai_decisions_disabled_flag_skips_without_invalidating_key(flag: bool | None) -> None:
- with (
- patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
- patch("posthog.temporal.ai_observability.evaluation_llm_judge.decision_evaluations_enabled", return_value=flag),
- patch("httpx.AsyncHTTPTransport.handle_async_request") as transport,
- ):
- spec.return_value.resolve.return_value = MagicMock(
- provider="openai",
- model="gpt-6-luna",
- is_byok=True,
- provider_key=MagicMock(encrypted_config={"api_key": "example-token"}),
- )
- result = call_llm_judge(
- evaluation={"team_id": 1, "evaluation_config": {"prompt": "Check the reply.", "judge_method": "decision"}},
- system_prompt="",
- user_prompt="Example reply.",
- allows_na=False,
- )
- assert result["skip_reason"] == "system_one_unavailable"
- assert "terminal_user_error" not in result
- assert "provider_key_state" not in result
- transport.assert_not_called()
-
-
[email protected]("flag", [False, None])
[email protected]("cached", [True, False])
-def test_openrouter_decision_model_with_disabled_flag_skips_without_disabling(flag: bool | None, cached: bool) -> None:
- cache.delete_many([MODEL_MODALITIES_CACHE_KEY, MODEL_MODALITIES_LAST_GOOD_CACHE_KEY])
- model = "example/decision"
- if cached:
- cache.set(MODEL_MODALITIES_LAST_GOOD_CACHE_KEY, {model: ["decisions"]}, timeout=None)
- key = MagicMock(provider="openrouter", encrypted_config={"api_key": "example-token"})
- try:
- with (
- patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
- patch(
- "posthog.temporal.ai_observability.evaluation_llm_judge.decision_evaluations_enabled", return_value=flag
- ),
- patch.object(OpenAIAdapter, "complete", side_effect=ModelNotFoundError(model)) as complete,
- patch("posthog.temporal.ai_observability.evaluation_llm_judge.DecisionClient.evaluate") as decide,
- patch("products.ai_observability.backend.llm.providers.openrouter.httpx.get") as catalogue,
- ):
- spec.return_value.resolve.return_value = MagicMock(
- provider="openrouter", model=model, provider_key=key, is_byok=True
- )
- catalogue.return_value.json.return_value = {
- "data": [{"id": model, "architecture": {"output_modalities": ["decisions"]}}]
- }
- result = call_llm_judge(
- evaluation={"team_id": 1, "evaluation_config": {"prompt": "Is the response polite?"}},
- system_prompt="",
- user_prompt="Hello!",
- allows_na=False,
- )
- assert result["skipped"] is True
- assert result["skip_reason"] == "system_one_unavailable"
- assert "terminal_user_error" not in result
- assert "provider_key_state" not in result
- assert complete.call_count == (0 if cached else 1)
- assert catalogue.call_count == (0 if cached else 1)
- decide.assert_not_called()
- finally:
- cache.delete_many([MODEL_MODALITIES_CACHE_KEY, MODEL_MODALITIES_LAST_GOOD_CACHE_KEY])
-
-
@pytest.mark.parametrize(
"provider,connection_config,base_url,model,usage",
[
@@ -698,11 +613,8 @@ def test_system_one_judge_emits_boolean_probability_without_reasoning(
),
patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),
patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
- patch("products.ai_observability.backend.llm.decisions.Team.objects.only") as teams,
- patch("products.ai_observability.backend.llm.decisions.get_feature_flag_or_none", return_value=True),
patch("httpx.AsyncHTTPTransport.handle_async_request", return_value=response) as request,
):
- teams.return_value.get.return_value = Team(id=1, organization_id=uuid.uuid4(), uuid=uuid.uuid4())
spec.return_value.resolve.return_value = resolved
result = call_llm_judge(
evaluation=evaluation,
@@ -795,7 +707,6 @@ def test_system_one_categorical_results_use_category_keys_without_boolean_probab
),
patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),
patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
- patch("posthog.temporal.ai_observability.evaluation_llm_judge.decision_evaluations_enabled", return_value=True),
patch("httpx.AsyncHTTPTransport.handle_async_request") as request,
):
spec.return_value.resolve.return_value = MagicMock(
@@ -878,7 +789,6 @@ def test_decision_numeric_scores_use_configured_bounds(
),
patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),
patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
- patch("posthog.temporal.ai_observability.evaluation_llm_judge.decision_evaluations_enabled", return_value=True),
patch(
"httpx.AsyncHTTPTransport.handle_async_request",
return_value=httpx.Response(
@@ -924,7 +834,6 @@ def test_system_one_numeric_requires_a_score_range(output_config: dict[str, floa
with (
patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),
patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
- patch("posthog.temporal.ai_observability.evaluation_llm_judge.decision_evaluations_enabled", return_value=True),
patch("httpx.AsyncHTTPTransport.handle_async_request") as request,
):
spec.return_value.resolve.return_value = MagicMock(
@@ -946,42 +855,6 @@ def test_system_one_numeric_requires_a_score_range(output_config: dict[str, floa
request.assert_not_called()
- "provider,base_url,flag",
- [
- ("system_one", "https://decisions.example.com/v1", False),
- ("system_one", "https://ai-gateway.us.posthog.com/v1", False),
- ],
-)
-def test_system_one_restricted_connection_does_not_send_evaluation_data(
- provider: str, base_url: str, flag: bool
-) -> None:
- with (
- override_settings(POSTHOG_INTERNAL_ORG_IDS=[]),
- patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
- patch("products.ai_observability.backend.llm.decisions.Team.objects.only") as teams,
- patch("products.ai_observability.backend.llm.decisions.get_feature_flag_or_none", return_value=flag),
- patch("httpx.AsyncHTTPTransport.handle_async_request") as request,
- ):
- teams.return_value.get.return_value = Team(id=1, organization_id=uuid.uuid4(), uuid=uuid.uuid4())
- spec.return_value.resolve.return_value = MagicMock(
- provider=provider,
- model="custom-model",
- provider_key=MagicMock(
- provider=provider, encrypted_config={"base_url": base_url, "api_key": "example-token"}
- ),
- is_byok=True,
- )
- result = call_llm_judge(
- evaluation={"team_id": 1, "evaluation_config": {"prompt": "Is this a greeting?"}},
- system_prompt="",
- user_prompt="Hello!",
- allows_na=False,
- )
- assert result["skipped"] is True
- request.assert_not_called()
-
-
@pytest.mark.parametrize(
"provider,status,encoding,expected_skip_reason",
[
@@ -1015,7 +888,6 @@ def test_provider_rejections_distinguish_blocked_endpoints_from_bad_inputs(
),
patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),
patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
- patch("posthog.temporal.ai_observability.evaluation_llm_judge.decision_evaluations_enabled", return_value=True),
patch("httpx.AsyncHTTPTransport.handle_async_request", return_value=response) as request,
):
spec.return_value.resolve.return_value = MagicMock(
@@ -1192,7 +1064,6 @@ def test_custom_provider_rate_limit_retries_without_disabling_the_evaluation(
with (
patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")}),
patch("posthog.temporal.ai_observability.evaluation_llm_judge.model_spec") as spec,
- patch("posthog.temporal.ai_observability.evaluation_llm_judge.decision_evaluations_enabled", return_value=True),
patch(
"httpx.AsyncHTTPTransport.handle_async_request",
side_effect=[
diff --git a/products/ai_observability/backend/api/evaluations.py b/products/ai_observability/backend/api/evaluations.py
index df4377acccf5..ca1b961c85b1 100644
--- a/products/ai_observability/backend/api/evaluations.py
+++ b/products/ai_observability/backend/api/evaluations.py
@@ -59,10 +59,10 @@ from ..facade import (
from ..facade.contracts import Contract
from ..hog import compile_ai_observability_hog
from ..llm import DEFAULT_MODEL_BY_PROVIDER
-from ..llm.decisions import decision_evaluations_enabled, is_decision_model
+from ..llm.decisions import is_decision_model
from ..llm.errors import ProviderConnectionError
-from ..llm.providers.openai import OPENAI_DECISIONS_BASE_URL, OpenAIConfig
-from ..llm.providers.openrouter import OPENROUTER_BASE_URL, is_non_chat_model
+from ..llm.providers.openai import OpenAIConfig
+from ..llm.providers.openrouter import is_non_chat_model
from ..models.evaluation_config import EvaluationConfig
from ..models.evaluation_configs import (
EVALUATION_TEST_LOOKBACK_DAYS,
@@ -572,12 +572,9 @@ class EvaluationSerializer(UserAccessControlSerializerMixin, serializers.ModelSe
model_provider,
model_configuration.get("model"),
decision_only=judge_method is None,
- openrouter_enabled=model_provider == "openrouter"
- and decision_evaluations_enabled(self.context["get_team"]().id, base_url=OPENROUTER_BASE_URL),
openai_enabled=judge_method == "decision"
and model_provider == LLMProvider.OPENAI
- and model_configuration.get("model") in OpenAIConfig.DECISION_MODELS
- and decision_evaluations_enabled(self.context["get_team"]().id, base_url=OPENAI_DECISIONS_BASE_URL),
+ and model_configuration.get("model") in OpenAIConfig.DECISION_MODELS,
)
except ProviderConnectionError as e:
raise serializers.ValidationError({"model_configuration": str(e)}) from e
@@ -738,13 +735,11 @@ class EvaluationSerializer(UserAccessControlSerializerMixin, serializers.ModelSe
if uses_decision_model:
return
if model and is_non_chat_model(model):
- guidance = (
- "Choose a chat model or a supported decision model."
- if decision_evaluations_enabled(self.context["get_team"]().id, base_url=OPENROUTER_BASE_URL)
- else "Choose a chat model."
- )
raise serializers.ValidationError(
- {"model_configuration": f"'{model}' is not supported as an evaluation judge. {guidance}"}
+ {
+ "model_configuration": f"'{model}' is not supported as an evaluation judge. "
+ "Choose a chat model or a supported decision model."
+ }
)
def _validate_can_run(self, data: dict) -> None:
diff --git a/products/ai_observability/backend/api/provider_keys.py b/products/ai_observability/backend/api/provider_keys.py
index da445f700a5d..a0d88dc9599b 100644
--- a/products/ai_observability/backend/api/provider_keys.py
+++ b/products/ai_observability/backend/api/provider_keys.py
@@ -26,7 +26,7 @@ from posthog.security.url_validation import strip_userinfo
from products.access_control.backend.presentation.access_control import AccessControlViewSetMixin
from ..llm.client import Client
-from ..llm.decisions import DecisionClient, decision_evaluations_enabled
+from ..llm.decisions import DecisionClient
from ..llm.providers.anthropic import (
DISALLOWED_FOUNDRY_ENDPOINT_MESSAGE,
error_field_for_validation_message as anthropic_error_field,
@@ -82,10 +82,6 @@ def _reload_model_config_dependents_on_commit(team_id: int, model_config_ids: li
def validate_provider_key(provider: str, api_key: str, *, team_id: int, **kwargs: str) -> tuple[str, str | None]:
"""Validate an API key for any supported provider using the unified client."""
- if provider == LLMProvider.SYSTEM_ONE and not decision_evaluations_enabled(
- team_id, base_url=kwargs.get("base_url", "")
- ):
- raise exceptions.PermissionDenied("System One evaluations are not available for this project.")
try:
return Client.validate_key(provider, api_key, **kwargs)
except Exception:
diff --git a/products/ai_observability/backend/api/proxy.py b/products/ai_observability/backend/api/proxy.py
index fc1de5ce3dad..fd2d2678fc1f 100644
--- a/products/ai_observability/backend/api/proxy.py
+++ b/products/ai_observability/backend/api/proxy.py
@@ -57,18 +57,13 @@ from products.ai_observability.backend.llm import (
OutputSchema,
get_playground_models,
)
-from products.ai_observability.backend.llm.decisions import decision_evaluations_enabled
from products.ai_observability.backend.llm.errors import (
ProviderConfigurationError,
ProviderHostUnresolvedError,
UnsupportedProviderError,
)
-from products.ai_observability.backend.llm.providers.openai import OPENAI_DECISIONS_BASE_URL, OpenAIConfig
-from products.ai_observability.backend.llm.providers.openrouter import (
- OPENROUTER_BASE_URL,
- decision_model_ids,
- non_chat_model_ids,
-)
+from products.ai_observability.backend.llm.providers.openai import OpenAIConfig
+from products.ai_observability.backend.llm.providers.openrouter import decision_model_ids, non_chat_model_ids
from products.ai_observability.backend.models.provider_keys import (
LLMProvider,
LLMProviderKey,
@@ -556,15 +551,11 @@ class LLMProxyViewSet(viewsets.ViewSet):
provider_display = PROVIDER_DISPLAY_NAMES.get(provider_key.provider, provider_key.provider.title())
decision_models: frozenset[str] = frozenset()
non_chat_models: frozenset[str] = frozenset()
- if provider_key.provider == LLMProvider.OPENROUTER and decision_evaluations_enabled(
- provider_key.team_id, base_url=OPENROUTER_BASE_URL
- ):
+ if provider_key.provider == LLMProvider.OPENROUTER:
decision_models = decision_model_ids() or frozenset()
non_chat_models = non_chat_model_ids() or frozenset()
models = list(dict.fromkeys([*models, *sorted(decision_models)]))
- elif provider_key.provider == LLMProvider.OPENAI and decision_evaluations_enabled(
- provider_key.team_id, base_url=OPENAI_DECISIONS_BASE_URL
- ):
+ elif provider_key.provider == LLMProvider.OPENAI:
decision_models = OpenAIConfig.DECISION_MODELS
models = list(dict.fromkeys([*models, *sorted(decision_models)]))
return Response(
diff --git a/products/ai_observability/backend/api/test/test_evaluations.py b/products/ai_observability/backend/api/test/test_evaluations.py
index 8f0504a2cb45..5fcc123c9561 100644
--- a/products/ai_observability/backend/api/test/test_evaluations.py
+++ b/products/ai_observability/backend/api/test/test_evaluations.py
@@ -161,7 +161,6 @@ class TestModelConfigurationSerializer(SimpleTestCase):
context={"get_team": lambda: Mock(id=1)},
)
with (
- patch("products.ai_observability.backend.api.evaluations.decision_evaluations_enabled", return_value=True),
patch(
"products.ai_observability.backend.llm.providers.openrouter._model_output_modalities",
return_value={"example/dual-model": ["text", "decisions"]},
@@ -217,10 +216,9 @@ class TestModelConfigurationSerializer(SimpleTestCase):
("unsupported_decision", "decision", "openai", ["text"], None),
("custom_chat", "llm", "system_one", ["decisions"], None),
("decision_only_chat", "llm", "openrouter", ["decisions"], None),
- ("openai_decision", "decision", "openai", ["text"], True, "gpt-6-luna", True),
- ("openai_legacy", None, "openai", ["text"], False, "gpt-6-luna", True),
- ("openai_chat", "llm", "openai", ["text"], False, "gpt-6-luna", True),
- ("openai_disabled", "decision", "openai", ["text"], None, "gpt-6-luna", False),
+ ("openai_decision", "decision", "openai", ["text"], True, "gpt-6-luna"),
+ ("openai_legacy", None, "openai", ["text"], False, "gpt-6-luna"),
+ ("openai_chat", "llm", "openai", ["text"], False, "gpt-6-luna"),
]
)
def test_judge_method_matches_model_capabilities(
@@ -231,7 +229,6 @@ class TestModelConfigurationSerializer(SimpleTestCase):
modalities: list[str],
expected: bool | None,
model: str = "example/model",
- flag: bool = True,
) -> None:
config = {"prompt": "Check the response."}
if method:
@@ -245,7 +242,6 @@ class TestModelConfigurationSerializer(SimpleTestCase):
)
serializer = EvaluationSerializer(instance=evaluation, partial=True, context={"get_team": lambda: Mock(id=1)})
with (
- patch("products.ai_observability.backend.api.evaluations.decision_evaluations_enabled", return_value=flag),
patch(
"products.ai_observability.backend.llm.providers.openrouter._model_output_modalities",
return_value={"example/model": modalities},
@@ -262,36 +258,6 @@ class TestModelConfigurationSerializer(SimpleTestCase):
validated = serializer.validate(data)
self.assertEqual(validated["evaluation_config"], config)
- @parameterized.expand(
- [
- ({"name": "Renamed decision evaluation"}, True),
- ({"enabled": False}, True),
- ({"enabled": True}, False),
- ({"output_config": {"allows_na": True}}, False),
- ]
- )
- def test_native_decision_flag_off_allows_rename_and_disable(self, data: dict[str, object], valid: bool) -> None:
- evaluation = Evaluation(
- evaluation_type="llm_judge",
- evaluation_config={"prompt": "Check the response.", "judge_method": "decision"},
- enabled=data.get("enabled") is not True,
- output_type="boolean",
- output_config={},
- model_configuration=LLMModelConfiguration(provider="openai", model="gpt-6-luna"),
- )
- serializer = EvaluationSerializer(
- instance=evaluation, data=data, partial=True, context={"get_team": lambda: Mock(id=1)}
- )
- with patch(
- "products.ai_observability.backend.api.evaluations.decision_evaluations_enabled", return_value=False
- ):
- self.assertEqual(serializer.is_valid(), valid, serializer.errors)
- if valid:
- for field, value in data.items():
- self.assertEqual(serializer.validated_data[field], value)
- else:
- self.assertIn("model_configuration", serializer.errors)
-
def test_judge_method_change_validates_the_stored_model(self) -> None:
evaluation = Evaluation(
evaluation_type="llm_judge",
@@ -320,13 +286,12 @@ class TestModelConfigurationSerializer(SimpleTestCase):
"selection_mode": mode,
"options": [{"key": f"option_{i}", "label": f"Option {i}"} for i in range(count)],
}
- with patch("products.ai_observability.backend.api.evaluations.decision_evaluations_enabled", return_value=True):
- if valid:
- assert serializer.validate({"output_config": config})["output_config"]["selection_mode"] == mode
- else:
- with self.assertRaises(ValidationError) as error:
- serializer.validate({"output_config": config})
- assert "at least two options" in str(error.exception)
+ if valid:
+ assert serializer.validate({"output_config": config})["output_config"]["selection_mode"] == mode
+ else:
+ with self.assertRaises(ValidationError) as error:
+ serializer.validate({"output_config": config})
+ assert "at least two options" in str(error.exception)
@parameterized.expand([("boolean", {}), ("categorical", {}), ("numeric", {"min": 0, "max": 10})])
def test_system_one_supports_evaluation_output_types(
@@ -378,7 +343,6 @@ class TestModelConfigurationSerializer(SimpleTestCase):
"products.ai_observability.backend.llm.providers.openrouter._model_output_modalities",
return_value={"typesafe/jev-1.13": ["decisions"]},
),
- patch("products.ai_observability.backend.api.evaluations.decision_evaluations_enabled", return_value=True),
):
self.assertEqual(serializer.is_valid(), valid, serializer.errors)
if valid:
@@ -388,17 +352,15 @@ class TestModelConfigurationSerializer(SimpleTestCase):
@parameterized.expand(
[
- (True, {"name": "Updated judge"}, True),
- (True, {"enabled": False}, True),
- (True, {"output_config": {"max": None}}, False),
- (False, {"output_config": {"max": None}}, True),
- (True, {"model_configuration": {"provider": "openrouter", "model": "openai/gpt-4o"}}, False),
- (False, {"model_configuration": {"provider": "openrouter", "model": "openai/gpt-4o"}}, True),
- (True, {"enabled": True}, False),
+ ({"name": "Updated judge"}, True),
+ ({"enabled": False}, True),
+ ({"output_config": {"max": None}}, False),
+ ({"model_configuration": {"provider": "openrouter", "model": "openai/gpt-4o"}}, False),
+ ({"enabled": True}, False),
]
)
def test_openrouter_catalogue_outage_only_blocks_model_validation(
- self, flag: bool, data: dict[str, object], valid: bool
+ self, data: dict[str, object], valid: bool
) -> None:
evaluation = Evaluation(
evaluation_type="llm_judge",
@@ -415,7 +377,6 @@ class TestModelConfigurationSerializer(SimpleTestCase):
patch(
"products.ai_observability.backend.llm.providers.openrouter._model_output_modalities", return_value=None
) as catalogue,
- patch("products.ai_observability.backend.api.evaluations.decision_evaluations_enabled", return_value=flag),
):
self.assertEqual(serializer.is_valid(), valid, serializer.errors)
if valid:
@@ -1146,14 +1107,13 @@ class TestEvaluationConfigsApi(APIBaseTest):
@parameterized.expand(
[
- ("decision_model_enabled", "typesafe/jev-1.13", True, 201),
- ("decision_model_disabled", "typesafe/jev-1.13", False, 400),
- ("decision_model_alias", "~typesafe/jev-latest", True, 201),
- ("chat_model", "openai/gpt-4o", False, 201),
- ("other_non_chat_model", "example/embedding", True, 400),
+ ("decision_model", "typesafe/jev-1.13", 201),
+ ("decision_model_alias", "~typesafe/jev-latest", 201),
+ ("chat_model", "openai/gpt-4o", 201),
+ ("other_non_chat_model", "example/embedding", 400),
]
)
- def test_llm_judge_creation_checks_openrouter_model_capabilities(self, _name, model, flag, expected_status):
+ def test_llm_judge_creation_checks_openrouter_model_capabilities(self, _name, model, expected_status):
with (
patch(
"products.ai_observability.backend.llm.providers.openrouter._model_output_modalities",
@@ -1163,7 +1123,6 @@ class TestEvaluationConfigsApi(APIBaseTest):
"example/embedding": ["embeddings"],
},
),
- patch("products.ai_observability.backend.api.evaluations.decision_evaluations_enabled", return_value=flag),
):
response = self.client.post(
f"/api/environments/{self.team.id}/evaluations/",
@@ -1181,10 +1140,7 @@ class TestEvaluationConfigsApi(APIBaseTest):
self.assertEqual(response.status_code, expected_status, response.json())
if expected_status == 400:
self.assertEqual(response.data["attr"], "model_configuration")
- if flag:
- self.assertIn("Choose a chat model or a supported decision model.", response.data["detail"])
- else:
- self.assertIn("Choose a chat model.", response.data["detail"])
+ self.assertIn("Choose a chat model or a supported decision model.", response.data["detail"])
@parameterized.expand([("omitted", False), ("null", True)])
def test_llm_judge_creation_requires_model_configuration(self, _name, include_null_configuration):
@@ -2469,8 +2425,7 @@ class TestReEnableValidatesRootCauseResolved(APIBaseTest):
self.assertTrue(eval_obj.enabled)
self.assertIsNone(eval_obj.status_reason)
- @parameterized.expand([(False, 400), (True, 200)])
- def test_re_enable_openrouter_decision_model_depends_on_rollout(self, flag, expected_status):
+ def test_re_enable_openrouter_decision_model(self):
key = LLMProviderKey.objects.create(
team=self.team,
provider="openrouter",
@@ -2488,7 +2443,6 @@ class TestReEnableValidatesRootCauseResolved(APIBaseTest):
"products.ai_observability.backend.llm.providers.openrouter._model_output_modalities",
return_value={"typesafe/jev-1.13": ["decisions"]},
),
- patch("products.ai_observability.backend.api.evaluations.decision_evaluations_enabled", return_value=flag),
):
response = self.client.patch(
f"/api/environments/{self.team.id}/evaluations/{eval_obj.id}/",
@@ -2496,9 +2450,9 @@ class TestReEnableValidatesRootCauseResolved(APIBaseTest):
format="json",
)
- self.assertEqual(response.status_code, expected_status, response.json())
+ self.assertEqual(response.status_code, 200, response.json())
eval_obj.refresh_from_db()
- self.assertEqual(eval_obj.enabled, flag)
+ self.assertTrue(eval_obj.enabled)
def test_allows_re_enable_when_model_not_found_with_new_model(self):
key = LLMProviderKey.objects.create(
diff --git a/products/ai_observability/backend/api/test/test_provider_keys.py b/products/ai_observability/backend/api/test/test_provider_keys.py
index 86c4834c317b..300f4f7d16e6 100644
--- a/products/ai_observability/backend/api/test/test_provider_keys.py
+++ b/products/ai_observability/backend/api/test/test_provider_keys.py
@@ -207,50 +207,6 @@ class TestLLMProviderKeyViewSet(APIBaseTest):
self.organization_membership.level = OrganizationMembership.Level.ADMIN
self.organization_membership.save()
- @parameterized.expand([("create",), ("update",), ("validate",), ("prevalidate",)])
- def test_system_one_connection_requires_enabled_feature(self, operation: str) -> None:
- gateway_url = "https://ai-gateway.us.posthog.com/v1"
- key = LLMProviderKey.objects.create(
- team=self.team,
- provider="system_one",
- name="Example connection",
- encrypted_config={"api_key": "example-token", "base_url": gateway_url},
- created_by=self.user,
- )
- base_url = f"/api/environments/{self.team.id}/llm_analytics/"
- with (
- self.settings(POSTHOG_INTERNAL_ORG_IDS=[]),
- patch(
- "products.ai_observability.backend.llm.decisions.get_feature_flag_or_none",
- return_value=False,
- ),
- patch("httpx.AsyncHTTPTransport.handle_async_request") as request,
- ):
- if operation == "create":
- response = self.client.post(
- f"{base_url}provider_keys/",
- {
- "provider": "system_one",
- "name": "Example connection",
- "api_key": "example-token",
- "base_url": gateway_url,
- "system_one_model": "custom-model",
- },
- )
- elif operation == "update":
- response = self.client.patch(f"{base_url}provider_keys/{key.id}/", {"api_key": "new-example-token"})
- elif operation == "validate":
- response = self.client.post(f"{base_url}provider_keys/{key.id}/validate/")
- else:
- response = self.client.post(
- f"{base_url}provider_key_validations/",
- {"provider": "system_one", "api_key": "example-token", "base_url": gateway_url},
- )
- self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN, response.data)
- request.assert_not_called()
- key.refresh_from_db()
- self.assertEqual(key.encrypted_config["api_key"], "example-token")
-
def test_unauthenticated_user_cannot_access_provider_keys(self):
self.client.logout()
response = self.client.get(f"/api/environments/{self.team.id}/llm_analytics/provider_keys/")
@@ -296,8 +252,7 @@ class TestLLMProviderKeyViewSet(APIBaseTest):
@patch("posthog.security.url_validation.resolve_host_ips", return_value={ip_address("8.8.8.8")})
@patch("httpx.AsyncHTTPTransport.handle_async_request")
- @patch("products.ai_observability.backend.llm.decisions.get_feature_flag_or_none", return_value=True)
- def test_custom_system_one_connection_round_trip(self, _flag: Mock, request: Mock, _dns: Mock) -> None:
+ def test_custom_system_one_connection_round_trip(self, request: Mock, _dns: Mock) -> None:
body = json.dumps(
{
"model": "custom-model",
diff --git a/products/ai_observability/backend/api/test/test_proxy.py b/products/ai_observability/backend/api/test/test_proxy.py
index 6549a769e914..efb3fe6ebb66 100644
--- a/products/ai_observability/backend/api/test/test_proxy.py
+++ b/products/ai_observability/backend/api/test/test_proxy.py
@@ -365,8 +365,7 @@ class TestPlaygroundModelEnforcement(APIBaseTest):
returned_ids = {m["id"] for m in response.json()}
assert returned_ids == PLAYGROUND_MODEL_IDS
- @parameterized.expand([(True,), (False,)])
- def test_openai_decision_models_use_the_existing_key(self, flag: bool) -> None:
+ def test_openai_decision_models_use_the_existing_key(self) -> None:
key = LLMProviderKey.objects.create(
team=self.team,
provider="openai",
@@ -375,23 +374,17 @@ class TestPlaygroundModelEnforcement(APIBaseTest):
encrypted_config={"api_key": "example-token"},
created_by=self.user,
)
- with (
- patch("products.ai_observability.backend.api.proxy.Client.list_models", return_value=["gpt-5-mini"]),
- patch("products.ai_observability.backend.api.proxy.decision_evaluations_enabled", return_value=flag),
- ):
+ with patch("products.ai_observability.backend.api.proxy.Client.list_models", return_value=["gpt-5-mini"]):
response = self.client.get("/api/llm_proxy/models/", {"provider_key_id": str(key.id)})
assert response.status_code == 200
models = {model["id"]: model for model in response.json()}
assert models["gpt-5-mini"]["supports_decisions"] is False
assert models["gpt-5-mini"]["supports_chat"] is True
- assert ("gpt-6-luna" in models) is flag
- if flag:
- assert models["gpt-6-luna"]["supports_decisions"] is True
- assert models["gpt-6-luna"]["supports_chat"] is True
- assert models["gpt-6-luna"]["provider"] == "OpenAI"
-
- @parameterized.expand([(True,), (False,)])
- def test_openrouter_decision_models_use_the_existing_key(self, flag: bool) -> None:
+ assert models["gpt-6-luna"]["supports_decisions"] is True
+ assert models["gpt-6-luna"]["supports_chat"] is True
+ assert models["gpt-6-luna"]["provider"] == "OpenAI"
+
+ def test_openrouter_decision_models_use_the_existing_key(self) -> None:
key = LLMProviderKey.objects.create(
team=self.team,
provider="openrouter",
@@ -404,7 +397,6 @@ class TestPlaygroundModelEnforcement(APIBaseTest):
"products.ai_observability.backend.api.proxy.Client.list_models",
return_value=["typesafe/jev-router", "example/dual-model"],
),
- patch("products.ai_observability.backend.api.proxy.decision_evaluations_enabled", return_value=flag),
patch(
"products.ai_observability.backend.api.proxy.decision_model_ids",
return_value=frozenset({"typesafe/jev-1.13", "example/dual-model"}),
@@ -412,7 +404,7 @@ class TestPlaygroundModelEnforcement(APIBaseTest):
patch(
"products.ai_observability.backend.api.proxy.non_chat_model_ids",
return_value=frozenset({"typesafe/jev-1.13"}),
- ) as non_chat_models,
+ ),
):
response = self.client.get("/api/llm_proxy/models/", {"provider_key_id": str(key.id)})
@@ -421,11 +413,7 @@ class TestPlaygroundModelEnforcement(APIBaseTest):
assert models["typesafe/jev-router"]["supports_decisions"] is False
assert models["typesafe/jev-router"]["supports_chat"] is True
assert models["example/dual-model"]["supports_chat"] is True
- assert models["example/dual-model"]["supports_decisions"] is flag
- assert ("typesafe/jev-1.13" in models) is flag
- if flag:
- assert models["typesafe/jev-1.13"]["provider"] == "OpenRouter"
- assert models["typesafe/jev-1.13"]["supports_decisions"] is True
- assert models["typesafe/jev-1.13"]["supports_chat"] is False
- else:
- non_chat_models.assert_not_called()
+ assert models["example/dual-model"]["supports_decisions"] is True
+ assert models["typesafe/jev-1.13"]["provider"] == "OpenRouter"
+ assert models["typesafe/jev-1.13"]["supports_decisions"] is True
+ assert models["typesafe/jev-1.13"]["supports_chat"] is False
diff --git a/products/ai_observability/backend/llm/decisions.py b/products/ai_observability/backend/llm/decisions.py
index 8dea28310178..aecc2da91d53 100644
--- a/products/ai_observability/backend/llm/decisions.py
+++ b/products/ai_observability/backend/llm/decisions.py
@@ -13,8 +13,6 @@ from posthog.llm.system_one import (
build_system_one_body,
parse_system_one_response,
)
-from posthog.models import Team
-from posthog.ph_client import get_feature_flag_or_none
from posthog.security.pinned_requests import SSRFBlockedError
from posthog.security.url_validation import UNRESOLVED_HOST_REASON, has_authority_bypass_chars, validate_url_and_pin_ips
@@ -43,7 +41,6 @@ def is_decision_model(
provider: str | None,
model: str | None,
*,
- openrouter_enabled: bool,
decision_only: bool = False,
openai_enabled: bool = False,
) -> bool:
@@ -51,8 +48,7 @@ def is_decision_model(
return True
if provider == "openai":
return openai_enabled and model in OpenAIConfig.DECISION_MODELS
- # Disabled projects keep the chat path independent of catalogue availability.
- if provider != "openrouter" or not model or not openrouter_enabled:
+ if provider != "openrouter" or not model:
return False
models = decision_model_ids(decision_only=decision_only)
if models is None:
@@ -60,25 +56,6 @@ def is_decision_model(
return model in models
-def decision_evaluations_enabled(team_id: int, *, base_url: str) -> bool:
- try:
- host = (urlsplit(base_url).hostname or "").encode("idna").decode("ascii").lower().rstrip(".")
- except (ValueError, UnicodeError):
- return False
- if not host:
- return False
- team = Team.objects.only("uuid", "organization_id").get(id=team_id)
- return (
- get_feature_flag_or_none(
- "llm-analytics-system-one-evaluations",
- str(team.uuid),
- groups={"organization": str(team.organization_id), "project": str(team.uuid)},
- send_feature_flag_events=False,
- )
- is True
- )
-
-
class DecisionRequestRejectedError(ProviderRequestRejectedError):
pass
diff --git a/products/ai_observability/backend/llm/test/test_decisions.py b/products/ai_observability/backend/llm/test/test_decisions.py
index d707df0a2ed0..4ce2ea0164a0 100644
--- a/products/ai_observability/backend/llm/test/test_decisions.py
+++ b/products/ai_observability/backend/llm/test/test_decisions.py
@@ -3,7 +3,6 @@ from collections.abc import Iterator
from contextlib import nullcontext
from ipaddress import ip_address
from typing import Literal
-from uuid import uuid4
import pytest
from unittest.mock import patch
@@ -13,7 +12,6 @@ from django.test import override_settings
import httpx
from posthog.llm.system_one import NoulAnswer, NoulQuestion, ScoreAnswer, ScoreQuestion
-from posthog.models import Team
from products.ai_observability.backend.llm.client import Client
from products.ai_observability.backend.llm.decisions import (
@@ -21,7 +19,6 @@ from products.ai_observability.backend.llm.decisions import (
DecisionEndpointBlockedError,
DecisionRateLimitError,
DecisionRequestRejectedError,
- decision_evaluations_enabled,
)
from products.ai_observability.backend.llm.errors import (
AuthenticationError,
@@ -49,38 +46,6 @@ def public_endpoint_dns() -> Iterator[None]:
yield
- "base_url,flag,enabled",
- [
- ("https://api.typesafe.ai/v1", True, True),
- ("", True, False),
- ("https://decisions.example.com/v1", True, True),
- ("https://decisions.example.com/v1", False, False),
- ("https://decisions.example.com/v1", None, False),
- ("https://ai-gateway.us.posthog.com/v1", True, True),
- ("https://ai-gateway.eu.posthog.com/v1", True, True),
- ("https://AI-GATEWAY.US.POSTHOG.COM.:443/v1", True, True),
- ("https://ai-gateway.us.posthog.com/v1", True, True),
- ("https://ai-gateway.us.posthog.com/v1", False, False),
- ("https://ai-gateway.us.posthog.com/v1", None, False),
- ],
-)
-def test_system_one_connections_require_flag_and_supported_endpoint(
- base_url: str, flag: bool | None, enabled: bool
-) -> None:
- team = Team(id=1, organization_id=uuid4(), uuid=uuid4())
- with (
- override_settings(POSTHOG_INTERNAL_ORG_IDS=[]),
- patch("products.ai_observability.backend.llm.decisions.Team.objects.only") as teams,
- patch(
- "products.ai_observability.backend.llm.decisions.get_feature_flag_or_none",
- side_effect=lambda *args, groups, **kwargs: flag if groups["project"] == str(team.uuid) else False,
- ),
- ):
- teams.return_value.get.return_value = team
- assert decision_evaluations_enabled(team.id, base_url=base_url) is enabled
-
-
@pytest.mark.parametrize(
"status, expected_state", [(200, "ok"), (401, "invalid"), (402, "error"), (403, "invalid"), (500, "error")]
)
diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.stories.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.stories.tsx
index e5b6b372b3e6..e84ceed4588f 100644
--- a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.stories.tsx
+++ b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.stories.tsx
@@ -1,8 +1,6 @@
import type { Meta, StoryObj } from '@storybook/react'
import { BindLogic } from 'kea'
-import { FEATURE_FLAGS } from 'lib/constants'
-
import { useStorybookMocks } from '~/mocks/browser'
import { llmEvaluationLogic } from '../llmEvaluationLogic'
@@ -14,7 +12,6 @@ const meta: Meta<{ method?: JudgeMethod; narrow?: boolean; empty?: boolean; nati
component: EvaluationModelPicker,
parameters: {
layout: 'padded',
- featureFlags: [FEATURE_FLAGS.LLM_ANALYTICS_SYSTEM_ONE_EVALUATIONS],
},
render: ({ method = 'llm', narrow = false, empty = false, nativeOpenAI = false }) => {
useStorybookMocks({
@@ -107,6 +104,5 @@ export const LLM: Story = { args: { method: 'llm' } }
export const Decision: Story = { args: { method: 'decision' } }
export const DecisionNarrow: Story = { args: { method: 'decision', narrow: true } }
export const NoDecisionModels: Story = { args: { method: 'decision', empty: true } }
-export const FlagOff: Story = { parameters: { featureFlags: [] } }
export const NativeOpenAI: Story = { args: { method: 'decision', nativeOpenAI: true } }
export const NativeOpenAINarrow: Story = { args: { method: 'decision', nativeOpenAI: true, narrow: true } }
diff --git a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx
index 6b16f8d7d15f..07b4b2de9cd1 100644
--- a/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx
+++ b/products/ai_observability/frontend/evaluations/components/EvaluationModelPicker.tsx
@@ -2,9 +2,7 @@ import { useActions, useValues } from 'kea'
import { LemonInput, LemonSegmentedButton } from '@posthog/lemon-ui'
-import { FEATURE_FLAGS } from 'lib/constants'
import { LemonField } from 'lib/lemon-ui/LemonField'
-import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
import { ByokModelPickerNotice } from '../../ByokModelPickerNotice'
import { getModelPickerFooterLink, ModelPicker } from '../../ModelPicker'
@@ -25,7 +23,6 @@ export function EvaluationModelPicker(): JSX.Element {
judgeModelGroups: groups,
} = useValues(llmEvaluationLogic)
const { selectModelFromPicker, setJudgeMethod, patchOutputConfig } = useActions(llmEvaluationLogic)
- const { featureFlags } = useValues(featureFlagLogic)
// Evals always run on the team's own provider key, so only BYOK models are offered.
const selectedModelName = byokModels.find(
@@ -65,9 +62,6 @@ export function EvaluationModelPicker(): JSX.Element {
Decision model
</span>
),
- disabledReason: !featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_SYSTEM_ONE_EVALUATIONS]
- ? 'Decision model evaluations are not enabled for this project.'
- : undefined,
},
]}
/>
diff --git a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
index b8e98ba85ec0..8afe8e5f46f4 100644
--- a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
+++ b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
@@ -15,10 +15,9 @@ import {
} from '@posthog/lemon-ui'
import { RestrictionScope, useRestrictedArea } from 'lib/components/RestrictedArea'
-import { FEATURE_FLAGS, TeamMembershipLevel } from 'lib/constants'
+import { TeamMembershipLevel } from 'lib/constants'
import { IconKey } from 'lib/lemon-ui/icons'
import { LemonTableColumns } from 'lib/lemon-ui/LemonTable'
-import { featureFlagLogic } from 'lib/logic/featureFlagLogic'
import { LLMProviderIcon, LLM_PROVIDER_SELECT_OPTIONS } from '../LLMProviderIcon'
import {
@@ -240,7 +239,6 @@ function KeyValidationStatus({
}
function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }): JSX.Element {
- const { featureFlags } = useValues(featureFlagLogic)
const { systemOneBaseUrl, systemOneModel } = useValues(llmProviderKeysLogic)
const { newKeyModalOpen, providerKeysLoading, preValidationResult, preValidationResultLoading, evaluationConfig } =
useValues(llmProviderKeysLogic)
@@ -435,11 +433,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
<LemonSelect
value={provider}
onChange={handleProviderChange}
- options={LLM_PROVIDER_SELECT_OPTIONS.filter(
- ({ value }) =>
- value !== 'system_one' ||
- featureFlags[FEATURE_FLAGS.LLM_ANALYTICS_SYSTEM_ONE_EVALUATIONS] === true
- )}
+ options={LLM_PROVIDER_SELECT_OPTIONS}
className="mt-1"
fullWidth
/>
diff --git a/products/ai_observability/frontend/settings/SystemOneConnectionFields.stories.tsx b/products/ai_observability/frontend/settings/SystemOneConnectionFields.stories.tsx
index 04831a61e3cf..ba7483ff4310 100644
--- a/products/ai_observability/frontend/settings/SystemOneConnectionFields.stories.tsx
+++ b/products/ai_observability/frontend/settings/SystemOneConnectionFields.stories.tsx
@@ -1,7 +1,5 @@
import type { Meta, StoryObj } from '@storybook/react'
-import { FEATURE_FLAGS } from 'lib/constants'
-
import { useStorybookMocks } from '~/mocks/browser'
import { LLMProviderKeysSettings } from './LLMProviderKeysSettings'
@@ -27,8 +25,4 @@ export default meta
type Story = StoryObj<typeof meta>
export const Default: Story = {}
-export const ProviderSettings: Story = {
- parameters: { featureFlags: [FEATURE_FLAGS.LLM_ANALYTICS_SYSTEM_ONE_EVALUATIONS] },
- render: () => <LLMProviderKeysSettings />,
-}
-export const ProviderSettingsFlagOff: Story = { render: () => <LLMProviderKeysSettings /> }
+export const ProviderSettings: Story = { render: () => <LLMProviderKeysSettings /> }