:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(aio): support claude on microsoft foundry as an evaluation judge (#117581)
posthog[bot]1fd65699206f
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.pyindex 617c13f12afc..1ff34d898afe 100644--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py@@ -57,6 +57,7 @@ from products.ai_observability.backend.llm.decisions import ()from products.ai_observability.backend.llm.errors import (AuthenticationError,+ ClaudeDeploymentOnAzureOpenAIError,ContentFilteredError,ContextWindowExceededError,ModelNotFoundError,@@ -909,7 +910,7 @@ def call_llm_judge({"error_type": "model_not_found", "provider": provider, "model": model},non_retryable=True,)- except UnsupportedModelError:+ except UnsupportedModelError as e:# A failed chat call can populate a cold catalogue; a disabled flag must not disable the evaluation.if (provider == "openrouter"@@ -925,7 +926,9 @@ def call_llm_judge(increment_user_errors("model_not_supported", provider=provider)return terminal_user_error_result(spec=require_user_error_spec("model_not_supported", is_byok=is_byok),- message=f"Model '{model}' does not support chat completions. Choose a chat model.",+ message=str(e)+ if isinstance(e, ClaudeDeploymentOnAzureOpenAIError)+ else f"Model '{model}' does not support chat completions. Choose a chat model.",allows_na=allows_na,output_type=output_type,provider=provider,diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.pyindex 28e5b957d4aa..87e5a12f5402 100644--- a/posthog/temporal/ai_observability/test_run_evaluation.py+++ b/posthog/temporal/ai_observability/test_run_evaluation.py@@ -35,6 +35,7 @@ from posthog.temporal.common.errors import NonReportableErrorfrom products.access_control.backend.models.access_control import AccessControlfrom products.ai_observability.backend.llm.errors import (AuthenticationError,+ ClaudeDeploymentOnAzureOpenAIError,ContentFilteredError,ContextWindowExceededError,ModelNotFoundError,@@ -3393,6 +3394,13 @@ class TestRunEvaluationWorkflow:None,id="model_not_supported",),+ pytest.param(+ ClaudeDeploymentOnAzureOpenAIError("gpt-5.4"),+ "model_not_supported",+ "model_not_supported",+ None,+ id="claude_deployment_on_azure_openai",+ ),],)@pytest.mark.django_db(transaction=True)@@ -3451,6 +3459,8 @@ class TestRunEvaluationWorkflow:assert result["key_id"] == str(provider_key.id)assert result["provider"] == "openai"assert result["model"] == "gpt-5.4"+ if isinstance(raised_exception, ClaudeDeploymentOnAzureOpenAIError):+ assert result["reasoning"] == str(raised_exception)if provider_key_state is None:assert "provider_key_state" not in resultelse:diff --git a/products/ai_observability/backend/api/provider_keys.py b/products/ai_observability/backend/api/provider_keys.pyindex be10f7df3234..26f9562fde06 100644--- a/products/ai_observability/backend/api/provider_keys.py+++ b/products/ai_observability/backend/api/provider_keys.py@@ -26,6 +26,10 @@ from products.access_control.backend.presentation.access_control import AccessCofrom ..llm.client import Clientfrom ..llm.decisions import DecisionClient, decision_evaluations_enabled+from ..llm.providers.anthropic import (+ DISALLOWED_FOUNDRY_ENDPOINT_MESSAGE,+ error_field_for_validation_message as anthropic_error_field,+)from ..llm.providers.azure_openai import (DEFAULT_API_VERSION,DISALLOWED_ENDPOINT_MESSAGE,@@ -91,13 +95,15 @@ def validate_provider_key(provider: str, api_key: str, *, team_id: int, **kwargsdef _validation_error_field(provider: str, error_message: str | None) -> str | None:"""Pick the form field a validation error belongs to, or None when it is unattributed.- Azure OpenAI and OpenAI-compatible providers may fail because of endpoint issues (unreachable,- wrong domain, 404), in which case the error is attributed to the endpoint field so the UI can- highlight the right input. Callers that must land the error on some field fall back to+ Azure OpenAI, Anthropic on Microsoft Foundry, and OpenAI-compatible providers may fail because+ of endpoint issues (unreachable, wrong domain, 404), in which case the error is attributed to the+ endpoint field so the UI can highlight the right input. Callers that must land the error on some field fall back to`api_key`; the pre-validation endpoint reports the field separately and keeps the None."""if provider == LLMProvider.AZURE_OPENAI:return azure_error_field(error_message)+ if provider == LLMProvider.ANTHROPIC:+ return anthropic_error_field(error_message)if provider == LLMProvider.OPENAI_COMPATIBLE:return openai_compatible_error_field(error_message)return None@@ -107,6 +113,7 @@ def _validation_error_field(provider: str, error_message: str | None) -> str | N# Fields for other providers are dropped on write so a stray value never persists._PROVIDER_CONFIG_FIELDS: dict[str, tuple[str, ...]] = {LLMProvider.AZURE_OPENAI: ("azure_endpoint", "api_version"),+ LLMProvider.ANTHROPIC: ("azure_endpoint",),LLMProvider.OPENAI_COMPATIBLE: ("base_url",),}_ALL_PROVIDER_CONFIG_FIELDS: tuple[str, ...] = tuple(@@ -124,7 +131,13 @@ class LLMProviderKeySerializer(serializers.ModelSerializer):)system_one_model_display = serializers.SerializerMethodField(help_text="Configured System One model ID.")api_key_masked = serializers.SerializerMethodField()- azure_endpoint = serializers.URLField(write_only=True, required=False, help_text="Azure OpenAI endpoint URL")+ azure_endpoint = serializers.URLField(+ write_only=True,+ required=False,+ help_text="Azure OpenAI endpoint URL. For an Anthropic key, the endpoint of the Microsoft Foundry "+ "resource that serves Claude, e.g. https://my-resource.services.ai.azure.com/. "+ "Omit it for a key from the Anthropic API.",+ )api_version = serializers.CharField(write_only=True, required=False, max_length=20, help_text="Azure OpenAI API version")@@ -174,7 +187,7 @@ class LLMProviderKeySerializer(serializers.ModelSerializer):return obj.encrypted_config.get("model") if obj.provider == LLMProvider.SYSTEM_ONE else Nonedef get_azure_endpoint_display(self, obj: LLMProviderKey) -> str | None:- if obj.provider != LLMProvider.AZURE_OPENAI:+ if obj.provider not in (LLMProvider.AZURE_OPENAI, LLMProvider.ANTHROPIC):return Nonereturn obj.encrypted_config.get("azure_endpoint")@@ -216,14 +229,19 @@ class LLMProviderKeySerializer(serializers.ModelSerializer):raise serializers.ValidationError("Invalid OpenAI API key format. Key should start with 'sk-' or 'sk-proj-'.")- elif provider == LLMProvider.ANTHROPIC:+ elif provider == LLMProvider.ANTHROPIC and not self._anthropic_uses_foundry():if not value.startswith("sk-ant-"):raise serializers.ValidationError("Invalid Anthropic API key format. Key should start with 'sk-ant-'.")- # Azure, Gemini, Together AI, OpenRouter, Fireworks, MiniMax, and OpenAI-compatible keys- # have no standard prefix, so no format validation is enforced here.+ # Azure, Microsoft Foundry, Gemini, Together AI, OpenRouter, Fireworks, MiniMax, and+ # OpenAI-compatible keys have no standard prefix, so no format validation is enforced here.return value+ def _anthropic_uses_foundry(self) -> bool:+ if self.initial_data.get("azure_endpoint"):+ return True+ return bool(self.instance and self.instance.encrypted_config.get("azure_endpoint"))+def validate(self, data):if self.instance is None and "api_key" not in data:raise serializers.ValidationError({"api_key": "API key is required when creating a new provider key."})@@ -263,6 +281,17 @@ class LLMProviderKeySerializer(serializers.ModelSerializer):if has_endpoint and not is_allowed_azure_endpoint(data["azure_endpoint"]):raise serializers.ValidationError({"azure_endpoint": f"{DISALLOWED_ENDPOINT_MESSAGE}."})+ if provider == LLMProvider.ANTHROPIC and data.get("azure_endpoint"):+ if not is_allowed_azure_endpoint(data["azure_endpoint"]):+ raise serializers.ValidationError({"azure_endpoint": f"{DISALLOWED_FOUNDRY_ENDPOINT_MESSAGE}."})+ # Same rule as the OpenAI-compatible base URL below: a stored key must never be sent+ # to a new endpoint unless the caller proves they hold it.+ current_endpoint = self.instance.encrypted_config.get("azure_endpoint") if self.instance else None+ if self.instance and data["azure_endpoint"] != current_endpoint and not data.get("api_key"):+ raise serializers.ValidationError(+ {"api_key": "Enter the API key again when you change the Microsoft Foundry endpoint."}+ )+if provider == LLMProvider.OPENAI_COMPATIBLE:has_base_url = bool(data.get("base_url"))has_existing_base_url = self.instance and self.instance.encrypted_config.get("base_url")@@ -395,6 +424,8 @@ class LLMProviderKeySerializer(serializers.ModelSerializer):encrypted_config["api_version"] = extra_kwargs.get("api_version", "")elif instance.provider == LLMProvider.OPENAI_COMPATIBLE:encrypted_config["base_url"] = extra_kwargs.get("base_url", "")+ elif instance.provider == LLMProvider.ANTHROPIC and extra_kwargs.get("azure_endpoint"):+ encrypted_config["azure_endpoint"] = extra_kwargs["azure_endpoint"]instance.encrypted_config = encrypted_configinstance.state = stateinstance.error_message = None@@ -410,9 +441,9 @@ class LLMProviderKeySerializer(serializers.ModelSerializer):instance.encrypted_config = configinstance.state = LLMProviderKey.State.UNKNOWNinstance.error_message = None- # No OpenAI-compatible branch here on purpose: `validate` rejects a base URL change- # that arrives without an API key, so every change to one runs through the branch- # above and is re-validated against the new endpoint before it is stored.+ # No OpenAI-compatible or Anthropic branch here on purpose: `validate` rejects a base URL+ # or Foundry endpoint change that arrives without an API key, so every change to one runs+ # through the branch above and is re-validated against the new endpoint before it is stored.return super().update(instance, validated_data)@@ -700,7 +731,11 @@ class LLMProviderKeyValidationViewSet(TeamAndOrgViewSetMixin, viewsets.ViewSet):)extra_kwargs = {}- if provider == LLMProvider.AZURE_OPENAI:+ if provider == LLMProvider.ANTHROPIC:+ azure_endpoint = request.data.get("azure_endpoint")+ if azure_endpoint:+ extra_kwargs["azure_endpoint"] = azure_endpoint+ elif provider == LLMProvider.AZURE_OPENAI:azure_endpoint = request.data.get("azure_endpoint")api_version = request.data.get("api_version")if azure_endpoint:diff --git a/products/ai_observability/backend/api/test/test_provider_keys.py b/products/ai_observability/backend/api/test/test_provider_keys.pyindex b07c60abaac5..775ae76fa0ac 100644--- a/products/ai_observability/backend/api/test/test_provider_keys.py+++ b/products/ai_observability/backend/api/test/test_provider_keys.py@@ -114,6 +114,32 @@ class TestProviderKeySerializer(SimpleTestCase):serializer = LLMProviderKeySerializer(key, data={"base_url": base_url}, partial=True)self.assertEqual(serializer.is_valid(), valid, serializer.errors)+ @parameterized.expand(+ [+ ("foundry_key_without_prefix", {"azure_endpoint": "https://contoso.services.ai.azure.com/"}, None),+ ("api_key_without_prefix", {}, "api_key"),+ ("non_azure_endpoint", {"azure_endpoint": "https://evil.example.com/"}, "azure_endpoint"),+ ]+ )+ def test_anthropic_key_validation(self, _name: str, extra: dict, error_field: str | None) -> None:+ serializer = LLMProviderKeySerializer(+ data={"provider": "anthropic", "name": "Claude", "api_key": "no-prefix-key", **extra}+ )+ self.assertEqual(serializer.is_valid(), error_field is None, serializer.errors)+ if error_field:+ self.assertIn(error_field, serializer.errors)++ @parameterized.expand([({}, False), ({"api_key": "no-prefix-key"}, True)])+ def test_foundry_endpoint_change_requires_the_api_key(self, extra: dict, valid: bool) -> None:+ key = LLMProviderKey(+ provider="anthropic",+ encrypted_config={"api_key": "foundry-key", "azure_endpoint": "https://old.services.ai.azure.com/"},+ )+ serializer = LLMProviderKeySerializer(+ key, data={"azure_endpoint": "https://new.services.ai.azure.com/", **extra}, partial=True+ )+ self.assertEqual(serializer.is_valid(), valid, serializer.errors)+def _setup_team():org = Organization.objects.create(name="test")@@ -770,6 +796,31 @@ class TestLLMProviderKeyViewSet(APIBaseTest):api_version="2024-10-21",)+ @patch("products.ai_observability.backend.api.provider_keys.validate_provider_key")+ def test_can_create_anthropic_key_on_microsoft_foundry(self, mock_validate):+ mock_validate.return_value = (LLMProviderKey.State.OK, None)++ response = self.client.post(+ f"/api/environments/{self.team.id}/llm_analytics/provider_keys/",+ {+ "provider": "anthropic",+ "name": "Foundry Claude",+ "api_key": "foundry-hex-123",+ "azure_endpoint": "https://contoso.services.ai.azure.com/",+ },+ )+ self.assertEqual(response.status_code, status.HTTP_201_CREATED)+ self.assertEqual(response.json()["azure_endpoint_display"], "https://contoso.services.ai.azure.com/")++ key = LLMProviderKey.objects.get(id=response.json()["id"])+ self.assertEqual(key.provider_extra_kwargs(), {"azure_endpoint": "https://contoso.services.ai.azure.com/"})+ mock_validate.assert_called_once_with(+ "anthropic",+ "foundry-hex-123",+ team_id=self.team.id,+ azure_endpoint="https://contoso.services.ai.azure.com/",+ )+def test_create_azure_openai_without_endpoint_fails(self):response = self.client.post(f"/api/environments/{self.team.id}/llm_analytics/provider_keys/",diff --git a/products/ai_observability/backend/llm/client.py b/products/ai_observability/backend/llm/client.pyindex 87e677d0edc9..6c6ea8313dd5 100644--- a/products/ai_observability/backend/llm/client.py+++ b/products/ai_observability/backend/llm/client.py@@ -102,7 +102,7 @@ class Client:def _get_provider(name: str, provider_key: "LLMProviderKey | None" = None) -> "Provider":- """Get provider by name. Azure and OpenAI-compatible read extra config from provider_key."""+ """Get provider by name. Anthropic, Azure and OpenAI-compatible read extra config from provider_key."""from typing import castfrom products.ai_observability.backend.llm.providers.anthropic import AnthropicAdapter@@ -119,7 +119,8 @@ def _get_provider(name: str, provider_key: "LLMProviderKey | None" = None) -> "Pcase "openai":return cast("Provider", OpenAIAdapter())case "anthropic":- return cast("Provider", AnthropicAdapter())+ config = provider_key.encrypted_config if provider_key else {}+ return cast("Provider", AnthropicAdapter(foundry_endpoint=config.get("azure_endpoint", "")))case "gemini":return cast("Provider", GeminiAdapter())case "together_ai":diff --git a/products/ai_observability/backend/llm/errors.py b/products/ai_observability/backend/llm/errors.pyindex cf561e350074..49713723e7ee 100644--- a/products/ai_observability/backend/llm/errors.py+++ b/products/ai_observability/backend/llm/errors.py@@ -18,6 +18,18 @@ class UnsupportedModelError(LLMError):super().__init__(f"Unsupported model: {model}")+class ClaudeDeploymentOnAzureOpenAIError(UnsupportedModelError):+ """Raised when an Azure OpenAI key names a Claude deployment, which only the Anthropic API serves."""++ def __init__(self, model: str):+ self.model = model+ LLMError.__init__(+ self,+ f"'{model}' looks like a Claude deployment. Azure OpenAI keys can't call Claude models. "+ "Add an Anthropic key with your Microsoft Foundry endpoint, then pick that key for this model.",+ )++class UnsupportedProviderError(LLMError):"""Raised when an unsupported provider is requested"""@@ -201,6 +213,8 @@ def user_facing_error_message(error: Exception | None) -> str:"""if isinstance(error, ModelNotFoundError):return f"Model '{error.model}' is not available. Pick a different model and try again."+ if isinstance(error, ClaudeDeploymentOnAzureOpenAIError):+ return str(error)if isinstance(error, UnsupportedModelError):return f"Model '{error.model}' is not supported. Pick a different model and try again."if isinstance(error, ModelPermissionError):diff --git a/products/ai_observability/backend/llm/providers/anthropic.py b/products/ai_observability/backend/llm/providers/anthropic.pyindex ed2fee95419b..83bec9ae74a7 100644--- a/products/ai_observability/backend/llm/providers/anthropic.py+++ b/products/ai_observability/backend/llm/providers/anthropic.py@@ -4,6 +4,7 @@ import uuidimport loggingfrom collections.abc import Generatorfrom typing import Any+from urllib.parse import urlparsefrom django.conf import settings@@ -12,6 +13,8 @@ import posthoganalyticsfrom anthropic.lib._parse._transform import transform_schemafrom anthropic.types import MessageParam, TextBlockParam, ThinkingConfigEnabledParamfrom posthoganalytics.ai.anthropic import Anthropic+from posthoganalytics.ai.anthropic.anthropic import WrappedMessages+from posthoganalytics.client import Client as PostHogClientfrom pydantic import BaseModelfrom products.ai_observability.backend.llm.errors import (@@ -21,14 +24,17 @@ from products.ai_observability.backend.llm.errors import (ModelNotFoundError,ModelPermissionError,OutputTokenLimitError,+ ProviderConfigurationError,ProviderConnectionError,QuotaExceededError,RateLimitError,StructuredOutputParseError,+ error_field_for_message,is_context_window_error_message,is_output_limit_error_message,stream_error_chunk,)+from products.ai_observability.backend.llm.providers.azure_openai import is_allowed_azure_endpointfrom products.ai_observability.backend.llm.types import (AnalyticsContext,CompletionRequest,@@ -39,6 +45,70 @@ from products.ai_observability.backend.llm.types import (logger = logging.getLogger(__name__)+# Shared by the adapter and the serializer so the error attributes to the endpoint field in the UI.+DISALLOWED_FOUNDRY_ENDPOINT_MESSAGE = (+ "Microsoft Foundry endpoint must be an https:// URL on an Azure domain (e.g. *.services.ai.azure.com)"+)++FOUNDRY_ENDPOINT_NOT_FOUND_MESSAGE = (+ "Microsoft Foundry endpoint not found. Check that the URL is the endpoint of your Foundry resource, "+ "for example https://my-resource.services.ai.azure.com/"+)++# Keep aligned with the `return` statements in `AnthropicAdapter._validate_foundry_key`.+_FOUNDRY_ERROR_FIELD_BY_PREFIX: tuple[tuple[str, str], ...] = (+ ("Microsoft Foundry endpoint must be", "azure_endpoint"),+ ("Microsoft Foundry endpoint not found", "azure_endpoint"),+ ("Could not connect to Microsoft Foundry", "azure_endpoint"),+ ("Invalid API key", "api_key"),+)++# Key validation is a single cheap call; don't inherit the long completion timeout.+FOUNDRY_VALIDATION_TIMEOUT = 10.0+++def error_field_for_validation_message(error_message: str | None) -> str | None:+ """Map an Anthropic `validate_key` error message to the UI form field that should be highlighted."""+ return error_field_for_message(_FOUNDRY_ERROR_FIELD_BY_PREFIX, error_message)+++def _names_a_missing_deployment(error: anthropic.APIStatusError) -> bool:+ """Return True when a Foundry 4xx is about the model or deployment, which means the key passed auth.++ Azure answers an unknown route with a generic 404 ("Resource not found") before it checks+ anything, so a 404 alone does not prove the endpoint or the key is right.+ """+ body = error.body if isinstance(error.body, dict) else {}+ nested = body.get("error")+ details = nested if isinstance(nested, dict) else body+ if details.get("code") == "DeploymentNotFound":+ return True+ message = str(details.get("message", "")).lower()+ return "deployment" in message or "model" in message+++def foundry_base_url(azure_endpoint: str) -> str:+ """Build the Anthropic API base URL of a Microsoft Foundry resource.++ Foundry serves Claude under `/anthropic/` on the resource host. Users paste the resource+ endpoint, the `/anthropic/` URL, or a full `/anthropic/v1/messages` URL, so keep only the host.+ """+ if not is_allowed_azure_endpoint(azure_endpoint):+ raise ProviderConfigurationError(f"{DISALLOWED_FOUNDRY_ENDPOINT_MESSAGE}.")+ host = urlparse(azure_endpoint).hostname+ return f"https://{host}/anthropic/"+++class _AnthropicFoundry(anthropic.AnthropicFoundry):+ """Foundry client that captures generations to PostHog, like the SDK's Bedrock and Vertex wrappers."""++ _ph_client: PostHogClient++ def __init__(self, posthog_client: PostHogClient, **kwargs: Any):+ super().__init__(**kwargs)+ self._ph_client = posthog_client+ self.messages = WrappedMessages(self) # type: ignore[assignment] # ty: ignore[invalid-assignment]+def _is_quota_or_billing_error(error: Exception) -> bool:error_message = str(error).lower()@@ -106,6 +176,28 @@ class AnthropicAdapter:name = "anthropic"+ def __init__(self, foundry_endpoint: str = ""):+ # Set when the key belongs to a Microsoft Foundry resource that serves Claude.+ self.foundry_endpoint = foundry_endpoint++ def _create_client(self, api_key: str, analytics: AnalyticsContext) -> Any:+ from products.ai_observability.backend.llm.providers._diagnostics import tagged_http_client++ posthog_client = posthoganalytics.default_client+ kwargs: dict[str, Any] = {+ "api_key": api_key,+ "timeout": AnthropicConfig.TIMEOUT,+ "http_client": tagged_http_client(timeout=AnthropicConfig.TIMEOUT),+ }+ if self.foundry_endpoint:+ kwargs["base_url"] = foundry_base_url(self.foundry_endpoint)+ if analytics.capture and posthog_client:+ return _AnthropicFoundry(posthog_client=posthog_client, **kwargs)+ return anthropic.AnthropicFoundry(**kwargs)+ if analytics.capture and posthog_client:+ return Anthropic(posthog_client=posthog_client, **kwargs)+ return anthropic.Anthropic(**kwargs)+def complete(self,request: CompletionRequest,@@ -115,25 +207,7 @@ class AnthropicAdapter:) -> CompletionResponse:"""Non-streaming completion with optional structured output."""effective_api_key = api_key or self._get_default_api_key()-- from products.ai_observability.backend.llm.providers._diagnostics import tagged_http_client-- posthog_client = posthoganalytics.default_client- client: Any- http_client = tagged_http_client(timeout=AnthropicConfig.TIMEOUT)- if analytics.capture and posthog_client:- client = Anthropic(- api_key=effective_api_key,- posthog_client=posthog_client,- timeout=AnthropicConfig.TIMEOUT,- http_client=http_client,- )- else:- client = anthropic.Anthropic(- api_key=effective_api_key,- timeout=AnthropicConfig.TIMEOUT,- http_client=http_client,- )+ client = self._create_client(effective_api_key, analytics)messages: Any = request.messagessystem_prompt = request.system or ""@@ -242,25 +316,7 @@ class AnthropicAdapter:"""Streaming completion."""effective_api_key = api_key or self._get_default_api_key()model_id = request.model-- from products.ai_observability.backend.llm.providers._diagnostics import tagged_http_client-- posthog_client = posthoganalytics.default_client- client: Any- http_client = tagged_http_client(timeout=AnthropicConfig.TIMEOUT)- if analytics.capture and posthog_client:- client = Anthropic(- api_key=effective_api_key,- posthog_client=posthog_client,- timeout=AnthropicConfig.TIMEOUT,- http_client=http_client,- )- else:- client = anthropic.Anthropic(- api_key=effective_api_key,- timeout=AnthropicConfig.TIMEOUT,- http_client=http_client,- )+ client = self._create_client(effective_api_key, analytics)reasoning_on = model_id in AnthropicConfig.SUPPORTED_MODELS_WITH_THINKING and request.thinking@@ -384,6 +440,10 @@ class AnthropicAdapter:"""Validate an Anthropic API key by making a lightweight API call."""from products.ai_observability.backend.models.provider_keys import LLMProviderKey+ azure_endpoint = kwargs.get("azure_endpoint")+ if azure_endpoint:+ return AnthropicAdapter._validate_foundry_key(api_key, azure_endpoint)+if not api_key.startswith("sk-ant-"):return (LLMProviderKey.State.INVALID, "Invalid key format (should start with 'sk-ant-')")try:@@ -404,6 +464,48 @@ class AnthropicAdapter:logger.exception(f"Anthropic key validation error: {e}")return (LLMProviderKey.State.ERROR, "Validation failed, please try again")+ @staticmethod+ def _validate_foundry_key(api_key: str, azure_endpoint: str) -> tuple[str, str | None]:+ """Validate a Microsoft Foundry key with a token count, which Foundry serves and does not bill.++ Foundry has no models endpoint, so the probe has to name a deployment. The default model ID+ is the default deployment name, but the user may have picked another name.+ """+ from products.ai_observability.backend.models.provider_keys import LLMProviderKey++ if not is_allowed_azure_endpoint(azure_endpoint):+ return (LLMProviderKey.State.INVALID, DISALLOWED_FOUNDRY_ENDPOINT_MESSAGE)+ try:+ client = anthropic.AnthropicFoundry(+ api_key=api_key,+ base_url=foundry_base_url(azure_endpoint),+ timeout=FOUNDRY_VALIDATION_TIMEOUT,+ max_retries=0,+ )+ client.messages.count_tokens(+ model=AnthropicConfig.DEFAULT_MODEL,+ messages=[{"role": "user", "content": "hi"}],+ )+ return (LLMProviderKey.State.OK, None)+ except (anthropic.AuthenticationError, anthropic.PermissionDeniedError):+ return (LLMProviderKey.State.INVALID, "Invalid API key")+ except anthropic.RateLimitError:+ return (LLMProviderKey.State.ERROR, "Rate limited, please try again later")+ except (anthropic.NotFoundError, anthropic.BadRequestError) as e:+ # The user picks the deployment later, in the model picker, so a missing default+ # deployment still counts as a working key.+ if _names_a_missing_deployment(e):+ return (LLMProviderKey.State.OK, None)+ if isinstance(e, anthropic.NotFoundError):+ return (LLMProviderKey.State.INVALID, FOUNDRY_ENDPOINT_NOT_FOUND_MESSAGE)+ logger.exception("Microsoft Foundry key validation error")+ return (LLMProviderKey.State.ERROR, "Validation failed, please try again")+ except anthropic.APIConnectionError:+ return (LLMProviderKey.State.ERROR, "Could not connect to Microsoft Foundry endpoint")+ except Exception:+ logger.exception("Microsoft Foundry key validation error")+ return (LLMProviderKey.State.ERROR, "Validation failed, please try again")+@staticmethoddef recommended_models() -> set[str]:return set(AnthropicConfig.SUPPORTED_MODELS)@@ -415,8 +517,10 @@ class AnthropicAdapter:Without a key, returns the curated SUPPORTED_MODELS list.With a key, returns SUPPORTED_MODELS first, then remaining Claude modelsfrom the API sorted by creation date (newest first).+ With a Microsoft Foundry endpoint, returns SUPPORTED_MODELS: Foundry has no models+ endpoint, and a new deployment takes the model ID as its default name."""- if not api_key:+ if not api_key or kwargs.get("azure_endpoint"):return AnthropicConfig.SUPPORTED_MODELSsupported = set(AnthropicConfig.SUPPORTED_MODELS)@@ -448,7 +552,7 @@ class AnthropicAdapter:def _build_analytics_kwargs(self, analytics: AnalyticsContext, client) -> dict:"""Build PostHog analytics kwargs if using instrumented client."""- if analytics.capture and isinstance(client, Anthropic):+ if analytics.capture and isinstance(client, Anthropic | _AnthropicFoundry):return {"posthog_distinct_id": analytics.distinct_id,"posthog_trace_id": analytics.trace_id or str(uuid.uuid4()),diff --git a/products/ai_observability/backend/llm/providers/azure_openai.py b/products/ai_observability/backend/llm/providers/azure_openai.pyindex c25bf8c98691..8c2db4402356 100644--- a/products/ai_observability/backend/llm/providers/azure_openai.py+++ b/products/ai_observability/backend/llm/providers/azure_openai.py@@ -15,7 +15,12 @@ from posthoganalytics.ai.openai import AzureOpenAI as WrappedAzureOpenAIfrom posthog.exceptions_capture import capture_exception-from products.ai_observability.backend.llm.errors import error_field_for_message+from products.ai_observability.backend.llm.errors import (+ ClaudeDeploymentOnAzureOpenAIError,+ LLMError,+ ModelNotFoundError,+ error_field_for_message,+)from products.ai_observability.backend.llm.providers.openai import OpenAIAdapterfrom products.ai_observability.backend.llm.types import AnalyticsContext@@ -159,6 +164,17 @@ class AzureOpenAIAdapter(OpenAIAdapter):http_client=http_client,)+ def _mapped_error(self, error: Exception, model: str) -> LLMError | None:+ mapped = super()._mapped_error(error, model)+ # A Foundry resource can host Claude next to OpenAI models, but Claude only answers on the+ # Anthropic API. The chat completions call then fails with a 404 or 400 that names neither+ # cause nor fix, so point the user at the Anthropic key instead.+ if "claude" in model.lower() and (+ isinstance(mapped, ModelNotFoundError) or (mapped is None and isinstance(error, openai.BadRequestError))+ ):+ return ClaudeDeploymentOnAzureOpenAIError(model)+ return mapped+@staticmethoddef validate_key(api_key: str, **kwargs: Any) -> tuple[str, str | None]:"""Validate an Azure OpenAI API key by listing deployments."""diff --git a/products/ai_observability/backend/llm/providers/test/test_anthropic.py b/products/ai_observability/backend/llm/providers/test/test_anthropic.pyindex a92ae874dd64..656bb0399ad2 100644--- a/products/ai_observability/backend/llm/providers/test/test_anthropic.py+++ b/products/ai_observability/backend/llm/providers/test/test_anthropic.py@@ -10,8 +10,13 @@ from products.ai_observability.backend.llm.errors import (ModelNotFoundError,ModelPermissionError,OutputTokenLimitError,+ ProviderConfigurationError,+)+from products.ai_observability.backend.llm.providers.anthropic import (+ AnthropicAdapter,+ AnthropicConfig,+ error_field_for_validation_message,)-from products.ai_observability.backend.llm.providers.anthropic import AnthropicAdapter, AnthropicConfigfrom products.ai_observability.backend.llm.types import AnalyticsContext, CompletionRequest, OutputSchema@@ -358,3 +363,133 @@ class TestAnthropicStreamErrorSurfacing:errors = [chunk.data["error"] for chunk in chunks if chunk.type == "error"]assert errors == ["The provider is rate limiting this key. Wait a moment, then try again."]+++def _make_status_error(+ error_class: type[anthropic.APIStatusError], status_code: int, body: dict | None = None+) -> anthropic.APIStatusError:+ body = body or {"error": {"message": "error"}}+ request = httpx.Request("POST", "https://my-resource.services.ai.azure.com/anthropic/v1/messages/count_tokens")+ response = httpx.Response(status_code=status_code, request=request, json=body)+ return error_class("error", response=response, body=body)+++class TestAnthropicFoundry:+ @parameterized.expand(+ [+ ("resource_endpoint", "https://my-resource.services.ai.azure.com/"),+ ("anthropic_base_url", "https://my-resource.services.ai.azure.com/anthropic/"),+ ("messages_url", "https://my-resource.services.ai.azure.com/anthropic/v1/messages"),+ ]+ )+ def test_completion_goes_to_the_foundry_anthropic_api(self, _name: str, endpoint: str):+ mock_response = MagicMock()+ mock_response.content = [MagicMock(text="yes")]+ mock_response.usage.input_tokens = 1+ mock_response.usage.output_tokens = 1++ with (+ patch("products.ai_observability.backend.llm.providers.anthropic.anthropic.AnthropicFoundry") as foundry,+ patch("products.ai_observability.backend.llm.providers.anthropic.anthropic.Anthropic") as direct,+ ):+ foundry.return_value.messages.create.return_value = mock_response+ AnthropicAdapter(foundry_endpoint=endpoint).complete(+ CompletionRequest(+ model="my-claude-deployment",+ messages=[{"role": "user", "content": "hi"}],+ provider="anthropic",+ ),+ api_key="foundry-key",+ analytics=AnalyticsContext(capture=False),+ )++ direct.assert_not_called()+ assert foundry.call_args.kwargs["api_key"] == "foundry-key"+ assert foundry.call_args.kwargs["base_url"] == "https://my-resource.services.ai.azure.com/anthropic/"+ assert foundry.return_value.messages.create.call_args.kwargs["model"] == "my-claude-deployment"++ @patch("products.ai_observability.backend.llm.providers.anthropic.anthropic.AnthropicFoundry")+ def test_completion_refuses_a_non_azure_endpoint(self, foundry):+ with pytest.raises(ProviderConfigurationError):+ AnthropicAdapter(foundry_endpoint="https://attacker.example.com/").complete(+ CompletionRequest(model="claude-haiku-4-5", messages=[], provider="anthropic"),+ api_key="foundry-key",+ analytics=AnalyticsContext(capture=False),+ )+ foundry.assert_not_called()++ @parameterized.expand(+ [+ ("accepted", None, "ok", None),+ (+ "default_deployment_missing",+ _make_status_error(+ anthropic.NotFoundError,+ 404,+ {"type": "error", "error": {"type": "not_found_error", "message": "model: claude-haiku-4-5"}},+ ),+ "ok",+ None,+ ),+ (+ "azure_deployment_not_found",+ _make_status_error(+ anthropic.NotFoundError,+ 404,+ {"error": {"code": "DeploymentNotFound", "message": "The API deployment does not exist."}},+ ),+ "ok",+ None,+ ),+ (+ "invalid_model_parameter",+ _make_status_error(anthropic.BadRequestError, 400, {"error": {"message": "Invalid model parameter"}}),+ "ok",+ None,+ ),+ (+ "unknown_route",+ _make_status_error(+ anthropic.NotFoundError, 404, {"error": {"code": "404", "message": "Resource not found"}}+ ),+ "invalid",+ "azure_endpoint",+ ),+ ("rejected_key", _make_status_error(anthropic.AuthenticationError, 401), "invalid", "api_key"),+ ("forbidden_key", _make_status_error(anthropic.PermissionDeniedError, 403), "invalid", "api_key"),+ (+ "unreachable",+ anthropic.APIConnectionError(request=httpx.Request("POST", "https://example.com")),+ "error",+ "azure_endpoint",+ ),+ ]+ )+ @patch("products.ai_observability.backend.llm.providers.anthropic.anthropic.AnthropicFoundry")+ def test_validate_key(self, _name, error, expected_state, expected_field, foundry):+ foundry.return_value.messages.count_tokens.side_effect = error++ state, message = AnthropicAdapter.validate_key(+ "no-sk-ant-prefix", azure_endpoint="https://my-resource.services.ai.azure.com/"+ )++ assert state == expected_state+ assert error_field_for_validation_message(message) == expected_field+ assert foundry.call_args.kwargs["base_url"] == "https://my-resource.services.ai.azure.com/anthropic/"++ @patch("products.ai_observability.backend.llm.providers.anthropic.anthropic.AnthropicFoundry")+ def test_validate_key_never_sends_the_key_to_a_non_azure_endpoint(self, foundry):+ state, message = AnthropicAdapter.validate_key("foundry-key", azure_endpoint="https://attacker.example.com/")++ foundry.assert_not_called()+ assert state == "invalid"+ assert error_field_for_validation_message(message) == "azure_endpoint"++ @patch("products.ai_observability.backend.llm.providers.anthropic.anthropic.Anthropic")+ def test_list_models_does_not_call_the_anthropic_api(self, direct):+ models = AnthropicAdapter.list_models(+ "foundry-key", azure_endpoint="https://my-resource.services.ai.azure.com/"+ )++ direct.assert_not_called()+ assert models == AnthropicConfig.SUPPORTED_MODELSdiff --git a/products/ai_observability/backend/llm/providers/test/test_azure_openai.py b/products/ai_observability/backend/llm/providers/test/test_azure_openai.pyindex c485bfd67d9e..040b2b4bf422 100644--- a/products/ai_observability/backend/llm/providers/test/test_azure_openai.py+++ b/products/ai_observability/backend/llm/providers/test/test_azure_openai.py@@ -2,8 +2,15 @@ import pytestfrom unittest.mock import MagicMock, patchimport httpx+import openaifrom parameterized import parameterized+from products.ai_observability.backend.llm.errors import (+ AuthenticationError,+ ClaudeDeploymentOnAzureOpenAIError,+ ModelNotFoundError,+ user_facing_error_message,+)from products.ai_observability.backend.llm.providers.azure_openai import (DEPLOYMENTS_LIST_API_VERSION,AzureOpenAIAdapter,@@ -315,3 +322,28 @@ class TestAzureOpenAICreateClient:assert mock_wrapped.call_args.kwargs["azure_endpoint"] == MOCK_ENDPOINTassert mock_wrapped.call_args.kwargs["timeout"] == 12.0assert mock_wrapped.call_args.kwargs["max_retries"] == 0+++def _make_openai_status_error(error_class: type[openai.APIStatusError], status_code: int) -> openai.APIStatusError:+ request = httpx.Request("POST", f"{MOCK_ENDPOINT}openai/deployments/x/chat/completions")+ response = httpx.Response(status_code=status_code, request=request, json={"error": {"message": "error"}})+ return error_class("error", response=response, body={"error": {"message": "error"}})+++class TestAzureOpenAIClaudeDeployment:+ @parameterized.expand(+ [+ ("claude_not_found", "claude-sonnet-4-5", openai.NotFoundError, 404, ClaudeDeploymentOnAzureOpenAIError),+ ("claude_bad_request", "my-Claude-judge", openai.BadRequestError, 400, ClaudeDeploymentOnAzureOpenAIError),+ ("gpt_not_found", "gpt-4o", openai.NotFoundError, 404, ModelNotFoundError),+ ("claude_rejected_key", "claude-sonnet-4-5", openai.AuthenticationError, 401, AuthenticationError),+ ]+ )+ def test_claude_deployment_points_at_the_anthropic_key(self, _name, model, error_class, status_code, expected):+ mapped = AzureOpenAIAdapter(azure_endpoint=MOCK_ENDPOINT)._mapped_error(+ _make_openai_status_error(error_class, status_code), model+ )++ assert type(mapped) is expected+ if expected is ClaudeDeploymentOnAzureOpenAIError:+ assert "Microsoft Foundry endpoint" in user_facing_error_message(mapped)diff --git a/products/ai_observability/backend/llm/test/test_client.py b/products/ai_observability/backend/llm/test/test_client.pyindex 8e64f07dbf56..05e7697b472b 100644--- a/products/ai_observability/backend/llm/test/test_client.py+++ b/products/ai_observability/backend/llm/test/test_client.py@@ -102,6 +102,21 @@ class TestProviderRouting(SimpleTestCase):assert isinstance(provider, OpenAICompatibleAdapter)assert provider.base_url == "https://api.example.com/v1"+ def test_get_provider_anthropic_reads_foundry_endpoint(self):+ from products.ai_observability.backend.llm.client import _get_provider+ from products.ai_observability.backend.llm.providers.anthropic import AnthropicAdapter++ mock_key = MagicMock()+ mock_key.encrypted_config = {+ "api_key": "irrelevant",+ "azure_endpoint": "https://contoso.services.ai.azure.com/",+ }++ provider = _get_provider("anthropic", mock_key)++ assert isinstance(provider, AnthropicAdapter)+ assert provider.foundry_endpoint == "https://contoso.services.ai.azure.com/"+def test_get_provider_openai_compatible_without_provider_key_has_no_base_url(self):from products.ai_observability.backend.llm.client import _get_providerfrom products.ai_observability.backend.llm.providers.openai_compatible import OpenAICompatibleAdapterdiff --git a/products/ai_observability/backend/models/provider_keys.py b/products/ai_observability/backend/models/provider_keys.pyindex d8050d261002..51a2f23c2fd9 100644--- a/products/ai_observability/backend/models/provider_keys.py+++ b/products/ai_observability/backend/models/provider_keys.py@@ -65,8 +65,8 @@ class LLMProviderKey(UUIDTModel):"""Return provider-specific kwargs from encrypted_config.Used when calling ``Client.validate_key`` / ``Client.list_models`` to pass- extra config (e.g. Azure's ``azure_endpoint`` and ``api_version``). Most- providers return an empty dict.+ extra config (e.g. Azure's ``azure_endpoint`` and ``api_version``, or the Microsoft+ Foundry ``azure_endpoint`` of an Anthropic key). Most providers return an empty dict."""if self.provider == LLMProvider.SYSTEM_ONE:return {@@ -79,6 +79,8 @@ class LLMProviderKey(UUIDTModel):}if self.provider == LLMProvider.OPENAI_COMPATIBLE:return {"base_url": self.encrypted_config.get("base_url", "")}+ if self.provider == LLMProvider.ANTHROPIC and self.encrypted_config.get("azure_endpoint"):+ return {"azure_endpoint": self.encrypted_config["azure_endpoint"]}return {}diff --git a/products/ai_observability/frontend/generated/api.schemas.ts b/products/ai_observability/frontend/generated/api.schemas.tsindex d21be1ebbbe5..4f1ad20157a7 100644--- a/products/ai_observability/frontend/generated/api.schemas.ts+++ b/products/ai_observability/frontend/generated/api.schemas.ts@@ -2969,7 +2969,7 @@ export interface LLMProviderKeyApi {* @nullable*/readonly system_one_model_display: string | null- /** Azure OpenAI endpoint URL */+ /** Azure OpenAI endpoint URL. For an Anthropic key, the endpoint of the Microsoft Foundry resource that serves Claude, e.g. https://my-resource.services.ai.azure.com/. Omit it for a key from the Anthropic API. */azure_endpoint?: string/*** Azure OpenAI API version@@ -3589,7 +3589,7 @@ export interface PatchedLLMProviderKeyApi {* @nullable*/readonly system_one_model_display?: string | null- /** Azure OpenAI endpoint URL */+ /** Azure OpenAI endpoint URL. For an Anthropic key, the endpoint of the Microsoft Foundry resource that serves Claude, e.g. https://my-resource.services.ai.azure.com/. Omit it for a key from the Anthropic API. */azure_endpoint?: string/*** Azure OpenAI API versiondiff --git a/products/ai_observability/frontend/generated/api.zod.ts b/products/ai_observability/frontend/generated/api.zod.tsindex 279164a67381..4ca36fa22143 100644--- a/products/ai_observability/frontend/generated/api.zod.ts+++ b/products/ai_observability/frontend/generated/api.zod.ts@@ -2159,7 +2159,12 @@ export const LlmAnalyticsProviderKeysCreateBody = /* @__PURE__ */ zod.object({.max(llmAnalyticsProviderKeysCreateBodySystemOneModelMax).optional().describe('Model ID served by the System One endpoint.'),- azure_endpoint: zod.url().optional().describe('Azure OpenAI endpoint URL'),+ azure_endpoint: zod+ .url()+ .optional()+ .describe(+ 'Azure OpenAI endpoint URL. For an Anthropic key, the endpoint of the Microsoft Foundry resource that serves Claude, e.g. https:\/\/my-resource.services.ai.azure.com\/. Omit it for a key from the Anthropic API.'+ ),api_version: zod.string().max(llmAnalyticsProviderKeysCreateBodyApiVersionMax)@@ -2206,7 +2211,12 @@ export const LlmAnalyticsProviderKeysUpdateBody = /* @__PURE__ */ zod.object({.max(llmAnalyticsProviderKeysUpdateBodySystemOneModelMax).optional().describe('Model ID served by the System One endpoint.'),- azure_endpoint: zod.url().optional().describe('Azure OpenAI endpoint URL'),+ azure_endpoint: zod+ .url()+ .optional()+ .describe(+ 'Azure OpenAI endpoint URL. For an Anthropic key, the endpoint of the Microsoft Foundry resource that serves Claude, e.g. https:\/\/my-resource.services.ai.azure.com\/. Omit it for a key from the Anthropic API.'+ ),api_version: zod.string().max(llmAnalyticsProviderKeysUpdateBodyApiVersionMax)@@ -2254,7 +2264,12 @@ export const LlmAnalyticsProviderKeysPartialUpdateBody = /* @__PURE__ */ zod.obj.max(llmAnalyticsProviderKeysPartialUpdateBodySystemOneModelMax).optional().describe('Model ID served by the System One endpoint.'),- azure_endpoint: zod.url().optional().describe('Azure OpenAI endpoint URL'),+ azure_endpoint: zod+ .url()+ .optional()+ .describe(+ 'Azure OpenAI endpoint URL. For an Anthropic key, the endpoint of the Microsoft Foundry resource that serves Claude, e.g. https:\/\/my-resource.services.ai.azure.com\/. Omit it for a key from the Anthropic API.'+ ),api_version: zod.string().max(llmAnalyticsProviderKeysPartialUpdateBodyApiVersionMax)@@ -2301,7 +2316,12 @@ export const LlmAnalyticsProviderKeysValidateCreateBody = /* @__PURE__ */ zod.ob.max(llmAnalyticsProviderKeysValidateCreateBodySystemOneModelMax).optional().describe('Model ID served by the System One endpoint.'),- azure_endpoint: zod.url().optional().describe('Azure OpenAI endpoint URL'),+ azure_endpoint: zod+ .url()+ .optional()+ .describe(+ 'Azure OpenAI endpoint URL. For an Anthropic key, the endpoint of the Microsoft Foundry resource that serves Claude, e.g. https:\/\/my-resource.services.ai.azure.com\/. Omit it for a key from the Anthropic API.'+ ),api_version: zod.string().max(llmAnalyticsProviderKeysValidateCreateBodyApiVersionMax)diff --git a/products/ai_observability/frontend/modelPickerLogic.test.ts b/products/ai_observability/frontend/modelPickerLogic.test.tsindex 2b3d7606a4dd..f511b858db92 100644--- a/products/ai_observability/frontend/modelPickerLogic.test.ts+++ b/products/ai_observability/frontend/modelPickerLogic.test.ts@@ -471,6 +471,14 @@ describe('modelPickerLogic', () => {results: [{ id: 'key-1', provider: 'openai', name: 'My OpenAI Key', state: 'ok' },{ id: 'key-azure', provider: 'azure_openai', name: 'My Azure Key', state: 'ok' },+ {+ id: 'key-foundry',+ provider: 'anthropic',+ name: 'My Foundry Key',+ state: 'ok',+ azure_endpoint_display: 'https://my-resource.services.ai.azure.com/',+ },+ { id: 'key-anthropic', provider: 'anthropic', name: 'My Anthropic Key', state: 'ok' },],},'/api/environments/:team_id/llm_analytics/evaluation_config/': {@@ -487,9 +495,11 @@ describe('modelPickerLogic', () => {logic.mount()await expectLogic(logic).toFinishAllListeners()- expect(logic.values.providerModelGroups.find((g) => g.providerKeyId === 'key-azure')).toMatchObject({- allowsCustomModel: true,- })+ const customModelKeyIds = logic.values.providerModelGroups+ .filter((g) => g.allowsCustomModel)+ .map((g) => g.providerKeyId)+ .sort()+ expect(customModelKeyIds).toEqual(['key-azure', 'key-foundry'])expect(logic.values.playgroundByokProviderModelGroups.map((g) => g.providerKeyId)).toEqual(['key-1'])expect(logic.values.playgroundByokProviderModelGroups.some((g) => g.allowsCustomModel)).toBe(false)})diff --git a/products/ai_observability/frontend/modelPickerLogic.ts b/products/ai_observability/frontend/modelPickerLogic.tsindex 4b271dc86841..cd12b23d87b8 100644--- a/products/ai_observability/frontend/modelPickerLogic.ts+++ b/products/ai_observability/frontend/modelPickerLogic.ts@@ -58,7 +58,7 @@ function getByokModelNotice(return null}const failedKeys = providerKeys.filter(- (key) => failedByokProviderKeyIds.includes(key.id) && !allowsCustomModel(key.provider)+ (key) => failedByokProviderKeyIds.includes(key.id) && !allowsCustomModel(key))if (failedKeys.length > 0) {return { kind: 'models-failed', keys: failedKeys }@@ -77,9 +77,9 @@ function getByokModelNotice(return providerKeys.length === 0 ? { kind: 'no-keys' } : { kind: 'no-usable-keys' }}-// Azure chat completions need a deployment name, and Azure does not always list deployments.-function allowsCustomModel(provider: LLMProvider): boolean {- return provider === 'azure_openai'+// Azure OpenAI and Claude on Microsoft Foundry need a deployment name, and Azure does not always list deployments.+function allowsCustomModel(key: LLMProviderKey): boolean {+ return key.provider === 'azure_openai' || (key.provider === 'anthropic' && !!key.azure_endpoint_display)}function providerKeyGroupLabel(key: LLMProviderKey, keysPerProvider: Record<string, number>, suffix = ''): string {@@ -414,7 +414,7 @@ export const modelPickerLogic = kea<modelPickerLogicType>([const groups: ProviderModelGroup[] = []for (const key of providerKeys) {const models = byKeyId[key.id] ?? []- const customModelKey = allowsCustomModel(key.provider) && key.state === 'ok'+ const customModelKey = allowsCustomModel(key) && key.state === 'ok'if (models.length > 0 || customModelKey) {groups.push({provider: key.provider,diff --git a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsxindex d3ab3e51cc92..110c4be30cde 100644--- a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx+++ b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx@@ -87,12 +87,12 @@ function formatDate(dateString: string | null): string {})}-function getKeyPlaceholder(provider: LLMProvider): string {+function getKeyPlaceholder(provider: LLMProvider, usesFoundry = false): string {switch (provider) {case 'openai':return 'sk-...'case 'anthropic':- return 'sk-ant-...'+ return usesFoundry ? 'Enter your Microsoft Foundry API key' : 'sk-ant-...'case 'gemini':return 'Enter your Gemini API key'case 'together_ai':@@ -122,22 +122,62 @@ function endpointErrorFieldFromResult(result: KeyValidationResult | null | undefreturn result.error_field === 'azure_endpoint' || result.error_field === 'base_url' ? 'endpoint' : 'key'}+function FoundryEndpointField({+ value,+ onChange,+ errorMessage,+ optional,+}: {+ value: string+ onChange: (value: string) => void+ errorMessage: string | null+ optional: boolean+}): JSX.Element {+ return (+ <div>+ <label className="text-sm font-medium">+ {optional ? 'Microsoft Foundry endpoint (optional)' : 'Microsoft Foundry endpoint'}+ </label>+ <LemonInput+ value={value}+ onChange={onChange}+ placeholder="https://my-resource.services.ai.azure.com/"+ className="mt-1"+ fullWidth+ status={errorMessage ? 'danger' : undefined}+ />+ {errorMessage ? (+ <p className="text-xs text-danger mt-1">{errorMessage}</p>+ ) : (+ <p className="text-xs text-muted mt-1">+ {optional+ ? 'Fill this in only if you run Claude on Microsoft Foundry. Leave it empty for a key from the Anthropic API.'+ : 'The endpoint URL of your Microsoft Foundry resource'}+ </p>+ )}+ </div>+ )+}+function KeyValidationStatus({result,isValidating,provider,suppressError = false,+ usesFoundry = false,}: {result: KeyValidationResult | nullisValidating: booleanprovider: LLMProvidersuppressError?: boolean+ usesFoundry?: boolean}): JSX.Element | null {if (isValidating) {return <p className="text-xs text-muted mt-1">Validating key...</p>}- const providerLabel = provider === 'openai_compatible' ? 'your provider' : LLM_PROVIDER_LABELS[provider]+ const providerLabel =+ provider === 'openai_compatible' ? 'your provider' : usesFoundry ? 'Microsoft' : LLM_PROVIDER_LABELS[provider]const bullets = (<ul className="text-xs text-muted mt-1 list-disc pl-4 space-y-0.5"><li>Your key will be encrypted and stored securely</li>@@ -185,7 +225,15 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }const [pendingSubmit, setPendingSubmit] = useState(false)const isAzure = provider === 'azure_openai'+ const isAnthropic = provider === 'anthropic'const isOpenAICompatible = provider === 'openai_compatible'+ const hasEndpointField = isAzure || isAnthropic || isOpenAICompatible+ // Anthropic keys take an optional Microsoft Foundry endpoint, so only send one that was entered.+ const endpointPayload = isAzure+ ? { azure_endpoint: azureEndpoint, api_version: apiVersion }+ : isAnthropic && azureEndpoint+ ? { azure_endpoint: azureEndpoint }+ : {}const keyValidated = preValidationResult?.state === 'ok'const isSystemOne = provider === 'system_one'const isValid =@@ -195,7 +243,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }(!isOpenAICompatible || baseUrl.length > 0)const validationFailed = !!preValidationResult && preValidationResult.state !== 'ok'const endpointErrorField =- (isAzure || isOpenAICompatible) && validationFailed ? endpointErrorFieldFromResult(preValidationResult) : null+ hasEndpointField && validationFailed ? endpointErrorFieldFromResult(preValidationResult) : null// Reset form when modal closesuseEffect(() => {@@ -220,10 +268,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }name,api_key: apiKey,set_as_active: provider !== 'system_one' && !evaluationConfig?.active_provider_key,- }- if (isAzure) {- payload.azure_endpoint = azureEndpoint- payload.api_version = apiVersion+ ...endpointPayload,}if (isOpenAICompatible) {payload.base_url = baseUrl@@ -240,6 +285,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }apiKey,provider,isAzure,+ isAnthropic,azureEndpoint,apiVersion,isOpenAICompatible,@@ -271,10 +317,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }name,api_key: apiKey,set_as_active: !evaluationConfig?.active_provider_key,- }- if (isAzure) {- payload.azure_endpoint = azureEndpoint- payload.api_version = apiVersion+ ...endpointPayload,}if (isOpenAICompatible) {payload.base_url = baseUrl@@ -285,7 +328,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }preValidateKey({apiKey,provider,- ...(isAzure ? { azure_endpoint: azureEndpoint, api_version: apiVersion } : {}),+ ...endpointPayload,...(isOpenAICompatible ? { base_url: baseUrl } : {}),})}@@ -299,7 +342,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }preValidateKey({apiKey,provider,- ...(isAzure ? { azure_endpoint: azureEndpoint, api_version: apiVersion } : {}),+ ...endpointPayload,...(isOpenAICompatible ? { base_url: baseUrl } : {}),})}@@ -312,14 +355,16 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }}}- // A result is only about the endpoint it was validated against, so editing the base URL has+ // A result is only about the endpoint it was validated against, so editing the endpoint has// to drop it. Otherwise a key validated against the old endpoint keeps showing as validated.- const handleBaseUrlChange = (value: string): void => {- setBaseUrl(value)- if (preValidationResult) {- clearPreValidation()+ const handleEndpointChange =+ (setEndpoint: (value: string) => void): ((value: string) => void) =>+ (value) => {+ setEndpoint(value)+ if (preValidationResult) {+ clearPreValidation()+ }}- }const handleProviderChange = (value: LLMProvider): void => {setProvider(value)@@ -405,13 +450,25 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }</div></>)}+ {isAnthropic && (+ <FoundryEndpointField+ optional+ value={azureEndpoint}+ onChange={handleEndpointChange(setAzureEndpoint)}+ errorMessage={+ endpointErrorField === 'endpoint'+ ? preValidationResult?.error_message || 'Invalid Microsoft Foundry endpoint'+ : null+ }+ />+ )}{isSystemOne && <SystemOneConnectionFields />}{isOpenAICompatible && (<div><label className="text-sm font-medium">Base URL</label><LemonInputvalue={baseUrl}- onChange={handleBaseUrlChange}+ onChange={handleEndpointChange(setBaseUrl)}placeholder="https://api.example.com/v1"className="mt-1"fullWidth@@ -446,13 +503,13 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }value={apiKey}onChange={handleApiKeyChange}onBlur={handleApiKeyBlur}- placeholder={getKeyPlaceholder(provider)}+ placeholder={getKeyPlaceholder(provider, isAnthropic && azureEndpoint.length > 0)}type="password"autoComplete="off"className="mt-1"fullWidthstatus={- validationFailed && ((!isAzure && !isOpenAICompatible) || endpointErrorField === 'key')+ validationFailed && (!hasEndpointField || endpointErrorField === 'key')? 'danger': undefined}@@ -462,6 +519,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }isValidating={preValidationResultLoading}provider={provider}suppressError={endpointErrorField === 'endpoint'}+ usesFoundry={isAnthropic && azureEndpoint.length > 0}/>{isSystemOne && (<p className="text-xs text-muted">@@ -491,6 +549,8 @@ function EditKeyModal({normalizeSystemOneBaseUrlForComparison(systemOneBaseUrl) !==normalizeSystemOneBaseUrlForComparison(keyToEdit.base_url_display ?? DEFAULT_SYSTEM_ONE_BASE_URL)const isOpenAICompatibleEdit = keyToEdit.provider === 'openai_compatible'+ // An Anthropic key keeps the API it was created for, so only Foundry keys show the endpoint.+ const isFoundryEdit = keyToEdit.provider === 'anthropic' && !!keyToEdit.azure_endpoint_displayconst [name, setName] = useState(keyToEdit.name)const [apiKey, setApiKey] = useState('')@@ -531,6 +591,9 @@ function EditKeyModal({if (baseUrlChanged) {payload.base_url = baseUrl}+ if (foundryEndpointChanged) {+ payload.azure_endpoint = azureEndpoint+ }updateProviderKey({ id: keyToEdit.id, payload })}@@ -543,6 +606,7 @@ function EditKeyModal({apiKey,provider: keyToEdit.provider,...(isAzureEdit ? { azure_endpoint: azureEndpoint, api_version: apiVersion } : {}),+ ...(isFoundryEdit ? { azure_endpoint: azureEndpoint } : {}),...(isOpenAICompatibleEdit ? { base_url: baseUrl } : {}),})}@@ -555,31 +619,48 @@ function EditKeyModal({}}- // A result is only about the endpoint it was validated against, so editing the base URL has+ // A result is only about the endpoint it was validated against, so editing the endpoint has// to drop it. Otherwise a key validated against the old endpoint keeps showing as validated.- const handleBaseUrlChange = (value: string): void => {- setBaseUrl(value)- if (preValidationResult) {- clearPreValidation()+ const handleEndpointChange =+ (setEndpoint: (value: string) => void): ((value: string) => void) =>+ (value) => {+ setEndpoint(value)+ if (preValidationResult) {+ clearPreValidation()+ }}- }const keyValidated = isSystemOne || apiKey.length === 0 || preValidationResult?.state === 'ok'const baseUrlMissing = isOpenAICompatibleEdit && baseUrl.length === 0+ const foundryEndpointMissing = isFoundryEdit && azureEndpoint.length === 0// The backend rejects a base URL change that arrives without a key, so the key can't stay// masked here: re-entering it is what proves the caller already has it.const baseUrlChanged = isOpenAICompatibleEdit && !baseUrlMissing && baseUrl !== (keyToEdit.base_url_display ?? '')+ const foundryEndpointChanged =+ isFoundryEdit && !foundryEndpointMissing && azureEndpoint !== (keyToEdit.azure_endpoint_display ?? '')const apiKeyRequiredReason =- baseUrlChanged && apiKey.length === 0 ? 'Enter the API key again to change the base URL' : null- const missingFieldReason = name.length === 0 ? 'Enter a name' : baseUrlMissing ? 'Enter a base URL' : null+ apiKey.length > 0+ ? null+ : baseUrlChanged+ ? 'Enter the API key again to change the base URL'+ : foundryEndpointChanged+ ? 'Enter the API key again to change the Microsoft Foundry endpoint'+ : null+ const missingFieldReason =+ name.length === 0+ ? 'Enter a name'+ : baseUrlMissing+ ? 'Enter a base URL'+ : foundryEndpointMissing+ ? 'Enter a Microsoft Foundry endpoint'+ : nullconst disabledReason = restrictionReason ?? missingFieldReason ?? apiKeyRequiredReasonconst isValid =keyValidated && !disabledReason && (!isSystemOne || (systemOneBaseUrl.length > 0 && systemOneModel.length > 0))const validationFailed = !!preValidationResult && preValidationResult.state !== 'ok'+ const hasEndpointField = isAzureEdit || isFoundryEdit || isOpenAICompatibleEditconst endpointErrorField =- (isAzureEdit || isOpenAICompatibleEdit) && validationFailed- ? endpointErrorFieldFromResult(preValidationResult)- : null+ hasEndpointField && validationFailed ? endpointErrorFieldFromResult(preValidationResult) : nullreturn (<LemonModal@@ -641,13 +722,25 @@ function EditKeyModal({</div></>)}+ {isFoundryEdit && (+ <FoundryEndpointField+ optional={false}+ value={azureEndpoint}+ onChange={handleEndpointChange(setAzureEndpoint)}+ errorMessage={+ endpointErrorField === 'endpoint'+ ? preValidationResult?.error_message || 'Invalid Microsoft Foundry endpoint'+ : null+ }+ />+ )}{isSystemOne && <SystemOneConnectionFields />}{isOpenAICompatibleEdit && (<div><label className="text-sm font-medium">Base URL</label><LemonInputvalue={baseUrl}- onChange={handleBaseUrlChange}+ onChange={handleEndpointChange(setBaseUrl)}placeholder="https://api.example.com/v1"className="mt-1"fullWidth@@ -680,8 +773,7 @@ function EditKeyModal({className="mt-1"fullWidthstatus={- validationFailed &&- ((!isAzureEdit && !isOpenAICompatibleEdit) || endpointErrorField === 'key')+ validationFailed && (!hasEndpointField || endpointErrorField === 'key')? 'danger': undefined}@@ -692,14 +784,13 @@ function EditKeyModal({isValidating={preValidationResultLoading}provider={keyToEdit.provider}suppressError={endpointErrorField === 'endpoint'}+ usesFoundry={isFoundryEdit}/>) : (<p className="text-xs text-muted mt-1">{isSystemOne && endpointChanged? 'The saved key will not be sent to the new endpoint. Enter its bearer token, or leave empty for no authentication.'- : baseUrlChanged- ? 'Enter the API key again to change the base URL'- : 'Leave empty to keep the current key'}+ : (apiKeyRequiredReason ?? 'Leave empty to keep the current key')}</p>)}</div>diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 9295173640d4..0a5773273d50 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -43064,7 +43064,7 @@ export namespace Schemas {* @nullable*/readonly system_one_model_display: string | null;- /** Azure OpenAI endpoint URL */+ /** Azure OpenAI endpoint URL. For an Anthropic key, the endpoint of the Microsoft Foundry resource that serves Claude, e.g. https://my-resource.services.ai.azure.com/. Omit it for a key from the Anthropic API. */azure_endpoint?: string;/*** Azure OpenAI API version@@ -83217,7 +83217,7 @@ export namespace Schemas {* @nullable*/readonly system_one_model_display?: string | null;- /** Azure OpenAI endpoint URL */+ /** Azure OpenAI endpoint URL. For an Anthropic key, the endpoint of the Microsoft Foundry resource that serves Claude, e.g. https://my-resource.services.ai.azure.com/. Omit it for a key from the Anthropic API. */azure_endpoint?: string;/*** Azure OpenAI API version