Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(aio): support claude on microsoft foundry as an evaluation judge (#117581)

posthog[bot]1fd65699206f
diff --git a/posthog/temporal/ai_observability/evaluation_llm_judge.py b/posthog/temporal/ai_observability/evaluation_llm_judge.py
index 617c13f12afc..1ff34d898afe 100644
--- a/posthog/temporal/ai_observability/evaluation_llm_judge.py
+++ b/posthog/temporal/ai_observability/evaluation_llm_judge.py
@@ -57,6 +57,7 @@ from products.ai_observability.backend.llm.decisions import (
)
from products.ai_observability.backend.llm.errors import (
AuthenticationError,
+ ClaudeDeploymentOnAzureOpenAIError,
ContentFilteredError,
ContextWindowExceededError,
ModelNotFoundError,
@@ -909,7 +910,7 @@ def call_llm_judge(
{"error_type": "model_not_found", "provider": provider, "model": model},
non_retryable=True,
)
- except UnsupportedModelError:
+ except UnsupportedModelError as e:
# A failed chat call can populate a cold catalogue; a disabled flag must not disable the evaluation.
if (
provider == "openrouter"
@@ -925,7 +926,9 @@ def call_llm_judge(
increment_user_errors("model_not_supported", provider=provider)
return terminal_user_error_result(
spec=require_user_error_spec("model_not_supported", is_byok=is_byok),
- message=f"Model '{model}' does not support chat completions. Choose a chat model.",
+ message=str(e)
+ if isinstance(e, ClaudeDeploymentOnAzureOpenAIError)
+ else f"Model '{model}' does not support chat completions. Choose a chat model.",
allows_na=allows_na,
output_type=output_type,
provider=provider,
diff --git a/posthog/temporal/ai_observability/test_run_evaluation.py b/posthog/temporal/ai_observability/test_run_evaluation.py
index 28e5b957d4aa..87e5a12f5402 100644
--- a/posthog/temporal/ai_observability/test_run_evaluation.py
+++ b/posthog/temporal/ai_observability/test_run_evaluation.py
@@ -35,6 +35,7 @@ from posthog.temporal.common.errors import NonReportableError
from products.access_control.backend.models.access_control import AccessControl
from products.ai_observability.backend.llm.errors import (
AuthenticationError,
+ ClaudeDeploymentOnAzureOpenAIError,
ContentFilteredError,
ContextWindowExceededError,
ModelNotFoundError,
@@ -3393,6 +3394,13 @@ class TestRunEvaluationWorkflow:
None,
id="model_not_supported",
),
+ pytest.param(
+ ClaudeDeploymentOnAzureOpenAIError("gpt-5.4"),
+ "model_not_supported",
+ "model_not_supported",
+ None,
+ id="claude_deployment_on_azure_openai",
+ ),
],
)
@pytest.mark.django_db(transaction=True)
@@ -3451,6 +3459,8 @@ class TestRunEvaluationWorkflow:
assert result["key_id"] == str(provider_key.id)
assert result["provider"] == "openai"
assert result["model"] == "gpt-5.4"
+ if isinstance(raised_exception, ClaudeDeploymentOnAzureOpenAIError):
+ assert result["reasoning"] == str(raised_exception)
if provider_key_state is None:
assert "provider_key_state" not in result
else:
diff --git a/products/ai_observability/backend/api/provider_keys.py b/products/ai_observability/backend/api/provider_keys.py
index be10f7df3234..26f9562fde06 100644
--- a/products/ai_observability/backend/api/provider_keys.py
+++ b/products/ai_observability/backend/api/provider_keys.py
@@ -26,6 +26,10 @@ from products.access_control.backend.presentation.access_control import AccessCo
from ..llm.client import Client
from ..llm.decisions import DecisionClient, decision_evaluations_enabled
+from ..llm.providers.anthropic import (
+ DISALLOWED_FOUNDRY_ENDPOINT_MESSAGE,
+ error_field_for_validation_message as anthropic_error_field,
+)
from ..llm.providers.azure_openai import (
DEFAULT_API_VERSION,
DISALLOWED_ENDPOINT_MESSAGE,
@@ -91,13 +95,15 @@ def validate_provider_key(provider: str, api_key: str, *, team_id: int, **kwargs
def _validation_error_field(provider: str, error_message: str | None) -> str | None:
"""Pick the form field a validation error belongs to, or None when it is unattributed.
- Azure OpenAI and OpenAI-compatible providers may fail because of endpoint issues (unreachable,
- wrong domain, 404), in which case the error is attributed to the endpoint field so the UI can
- highlight the right input. Callers that must land the error on some field fall back to
+ Azure OpenAI, Anthropic on Microsoft Foundry, and OpenAI-compatible providers may fail because
+ of endpoint issues (unreachable, wrong domain, 404), in which case the error is attributed to the
+ endpoint field so the UI can highlight the right input. Callers that must land the error on some field fall back to
`api_key`; the pre-validation endpoint reports the field separately and keeps the None.
"""
if provider == LLMProvider.AZURE_OPENAI:
return azure_error_field(error_message)
+ if provider == LLMProvider.ANTHROPIC:
+ return anthropic_error_field(error_message)
if provider == LLMProvider.OPENAI_COMPATIBLE:
return openai_compatible_error_field(error_message)
return None
@@ -107,6 +113,7 @@ def _validation_error_field(provider: str, error_message: str | None) -> str | N
# Fields for other providers are dropped on write so a stray value never persists.
_PROVIDER_CONFIG_FIELDS: dict[str, tuple[str, ...]] = {
LLMProvider.AZURE_OPENAI: ("azure_endpoint", "api_version"),
+ LLMProvider.ANTHROPIC: ("azure_endpoint",),
LLMProvider.OPENAI_COMPATIBLE: ("base_url",),
}
_ALL_PROVIDER_CONFIG_FIELDS: tuple[str, ...] = tuple(
@@ -124,7 +131,13 @@ class LLMProviderKeySerializer(serializers.ModelSerializer):
)
system_one_model_display = serializers.SerializerMethodField(help_text="Configured System One model ID.")
api_key_masked = serializers.SerializerMethodField()
- azure_endpoint = serializers.URLField(write_only=True, required=False, help_text="Azure OpenAI endpoint URL")
+ azure_endpoint = serializers.URLField(
+ write_only=True,
+ required=False,
+ help_text="Azure OpenAI endpoint URL. For an Anthropic key, the endpoint of the Microsoft Foundry "
+ "resource that serves Claude, e.g. https://my-resource.services.ai.azure.com/. "
+ "Omit it for a key from the Anthropic API.",
+ )
api_version = serializers.CharField(
write_only=True, required=False, max_length=20, help_text="Azure OpenAI API version"
)
@@ -174,7 +187,7 @@ class LLMProviderKeySerializer(serializers.ModelSerializer):
return obj.encrypted_config.get("model") if obj.provider == LLMProvider.SYSTEM_ONE else None
def get_azure_endpoint_display(self, obj: LLMProviderKey) -> str | None:
- if obj.provider != LLMProvider.AZURE_OPENAI:
+ if obj.provider not in (LLMProvider.AZURE_OPENAI, LLMProvider.ANTHROPIC):
return None
return obj.encrypted_config.get("azure_endpoint")
@@ -216,14 +229,19 @@ class LLMProviderKeySerializer(serializers.ModelSerializer):
raise serializers.ValidationError(
"Invalid OpenAI API key format. Key should start with 'sk-' or 'sk-proj-'."
)
- elif provider == LLMProvider.ANTHROPIC:
+ elif provider == LLMProvider.ANTHROPIC and not self._anthropic_uses_foundry():
if not value.startswith("sk-ant-"):
raise serializers.ValidationError("Invalid Anthropic API key format. Key should start with 'sk-ant-'.")
- # Azure, Gemini, Together AI, OpenRouter, Fireworks, MiniMax, and OpenAI-compatible keys
- # have no standard prefix, so no format validation is enforced here.
+ # Azure, Microsoft Foundry, Gemini, Together AI, OpenRouter, Fireworks, MiniMax, and
+ # OpenAI-compatible keys have no standard prefix, so no format validation is enforced here.
return value
+ def _anthropic_uses_foundry(self) -> bool:
+ if self.initial_data.get("azure_endpoint"):
+ return True
+ return bool(self.instance and self.instance.encrypted_config.get("azure_endpoint"))
+
def validate(self, data):
if self.instance is None and "api_key" not in data:
raise serializers.ValidationError({"api_key": "API key is required when creating a new provider key."})
@@ -263,6 +281,17 @@ class LLMProviderKeySerializer(serializers.ModelSerializer):
if has_endpoint and not is_allowed_azure_endpoint(data["azure_endpoint"]):
raise serializers.ValidationError({"azure_endpoint": f"{DISALLOWED_ENDPOINT_MESSAGE}."})
+ if provider == LLMProvider.ANTHROPIC and data.get("azure_endpoint"):
+ if not is_allowed_azure_endpoint(data["azure_endpoint"]):
+ raise serializers.ValidationError({"azure_endpoint": f"{DISALLOWED_FOUNDRY_ENDPOINT_MESSAGE}."})
+ # Same rule as the OpenAI-compatible base URL below: a stored key must never be sent
+ # to a new endpoint unless the caller proves they hold it.
+ current_endpoint = self.instance.encrypted_config.get("azure_endpoint") if self.instance else None
+ if self.instance and data["azure_endpoint"] != current_endpoint and not data.get("api_key"):
+ raise serializers.ValidationError(
+ {"api_key": "Enter the API key again when you change the Microsoft Foundry endpoint."}
+ )
+
if provider == LLMProvider.OPENAI_COMPATIBLE:
has_base_url = bool(data.get("base_url"))
has_existing_base_url = self.instance and self.instance.encrypted_config.get("base_url")
@@ -395,6 +424,8 @@ class LLMProviderKeySerializer(serializers.ModelSerializer):
encrypted_config["api_version"] = extra_kwargs.get("api_version", "")
elif instance.provider == LLMProvider.OPENAI_COMPATIBLE:
encrypted_config["base_url"] = extra_kwargs.get("base_url", "")
+ elif instance.provider == LLMProvider.ANTHROPIC and extra_kwargs.get("azure_endpoint"):
+ encrypted_config["azure_endpoint"] = extra_kwargs["azure_endpoint"]
instance.encrypted_config = encrypted_config
instance.state = state
instance.error_message = None
@@ -410,9 +441,9 @@ class LLMProviderKeySerializer(serializers.ModelSerializer):
instance.encrypted_config = config
instance.state = LLMProviderKey.State.UNKNOWN
instance.error_message = None
- # No OpenAI-compatible branch here on purpose: `validate` rejects a base URL change
- # that arrives without an API key, so every change to one runs through the branch
- # above and is re-validated against the new endpoint before it is stored.
+ # No OpenAI-compatible or Anthropic branch here on purpose: `validate` rejects a base URL
+ # or Foundry endpoint change that arrives without an API key, so every change to one runs
+ # through the branch above and is re-validated against the new endpoint before it is stored.
return super().update(instance, validated_data)
@@ -700,7 +731,11 @@ class LLMProviderKeyValidationViewSet(TeamAndOrgViewSetMixin, viewsets.ViewSet):
)
extra_kwargs = {}
- if provider == LLMProvider.AZURE_OPENAI:
+ if provider == LLMProvider.ANTHROPIC:
+ azure_endpoint = request.data.get("azure_endpoint")
+ if azure_endpoint:
+ extra_kwargs["azure_endpoint"] = azure_endpoint
+ elif provider == LLMProvider.AZURE_OPENAI:
azure_endpoint = request.data.get("azure_endpoint")
api_version = request.data.get("api_version")
if azure_endpoint:
diff --git a/products/ai_observability/backend/api/test/test_provider_keys.py b/products/ai_observability/backend/api/test/test_provider_keys.py
index b07c60abaac5..775ae76fa0ac 100644
--- a/products/ai_observability/backend/api/test/test_provider_keys.py
+++ b/products/ai_observability/backend/api/test/test_provider_keys.py
@@ -114,6 +114,32 @@ class TestProviderKeySerializer(SimpleTestCase):
serializer = LLMProviderKeySerializer(key, data={"base_url": base_url}, partial=True)
self.assertEqual(serializer.is_valid(), valid, serializer.errors)
+ @parameterized.expand(
+ [
+ ("foundry_key_without_prefix", {"azure_endpoint": "https://contoso.services.ai.azure.com/"}, None),
+ ("api_key_without_prefix", {}, "api_key"),
+ ("non_azure_endpoint", {"azure_endpoint": "https://evil.example.com/"}, "azure_endpoint"),
+ ]
+ )
+ def test_anthropic_key_validation(self, _name: str, extra: dict, error_field: str | None) -> None:
+ serializer = LLMProviderKeySerializer(
+ data={"provider": "anthropic", "name": "Claude", "api_key": "no-prefix-key", **extra}
+ )
+ self.assertEqual(serializer.is_valid(), error_field is None, serializer.errors)
+ if error_field:
+ self.assertIn(error_field, serializer.errors)
+
+ @parameterized.expand([({}, False), ({"api_key": "no-prefix-key"}, True)])
+ def test_foundry_endpoint_change_requires_the_api_key(self, extra: dict, valid: bool) -> None:
+ key = LLMProviderKey(
+ provider="anthropic",
+ encrypted_config={"api_key": "foundry-key", "azure_endpoint": "https://old.services.ai.azure.com/"},
+ )
+ serializer = LLMProviderKeySerializer(
+ key, data={"azure_endpoint": "https://new.services.ai.azure.com/", **extra}, partial=True
+ )
+ self.assertEqual(serializer.is_valid(), valid, serializer.errors)
+
def _setup_team():
org = Organization.objects.create(name="test")
@@ -770,6 +796,31 @@ class TestLLMProviderKeyViewSet(APIBaseTest):
api_version="2024-10-21",
)
+ @patch("products.ai_observability.backend.api.provider_keys.validate_provider_key")
+ def test_can_create_anthropic_key_on_microsoft_foundry(self, mock_validate):
+ mock_validate.return_value = (LLMProviderKey.State.OK, None)
+
+ response = self.client.post(
+ f"/api/environments/{self.team.id}/llm_analytics/provider_keys/",
+ {
+ "provider": "anthropic",
+ "name": "Foundry Claude",
+ "api_key": "foundry-hex-123",
+ "azure_endpoint": "https://contoso.services.ai.azure.com/",
+ },
+ )
+ self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+ self.assertEqual(response.json()["azure_endpoint_display"], "https://contoso.services.ai.azure.com/")
+
+ key = LLMProviderKey.objects.get(id=response.json()["id"])
+ self.assertEqual(key.provider_extra_kwargs(), {"azure_endpoint": "https://contoso.services.ai.azure.com/"})
+ mock_validate.assert_called_once_with(
+ "anthropic",
+ "foundry-hex-123",
+ team_id=self.team.id,
+ azure_endpoint="https://contoso.services.ai.azure.com/",
+ )
+
def test_create_azure_openai_without_endpoint_fails(self):
response = self.client.post(
f"/api/environments/{self.team.id}/llm_analytics/provider_keys/",
diff --git a/products/ai_observability/backend/llm/client.py b/products/ai_observability/backend/llm/client.py
index 87e677d0edc9..6c6ea8313dd5 100644
--- a/products/ai_observability/backend/llm/client.py
+++ b/products/ai_observability/backend/llm/client.py
@@ -102,7 +102,7 @@ class Client:
def _get_provider(name: str, provider_key: "LLMProviderKey | None" = None) -> "Provider":
- """Get provider by name. Azure and OpenAI-compatible read extra config from provider_key."""
+ """Get provider by name. Anthropic, Azure and OpenAI-compatible read extra config from provider_key."""
from typing import cast
from products.ai_observability.backend.llm.providers.anthropic import AnthropicAdapter
@@ -119,7 +119,8 @@ def _get_provider(name: str, provider_key: "LLMProviderKey | None" = None) -> "P
case "openai":
return cast("Provider", OpenAIAdapter())
case "anthropic":
- return cast("Provider", AnthropicAdapter())
+ config = provider_key.encrypted_config if provider_key else {}
+ return cast("Provider", AnthropicAdapter(foundry_endpoint=config.get("azure_endpoint", "")))
case "gemini":
return cast("Provider", GeminiAdapter())
case "together_ai":
diff --git a/products/ai_observability/backend/llm/errors.py b/products/ai_observability/backend/llm/errors.py
index cf561e350074..49713723e7ee 100644
--- a/products/ai_observability/backend/llm/errors.py
+++ b/products/ai_observability/backend/llm/errors.py
@@ -18,6 +18,18 @@ class UnsupportedModelError(LLMError):
super().__init__(f"Unsupported model: {model}")
+class ClaudeDeploymentOnAzureOpenAIError(UnsupportedModelError):
+ """Raised when an Azure OpenAI key names a Claude deployment, which only the Anthropic API serves."""
+
+ def __init__(self, model: str):
+ self.model = model
+ LLMError.__init__(
+ self,
+ f"'{model}' looks like a Claude deployment. Azure OpenAI keys can't call Claude models. "
+ "Add an Anthropic key with your Microsoft Foundry endpoint, then pick that key for this model.",
+ )
+
+
class UnsupportedProviderError(LLMError):
"""Raised when an unsupported provider is requested"""
@@ -201,6 +213,8 @@ def user_facing_error_message(error: Exception | None) -> str:
"""
if isinstance(error, ModelNotFoundError):
return f"Model '{error.model}' is not available. Pick a different model and try again."
+ if isinstance(error, ClaudeDeploymentOnAzureOpenAIError):
+ return str(error)
if isinstance(error, UnsupportedModelError):
return f"Model '{error.model}' is not supported. Pick a different model and try again."
if isinstance(error, ModelPermissionError):
diff --git a/products/ai_observability/backend/llm/providers/anthropic.py b/products/ai_observability/backend/llm/providers/anthropic.py
index ed2fee95419b..83bec9ae74a7 100644
--- a/products/ai_observability/backend/llm/providers/anthropic.py
+++ b/products/ai_observability/backend/llm/providers/anthropic.py
@@ -4,6 +4,7 @@ import uuid
import logging
from collections.abc import Generator
from typing import Any
+from urllib.parse import urlparse
from django.conf import settings
@@ -12,6 +13,8 @@ import posthoganalytics
from anthropic.lib._parse._transform import transform_schema
from anthropic.types import MessageParam, TextBlockParam, ThinkingConfigEnabledParam
from posthoganalytics.ai.anthropic import Anthropic
+from posthoganalytics.ai.anthropic.anthropic import WrappedMessages
+from posthoganalytics.client import Client as PostHogClient
from pydantic import BaseModel
from products.ai_observability.backend.llm.errors import (
@@ -21,14 +24,17 @@ from products.ai_observability.backend.llm.errors import (
ModelNotFoundError,
ModelPermissionError,
OutputTokenLimitError,
+ ProviderConfigurationError,
ProviderConnectionError,
QuotaExceededError,
RateLimitError,
StructuredOutputParseError,
+ error_field_for_message,
is_context_window_error_message,
is_output_limit_error_message,
stream_error_chunk,
)
+from products.ai_observability.backend.llm.providers.azure_openai import is_allowed_azure_endpoint
from products.ai_observability.backend.llm.types import (
AnalyticsContext,
CompletionRequest,
@@ -39,6 +45,70 @@ from products.ai_observability.backend.llm.types import (
logger = logging.getLogger(__name__)
+# Shared by the adapter and the serializer so the error attributes to the endpoint field in the UI.
+DISALLOWED_FOUNDRY_ENDPOINT_MESSAGE = (
+ "Microsoft Foundry endpoint must be an https:// URL on an Azure domain (e.g. *.services.ai.azure.com)"
+)
+
+FOUNDRY_ENDPOINT_NOT_FOUND_MESSAGE = (
+ "Microsoft Foundry endpoint not found. Check that the URL is the endpoint of your Foundry resource, "
+ "for example https://my-resource.services.ai.azure.com/"
+)
+
+# Keep aligned with the `return` statements in `AnthropicAdapter._validate_foundry_key`.
+_FOUNDRY_ERROR_FIELD_BY_PREFIX: tuple[tuple[str, str], ...] = (
+ ("Microsoft Foundry endpoint must be", "azure_endpoint"),
+ ("Microsoft Foundry endpoint not found", "azure_endpoint"),
+ ("Could not connect to Microsoft Foundry", "azure_endpoint"),
+ ("Invalid API key", "api_key"),
+)
+
+# Key validation is a single cheap call; don't inherit the long completion timeout.
+FOUNDRY_VALIDATION_TIMEOUT = 10.0
+
+
+def error_field_for_validation_message(error_message: str | None) -> str | None:
+ """Map an Anthropic `validate_key` error message to the UI form field that should be highlighted."""
+ return error_field_for_message(_FOUNDRY_ERROR_FIELD_BY_PREFIX, error_message)
+
+
+def _names_a_missing_deployment(error: anthropic.APIStatusError) -> bool:
+ """Return True when a Foundry 4xx is about the model or deployment, which means the key passed auth.
+
+ Azure answers an unknown route with a generic 404 ("Resource not found") before it checks
+ anything, so a 404 alone does not prove the endpoint or the key is right.
+ """
+ body = error.body if isinstance(error.body, dict) else {}
+ nested = body.get("error")
+ details = nested if isinstance(nested, dict) else body
+ if details.get("code") == "DeploymentNotFound":
+ return True
+ message = str(details.get("message", "")).lower()
+ return "deployment" in message or "model" in message
+
+
+def foundry_base_url(azure_endpoint: str) -> str:
+ """Build the Anthropic API base URL of a Microsoft Foundry resource.
+
+ Foundry serves Claude under `/anthropic/` on the resource host. Users paste the resource
+ endpoint, the `/anthropic/` URL, or a full `/anthropic/v1/messages` URL, so keep only the host.
+ """
+ if not is_allowed_azure_endpoint(azure_endpoint):
+ raise ProviderConfigurationError(f"{DISALLOWED_FOUNDRY_ENDPOINT_MESSAGE}.")
+ host = urlparse(azure_endpoint).hostname
+ return f"https://{host}/anthropic/"
+
+
+class _AnthropicFoundry(anthropic.AnthropicFoundry):
+ """Foundry client that captures generations to PostHog, like the SDK's Bedrock and Vertex wrappers."""
+
+ _ph_client: PostHogClient
+
+ def __init__(self, posthog_client: PostHogClient, **kwargs: Any):
+ super().__init__(**kwargs)
+ self._ph_client = posthog_client
+ self.messages = WrappedMessages(self) # type: ignore[assignment] # ty: ignore[invalid-assignment]
+
def _is_quota_or_billing_error(error: Exception) -> bool:
error_message = str(error).lower()
@@ -106,6 +176,28 @@ class AnthropicAdapter:
name = "anthropic"
+ def __init__(self, foundry_endpoint: str = ""):
+ # Set when the key belongs to a Microsoft Foundry resource that serves Claude.
+ self.foundry_endpoint = foundry_endpoint
+
+ def _create_client(self, api_key: str, analytics: AnalyticsContext) -> Any:
+ from products.ai_observability.backend.llm.providers._diagnostics import tagged_http_client
+
+ posthog_client = posthoganalytics.default_client
+ kwargs: dict[str, Any] = {
+ "api_key": api_key,
+ "timeout": AnthropicConfig.TIMEOUT,
+ "http_client": tagged_http_client(timeout=AnthropicConfig.TIMEOUT),
+ }
+ if self.foundry_endpoint:
+ kwargs["base_url"] = foundry_base_url(self.foundry_endpoint)
+ if analytics.capture and posthog_client:
+ return _AnthropicFoundry(posthog_client=posthog_client, **kwargs)
+ return anthropic.AnthropicFoundry(**kwargs)
+ if analytics.capture and posthog_client:
+ return Anthropic(posthog_client=posthog_client, **kwargs)
+ return anthropic.Anthropic(**kwargs)
+
def complete(
self,
request: CompletionRequest,
@@ -115,25 +207,7 @@ class AnthropicAdapter:
) -> CompletionResponse:
"""Non-streaming completion with optional structured output."""
effective_api_key = api_key or self._get_default_api_key()
-
- from products.ai_observability.backend.llm.providers._diagnostics import tagged_http_client
-
- posthog_client = posthoganalytics.default_client
- client: Any
- http_client = tagged_http_client(timeout=AnthropicConfig.TIMEOUT)
- if analytics.capture and posthog_client:
- client = Anthropic(
- api_key=effective_api_key,
- posthog_client=posthog_client,
- timeout=AnthropicConfig.TIMEOUT,
- http_client=http_client,
- )
- else:
- client = anthropic.Anthropic(
- api_key=effective_api_key,
- timeout=AnthropicConfig.TIMEOUT,
- http_client=http_client,
- )
+ client = self._create_client(effective_api_key, analytics)
messages: Any = request.messages
system_prompt = request.system or ""
@@ -242,25 +316,7 @@ class AnthropicAdapter:
"""Streaming completion."""
effective_api_key = api_key or self._get_default_api_key()
model_id = request.model
-
- from products.ai_observability.backend.llm.providers._diagnostics import tagged_http_client
-
- posthog_client = posthoganalytics.default_client
- client: Any
- http_client = tagged_http_client(timeout=AnthropicConfig.TIMEOUT)
- if analytics.capture and posthog_client:
- client = Anthropic(
- api_key=effective_api_key,
- posthog_client=posthog_client,
- timeout=AnthropicConfig.TIMEOUT,
- http_client=http_client,
- )
- else:
- client = anthropic.Anthropic(
- api_key=effective_api_key,
- timeout=AnthropicConfig.TIMEOUT,
- http_client=http_client,
- )
+ client = self._create_client(effective_api_key, analytics)
reasoning_on = model_id in AnthropicConfig.SUPPORTED_MODELS_WITH_THINKING and request.thinking
@@ -384,6 +440,10 @@ class AnthropicAdapter:
"""Validate an Anthropic API key by making a lightweight API call."""
from products.ai_observability.backend.models.provider_keys import LLMProviderKey
+ azure_endpoint = kwargs.get("azure_endpoint")
+ if azure_endpoint:
+ return AnthropicAdapter._validate_foundry_key(api_key, azure_endpoint)
+
if not api_key.startswith("sk-ant-"):
return (LLMProviderKey.State.INVALID, "Invalid key format (should start with 'sk-ant-')")
try:
@@ -404,6 +464,48 @@ class AnthropicAdapter:
logger.exception(f"Anthropic key validation error: {e}")
return (LLMProviderKey.State.ERROR, "Validation failed, please try again")
+ @staticmethod
+ def _validate_foundry_key(api_key: str, azure_endpoint: str) -> tuple[str, str | None]:
+ """Validate a Microsoft Foundry key with a token count, which Foundry serves and does not bill.
+
+ Foundry has no models endpoint, so the probe has to name a deployment. The default model ID
+ is the default deployment name, but the user may have picked another name.
+ """
+ from products.ai_observability.backend.models.provider_keys import LLMProviderKey
+
+ if not is_allowed_azure_endpoint(azure_endpoint):
+ return (LLMProviderKey.State.INVALID, DISALLOWED_FOUNDRY_ENDPOINT_MESSAGE)
+ try:
+ client = anthropic.AnthropicFoundry(
+ api_key=api_key,
+ base_url=foundry_base_url(azure_endpoint),
+ timeout=FOUNDRY_VALIDATION_TIMEOUT,
+ max_retries=0,
+ )
+ client.messages.count_tokens(
+ model=AnthropicConfig.DEFAULT_MODEL,
+ messages=[{"role": "user", "content": "hi"}],
+ )
+ return (LLMProviderKey.State.OK, None)
+ except (anthropic.AuthenticationError, anthropic.PermissionDeniedError):
+ return (LLMProviderKey.State.INVALID, "Invalid API key")
+ except anthropic.RateLimitError:
+ return (LLMProviderKey.State.ERROR, "Rate limited, please try again later")
+ except (anthropic.NotFoundError, anthropic.BadRequestError) as e:
+ # The user picks the deployment later, in the model picker, so a missing default
+ # deployment still counts as a working key.
+ if _names_a_missing_deployment(e):
+ return (LLMProviderKey.State.OK, None)
+ if isinstance(e, anthropic.NotFoundError):
+ return (LLMProviderKey.State.INVALID, FOUNDRY_ENDPOINT_NOT_FOUND_MESSAGE)
+ logger.exception("Microsoft Foundry key validation error")
+ return (LLMProviderKey.State.ERROR, "Validation failed, please try again")
+ except anthropic.APIConnectionError:
+ return (LLMProviderKey.State.ERROR, "Could not connect to Microsoft Foundry endpoint")
+ except Exception:
+ logger.exception("Microsoft Foundry key validation error")
+ return (LLMProviderKey.State.ERROR, "Validation failed, please try again")
+
@staticmethod
def recommended_models() -> set[str]:
return set(AnthropicConfig.SUPPORTED_MODELS)
@@ -415,8 +517,10 @@ class AnthropicAdapter:
Without a key, returns the curated SUPPORTED_MODELS list.
With a key, returns SUPPORTED_MODELS first, then remaining Claude models
from the API sorted by creation date (newest first).
+ With a Microsoft Foundry endpoint, returns SUPPORTED_MODELS: Foundry has no models
+ endpoint, and a new deployment takes the model ID as its default name.
"""
- if not api_key:
+ if not api_key or kwargs.get("azure_endpoint"):
return AnthropicConfig.SUPPORTED_MODELS
supported = set(AnthropicConfig.SUPPORTED_MODELS)
@@ -448,7 +552,7 @@ class AnthropicAdapter:
def _build_analytics_kwargs(self, analytics: AnalyticsContext, client) -> dict:
"""Build PostHog analytics kwargs if using instrumented client."""
- if analytics.capture and isinstance(client, Anthropic):
+ if analytics.capture and isinstance(client, Anthropic | _AnthropicFoundry):
return {
"posthog_distinct_id": analytics.distinct_id,
"posthog_trace_id": analytics.trace_id or str(uuid.uuid4()),
diff --git a/products/ai_observability/backend/llm/providers/azure_openai.py b/products/ai_observability/backend/llm/providers/azure_openai.py
index c25bf8c98691..8c2db4402356 100644
--- a/products/ai_observability/backend/llm/providers/azure_openai.py
+++ b/products/ai_observability/backend/llm/providers/azure_openai.py
@@ -15,7 +15,12 @@ from posthoganalytics.ai.openai import AzureOpenAI as WrappedAzureOpenAI
from posthog.exceptions_capture import capture_exception
-from products.ai_observability.backend.llm.errors import error_field_for_message
+from products.ai_observability.backend.llm.errors import (
+ ClaudeDeploymentOnAzureOpenAIError,
+ LLMError,
+ ModelNotFoundError,
+ error_field_for_message,
+)
from products.ai_observability.backend.llm.providers.openai import OpenAIAdapter
from products.ai_observability.backend.llm.types import AnalyticsContext
@@ -159,6 +164,17 @@ class AzureOpenAIAdapter(OpenAIAdapter):
http_client=http_client,
)
+ def _mapped_error(self, error: Exception, model: str) -> LLMError | None:
+ mapped = super()._mapped_error(error, model)
+ # A Foundry resource can host Claude next to OpenAI models, but Claude only answers on the
+ # Anthropic API. The chat completions call then fails with a 404 or 400 that names neither
+ # cause nor fix, so point the user at the Anthropic key instead.
+ if "claude" in model.lower() and (
+ isinstance(mapped, ModelNotFoundError) or (mapped is None and isinstance(error, openai.BadRequestError))
+ ):
+ return ClaudeDeploymentOnAzureOpenAIError(model)
+ return mapped
+
@staticmethod
def validate_key(api_key: str, **kwargs: Any) -> tuple[str, str | None]:
"""Validate an Azure OpenAI API key by listing deployments."""
diff --git a/products/ai_observability/backend/llm/providers/test/test_anthropic.py b/products/ai_observability/backend/llm/providers/test/test_anthropic.py
index a92ae874dd64..656bb0399ad2 100644
--- a/products/ai_observability/backend/llm/providers/test/test_anthropic.py
+++ b/products/ai_observability/backend/llm/providers/test/test_anthropic.py
@@ -10,8 +10,13 @@ from products.ai_observability.backend.llm.errors import (
ModelNotFoundError,
ModelPermissionError,
OutputTokenLimitError,
+ ProviderConfigurationError,
+)
+from products.ai_observability.backend.llm.providers.anthropic import (
+ AnthropicAdapter,
+ AnthropicConfig,
+ error_field_for_validation_message,
)
-from products.ai_observability.backend.llm.providers.anthropic import AnthropicAdapter, AnthropicConfig
from products.ai_observability.backend.llm.types import AnalyticsContext, CompletionRequest, OutputSchema
@@ -358,3 +363,133 @@ class TestAnthropicStreamErrorSurfacing:
errors = [chunk.data["error"] for chunk in chunks if chunk.type == "error"]
assert errors == ["The provider is rate limiting this key. Wait a moment, then try again."]
+
+
+def _make_status_error(
+ error_class: type[anthropic.APIStatusError], status_code: int, body: dict | None = None
+) -> anthropic.APIStatusError:
+ body = body or {"error": {"message": "error"}}
+ request = httpx.Request("POST", "https://my-resource.services.ai.azure.com/anthropic/v1/messages/count_tokens")
+ response = httpx.Response(status_code=status_code, request=request, json=body)
+ return error_class("error", response=response, body=body)
+
+
+class TestAnthropicFoundry:
+ @parameterized.expand(
+ [
+ ("resource_endpoint", "https://my-resource.services.ai.azure.com/"),
+ ("anthropic_base_url", "https://my-resource.services.ai.azure.com/anthropic/"),
+ ("messages_url", "https://my-resource.services.ai.azure.com/anthropic/v1/messages"),
+ ]
+ )
+ def test_completion_goes_to_the_foundry_anthropic_api(self, _name: str, endpoint: str):
+ mock_response = MagicMock()
+ mock_response.content = [MagicMock(text="yes")]
+ mock_response.usage.input_tokens = 1
+ mock_response.usage.output_tokens = 1
+
+ with (
+ patch("products.ai_observability.backend.llm.providers.anthropic.anthropic.AnthropicFoundry") as foundry,
+ patch("products.ai_observability.backend.llm.providers.anthropic.anthropic.Anthropic") as direct,
+ ):
+ foundry.return_value.messages.create.return_value = mock_response
+ AnthropicAdapter(foundry_endpoint=endpoint).complete(
+ CompletionRequest(
+ model="my-claude-deployment",
+ messages=[{"role": "user", "content": "hi"}],
+ provider="anthropic",
+ ),
+ api_key="foundry-key",
+ analytics=AnalyticsContext(capture=False),
+ )
+
+ direct.assert_not_called()
+ assert foundry.call_args.kwargs["api_key"] == "foundry-key"
+ assert foundry.call_args.kwargs["base_url"] == "https://my-resource.services.ai.azure.com/anthropic/"
+ assert foundry.return_value.messages.create.call_args.kwargs["model"] == "my-claude-deployment"
+
+ @patch("products.ai_observability.backend.llm.providers.anthropic.anthropic.AnthropicFoundry")
+ def test_completion_refuses_a_non_azure_endpoint(self, foundry):
+ with pytest.raises(ProviderConfigurationError):
+ AnthropicAdapter(foundry_endpoint="https://attacker.example.com/").complete(
+ CompletionRequest(model="claude-haiku-4-5", messages=[], provider="anthropic"),
+ api_key="foundry-key",
+ analytics=AnalyticsContext(capture=False),
+ )
+ foundry.assert_not_called()
+
+ @parameterized.expand(
+ [
+ ("accepted", None, "ok", None),
+ (
+ "default_deployment_missing",
+ _make_status_error(
+ anthropic.NotFoundError,
+ 404,
+ {"type": "error", "error": {"type": "not_found_error", "message": "model: claude-haiku-4-5"}},
+ ),
+ "ok",
+ None,
+ ),
+ (
+ "azure_deployment_not_found",
+ _make_status_error(
+ anthropic.NotFoundError,
+ 404,
+ {"error": {"code": "DeploymentNotFound", "message": "The API deployment does not exist."}},
+ ),
+ "ok",
+ None,
+ ),
+ (
+ "invalid_model_parameter",
+ _make_status_error(anthropic.BadRequestError, 400, {"error": {"message": "Invalid model parameter"}}),
+ "ok",
+ None,
+ ),
+ (
+ "unknown_route",
+ _make_status_error(
+ anthropic.NotFoundError, 404, {"error": {"code": "404", "message": "Resource not found"}}
+ ),
+ "invalid",
+ "azure_endpoint",
+ ),
+ ("rejected_key", _make_status_error(anthropic.AuthenticationError, 401), "invalid", "api_key"),
+ ("forbidden_key", _make_status_error(anthropic.PermissionDeniedError, 403), "invalid", "api_key"),
+ (
+ "unreachable",
+ anthropic.APIConnectionError(request=httpx.Request("POST", "https://example.com")),
+ "error",
+ "azure_endpoint",
+ ),
+ ]
+ )
+ @patch("products.ai_observability.backend.llm.providers.anthropic.anthropic.AnthropicFoundry")
+ def test_validate_key(self, _name, error, expected_state, expected_field, foundry):
+ foundry.return_value.messages.count_tokens.side_effect = error
+
+ state, message = AnthropicAdapter.validate_key(
+ "no-sk-ant-prefix", azure_endpoint="https://my-resource.services.ai.azure.com/"
+ )
+
+ assert state == expected_state
+ assert error_field_for_validation_message(message) == expected_field
+ assert foundry.call_args.kwargs["base_url"] == "https://my-resource.services.ai.azure.com/anthropic/"
+
+ @patch("products.ai_observability.backend.llm.providers.anthropic.anthropic.AnthropicFoundry")
+ def test_validate_key_never_sends_the_key_to_a_non_azure_endpoint(self, foundry):
+ state, message = AnthropicAdapter.validate_key("foundry-key", azure_endpoint="https://attacker.example.com/")
+
+ foundry.assert_not_called()
+ assert state == "invalid"
+ assert error_field_for_validation_message(message) == "azure_endpoint"
+
+ @patch("products.ai_observability.backend.llm.providers.anthropic.anthropic.Anthropic")
+ def test_list_models_does_not_call_the_anthropic_api(self, direct):
+ models = AnthropicAdapter.list_models(
+ "foundry-key", azure_endpoint="https://my-resource.services.ai.azure.com/"
+ )
+
+ direct.assert_not_called()
+ assert models == AnthropicConfig.SUPPORTED_MODELS
diff --git a/products/ai_observability/backend/llm/providers/test/test_azure_openai.py b/products/ai_observability/backend/llm/providers/test/test_azure_openai.py
index c485bfd67d9e..040b2b4bf422 100644
--- a/products/ai_observability/backend/llm/providers/test/test_azure_openai.py
+++ b/products/ai_observability/backend/llm/providers/test/test_azure_openai.py
@@ -2,8 +2,15 @@ import pytest
from unittest.mock import MagicMock, patch
import httpx
+import openai
from parameterized import parameterized
+from products.ai_observability.backend.llm.errors import (
+ AuthenticationError,
+ ClaudeDeploymentOnAzureOpenAIError,
+ ModelNotFoundError,
+ user_facing_error_message,
+)
from products.ai_observability.backend.llm.providers.azure_openai import (
DEPLOYMENTS_LIST_API_VERSION,
AzureOpenAIAdapter,
@@ -315,3 +322,28 @@ class TestAzureOpenAICreateClient:
assert mock_wrapped.call_args.kwargs["azure_endpoint"] == MOCK_ENDPOINT
assert mock_wrapped.call_args.kwargs["timeout"] == 12.0
assert mock_wrapped.call_args.kwargs["max_retries"] == 0
+
+
+def _make_openai_status_error(error_class: type[openai.APIStatusError], status_code: int) -> openai.APIStatusError:
+ request = httpx.Request("POST", f"{MOCK_ENDPOINT}openai/deployments/x/chat/completions")
+ response = httpx.Response(status_code=status_code, request=request, json={"error": {"message": "error"}})
+ return error_class("error", response=response, body={"error": {"message": "error"}})
+
+
+class TestAzureOpenAIClaudeDeployment:
+ @parameterized.expand(
+ [
+ ("claude_not_found", "claude-sonnet-4-5", openai.NotFoundError, 404, ClaudeDeploymentOnAzureOpenAIError),
+ ("claude_bad_request", "my-Claude-judge", openai.BadRequestError, 400, ClaudeDeploymentOnAzureOpenAIError),
+ ("gpt_not_found", "gpt-4o", openai.NotFoundError, 404, ModelNotFoundError),
+ ("claude_rejected_key", "claude-sonnet-4-5", openai.AuthenticationError, 401, AuthenticationError),
+ ]
+ )
+ def test_claude_deployment_points_at_the_anthropic_key(self, _name, model, error_class, status_code, expected):
+ mapped = AzureOpenAIAdapter(azure_endpoint=MOCK_ENDPOINT)._mapped_error(
+ _make_openai_status_error(error_class, status_code), model
+ )
+
+ assert type(mapped) is expected
+ if expected is ClaudeDeploymentOnAzureOpenAIError:
+ assert "Microsoft Foundry endpoint" in user_facing_error_message(mapped)
diff --git a/products/ai_observability/backend/llm/test/test_client.py b/products/ai_observability/backend/llm/test/test_client.py
index 8e64f07dbf56..05e7697b472b 100644
--- a/products/ai_observability/backend/llm/test/test_client.py
+++ b/products/ai_observability/backend/llm/test/test_client.py
@@ -102,6 +102,21 @@ class TestProviderRouting(SimpleTestCase):
assert isinstance(provider, OpenAICompatibleAdapter)
assert provider.base_url == "https://api.example.com/v1"
+ def test_get_provider_anthropic_reads_foundry_endpoint(self):
+ from products.ai_observability.backend.llm.client import _get_provider
+ from products.ai_observability.backend.llm.providers.anthropic import AnthropicAdapter
+
+ mock_key = MagicMock()
+ mock_key.encrypted_config = {
+ "api_key": "irrelevant",
+ "azure_endpoint": "https://contoso.services.ai.azure.com/",
+ }
+
+ provider = _get_provider("anthropic", mock_key)
+
+ assert isinstance(provider, AnthropicAdapter)
+ assert provider.foundry_endpoint == "https://contoso.services.ai.azure.com/"
+
def test_get_provider_openai_compatible_without_provider_key_has_no_base_url(self):
from products.ai_observability.backend.llm.client import _get_provider
from products.ai_observability.backend.llm.providers.openai_compatible import OpenAICompatibleAdapter
diff --git a/products/ai_observability/backend/models/provider_keys.py b/products/ai_observability/backend/models/provider_keys.py
index d8050d261002..51a2f23c2fd9 100644
--- a/products/ai_observability/backend/models/provider_keys.py
+++ b/products/ai_observability/backend/models/provider_keys.py
@@ -65,8 +65,8 @@ class LLMProviderKey(UUIDTModel):
"""Return provider-specific kwargs from encrypted_config.
Used when calling ``Client.validate_key`` / ``Client.list_models`` to pass
- extra config (e.g. Azure's ``azure_endpoint`` and ``api_version``). Most
- providers return an empty dict.
+ extra config (e.g. Azure's ``azure_endpoint`` and ``api_version``, or the Microsoft
+ Foundry ``azure_endpoint`` of an Anthropic key). Most providers return an empty dict.
"""
if self.provider == LLMProvider.SYSTEM_ONE:
return {
@@ -79,6 +79,8 @@ class LLMProviderKey(UUIDTModel):
}
if self.provider == LLMProvider.OPENAI_COMPATIBLE:
return {"base_url": self.encrypted_config.get("base_url", "")}
+ if self.provider == LLMProvider.ANTHROPIC and self.encrypted_config.get("azure_endpoint"):
+ return {"azure_endpoint": self.encrypted_config["azure_endpoint"]}
return {}
diff --git a/products/ai_observability/frontend/generated/api.schemas.ts b/products/ai_observability/frontend/generated/api.schemas.ts
index d21be1ebbbe5..4f1ad20157a7 100644
--- a/products/ai_observability/frontend/generated/api.schemas.ts
+++ b/products/ai_observability/frontend/generated/api.schemas.ts
@@ -2969,7 +2969,7 @@ export interface LLMProviderKeyApi {
* @nullable
*/
readonly system_one_model_display: string | null
- /** Azure OpenAI endpoint URL */
+ /** Azure OpenAI endpoint URL. For an Anthropic key, the endpoint of the Microsoft Foundry resource that serves Claude, e.g. https://my-resource.services.ai.azure.com/. Omit it for a key from the Anthropic API. */
azure_endpoint?: string
/**
* Azure OpenAI API version
@@ -3589,7 +3589,7 @@ export interface PatchedLLMProviderKeyApi {
* @nullable
*/
readonly system_one_model_display?: string | null
- /** Azure OpenAI endpoint URL */
+ /** Azure OpenAI endpoint URL. For an Anthropic key, the endpoint of the Microsoft Foundry resource that serves Claude, e.g. https://my-resource.services.ai.azure.com/. Omit it for a key from the Anthropic API. */
azure_endpoint?: string
/**
* Azure OpenAI API version
diff --git a/products/ai_observability/frontend/generated/api.zod.ts b/products/ai_observability/frontend/generated/api.zod.ts
index 279164a67381..4ca36fa22143 100644
--- a/products/ai_observability/frontend/generated/api.zod.ts
+++ b/products/ai_observability/frontend/generated/api.zod.ts
@@ -2159,7 +2159,12 @@ export const LlmAnalyticsProviderKeysCreateBody = /* @__PURE__ */ zod.object({
.max(llmAnalyticsProviderKeysCreateBodySystemOneModelMax)
.optional()
.describe('Model ID served by the System One endpoint.'),
- azure_endpoint: zod.url().optional().describe('Azure OpenAI endpoint URL'),
+ azure_endpoint: zod
+ .url()
+ .optional()
+ .describe(
+ 'Azure OpenAI endpoint URL. For an Anthropic key, the endpoint of the Microsoft Foundry resource that serves Claude, e.g. https:\/\/my-resource.services.ai.azure.com\/. Omit it for a key from the Anthropic API.'
+ ),
api_version: zod
.string()
.max(llmAnalyticsProviderKeysCreateBodyApiVersionMax)
@@ -2206,7 +2211,12 @@ export const LlmAnalyticsProviderKeysUpdateBody = /* @__PURE__ */ zod.object({
.max(llmAnalyticsProviderKeysUpdateBodySystemOneModelMax)
.optional()
.describe('Model ID served by the System One endpoint.'),
- azure_endpoint: zod.url().optional().describe('Azure OpenAI endpoint URL'),
+ azure_endpoint: zod
+ .url()
+ .optional()
+ .describe(
+ 'Azure OpenAI endpoint URL. For an Anthropic key, the endpoint of the Microsoft Foundry resource that serves Claude, e.g. https:\/\/my-resource.services.ai.azure.com\/. Omit it for a key from the Anthropic API.'
+ ),
api_version: zod
.string()
.max(llmAnalyticsProviderKeysUpdateBodyApiVersionMax)
@@ -2254,7 +2264,12 @@ export const LlmAnalyticsProviderKeysPartialUpdateBody = /* @__PURE__ */ zod.obj
.max(llmAnalyticsProviderKeysPartialUpdateBodySystemOneModelMax)
.optional()
.describe('Model ID served by the System One endpoint.'),
- azure_endpoint: zod.url().optional().describe('Azure OpenAI endpoint URL'),
+ azure_endpoint: zod
+ .url()
+ .optional()
+ .describe(
+ 'Azure OpenAI endpoint URL. For an Anthropic key, the endpoint of the Microsoft Foundry resource that serves Claude, e.g. https:\/\/my-resource.services.ai.azure.com\/. Omit it for a key from the Anthropic API.'
+ ),
api_version: zod
.string()
.max(llmAnalyticsProviderKeysPartialUpdateBodyApiVersionMax)
@@ -2301,7 +2316,12 @@ export const LlmAnalyticsProviderKeysValidateCreateBody = /* @__PURE__ */ zod.ob
.max(llmAnalyticsProviderKeysValidateCreateBodySystemOneModelMax)
.optional()
.describe('Model ID served by the System One endpoint.'),
- azure_endpoint: zod.url().optional().describe('Azure OpenAI endpoint URL'),
+ azure_endpoint: zod
+ .url()
+ .optional()
+ .describe(
+ 'Azure OpenAI endpoint URL. For an Anthropic key, the endpoint of the Microsoft Foundry resource that serves Claude, e.g. https:\/\/my-resource.services.ai.azure.com\/. Omit it for a key from the Anthropic API.'
+ ),
api_version: zod
.string()
.max(llmAnalyticsProviderKeysValidateCreateBodyApiVersionMax)
diff --git a/products/ai_observability/frontend/modelPickerLogic.test.ts b/products/ai_observability/frontend/modelPickerLogic.test.ts
index 2b3d7606a4dd..f511b858db92 100644
--- a/products/ai_observability/frontend/modelPickerLogic.test.ts
+++ b/products/ai_observability/frontend/modelPickerLogic.test.ts
@@ -471,6 +471,14 @@ describe('modelPickerLogic', () => {
results: [
{ id: 'key-1', provider: 'openai', name: 'My OpenAI Key', state: 'ok' },
{ id: 'key-azure', provider: 'azure_openai', name: 'My Azure Key', state: 'ok' },
+ {
+ id: 'key-foundry',
+ provider: 'anthropic',
+ name: 'My Foundry Key',
+ state: 'ok',
+ azure_endpoint_display: 'https://my-resource.services.ai.azure.com/',
+ },
+ { id: 'key-anthropic', provider: 'anthropic', name: 'My Anthropic Key', state: 'ok' },
],
},
'/api/environments/:team_id/llm_analytics/evaluation_config/': {
@@ -487,9 +495,11 @@ describe('modelPickerLogic', () => {
logic.mount()
await expectLogic(logic).toFinishAllListeners()
- expect(logic.values.providerModelGroups.find((g) => g.providerKeyId === 'key-azure')).toMatchObject({
- allowsCustomModel: true,
- })
+ const customModelKeyIds = logic.values.providerModelGroups
+ .filter((g) => g.allowsCustomModel)
+ .map((g) => g.providerKeyId)
+ .sort()
+ expect(customModelKeyIds).toEqual(['key-azure', 'key-foundry'])
expect(logic.values.playgroundByokProviderModelGroups.map((g) => g.providerKeyId)).toEqual(['key-1'])
expect(logic.values.playgroundByokProviderModelGroups.some((g) => g.allowsCustomModel)).toBe(false)
})
diff --git a/products/ai_observability/frontend/modelPickerLogic.ts b/products/ai_observability/frontend/modelPickerLogic.ts
index 4b271dc86841..cd12b23d87b8 100644
--- a/products/ai_observability/frontend/modelPickerLogic.ts
+++ b/products/ai_observability/frontend/modelPickerLogic.ts
@@ -58,7 +58,7 @@ function getByokModelNotice(
return null
}
const failedKeys = providerKeys.filter(
- (key) => failedByokProviderKeyIds.includes(key.id) && !allowsCustomModel(key.provider)
+ (key) => failedByokProviderKeyIds.includes(key.id) && !allowsCustomModel(key)
)
if (failedKeys.length > 0) {
return { kind: 'models-failed', keys: failedKeys }
@@ -77,9 +77,9 @@ function getByokModelNotice(
return providerKeys.length === 0 ? { kind: 'no-keys' } : { kind: 'no-usable-keys' }
}
-// Azure chat completions need a deployment name, and Azure does not always list deployments.
-function allowsCustomModel(provider: LLMProvider): boolean {
- return provider === 'azure_openai'
+// Azure OpenAI and Claude on Microsoft Foundry need a deployment name, and Azure does not always list deployments.
+function allowsCustomModel(key: LLMProviderKey): boolean {
+ return key.provider === 'azure_openai' || (key.provider === 'anthropic' && !!key.azure_endpoint_display)
}
function providerKeyGroupLabel(key: LLMProviderKey, keysPerProvider: Record<string, number>, suffix = ''): string {
@@ -414,7 +414,7 @@ export const modelPickerLogic = kea<modelPickerLogicType>([
const groups: ProviderModelGroup[] = []
for (const key of providerKeys) {
const models = byKeyId[key.id] ?? []
- const customModelKey = allowsCustomModel(key.provider) && key.state === 'ok'
+ const customModelKey = allowsCustomModel(key) && key.state === 'ok'
if (models.length > 0 || customModelKey) {
groups.push({
provider: key.provider,
diff --git a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
index d3ab3e51cc92..110c4be30cde 100644
--- a/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
+++ b/products/ai_observability/frontend/settings/LLMProviderKeysSettings.tsx
@@ -87,12 +87,12 @@ function formatDate(dateString: string | null): string {
})
}
-function getKeyPlaceholder(provider: LLMProvider): string {
+function getKeyPlaceholder(provider: LLMProvider, usesFoundry = false): string {
switch (provider) {
case 'openai':
return 'sk-...'
case 'anthropic':
- return 'sk-ant-...'
+ return usesFoundry ? 'Enter your Microsoft Foundry API key' : 'sk-ant-...'
case 'gemini':
return 'Enter your Gemini API key'
case 'together_ai':
@@ -122,22 +122,62 @@ function endpointErrorFieldFromResult(result: KeyValidationResult | null | undef
return result.error_field === 'azure_endpoint' || result.error_field === 'base_url' ? 'endpoint' : 'key'
}
+function FoundryEndpointField({
+ value,
+ onChange,
+ errorMessage,
+ optional,
+}: {
+ value: string
+ onChange: (value: string) => void
+ errorMessage: string | null
+ optional: boolean
+}): JSX.Element {
+ return (
+ <div>
+ <label className="text-sm font-medium">
+ {optional ? 'Microsoft Foundry endpoint (optional)' : 'Microsoft Foundry endpoint'}
+ </label>
+ <LemonInput
+ value={value}
+ onChange={onChange}
+ placeholder="https://my-resource.services.ai.azure.com/"
+ className="mt-1"
+ fullWidth
+ status={errorMessage ? 'danger' : undefined}
+ />
+ {errorMessage ? (
+ <p className="text-xs text-danger mt-1">{errorMessage}</p>
+ ) : (
+ <p className="text-xs text-muted mt-1">
+ {optional
+ ? 'Fill this in only if you run Claude on Microsoft Foundry. Leave it empty for a key from the Anthropic API.'
+ : 'The endpoint URL of your Microsoft Foundry resource'}
+ </p>
+ )}
+ </div>
+ )
+}
+
function KeyValidationStatus({
result,
isValidating,
provider,
suppressError = false,
+ usesFoundry = false,
}: {
result: KeyValidationResult | null
isValidating: boolean
provider: LLMProvider
suppressError?: boolean
+ usesFoundry?: boolean
}): JSX.Element | null {
if (isValidating) {
return <p className="text-xs text-muted mt-1">Validating key...</p>
}
- const providerLabel = provider === 'openai_compatible' ? 'your provider' : LLM_PROVIDER_LABELS[provider]
+ const providerLabel =
+ provider === 'openai_compatible' ? 'your provider' : usesFoundry ? 'Microsoft' : LLM_PROVIDER_LABELS[provider]
const bullets = (
<ul className="text-xs text-muted mt-1 list-disc pl-4 space-y-0.5">
<li>Your key will be encrypted and stored securely</li>
@@ -185,7 +225,15 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
const [pendingSubmit, setPendingSubmit] = useState(false)
const isAzure = provider === 'azure_openai'
+ const isAnthropic = provider === 'anthropic'
const isOpenAICompatible = provider === 'openai_compatible'
+ const hasEndpointField = isAzure || isAnthropic || isOpenAICompatible
+ // Anthropic keys take an optional Microsoft Foundry endpoint, so only send one that was entered.
+ const endpointPayload = isAzure
+ ? { azure_endpoint: azureEndpoint, api_version: apiVersion }
+ : isAnthropic && azureEndpoint
+ ? { azure_endpoint: azureEndpoint }
+ : {}
const keyValidated = preValidationResult?.state === 'ok'
const isSystemOne = provider === 'system_one'
const isValid =
@@ -195,7 +243,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
(!isOpenAICompatible || baseUrl.length > 0)
const validationFailed = !!preValidationResult && preValidationResult.state !== 'ok'
const endpointErrorField =
- (isAzure || isOpenAICompatible) && validationFailed ? endpointErrorFieldFromResult(preValidationResult) : null
+ hasEndpointField && validationFailed ? endpointErrorFieldFromResult(preValidationResult) : null
// Reset form when modal closes
useEffect(() => {
@@ -220,10 +268,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
name,
api_key: apiKey,
set_as_active: provider !== 'system_one' && !evaluationConfig?.active_provider_key,
- }
- if (isAzure) {
- payload.azure_endpoint = azureEndpoint
- payload.api_version = apiVersion
+ ...endpointPayload,
}
if (isOpenAICompatible) {
payload.base_url = baseUrl
@@ -240,6 +285,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
apiKey,
provider,
isAzure,
+ isAnthropic,
azureEndpoint,
apiVersion,
isOpenAICompatible,
@@ -271,10 +317,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
name,
api_key: apiKey,
set_as_active: !evaluationConfig?.active_provider_key,
- }
- if (isAzure) {
- payload.azure_endpoint = azureEndpoint
- payload.api_version = apiVersion
+ ...endpointPayload,
}
if (isOpenAICompatible) {
payload.base_url = baseUrl
@@ -285,7 +328,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
preValidateKey({
apiKey,
provider,
- ...(isAzure ? { azure_endpoint: azureEndpoint, api_version: apiVersion } : {}),
+ ...endpointPayload,
...(isOpenAICompatible ? { base_url: baseUrl } : {}),
})
}
@@ -299,7 +342,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
preValidateKey({
apiKey,
provider,
- ...(isAzure ? { azure_endpoint: azureEndpoint, api_version: apiVersion } : {}),
+ ...endpointPayload,
...(isOpenAICompatible ? { base_url: baseUrl } : {}),
})
}
@@ -312,14 +355,16 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
}
}
- // A result is only about the endpoint it was validated against, so editing the base URL has
+ // A result is only about the endpoint it was validated against, so editing the endpoint has
// to drop it. Otherwise a key validated against the old endpoint keeps showing as validated.
- const handleBaseUrlChange = (value: string): void => {
- setBaseUrl(value)
- if (preValidationResult) {
- clearPreValidation()
+ const handleEndpointChange =
+ (setEndpoint: (value: string) => void): ((value: string) => void) =>
+ (value) => {
+ setEndpoint(value)
+ if (preValidationResult) {
+ clearPreValidation()
+ }
}
- }
const handleProviderChange = (value: LLMProvider): void => {
setProvider(value)
@@ -405,13 +450,25 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
</div>
</>
)}
+ {isAnthropic && (
+ <FoundryEndpointField
+ optional
+ value={azureEndpoint}
+ onChange={handleEndpointChange(setAzureEndpoint)}
+ errorMessage={
+ endpointErrorField === 'endpoint'
+ ? preValidationResult?.error_message || 'Invalid Microsoft Foundry endpoint'
+ : null
+ }
+ />
+ )}
{isSystemOne && <SystemOneConnectionFields />}
{isOpenAICompatible && (
<div>
<label className="text-sm font-medium">Base URL</label>
<LemonInput
value={baseUrl}
- onChange={handleBaseUrlChange}
+ onChange={handleEndpointChange(setBaseUrl)}
placeholder="https://api.example.com/v1"
className="mt-1"
fullWidth
@@ -446,13 +503,13 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
value={apiKey}
onChange={handleApiKeyChange}
onBlur={handleApiKeyBlur}
- placeholder={getKeyPlaceholder(provider)}
+ placeholder={getKeyPlaceholder(provider, isAnthropic && azureEndpoint.length > 0)}
type="password"
autoComplete="off"
className="mt-1"
fullWidth
status={
- validationFailed && ((!isAzure && !isOpenAICompatible) || endpointErrorField === 'key')
+ validationFailed && (!hasEndpointField || endpointErrorField === 'key')
? 'danger'
: undefined
}
@@ -462,6 +519,7 @@ function AddKeyModal({ restrictionReason }: { restrictionReason: string | null }
isValidating={preValidationResultLoading}
provider={provider}
suppressError={endpointErrorField === 'endpoint'}
+ usesFoundry={isAnthropic && azureEndpoint.length > 0}
/>
{isSystemOne && (
<p className="text-xs text-muted">
@@ -491,6 +549,8 @@ function EditKeyModal({
normalizeSystemOneBaseUrlForComparison(systemOneBaseUrl) !==
normalizeSystemOneBaseUrlForComparison(keyToEdit.base_url_display ?? DEFAULT_SYSTEM_ONE_BASE_URL)
const isOpenAICompatibleEdit = keyToEdit.provider === 'openai_compatible'
+ // An Anthropic key keeps the API it was created for, so only Foundry keys show the endpoint.
+ const isFoundryEdit = keyToEdit.provider === 'anthropic' && !!keyToEdit.azure_endpoint_display
const [name, setName] = useState(keyToEdit.name)
const [apiKey, setApiKey] = useState('')
@@ -531,6 +591,9 @@ function EditKeyModal({
if (baseUrlChanged) {
payload.base_url = baseUrl
}
+ if (foundryEndpointChanged) {
+ payload.azure_endpoint = azureEndpoint
+ }
updateProviderKey({ id: keyToEdit.id, payload })
}
@@ -543,6 +606,7 @@ function EditKeyModal({
apiKey,
provider: keyToEdit.provider,
...(isAzureEdit ? { azure_endpoint: azureEndpoint, api_version: apiVersion } : {}),
+ ...(isFoundryEdit ? { azure_endpoint: azureEndpoint } : {}),
...(isOpenAICompatibleEdit ? { base_url: baseUrl } : {}),
})
}
@@ -555,31 +619,48 @@ function EditKeyModal({
}
}
- // A result is only about the endpoint it was validated against, so editing the base URL has
+ // A result is only about the endpoint it was validated against, so editing the endpoint has
// to drop it. Otherwise a key validated against the old endpoint keeps showing as validated.
- const handleBaseUrlChange = (value: string): void => {
- setBaseUrl(value)
- if (preValidationResult) {
- clearPreValidation()
+ const handleEndpointChange =
+ (setEndpoint: (value: string) => void): ((value: string) => void) =>
+ (value) => {
+ setEndpoint(value)
+ if (preValidationResult) {
+ clearPreValidation()
+ }
}
- }
const keyValidated = isSystemOne || apiKey.length === 0 || preValidationResult?.state === 'ok'
const baseUrlMissing = isOpenAICompatibleEdit && baseUrl.length === 0
+ const foundryEndpointMissing = isFoundryEdit && azureEndpoint.length === 0
// The backend rejects a base URL change that arrives without a key, so the key can't stay
// masked here: re-entering it is what proves the caller already has it.
const baseUrlChanged = isOpenAICompatibleEdit && !baseUrlMissing && baseUrl !== (keyToEdit.base_url_display ?? '')
+ const foundryEndpointChanged =
+ isFoundryEdit && !foundryEndpointMissing && azureEndpoint !== (keyToEdit.azure_endpoint_display ?? '')
const apiKeyRequiredReason =
- baseUrlChanged && apiKey.length === 0 ? 'Enter the API key again to change the base URL' : null
- const missingFieldReason = name.length === 0 ? 'Enter a name' : baseUrlMissing ? 'Enter a base URL' : null
+ apiKey.length > 0
+ ? null
+ : baseUrlChanged
+ ? 'Enter the API key again to change the base URL'
+ : foundryEndpointChanged
+ ? 'Enter the API key again to change the Microsoft Foundry endpoint'
+ : null
+ const missingFieldReason =
+ name.length === 0
+ ? 'Enter a name'
+ : baseUrlMissing
+ ? 'Enter a base URL'
+ : foundryEndpointMissing
+ ? 'Enter a Microsoft Foundry endpoint'
+ : null
const disabledReason = restrictionReason ?? missingFieldReason ?? apiKeyRequiredReason
const isValid =
keyValidated && !disabledReason && (!isSystemOne || (systemOneBaseUrl.length > 0 && systemOneModel.length > 0))
const validationFailed = !!preValidationResult && preValidationResult.state !== 'ok'
+ const hasEndpointField = isAzureEdit || isFoundryEdit || isOpenAICompatibleEdit
const endpointErrorField =
- (isAzureEdit || isOpenAICompatibleEdit) && validationFailed
- ? endpointErrorFieldFromResult(preValidationResult)
- : null
+ hasEndpointField && validationFailed ? endpointErrorFieldFromResult(preValidationResult) : null
return (
<LemonModal
@@ -641,13 +722,25 @@ function EditKeyModal({
</div>
</>
)}
+ {isFoundryEdit && (
+ <FoundryEndpointField
+ optional={false}
+ value={azureEndpoint}
+ onChange={handleEndpointChange(setAzureEndpoint)}
+ errorMessage={
+ endpointErrorField === 'endpoint'
+ ? preValidationResult?.error_message || 'Invalid Microsoft Foundry endpoint'
+ : null
+ }
+ />
+ )}
{isSystemOne && <SystemOneConnectionFields />}
{isOpenAICompatibleEdit && (
<div>
<label className="text-sm font-medium">Base URL</label>
<LemonInput
value={baseUrl}
- onChange={handleBaseUrlChange}
+ onChange={handleEndpointChange(setBaseUrl)}
placeholder="https://api.example.com/v1"
className="mt-1"
fullWidth
@@ -680,8 +773,7 @@ function EditKeyModal({
className="mt-1"
fullWidth
status={
- validationFailed &&
- ((!isAzureEdit && !isOpenAICompatibleEdit) || endpointErrorField === 'key')
+ validationFailed && (!hasEndpointField || endpointErrorField === 'key')
? 'danger'
: undefined
}
@@ -692,14 +784,13 @@ function EditKeyModal({
isValidating={preValidationResultLoading}
provider={keyToEdit.provider}
suppressError={endpointErrorField === 'endpoint'}
+ usesFoundry={isFoundryEdit}
/>
) : (
<p className="text-xs text-muted mt-1">
{isSystemOne && endpointChanged
? 'The saved key will not be sent to the new endpoint. Enter its bearer token, or leave empty for no authentication.'
- : baseUrlChanged
- ? 'Enter the API key again to change the base URL'
- : 'Leave empty to keep the current key'}
+ : (apiKeyRequiredReason ?? 'Leave empty to keep the current key')}
</p>
)}
</div>
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 9295173640d4..0a5773273d50 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -43064,7 +43064,7 @@ export namespace Schemas {
* @nullable
*/
readonly system_one_model_display: string | null;
- /** Azure OpenAI endpoint URL */
+ /** Azure OpenAI endpoint URL. For an Anthropic key, the endpoint of the Microsoft Foundry resource that serves Claude, e.g. https://my-resource.services.ai.azure.com/. Omit it for a key from the Anthropic API. */
azure_endpoint?: string;
/**
* Azure OpenAI API version
@@ -83217,7 +83217,7 @@ export namespace Schemas {
* @nullable
*/
readonly system_one_model_display?: string | null;
- /** Azure OpenAI endpoint URL */
+ /** Azure OpenAI endpoint URL. For an Anthropic key, the endpoint of the Microsoft Foundry resource that serves Claude, e.g. https://my-resource.services.ai.azure.com/. Omit it for a key from the Anthropic API. */
azure_endpoint?: string;
/**
* Azure OpenAI API version