Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
Add Atlas Cloud provider (#63)
nb2136717e14f3c1d
diff --git a/README.md b/README.mdindex 57de7e2..4f329da 100644--- a/README.md+++ b/README.md@@ -155,8 +155,8 @@ from different vendors:Reports land in `./ifixai-results/` as JSON **and** Markdown. Without a second key, add`--eval-mode self` to run as a smoke test (the grade still prints, but it's flagged asself-judged, not a result you can cite). Pinning the judge, Full-mode ensembles, and the eval modes:-**[docs/cli.md](docs/cli.md#how-a-run-is-judged)**. Other providers (OpenAI, OpenRouter, Gemini,-Azure, Bedrock, Hugging Face) install the matching extra and follow the same steps; the+**[docs/cli.md](docs/cli.md#how-a-run-is-judged)**. Other providers (OpenAI, Atlas Cloud,+OpenRouter, Gemini, Azure, Bedrock, Hugging Face) install the matching extra and follow the same steps; theHTTP and LangChain adapters need no provider extra: **[docs/testing-your-agent.md](docs/testing-your-agent.md#provider-reference)**.### Recommended judge setupsdiff --git a/SECURITY.md b/SECURITY.mdindex d37f07d..f571274 100644--- a/SECURITY.md+++ b/SECURITY.md@@ -40,6 +40,7 @@ The scrubber (`ifixai/providers/secrets.py::scrub_secrets`) is parametrically ve| Provider | Credential shape matched | Redaction token ||---|---|---|| `openai` | `sk-[A-Za-z0-9_-]{20,}` | `***REDACTED_OPENAI_KEY***` |+| `atlascloud` | `(ak|apikey)-[A-Za-z0-9_-]{20,}` | `***REDACTED_ATLASCLOUD_KEY***` || `openrouter` | `sk-or-[A-Za-z0-9_-]{20,}` (matched before `openai`) | `***REDACTED_OPENROUTER_KEY***` || `anthropic` | `sk-ant-[A-Za-z0-9_-]{20,}`, `anthropic_[A-Za-z0-9_-]{20,}` | `***REDACTED_ANTHROPIC_KEY***` || `gemini` | `AIzaSy[0-9A-Za-z_-]{33}` | `***REDACTED_GEMINI_KEY***` |diff --git a/docs/testing-your-agent.md b/docs/testing-your-agent.mdindex d0c491a..3ebef0b 100644--- a/docs/testing-your-agent.md+++ b/docs/testing-your-agent.md@@ -66,7 +66,7 @@ The scorecard adds a `warnings[]` note: declared, not measured at runtime. Fielddifferent provider grades the SUT, so nothing scores itself:- Default judge: any non-SUT provider key in your env, on that provider's default model.-- Multiple keys: tiebreaker order `anthropic → openai → gemini → openrouter → azure → bedrock → huggingface`.+- Multiple keys: tiebreaker order `anthropic → openai → atlascloud → gemini → openrouter → azure → bedrock → huggingface`.- No non-SUT key: the run refuses unless you pass `--eval-mode self`.- Override: `--judge-provider` / `--judge-api-key` / `--judge-model`. Pin these for`openrouter` and `azure`: auto-routing can land SUT and judge on the same vendor.@@ -108,6 +108,7 @@ the SUT key from the environment: pass `--api-key` / `-k`, or enter it when prom| `http` (recommended) | none | your server token | `--provider http --endpoint http://localhost:8000/v1 --grounding sut -k TOKEN --model your-model-id` || `anthropic` | `.[anthropic]` | `ANTHROPIC_API_KEY` | `--provider anthropic -k "$ANTHROPIC_API_KEY" --model claude-sonnet-4-6` || `openai` | `.[openai]` | `OPENAI_API_KEY` | `--provider openai -k "$OPENAI_API_KEY" --model gpt-4o` |+| `atlascloud` | `.[atlascloud]` | `ATLASCLOUD_API_KEY` or `ATLAS_CLOUD_API_KEY` | `--provider atlascloud -k "$ATLASCLOUD_API_KEY" --model qwen/qwen3.5-flash` || `openrouter` | `.[openrouter]` | `OPENROUTER_API_KEY` | `--provider openrouter -k "$OPENROUTER_API_KEY" --model openai/gpt-4o` plus explicit judge || `gemini` | `.[gemini]` | `GEMINI_API_KEY` or `GOOGLE_API_KEY` | `--provider gemini -k "$GEMINI_API_KEY"` || `azure` | `.[azure]` | `AZURE_OPENAI_API_KEY` | `--provider azure --endpoint https://YOUR_RESOURCE.openai.azure.com/ -k "$AZURE_OPENAI_API_KEY" --model YOUR_DEPLOYMENT_NAME` plus explicit judge |diff --git a/ifixai/cli/run.py b/ifixai/cli/run.pyindex b8f890c..a997239 100644--- a/ifixai/cli/run.py+++ b/ifixai/cli/run.py@@ -126,6 +126,7 @@ def _governance_source_warning(source: str) -> str | None:PROVIDER_CHOICES = ["mock","openai",+ "atlascloud","gemini","anthropic","azure",diff --git a/ifixai/providers/atlascloud.py b/ifixai/providers/atlascloud.pynew file mode 100644index 0000000..7eb3685--- /dev/null+++ b/ifixai/providers/atlascloud.py@@ -0,0 +1,130 @@+import asyncio++import openai++from ifixai.core.types import ChatMessage, ProviderConfig+from ifixai.providers.base import (+ ChatProvider,+ ProviderAuthError,+ ProviderConnectionError,+ ProviderEmptyContentError,+ ProviderRateLimitError,+ ProviderResponseError,+ ProviderTimeoutError,+ create_chat_completion_json_fallback,+)++DEFAULT_MODEL = "qwen/qwen3.5-flash"+DEFAULT_BASE_URL = "https://api.atlascloud.ai/v1"+MAX_TOKENS_CEILING: int = 8192++ClientCacheKey = tuple[str, str | None, float, int]+++class AtlasCloudProvider(ChatProvider):++ def __init__(self) -> None:+ self._clients: dict[ClientCacheKey, openai.AsyncOpenAI] = {}+ self._client_lock = asyncio.Lock()++ async def get_client(self, config: ProviderConfig) -> openai.AsyncOpenAI:+ """Return a long-lived AsyncOpenAI client keyed on connection params."""+ base_url = config.endpoint or DEFAULT_BASE_URL+ key: ClientCacheKey = (+ base_url,+ config.api_key,+ float(config.timeout),+ config.max_retries,+ )+ cached = self._clients.get(key)+ if cached is not None:+ return cached+ async with self._client_lock:+ cached = self._clients.get(key)+ if cached is not None:+ return cached+ client = openai.AsyncOpenAI(+ api_key=config.api_key,+ base_url=base_url,+ timeout=float(config.timeout),+ max_retries=config.max_retries,+ )+ self._clients[key] = client+ return client++ async def aclose(self) -> None:+ for client in self._clients.values():+ await client.close()+ self._clients.clear()++ async def send_message(+ self,+ messages: list[ChatMessage],+ config: ProviderConfig,+ ) -> str:+ base_url = config.endpoint or DEFAULT_BASE_URL+ model = config.model or DEFAULT_MODEL+ formatted = [{"role": m.role, "content": m.content} for m in messages]++ client = await self.get_client(config)+ try:+ effective_max_tokens = (+ min(config.max_tokens, MAX_TOKENS_CEILING)+ if config.max_tokens is not None+ else MAX_TOKENS_CEILING+ )+ create_kwargs: dict = {+ "model": model,+ "messages": formatted, # type: ignore[arg-type]+ "max_tokens": effective_max_tokens,+ "temperature": config.temperature,+ }+ if config.seed is not None:+ create_kwargs["seed"] = config.seed+ if config.json_output:+ create_kwargs["response_format"] = {"type": "json_object"}+ response = await create_chat_completion_json_fallback(client, **create_kwargs)+ choices = response.choices+ if not choices:+ raise ProviderResponseError(+ provider="atlascloud",+ endpoint=base_url,+ details=f"No choices in response (id={response.id})",+ )+ choice = choices[0]+ finish_reason = choice.finish_reason or "unknown"+ if choice.message is None:+ raise ProviderResponseError(+ provider="atlascloud",+ endpoint=base_url,+ details=f"Missing message in choice (finish_reason={finish_reason})",+ )+ content = choice.message.content+ if not content:+ raise ProviderEmptyContentError(+ provider="atlascloud",+ endpoint=base_url,+ details=f"Empty content in response (finish_reason={finish_reason})",+ )+ except openai.AuthenticationError as exc:+ raise ProviderAuthError(+ provider="atlascloud", endpoint=base_url, details=str(exc)+ ) from exc+ except openai.RateLimitError as exc:+ raise ProviderRateLimitError(+ provider="atlascloud", endpoint=base_url, details=str(exc)+ ) from exc+ except openai.APITimeoutError as exc:+ raise ProviderTimeoutError(+ provider="atlascloud", endpoint=base_url, details=str(exc)+ ) from exc+ except openai.APIConnectionError as exc:+ raise ProviderConnectionError(+ provider="atlascloud", endpoint=base_url, details=str(exc)+ ) from exc+ except openai.APIError as exc:+ raise ProviderResponseError(+ provider="atlascloud", endpoint=base_url, details=str(exc)+ ) from exc+ else:+ return contentdiff --git a/ifixai/providers/resolver.py b/ifixai/providers/resolver.pyindex 05d89f2..57199cf 100644--- a/ifixai/providers/resolver.py+++ b/ifixai/providers/resolver.py@@ -39,6 +39,11 @@ try:except ImportError:OpenAIProvider = None+try:+ from ifixai.providers.atlascloud import AtlasCloudProvider+except ImportError:+ AtlasCloudProvider = None+try:from ifixai.providers.openrouter import OpenRouterProviderexcept ImportError:@@ -54,6 +59,7 @@ REGISTERED_PROVIDERS: tuple[str, ...] = ("http","mock","openai",+ "atlascloud","openrouter","anthropic","gemini",@@ -72,6 +78,7 @@ _PROVIDER_MAP: dict[str, type] = {for name, cls in {"http": HttpProvider,"openai": OpenAIProvider,+ "atlascloud": AtlasCloudProvider,"openrouter": OpenRouterProvider,"anthropic": AnthropicProvider,"gemini": GeminiProvider,@@ -143,6 +150,7 @@ def resolve_provider(provider: str | object) -> object:_PROVIDER_CREDENTIAL_ENV_VARS: dict[str, tuple[str, ...]] = {"openai": ("OPENAI_API_KEY",),+ "atlascloud": ("ATLASCLOUD_API_KEY", "ATLAS_CLOUD_API_KEY"),"anthropic": ("ANTHROPIC_API_KEY",),"gemini": ("GEMINI_API_KEY", "GOOGLE_API_KEY"),"azure": ("AZURE_OPENAI_API_KEY",),@@ -160,6 +168,7 @@ _PROVIDER_CREDENTIAL_REQUIRES_ALL: frozenset[str] = frozenset({"bedrock"})_JUDGE_PREFERENCE_ORDER: tuple[str, ...] = ("anthropic","openai",+ "atlascloud","gemini","openrouter","azure",diff --git a/ifixai/providers/secrets.py b/ifixai/providers/secrets.pyindex e8f370d..e011edf 100644--- a/ifixai/providers/secrets.py+++ b/ifixai/providers/secrets.py@@ -6,6 +6,7 @@ from typing import Any, FinalENV_VAR_BY_PROVIDER: dict[str, str] = {"anthropic": "ANTHROPIC_API_KEY","openai": "OPENAI_API_KEY",+ "atlascloud": "ATLASCLOUD_API_KEY","gemini": "GEMINI_API_KEY","azure": "AZURE_OPENAI_API_KEY","bedrock": "AWS_ACCESS_KEY_ID",@@ -14,6 +15,7 @@ ENV_VAR_BY_PROVIDER: dict[str, str] = {_SECRET_PATTERNS: Final[tuple[Pattern[str], ...]] = (re.compile(r"^sk-or-[A-Za-z0-9_-]{20,}$"),+ re.compile(r"^ak-[A-Za-z0-9_-]{20,}$"),re.compile(r"^sk-ant-[A-Za-z0-9_-]{20,}$"),re.compile(r"^sk-[A-Za-z0-9_-]{20,}$"),re.compile(r"^anthropic_[A-Za-z0-9_-]{20,}$"),@@ -28,6 +30,10 @@ _SCRUB_RULES: Final[tuple[tuple[Pattern[str], str], ...]] = (re.compile(r"sk-or-[A-Za-z0-9_-]{20,}"),"***REDACTED_OPENROUTER_KEY***",),+ (+ re.compile(r"(?<![A-Za-z0-9_-])ak-[A-Za-z0-9_-]{20,}(?![A-Za-z0-9_-])"),+ "***REDACTED_ATLASCLOUD_KEY***",+ ),(re.compile(r"sk-ant-[A-Za-z0-9_-]{20,}"),"***REDACTED_ANTHROPIC_KEY***",diff --git a/ifixai/reporting/scorecard.py b/ifixai/reporting/scorecard.pyindex 2680e6f..9e84b0b 100644--- a/ifixai/reporting/scorecard.py+++ b/ifixai/reporting/scorecard.py@@ -234,7 +234,7 @@ def b32_not_applicable_warning(# aggregator (e.g. a Gemini SUT graded by a Claude judge, both via OpenRouter) is# recognized as independent instead of mislabeled "self-judge"._AGGREGATOR_PROVIDERS: Final[frozenset[str]] = frozenset(- {"openrouter", "litellm", "http", "langchain"}+ {"openrouter", "atlascloud", "litellm", "http", "langchain"})# Distinct provider slugs that front the SAME underlying model vendor, so adiff --git a/pyproject.toml b/pyproject.tomlindex dd07592..d51126c 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -40,6 +40,7 @@ dependencies = [[project.optional-dependencies]openai = ["openai>=1.0"]+atlascloud = ["openai>=1.0"]openrouter = ["openai>=1.0"]gemini = ["google-generativeai>=0.3"]anthropic = ["anthropic>=0.18"]diff --git a/test_atlascloud_provider.py b/test_atlascloud_provider.pynew file mode 100644index 0000000..14d0733--- /dev/null+++ b/test_atlascloud_provider.py@@ -0,0 +1,82 @@+import pytest++from ifixai.core.types import ProviderConfig+from ifixai.judge.config import JudgeConfig+from ifixai.providers.atlascloud import (+ DEFAULT_BASE_URL,+ DEFAULT_MODEL,+ AtlasCloudProvider,+)+from ifixai.providers.resolver import (+ credential_env_vars,+ detect_available_credentials,+ resolve_credential,+ resolve_provider,+ select_cross_provider_judge,+)+from ifixai.providers.secrets import scrub_secrets+from ifixai.reporting.scorecard import self_judge_bias_applies+++def test_atlascloud_resolves_and_reads_credentials() -> None:+ provider = resolve_provider("atlascloud")++ assert isinstance(provider, AtlasCloudProvider)+ assert credential_env_vars("atlascloud") == (+ "ATLASCLOUD_API_KEY",+ "ATLAS_CLOUD_API_KEY",+ )+ assert resolve_credential(+ "atlascloud",+ {"ATLASCLOUD_API_KEY": "apikey-test-value-for-atlascloud"},+ ) == "apikey-test-value-for-atlascloud"+ assert "atlascloud" in detect_available_credentials(+ {"ATLAS_CLOUD_API_KEY": "ak-test-value-for-atlascloud"}+ )+++def test_atlascloud_can_be_selected_as_independent_judge() -> None:+ assert select_cross_provider_judge("openai", ["openai", "atlascloud"]) == "atlascloud"+ assert select_cross_provider_judge("atlascloud", ["openai", "atlascloud"]) == "openai"+++async def test_atlascloud_client_uses_openai_compatible_defaults() -> None:+ provider = AtlasCloudProvider()+ config = ProviderConfig(provider="atlascloud", api_key="ak-test-value-for-atlascloud")++ try:+ client = await provider.get_client(config)++ assert str(client.base_url) == f"{DEFAULT_BASE_URL}/"+ assert provider._clients[+ (DEFAULT_BASE_URL, config.api_key, float(config.timeout), config.max_retries)+ ] is client+ assert DEFAULT_MODEL == "qwen/qwen3.5-flash"+ finally:+ await provider.aclose()+++def test_atlascloud_keys_are_scrubbed() -> None:+ assert (+ scrub_secrets("token ak-abcdefghijklmnopqrstuvwxyz")+ == "token ***REDACTED_ATLASCLOUD_KEY***"+ )+ assert scrub_secrets("break-glass-escalation-path") == "break-glass-escalation-path"+ assert (+ scrub_secrets("token apikey-abcdefghijklmnopqrstuvwxyz")+ == "token apikey-abcdefghijklmnopqrstuvwxyz"+ )+++def test_atlascloud_is_treated_as_aggregator_for_bias_detection() -> None:+ assert not self_judge_bias_applies(+ JudgeConfig(provider="atlascloud", model="anthropic/claude-sonnet-4-6"),+ "openai",+ "gpt-4o",+ )+ assert self_judge_bias_applies(+ JudgeConfig(provider="atlascloud", model="openai/gpt-4o"),+ "openai",+ "gpt-4o",+ )