Personal AI, On Personal Devices
Python
"""Cloud inference engine.
OpenAI, Anthropic, Google, MiniMax, DeepSeek, and Atlas Cloud API backends.
"""
from __future__ import annotations
import json
import logging
import os
import time
import uuid
from collections.abc import AsyncIterator, Sequence
from typing import Any, Dict, List, Tuple
import httpx
from openjarvis.core.registry import EngineRegistry
from openjarvis.core.types import Message
from openjarvis.engine._base import (
EngineConnectionError,
InferenceEngine,
messages_to_dicts,
)
from openjarvis.engine._stubs import StreamChunk
logger = logging.getLogger(__name__)
# Pricing per million tokens (input, output)
PRICING: Dict[str, tuple[float, float]] = {
"gpt-4o": (2.50, 10.00),
"gpt-4o-mini": (0.15, 0.60),
"gpt-5": (10.00, 30.00),
"gpt-5.4": (15.00, 60.00),
"gpt-5-mini": (0.25, 2.00),
"o3-mini": (1.10, 4.40),
"claude-sonnet-4-20250514": (3.00, 15.00),
"claude-opus-4-20250514": (15.00, 75.00),
"claude-haiku-3-5-20241022": (0.80, 4.00),
"claude-opus-4-6": (5.00, 25.00),
"claude-sonnet-4-6": (3.00, 15.00),
"claude-haiku-4-5": (1.00, 5.00),
"gemini-2.5-pro": (1.25, 10.00),
"gemini-2.5-flash": (0.30, 2.50),
"gemini-3-pro": (2.00, 12.00),
"gemini-3-flash": (0.50, 3.00),
"gemini-3.1-pro-preview": (2.50, 15.00),
"gemini-3.1-flash-lite-preview": (0.30, 2.50),
"gemini-3-flash-preview": (0.50, 3.00),
"claude-haiku-4-5-20251001": (1.00, 5.00),
"MiniMax-M3": (0.30, 1.20),
"MiniMax-M2.7": (0.30, 1.20),
"MiniMax-M2.7-highspeed": (0.60, 2.40),
"MiniMax-M2.5": (0.30, 1.20),
"MiniMax-M2.5-highspeed": (0.60, 2.40),
"deepseek-v4-flash": (0.27, 1.10),
"deepseek-v4-pro": (0.55, 2.19),
# Atlas Cloud — an aggregator, so keys carry the "atlascloud/" routing
# prefix (same convention as OpenRouter below). Rates are USD per 1M
# tokens, read from the gateway's own catalog: GET
# https://api.atlascloud.ai/v1/models returns pricing.prompt /
# pricing.completion in USD *per token*; multiply by 1e6.
"atlascloud/openai/gpt-4.1-mini": (0.40, 1.60),
"atlascloud/openai/gpt-5.4-mini": (0.75, 4.50),
"atlascloud/deepseek-ai/deepseek-v3.2": (0.26, 0.38),
"atlascloud/Qwen/Qwen3-235B-A22B-Instruct-2507": (0.20, 0.88),
"atlascloud/qwen/qwen3.5-35b-a3b": (0.225, 1.80),
"atlascloud/zai-org/GLM-4.6": (0.60, 2.20),
"atlascloud/moonshotai/kimi-k2.5": (0.49, 2.50),
"atlascloud/minimaxai/minimax-m2.5": (0.295, 1.20),
}
_MINIMAX_M3_LONG_CONTEXT_THRESHOLD = 512_000
_MINIMAX_M3_LONG_CONTEXT_PRICING = (0.60, 2.40)
# Well-known model IDs per provider
_OPENAI_MODELS = [
"gpt-4o",
"gpt-4o-mini",
"gpt-5",
"gpt-5.4",
"gpt-5-mini",
"o3-mini",
]
_ANTHROPIC_MODELS = [
"claude-sonnet-4-20250514",
"claude-opus-4-20250514",
"claude-haiku-3-5-20241022",
"claude-opus-4-6",
"claude-sonnet-4-6",
"claude-haiku-4-5",
"claude-haiku-4-5-20251001",
]
_ANTHROPIC_FIXED_SAMPLING_PREFIXES = (
"claude-opus-4-7",
"claude-opus-4-8",
"claude-opus-5",
"claude-sonnet-5",
"claude-fable-5",
"claude-haiku-5-5",
)
_GOOGLE_MODELS = [
"gemini-2.5-pro",
"gemini-2.5-flash",
"gemini-3-pro",
"gemini-3-flash",
"gemini-3.1-pro-preview",
"gemini-3.1-flash-lite-preview",
"gemini-3-flash-preview",
]
_MINIMAX_MODELS = [
"MiniMax-M3",
"MiniMax-M2.7",
"MiniMax-M2.7-highspeed",
"MiniMax-M2.5",
"MiniMax-M2.5-highspeed",
]
_DEEPSEEK_MODELS = [
"deepseek-v4-flash",
"deepseek-v4-pro",
]
# OpenRouter models — prefixed with "openrouter/" so they can be identified
_OPENROUTER_POPULAR = [
"openrouter/auto",
"openrouter/openai/gpt-4o",
"openrouter/anthropic/claude-sonnet-4",
"openrouter/google/gemini-2.5-pro",
"openrouter/meta-llama/llama-3.3-70b-instruct",
"openrouter/mistralai/mistral-large",
"openrouter/deepseek/deepseek-r1",
"openrouter/qwen/qwen3-235b-a22b",
]
# Atlas Cloud models — prefixed with "atlascloud/" so they can be identified.
# Atlas Cloud is an OpenAI-compatible aggregator whose own model IDs are
# already "vendor/model", so the prefix is what keeps
# "atlascloud/anthropic/claude-*" from being routed to the Anthropic SDK.
# The gateway serves many more than these — GET /v1/models lists what is
# currently routable, and any ID from there works as "atlascloud/<id>".
_ATLASCLOUD_POPULAR = [
"atlascloud/openai/gpt-4.1-mini",
"atlascloud/openai/gpt-5.4-mini",
"atlascloud/deepseek-ai/deepseek-v3.2",
"atlascloud/Qwen/Qwen3-235B-A22B-Instruct-2507",
"atlascloud/qwen/qwen3.5-35b-a3b",
"atlascloud/zai-org/GLM-4.6",
"atlascloud/moonshotai/kimi-k2.5",
"atlascloud/minimaxai/minimax-m2.5",
]
# Codex models — prefixed with "codex/" for ChatGPT Plus/Pro subscribers.
# Uses the Responses API at chatgpt.com, not the standard OpenAI API.
_CODEX_MODELS = [
"codex/gpt-4o",
"codex/gpt-4o-mini",
"codex/o3-mini",
"codex/gpt-5-mini",
"codex/gpt-5-mini-2025-08-07",
]
def _is_minimax_model(model: str) -> bool:
return model.lower().startswith("minimax")
def _is_deepseek_model(model: str) -> bool:
return model.lower().startswith("deepseek")
def _is_openrouter_model(model: str) -> bool:
return model.startswith("openrouter/")
def _is_atlascloud_model(model: str) -> bool:
return model.startswith("atlascloud/")
def _is_codex_model(model: str) -> bool:
return model.startswith("codex/")
def _is_anthropic_model(model: str) -> bool:
# Aggregator-prefixed IDs are excluded: "atlascloud/anthropic/claude-*"
# contains "claude" but must go to the aggregator, not the Anthropic SDK.
return (
"claude" in model.lower()
and not _is_openrouter_model(model)
and not _is_atlascloud_model(model)
)
def _is_google_model(model: str) -> bool:
return (
"gemini" in model.lower()
and not _is_openrouter_model(model)
and not _is_atlascloud_model(model)
)
# Positive prefix predicate for genuine OpenAI models. Kept in sync with
# ``server/cloud_router.py:_OPENAI_PREFIXES`` so local-vs-cloud classification
# agrees across the codebase. Used by ``_client_for_model``/``can_serve`` so the
# cloud engine never claims it can serve an unrecognized (e.g. local Ollama)
# model name just because an OpenAI key happens to be present (see #335).
_OPENAI_PREFIXES = ("gpt-", "chatgpt-", "o1", "o3", "o4")
def _is_openai_model(model: str) -> bool:
"""True only for genuine OpenAI models (gpt-*, chatgpt-*, o1/o3/o4 series).
Defined positively so that an unrecognized model name (a local Ollama model
like ``qwen3.5:0.8b``, or a typo) is NOT treated as an OpenAI model. This is
the routing surface ``can_serve`` relies on; ``generate``/``stream`` keep
their OpenAI fall-through so an explicitly-requested unknown cloud model
still errors loudly at call time.
Caveat: a user may repoint the OpenAI client at an OpenAI-compatible server
(vLLM/LM Studio) via ``OPENAI_BASE_URL`` and legitimately serve non-gpt
names. That path is undocumented/untested in this engine; if it is added,
this predicate (or ``_client_for_model``) should treat a configured custom
base_url as "serves anything".
"""
m = model.lower()
if m in (name.lower() for name in _OPENAI_MODELS):
return True
return m.startswith(_OPENAI_PREFIXES)
def _is_openai_reasoning_model(model: str) -> bool:
"""Check if model is an OpenAI reasoning model that restricts temperature."""
m = model.lower()
# o1/o3 series and gpt-5-mini (all variants) are reasoning models
if m.startswith(("o1", "o3")):
return True
return m == "gpt-5-mini" or m.startswith("gpt-5-mini-")
def _is_unsupported_temperature_error(exc: Exception) -> bool:
"""True if an OpenAI 400 says the model rejects a non-default temperature.
Some models (e.g. gpt-5) only accept the default temperature and return
``code: unsupported_value`` for ``param: temperature`` (see #426, #1019). We
can't enumerate every such model up front, so detect the error and retry
without temperature — mirroring the tools-400 retry in the local engines.
"""
message = str(exc).lower()
if "temperature" not in message:
return False
return (
"unsupported_value" in message
or "unsupported value" in message
or "only the default" in message
or "does not support" in message
or "unsupported parameter" in message
or "is not supported" in message
)
def _chat_completion_with_temperature_retry(
client: Any,
create_kwargs: Dict[str, Any],
) -> Any:
"""Invoke client.chat.completions.create, retrying without temperature on 400."""
try:
return client.chat.completions.create(**create_kwargs)
except Exception as exc:
if "temperature" in create_kwargs and _is_unsupported_temperature_error(exc):
create_kwargs.pop("temperature", None)
return client.chat.completions.create(**create_kwargs)
raise
def estimate_cost(model: str, prompt_tokens: int, completion_tokens: int) -> float:
"""Estimate USD cost based on the hardcoded pricing table."""
# Try exact match first, then prefix match
prices = PRICING.get(model)
if prices is None:
for key, val in PRICING.items():
if model.startswith(key):
prices = val
break
if prices is None:
return 0.0
if (
model.startswith("MiniMax-M3")
and prompt_tokens > _MINIMAX_M3_LONG_CONTEXT_THRESHOLD
):
prices = _MINIMAX_M3_LONG_CONTEXT_PRICING
input_cost = (prompt_tokens / 1_000_000) * prices[0]
output_cost = (completion_tokens / 1_000_000) * prices[1]
return input_cost + output_cost
def _first_choice_or_raise(resp: Any, *, provider: str, model: str) -> Any:
"""Return the first completion choice or surface the provider error."""
choices = getattr(resp, "choices", None)
if choices:
return choices[0]
error = getattr(resp, "error", None)
if isinstance(error, dict):
detail = str(error.get("message") or error)
elif error is not None:
detail = str(getattr(error, "message", None) or error)
else:
detail = ""
message = f"{provider} returned no choices for model {model!r}"
raise EngineConnectionError(
message + (f": {detail}" if detail else " and no error message")
)
def _serialize_anthropic_block(block: Any) -> Dict[str, Any]:
"""Turn an Anthropic content block into a JSON-safe dict for tracing.
Handles every block type Anthropic returns today (text, tool_use,
server_tool_use, web_search_tool_result, tool_result, thinking). Nested
content (e.g. ``web_search_tool_result.content`` is itself a list of
citation/result blocks) recurses so the trace has the full payload, not
a truncated summary.
"""
out: Dict[str, Any] = {
"type": getattr(block, "type", None) or type(block).__name__,
}
for attr in (
"id",
"name",
"input",
"text",
"thinking",
"signature",
"tool_use_id",
"content",
):
if not hasattr(block, attr):
continue
val = getattr(block, attr)
if attr == "content" and isinstance(val, list):
out[attr] = [_serialize_anthropic_block(b) for b in val]
elif hasattr(val, "model_dump"):
try:
out[attr] = val.model_dump()
except Exception:
out[attr] = str(val)
else:
out[attr] = val
return out
def _annotate_anthropic_cache(messages: list[dict]) -> list[dict]:
"""Add cache_control to system message for Anthropic prompt caching."""
result = []
for msg in messages:
if msg.get("role") == "system":
content = msg["content"]
if isinstance(content, str):
content = [
{
"type": "text",
"text": content,
"cache_control": {"type": "ephemeral"},
}
]
elif isinstance(content, list):
content = [
{**block, "cache_control": {"type": "ephemeral"}}
for block in content
]
result.append({**msg, "content": content})
else:
result.append(msg)
return result
def _convert_tools_to_anthropic(
openai_tools: List[Dict[str, Any]],
) -> List[Dict[str, Any]]:
"""Convert OpenAI function-calling tools to Anthropic tool format."""
result = []
for tool in openai_tools:
func = tool.get("function", {})
result.append(
{
"name": func.get("name", ""),
"description": func.get("description", ""),
"input_schema": func.get("parameters", {}),
}
)
return result
def _convert_tools_to_google(
openai_tools: List[Dict[str, Any]],
) -> List[Dict[str, Any]]:
"""Convert OpenAI function-calling tools to Google function declarations."""
declarations = []
for tool in openai_tools:
func = tool.get("function", {})
declarations.append(
{
"name": func.get("name", ""),
"description": func.get("description", ""),
"parameters": func.get("parameters", {}),
}
)
return declarations
@EngineRegistry.register("cloud")
class CloudEngine(InferenceEngine):
"""Cloud inference via OpenAI, Anthropic, Google, MiniMax, DeepSeek, and
Atlas Cloud SDKs."""
engine_id = "cloud"
is_cloud = True
def __init__(self) -> None:
self._openai_client: Any = None
self._anthropic_client: Any = None
self._google_client: Any = None
self._openrouter_client: Any = None
self._atlascloud_client: Any = None
self._minimax_client: Any = None
self._deepseek_client: Any = None
self._codex_client: Any = None
# Gemini thought_signatures: tool_call_id -> signature bytes
self._thought_sigs: Dict[str, bytes] = {}
self._init_clients()
def _init_clients(self) -> None:
if os.environ.get("OPENAI_API_KEY"):
try:
import openai
self._openai_client = openai.OpenAI()
except ImportError:
pass
if os.environ.get("ANTHROPIC_API_KEY"):
try:
import anthropic
self._anthropic_client = anthropic.Anthropic()
except ImportError:
pass
gemini_key = os.environ.get("GEMINI_API_KEY") or os.environ.get(
"GOOGLE_API_KEY"
)
if gemini_key:
try:
from google import genai
self._google_client = genai.Client(api_key=gemini_key)
except ImportError:
pass
openrouter_key = os.environ.get("OPENROUTER_API_KEY")
if openrouter_key:
try:
import openai
self._openrouter_client = openai.OpenAI(
base_url="https://openrouter.ai/api/v1",
api_key=openrouter_key,
)
except ImportError:
pass
atlascloud_key = os.environ.get("ATLASCLOUD_API_KEY")
if atlascloud_key:
try:
import openai
self._atlascloud_client = openai.OpenAI(
base_url="https://api.atlascloud.ai/v1",
api_key=atlascloud_key,
)
except ImportError:
pass
minimax_key = os.environ.get("MINIMAX_API_KEY")
if minimax_key:
try:
import openai
self._minimax_client = openai.OpenAI(
base_url="https://api.minimax.io/v1",
api_key=minimax_key,
)
except ImportError:
pass
deepseek_key = os.environ.get("DEEPSEEK_API_KEY")
if deepseek_key:
try:
import openai
self._deepseek_client = openai.OpenAI(
base_url="https://api.deepseek.com/v1",
api_key=deepseek_key,
)
except ImportError:
pass
# Codex — uses the OpenAI Responses API.
# Supports both standard API keys (api.openai.com) and ChatGPT
# OAuth tokens (chatgpt.com) via OPENAI_CODEX_BASE_URL override.
codex_token = os.environ.get("OPENAI_CODEX_API_KEY")
if codex_token:
codex_url = os.environ.get(
"OPENAI_CODEX_BASE_URL",
"https://api.openai.com/v1",
).rstrip("/")
if not codex_url.endswith("/responses"):
codex_url += "/responses"
self._codex_client = {
"token": codex_token,
"url": codex_url,
}
def _prepare_anthropic_messages(
self,
messages: Sequence[Message],
) -> Tuple[str, List[Dict[str, Any]]]:
"""Extract system text and convert messages to Anthropic format."""
system_text = ""
chat_msgs: List[Dict[str, Any]] = []
for m in messages:
if m.role.value == "system":
system_text = m.content
elif m.role.value == "tool":
tool_result_block = {
"type": "tool_result",
"tool_use_id": m.tool_call_id or "",
"content": m.content,
}
if (
chat_msgs
and chat_msgs[-1]["role"] == "user"
and isinstance(chat_msgs[-1]["content"], list)
and chat_msgs[-1]["content"]
and chat_msgs[-1]["content"][-1].get("type") == "tool_result"
):
chat_msgs[-1]["content"].append(tool_result_block)
else:
chat_msgs.append(
{
"role": "user",
"content": [tool_result_block],
}
)
elif m.role.value == "assistant" and m.tool_calls:
content_blocks: List[Dict[str, Any]] = []
if m.content:
content_blocks.append({"type": "text", "text": m.content})
for tc in m.tool_calls:
args = tc.arguments
if isinstance(args, str):
try:
args = json.loads(args)
except (json.JSONDecodeError, TypeError):
args = {"input": args}
content_blocks.append(
{
"type": "tool_use",
"id": tc.id,
"name": tc.name,
"input": args if isinstance(args, dict) else {},
}
)
chat_msgs.append({"role": "assistant", "content": content_blocks})
else:
chat_msgs.append({"role": m.role.value, "content": m.content})
return system_text, chat_msgs
@staticmethod
def _codex_build_input(
messages: Sequence[Message],
) -> tuple[str, List[Dict[str, Any]]]:
"""Convert Message list to Codex Responses API format.
Returns (system_instructions, input_messages).
"""
instructions = ""
input_msgs: List[Dict[str, Any]] = []
for m in messages:
if m.role.value == "system":
instructions = m.content
elif m.role.value in ("user", "assistant"):
input_msgs.append(
{
"role": m.role.value,
"content": [{"type": "input_text", "text": m.content}],
}
)
return instructions, input_msgs
def _generate_codex(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float,
max_tokens: int,
**kwargs: Any,
) -> Dict[str, Any]:
"""Generate via Codex Responses API (ChatGPT Plus/Pro)."""
if self._codex_client is None:
raise EngineConnectionError(
"Codex client not available — set OPENAI_CODEX_API_KEY"
)
actual_model = model.removeprefix("codex/")
instructions, input_msgs = self._codex_build_input(messages)
body: Dict[str, Any] = {
"model": actual_model,
"input": input_msgs,
"store": False,
"stream": False,
}
if instructions:
body["instructions"] = instructions
headers = {
"Authorization": f"Bearer {self._codex_client['token']}",
"Content-Type": "application/json",
"OpenAI-Beta": "responses=experimental",
}
t0 = time.monotonic()
resp = httpx.post(
self._codex_client["url"],
json=body,
headers=headers,
timeout=120.0,
)
elapsed = time.monotonic() - t0
resp.raise_for_status()
data = resp.json()
# Extract text from Responses API output.
# The output array contains items of type "reasoning" and "message";
# we want the "message" item's content blocks.
content = data.get("output_text", "")
if not content:
for item in data.get("output", []):
if item.get("type") not in ("message", None):
continue
for block in item.get("content", []):
if block.get("type") == "output_text":
content = block.get("text", "")
break
if content:
break
usage_data = data.get("usage", {})
prompt_tokens = usage_data.get("input_tokens", 0)
completion_tokens = usage_data.get("output_tokens", 0)
return {
"content": content,
"usage": {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": prompt_tokens + completion_tokens,
},
"model": actual_model,
"finish_reason": "stop",
"cost_usd": 0.0,
"ttft": elapsed,
}
def _generate_openai(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float,
max_tokens: int,
**kwargs: Any,
) -> Dict[str, Any]:
if self._openai_client is None:
raise EngineConnectionError(
"OpenAI client not available — set "
"OPENAI_API_KEY and install "
"openjarvis[inference-cloud]"
)
# Extract response_format before spreading kwargs into create_kwargs
response_format = kwargs.pop("response_format", None)
create_kwargs: Dict[str, Any] = {
"model": model,
"messages": messages_to_dicts(messages),
"max_completion_tokens": max_tokens,
**kwargs,
}
if not _is_openai_reasoning_model(model):
create_kwargs["temperature"] = temperature
# Apply structured output / JSON mode
if response_format is not None:
from openjarvis.engine._stubs import ResponseFormat
if isinstance(response_format, ResponseFormat):
if response_format.type == "json_schema" and response_format.schema:
create_kwargs["response_format"] = {
"type": "json_schema",
"json_schema": {
"name": "response",
"schema": response_format.schema,
},
}
else:
create_kwargs["response_format"] = {"type": "json_object"}
else:
# Raw dict pass-through for backward compatibility
create_kwargs["response_format"] = response_format
t0 = time.monotonic()
resp = _chat_completion_with_temperature_retry(
self._openai_client, create_kwargs
)
elapsed = time.monotonic() - t0
choice = _first_choice_or_raise(resp, provider="OpenAI", model=model)
usage = resp.usage
prompt_tokens = usage.prompt_tokens if usage else 0
completion_tokens = usage.completion_tokens if usage else 0
result = {
"content": choice.message.content or "",
"usage": {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": (usage.total_tokens if usage else 0),
},
"model": resp.model,
"finish_reason": choice.finish_reason or "stop",
"cost_usd": estimate_cost(model, prompt_tokens, completion_tokens),
"ttft": elapsed,
}
# Extract tool_calls if present
if hasattr(choice.message, "tool_calls") and choice.message.tool_calls:
result["tool_calls"] = [
{
"id": tc.id,
"name": tc.function.name,
"arguments": tc.function.arguments,
}
for tc in choice.message.tool_calls
]
return result
def _generate_anthropic(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float,
max_tokens: int,
**kwargs: Any,
) -> Dict[str, Any]:
if self._anthropic_client is None:
raise EngineConnectionError(
"Anthropic client not available — set "
"ANTHROPIC_API_KEY and install "
"openjarvis[inference-cloud]"
)
system_text, chat_msgs = self._prepare_anthropic_messages(messages)
create_kwargs: Dict[str, Any] = {
"model": model,
"messages": chat_msgs,
"max_tokens": max_tokens,
}
if not model.startswith(_ANTHROPIC_FIXED_SAMPLING_PREFIXES):
# Anthropic SDK v1 moved legacy sampling parameters to extra_body.
create_kwargs["extra_body"] = {"temperature": temperature}
if system_text:
create_kwargs["system"] = system_text
# Convert and pass tools in Anthropic format
raw_tools = kwargs.pop("tools", None)
if raw_tools:
create_kwargs["tools"] = _convert_tools_to_anthropic(raw_tools)
# Apply structured output via Anthropic's tool_choice pattern
response_format = kwargs.pop("response_format", None)
if response_format is not None:
from openjarvis.engine._stubs import ResponseFormat
if isinstance(response_format, ResponseFormat):
json_tool = {
"name": "json_output",
"description": "Output structured JSON response",
"input_schema": response_format.schema or {"type": "object"},
}
if "tools" not in create_kwargs:
create_kwargs["tools"] = [json_tool]
else:
create_kwargs["tools"].append(json_tool)
create_kwargs["tool_choice"] = {
"type": "tool",
"name": "json_output",
}
t0 = time.monotonic()
resp = self._anthropic_client.messages.create(**create_kwargs)
elapsed = time.monotonic() - t0
# Walk every block in resp.content. Anthropic returns several kinds:
# - text (plain assistant text)
# - tool_use (model wants the caller to run a tool)
# - server_tool_use (model invoked a server-side tool,
# e.g. web_search; carries the actual
# query the model issued)
# - web_search_tool_result (server-tool result body)
# - tool_result (caller-side tool result echo)
# - thinking (Opus reasoning trace)
# ``content_blocks`` keeps the full serialized list for trace
# observability. ``tool_calls`` is the narrow caller-executable
# surface — only ``tool_use`` blocks (server_tool_use lives in
# content_blocks since Anthropic already ran it server-side).
# ``tool_results`` aggregates both result kinds.
content_parts: list[str] = []
tool_calls: list[Dict[str, Any]] = []
tool_results: list[Dict[str, Any]] = []
content_blocks: list[Dict[str, Any]] = []
for block in resp.content:
btype = getattr(block, "type", None) or type(block).__name__
serialized = _serialize_anthropic_block(block)
content_blocks.append(serialized)
if btype == "tool_use":
block_id = getattr(block, "id", None)
if not block_id:
logger.warning(
"Anthropic tool_use block without an id; skipping. "
"Round-trip into the next assistant turn would fail "
"Anthropic's tool_use_id matching."
)
continue
tool_calls.append(
{
"id": block_id,
"name": getattr(block, "name", ""),
"arguments": json.dumps(getattr(block, "input", None))
if isinstance(getattr(block, "input", None), dict)
else str(getattr(block, "input", "")),
}
)
elif btype in ("web_search_tool_result", "tool_result"):
tool_results.append(serialized)
elif hasattr(block, "text"):
content_parts.append(block.text)
content = "\n".join(content_parts) if content_parts else ""
prompt_tokens = resp.usage.input_tokens if resp.usage else 0
completion_tokens = resp.usage.output_tokens if resp.usage else 0
result: Dict[str, Any] = {
"content": content,
"usage": {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": prompt_tokens + completion_tokens,
},
"model": resp.model,
"finish_reason": resp.stop_reason or "stop",
"cost_usd": estimate_cost(model, prompt_tokens, completion_tokens),
"ttft": elapsed,
"content_blocks": content_blocks,
}
if tool_calls:
result["tool_calls"] = tool_calls
if tool_results:
result["tool_results"] = tool_results
return result
def _generate_google(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float,
max_tokens: int,
**kwargs: Any,
) -> Dict[str, Any]:
if self._google_client is None:
raise EngineConnectionError(
"Google client not available — set "
"GEMINI_API_KEY or GOOGLE_API_KEY and install "
"openjarvis[inference-google]"
)
# Build contents from messages, converting tool roles for Gemini
system_text = ""
contents: List[Dict[str, Any]] = []
for m in messages:
if m.role.value == "system":
system_text = m.content
elif m.role.value == "tool":
# Gemini expects function responses as role="user" with
# function_response parts
fn_resp_part = {
"function_response": {
"name": m.name or "unknown",
"response": {"result": m.content},
}
}
# Merge consecutive tool results into a single user message
if (
contents
and contents[-1]["role"] == "user"
and contents[-1]["parts"]
and "function_response" in contents[-1]["parts"][-1]
):
contents[-1]["parts"].append(fn_resp_part)
else:
contents.append({"role": "user", "parts": [fn_resp_part]})
elif m.role.value == "assistant" and m.tool_calls:
# Convert assistant tool_calls to function_call parts
parts: List[Dict[str, Any]] = []
if m.content:
parts.append({"text": m.content})
for tc in m.tool_calls:
args = tc.arguments
if isinstance(args, str):
try:
args = json.loads(args)
except (json.JSONDecodeError, TypeError):
args = {"input": args}
fc_part: Dict[str, Any] = {
"function_call": {
"name": tc.name,
"args": args if isinstance(args, dict) else {},
}
}
# Replay thought_signature for Gemini reasoning models
sig = self._thought_sigs.get(tc.id)
if sig is not None:
fc_part["thought_signature"] = sig
parts.append(fc_part)
contents.append({"role": "model", "parts": parts})
elif m.role.value == "assistant":
contents.append({"role": "model", "parts": [{"text": m.content}]})
else:
contents.append({"role": "user", "parts": [{"text": m.content}]})
from google.genai import types as genai_types
config = genai_types.GenerateContentConfig(
temperature=temperature,
max_output_tokens=max_tokens,
)
if system_text:
config.system_instruction = system_text
# Convert and pass tools in Google format
raw_tools = kwargs.pop("tools", None)
if raw_tools:
declarations = _convert_tools_to_google(raw_tools)
config.tools = [{"function_declarations": declarations}]
# Apply structured output / JSON mode for Google
response_format = kwargs.pop("response_format", None)
if response_format is not None:
from openjarvis.engine._stubs import ResponseFormat
if isinstance(response_format, ResponseFormat):
config.response_mime_type = "application/json"
if response_format.schema:
config.response_schema = response_format.schema
t0 = time.monotonic()
resp = self._google_client.models.generate_content(
model=model,
contents=contents,
config=config,
)
elapsed = time.monotonic() - t0
# Extract text and function_call parts from response
text_parts: list[str] = []
tool_calls: list[Dict[str, Any]] = []
candidates = getattr(resp, "candidates", None)
if candidates:
parts = getattr(candidates[0].content, "parts", [])
for part in parts:
if hasattr(part, "function_call") and part.function_call:
fc = part.function_call
fc_args = dict(fc.args) if hasattr(fc.args, "items") else {}
tc_dict: Dict[str, Any] = {
"id": f"google_{fc.name}",
"name": fc.name,
"arguments": json.dumps(fc_args),
}
# Preserve thought_signature for Gemini reasoning models
sig = getattr(part, "thought_signature", None)
if sig is not None:
tc_dict["thought_signature"] = sig
self._thought_sigs[tc_dict["id"]] = sig
tool_calls.append(tc_dict)
elif hasattr(part, "text") and part.text:
text_parts.append(part.text)
# Guard against resp.text ValueError when only function_call parts
if text_parts:
content = "\n".join(text_parts)
else:
try:
content = resp.text or ""
except (ValueError, AttributeError):
content = ""
um = resp.usage_metadata
prompt_tokens = getattr(um, "prompt_token_count", 0) if um else 0
completion_tokens = getattr(um, "candidates_token_count", 0) if um else 0
result: Dict[str, Any] = {
"content": content,
"usage": {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": prompt_tokens + completion_tokens,
},
"model": model,
"finish_reason": "stop",
"cost_usd": estimate_cost(model, prompt_tokens, completion_tokens),
"ttft": elapsed,
}
if tool_calls:
result["tool_calls"] = tool_calls
return result
def _generate_openrouter(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float,
max_tokens: int,
**kwargs: Any,
) -> Dict[str, Any]:
if self._openrouter_client is None:
raise EngineConnectionError(
"OpenRouter client not available — set OPENROUTER_API_KEY"
)
# Strip the "openrouter/" prefix to get the actual model ID
actual_model = model.removeprefix("openrouter/")
kwargs.pop("response_format", None)
create_kwargs: Dict[str, Any] = {
"model": actual_model,
"messages": messages_to_dicts(messages),
"max_tokens": max_tokens,
"temperature": temperature,
}
# Forward tools / tool_choice (OpenRouter is OpenAI-compatible).
tools = kwargs.pop("tools", None)
if tools:
create_kwargs["tools"] = tools
tool_choice = kwargs.pop("tool_choice", None)
if tool_choice is not None:
create_kwargs["tool_choice"] = tool_choice
t0 = time.monotonic()
resp = _chat_completion_with_temperature_retry(
self._openrouter_client, create_kwargs
)
elapsed = time.monotonic() - t0
choice = _first_choice_or_raise(resp, provider="OpenRouter", model=actual_model)
usage = resp.usage
prompt_tokens = usage.prompt_tokens if usage else 0
completion_tokens = usage.completion_tokens if usage else 0
result: Dict[str, Any] = {
"content": choice.message.content or "",
"usage": {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": (usage.total_tokens if usage else 0),
},
"model": resp.model,
"finish_reason": choice.finish_reason or "stop",
"ttft": elapsed,
}
if getattr(choice.message, "tool_calls", None):
result["tool_calls"] = [
{
"id": tc.id,
"type": tc.type,
"function": {
"name": tc.function.name,
"arguments": tc.function.arguments,
},
}
for tc in choice.message.tool_calls
]
return result
def _generate_atlascloud(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float,
max_tokens: int,
**kwargs: Any,
) -> Dict[str, Any]:
if self._atlascloud_client is None:
raise EngineConnectionError(
"Atlas Cloud client not available — set ATLASCLOUD_API_KEY"
)
# Strip the "atlascloud/" prefix to get the gateway's own model ID.
actual_model = model.removeprefix("atlascloud/")
create_kwargs: Dict[str, Any] = {
"model": actual_model,
"messages": messages_to_dicts(messages),
"max_tokens": max_tokens,
"temperature": temperature,
}
# Atlas Cloud is OpenAI-compatible: response_format, tools and
# tool_choice all pass straight through.
response_format = kwargs.pop("response_format", None)
if response_format is not None:
create_kwargs["response_format"] = response_format
tools = kwargs.pop("tools", None)
if tools:
create_kwargs["tools"] = tools
tool_choice = kwargs.pop("tool_choice", None)
if tool_choice is not None:
create_kwargs["tool_choice"] = tool_choice
t0 = time.monotonic()
resp = _chat_completion_with_temperature_retry(
self._atlascloud_client, create_kwargs
)
elapsed = time.monotonic() - t0
choice = _first_choice_or_raise(
resp, provider="Atlas Cloud", model=actual_model
)
usage = resp.usage
prompt_tokens = usage.prompt_tokens if usage else 0
completion_tokens = usage.completion_tokens if usage else 0
result: Dict[str, Any] = {
"content": choice.message.content or "",
"usage": {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": (usage.total_tokens if usage else 0),
},
"model": resp.model,
"finish_reason": choice.finish_reason or "stop",
# Cost is keyed on the prefixed ID so PRICING stays unambiguous
# between "atlascloud/minimaxai/minimax-m2.5" and MiniMax direct.
"cost_usd": estimate_cost(model, prompt_tokens, completion_tokens),
"ttft": elapsed,
}
if getattr(choice.message, "tool_calls", None):
result["tool_calls"] = [
{
"id": tc.id,
"type": tc.type,
"function": {
"name": tc.function.name,
"arguments": tc.function.arguments,
},
}
for tc in choice.message.tool_calls
]
return result
def _generate_minimax(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float,
max_tokens: int,
**kwargs: Any,
) -> Dict[str, Any]:
if self._minimax_client is None:
raise EngineConnectionError(
"MiniMax client not available — set MINIMAX_API_KEY"
)
# MiniMax requires temperature in (0.0, 1.0]; clamp zero
temperature = max(temperature, 0.01)
temperature = min(temperature, 1.0)
kwargs.pop("response_format", None)
create_kwargs: Dict[str, Any] = {
"model": model,
"messages": messages_to_dicts(messages),
"max_tokens": max_tokens,
"temperature": temperature,
}
t0 = time.monotonic()
resp = _chat_completion_with_temperature_retry(
self._minimax_client, create_kwargs
)
elapsed = time.monotonic() - t0
choice = _first_choice_or_raise(resp, provider="MiniMax", model=model)
usage = resp.usage
prompt_tokens = usage.prompt_tokens if usage else 0
completion_tokens = usage.completion_tokens if usage else 0
result: Dict[str, Any] = {
"content": choice.message.content or "",
"usage": {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": (usage.total_tokens if usage else 0),
},
"model": resp.model,
"finish_reason": choice.finish_reason or "stop",
"cost_usd": estimate_cost(model, prompt_tokens, completion_tokens),
"ttft": elapsed,
}
if hasattr(choice.message, "tool_calls") and choice.message.tool_calls:
result["tool_calls"] = [
{
"id": tc.id,
"name": tc.function.name,
"arguments": tc.function.arguments,
}
for tc in choice.message.tool_calls
]
return result
def _generate_deepseek(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float,
max_tokens: int,
**kwargs: Any,
) -> Dict[str, Any]:
if self._deepseek_client is None:
raise EngineConnectionError(
"DeepSeek client not available — set DEEPSEEK_API_KEY"
)
kwargs.pop("response_format", None)
create_kwargs: Dict[str, Any] = {
"model": model,
"messages": messages_to_dicts(messages),
"max_tokens": max_tokens,
"temperature": temperature,
}
t0 = time.monotonic()
resp = _chat_completion_with_temperature_retry(
self._deepseek_client, create_kwargs
)
elapsed = time.monotonic() - t0
choice = _first_choice_or_raise(resp, provider="DeepSeek", model=model)
usage = resp.usage
prompt_tokens = usage.prompt_tokens if usage else 0
completion_tokens = usage.completion_tokens if usage else 0
result: Dict[str, Any] = {
"content": choice.message.content or "",
"usage": {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": (usage.total_tokens if usage else 0),
},
"model": resp.model,
"finish_reason": choice.finish_reason or "stop",
"cost_usd": estimate_cost(model, prompt_tokens, completion_tokens),
"ttft": elapsed,
}
if hasattr(choice.message, "tool_calls") and choice.message.tool_calls:
result["tool_calls"] = [
{
"id": tc.id,
"name": tc.function.name,
"arguments": tc.function.arguments,
}
for tc in choice.message.tool_calls
]
return result
def generate(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float = 0.7,
max_tokens: int = 1024,
**kwargs: Any,
) -> Dict[str, Any]:
kw = dict(
model=model,
temperature=temperature,
max_tokens=max_tokens,
**kwargs,
)
if _is_codex_model(model):
return self._generate_codex(messages, **kw)
if _is_openrouter_model(model):
return self._generate_openrouter(messages, **kw)
if _is_atlascloud_model(model):
return self._generate_atlascloud(messages, **kw)
if _is_minimax_model(model):
return self._generate_minimax(messages, **kw)
if _is_deepseek_model(model):
return self._generate_deepseek(messages, **kw)
if _is_anthropic_model(model):
return self._generate_anthropic(messages, **kw)
if _is_google_model(model):
return self._generate_google(messages, **kw)
return self._generate_openai(messages, **kw)
async def stream(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float = 0.7,
max_tokens: int = 1024,
**kwargs: Any,
) -> AsyncIterator[str]:
kw = dict(
model=model,
temperature=temperature,
max_tokens=max_tokens,
**kwargs,
)
if _is_codex_model(model):
async for token in self._stream_codex(messages, **kw):
yield token
elif _is_openrouter_model(model):
async for token in self._stream_openrouter(messages, **kw):
yield token
elif _is_atlascloud_model(model):
async for token in self._stream_atlascloud(messages, **kw):
yield token
elif _is_minimax_model(model):
async for token in self._stream_minimax(messages, **kw):
yield token
elif _is_deepseek_model(model):
async for token in self._stream_deepseek(messages, **kw):
yield token
elif _is_anthropic_model(model):
async for token in self._stream_anthropic(messages, **kw):
yield token
elif _is_google_model(model):
async for token in self._stream_google(messages, **kw):
yield token
else:
async for token in self._stream_openai(messages, **kw):
yield token
async def _stream_codex(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float,
max_tokens: int,
**kwargs: Any,
) -> AsyncIterator[str]:
"""Stream via Codex Responses API (SSE)."""
if self._codex_client is None:
raise EngineConnectionError("Codex client not available")
actual_model = model.removeprefix("codex/")
instructions, input_msgs = self._codex_build_input(messages)
body: Dict[str, Any] = {
"model": actual_model,
"input": input_msgs,
"store": False,
"stream": True,
}
if instructions:
body["instructions"] = instructions
headers = {
"Authorization": f"Bearer {self._codex_client['token']}",
"Content-Type": "application/json",
"Accept": "text/event-stream",
"OpenAI-Beta": "responses=experimental",
}
async with httpx.AsyncClient() as client:
async with client.stream(
"POST",
self._codex_client["url"],
json=body,
headers=headers,
timeout=120.0,
) as resp:
resp.raise_for_status()
async for line in resp.aiter_lines():
if not line.startswith("data: "):
continue
payload = line[6:]
if payload == "[DONE]":
break
try:
event = json.loads(payload)
except (json.JSONDecodeError, ValueError):
continue
etype = event.get("type", "")
if etype == "response.output_text.delta":
delta = event.get("delta", "")
if delta:
yield delta
async def _stream_openai(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float,
max_tokens: int,
**kwargs: Any,
) -> AsyncIterator[str]:
if self._openai_client is None:
raise EngineConnectionError("OpenAI client not available")
create_kwargs: Dict[str, Any] = {
"model": model,
"messages": messages_to_dicts(messages),
"max_completion_tokens": max_tokens,
"stream": True,
**kwargs,
}
if not _is_openai_reasoning_model(model):
create_kwargs["temperature"] = temperature
resp = _chat_completion_with_temperature_retry(
self._openai_client, create_kwargs
)
for chunk in resp:
delta = chunk.choices[0].delta if chunk.choices else None
if delta and delta.content:
yield delta.content
async def _stream_anthropic(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float,
max_tokens: int,
**kwargs: Any,
) -> AsyncIterator[str]:
if self._anthropic_client is None:
raise EngineConnectionError("Anthropic client not available")
system_text = ""
chat_msgs: List[Dict[str, Any]] = []
for m in messages:
if m.role.value == "system":
system_text = m.content
else:
chat_msgs.append({"role": m.role.value, "content": m.content})
create_kwargs: Dict[str, Any] = {
"model": model,
"messages": chat_msgs,
"max_tokens": max_tokens,
}
if not model.startswith(_ANTHROPIC_FIXED_SAMPLING_PREFIXES):
# Anthropic SDK v1 moved legacy sampling parameters to extra_body.
create_kwargs["extra_body"] = {"temperature": temperature}
if system_text:
create_kwargs["system"] = system_text
with self._anthropic_client.messages.stream(**create_kwargs) as stream:
for text in stream.text_stream:
yield text
async def _stream_google(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float,
max_tokens: int,
**kwargs: Any,
) -> AsyncIterator[str]:
if self._google_client is None:
raise EngineConnectionError("Google client not available")
system_text = ""
contents: List[Dict[str, Any]] = []
for m in messages:
if m.role.value == "system":
system_text = m.content
elif m.role.value == "assistant":
contents.append({"role": "model", "parts": [{"text": m.content}]})
else:
contents.append({"role": "user", "parts": [{"text": m.content}]})
from google.genai import types as genai_types
config = genai_types.GenerateContentConfig(
temperature=temperature,
max_output_tokens=max_tokens,
)
if system_text:
config.system_instruction = system_text
for chunk in self._google_client.models.generate_content_stream(
model=model,
contents=contents,
config=config,
):
if chunk.text:
yield chunk.text
async def _stream_full_google(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float,
max_tokens: int,
**kwargs: Any,
) -> AsyncIterator[StreamChunk]:
"""Stream Google text and function-call parts as full chunks."""
if self._google_client is None:
raise EngineConnectionError("Google client not available")
system_text = ""
contents: List[Dict[str, Any]] = []
for message in messages:
if message.role.value == "system":
system_text = message.content
elif message.role.value == "tool":
function_response = {
"function_response": {
"name": message.name or "unknown",
"response": {"result": message.content},
}
}
if (
contents
and contents[-1]["role"] == "user"
and contents[-1]["parts"]
and "function_response" in contents[-1]["parts"][-1]
):
contents[-1]["parts"].append(function_response)
else:
contents.append({"role": "user", "parts": [function_response]})
elif message.role.value == "assistant" and message.tool_calls:
parts: List[Dict[str, Any]] = []
if message.content:
parts.append({"text": message.content})
for tool_call in message.tool_calls:
args = tool_call.arguments
if isinstance(args, str):
try:
args = json.loads(args)
except (json.JSONDecodeError, TypeError):
args = {"input": args}
function_call_part: Dict[str, Any] = {
"function_call": {
"name": tool_call.name,
"args": args if isinstance(args, dict) else {},
}
}
signature = self._thought_sigs.get(tool_call.id)
if signature is not None:
function_call_part["thought_signature"] = signature
parts.append(function_call_part)
contents.append({"role": "model", "parts": parts})
elif message.role.value == "assistant":
contents.append({"role": "model", "parts": [{"text": message.content}]})
else:
contents.append({"role": "user", "parts": [{"text": message.content}]})
from google.genai import types as genai_types
config = genai_types.GenerateContentConfig(
temperature=temperature,
max_output_tokens=max_tokens,
)
if system_text:
config.system_instruction = system_text
tools = kwargs.pop("tools", None)
if tools:
config.tools = [{"function_declarations": _convert_tools_to_google(tools)}]
tool_call_count = 0
stream_id = uuid.uuid4().hex
final_usage: Dict[str, Any] | None = None
for chunk in self._google_client.models.generate_content_stream(
model=model,
contents=contents,
config=config,
):
usage_metadata = getattr(chunk, "usage_metadata", None)
if usage_metadata is not None:
prompt_tokens = getattr(usage_metadata, "prompt_token_count", 0) or 0
completion_tokens = (
getattr(usage_metadata, "candidates_token_count", 0) or 0
)
final_usage = {
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": prompt_tokens + completion_tokens,
}
candidates = getattr(chunk, "candidates", None)
parts = []
if candidates:
parts = getattr(candidates[0].content, "parts", []) or []
if parts:
text_found = False
calls: List[Dict[str, Any]] = []
for part in parts:
text = getattr(part, "text", None)
if text:
text_found = True
yield StreamChunk(content=text)
function_call = getattr(part, "function_call", None)
if function_call:
name = getattr(function_call, "name", "")
raw_args = getattr(function_call, "args", {})
args = dict(raw_args) if hasattr(raw_args, "items") else {}
# Gemini emits complete function-call parts, so each part is
# a distinct invocation. The same function may legitimately
# be called more than once in a parallel response.
tool_index = tool_call_count
# The engine is shared across server requests, and saved
# thought signatures are keyed by tool-call ID. Include a
# per-stream nonce so concurrent conversations cannot
# overwrite each other's signatures.
tool_id = f"google_{stream_id}_{tool_index}"
tool_call_count += 1
tool_call = {
"index": tool_index,
"id": tool_id,
"type": "function",
"function": {
"name": name,
"arguments": json.dumps(args),
},
}
calls.append(tool_call)
signature = getattr(part, "thought_signature", None)
if signature is not None:
tool_call["thought_signature"] = signature
self._thought_sigs[tool_id] = signature
if calls:
yield StreamChunk(tool_calls=calls)
if text_found:
continue
try:
text = chunk.text
except (AttributeError, ValueError):
text = None
if text:
yield StreamChunk(content=text)
yield StreamChunk(
finish_reason="tool_calls" if tool_call_count else "stop",
usage=final_usage,
)
async def _stream_openrouter(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float,
max_tokens: int,
**kwargs: Any,
) -> AsyncIterator[str]:
if self._openrouter_client is None:
raise EngineConnectionError("OpenRouter client not available")
actual_model = model.removeprefix("openrouter/")
create_kwargs: Dict[str, Any] = {
"model": actual_model,
"messages": messages_to_dicts(messages),
"max_tokens": max_tokens,
"temperature": temperature,
"stream": True,
}
# Forward tools / tool_choice (OpenRouter is OpenAI-compatible).
tools = kwargs.pop("tools", None)
if tools:
create_kwargs["tools"] = tools
tool_choice = kwargs.pop("tool_choice", None)
if tool_choice is not None:
create_kwargs["tool_choice"] = tool_choice
resp = _chat_completion_with_temperature_retry(
self._openrouter_client, create_kwargs
)
for chunk in resp:
delta = chunk.choices[0].delta if chunk.choices else None
if delta and delta.content:
yield delta.content
async def _stream_atlascloud(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float,
max_tokens: int,
**kwargs: Any,
) -> AsyncIterator[str]:
if self._atlascloud_client is None:
raise EngineConnectionError("Atlas Cloud client not available")
actual_model = model.removeprefix("atlascloud/")
create_kwargs: Dict[str, Any] = {
"model": actual_model,
"messages": messages_to_dicts(messages),
"max_tokens": max_tokens,
"temperature": temperature,
"stream": True,
}
# Forward tools / tool_choice (Atlas Cloud is OpenAI-compatible).
tools = kwargs.pop("tools", None)
if tools:
create_kwargs["tools"] = tools
tool_choice = kwargs.pop("tool_choice", None)
if tool_choice is not None:
create_kwargs["tool_choice"] = tool_choice
resp = _chat_completion_with_temperature_retry(
self._atlascloud_client, create_kwargs
)
for chunk in resp:
delta = chunk.choices[0].delta if chunk.choices else None
if delta and delta.content:
yield delta.content
async def _stream_minimax(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float,
max_tokens: int,
**kwargs: Any,
) -> AsyncIterator[str]:
if self._minimax_client is None:
raise EngineConnectionError("MiniMax client not available")
temperature = max(temperature, 0.01)
temperature = min(temperature, 1.0)
create_kwargs: Dict[str, Any] = {
"model": model,
"messages": messages_to_dicts(messages),
"max_tokens": max_tokens,
"temperature": temperature,
"stream": True,
}
resp = _chat_completion_with_temperature_retry(
self._minimax_client, create_kwargs
)
for chunk in resp:
delta = chunk.choices[0].delta if chunk.choices else None
if delta and delta.content:
yield delta.content
async def _stream_deepseek(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float,
max_tokens: int,
**kwargs: Any,
) -> AsyncIterator[str]:
if self._deepseek_client is None:
raise EngineConnectionError("DeepSeek client not available")
create_kwargs: Dict[str, Any] = {
"model": model,
"messages": messages_to_dicts(messages),
"max_tokens": max_tokens,
"temperature": temperature,
"stream": True,
}
resp = _chat_completion_with_temperature_retry(
self._deepseek_client, create_kwargs
)
for chunk in resp:
delta = chunk.choices[0].delta if chunk.choices else None
if delta and delta.content:
yield delta.content
# -- stream_full: rich streaming with tool_calls support ----------------
async def _stream_full_openai(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float,
max_tokens: int,
**kwargs: Any,
) -> AsyncIterator[StreamChunk]:
"""Yield StreamChunks from an OpenAI-compatible streaming response.
Works for OpenAI, OpenRouter, Atlas Cloud, MiniMax, DeepSeek, and Codex.
"""
if _is_codex_model(model):
# Codex uses Responses API — fall back to base stream_full wrapper
async for chunk in super().stream_full(
messages,
model=model,
temperature=temperature,
max_tokens=max_tokens,
**kwargs,
):
yield chunk
return
if _is_openrouter_model(model):
client = self._openrouter_client
if client is None:
raise EngineConnectionError("OpenRouter client not available")
actual_model = model.removeprefix("openrouter/")
create_kwargs: Dict[str, Any] = {
"model": actual_model,
"messages": messages_to_dicts(messages),
"max_tokens": max_tokens,
"temperature": temperature,
"stream": True,
**kwargs,
}
elif _is_atlascloud_model(model):
client = self._atlascloud_client
if client is None:
raise EngineConnectionError("Atlas Cloud client not available")
create_kwargs = {
"model": model.removeprefix("atlascloud/"),
"messages": messages_to_dicts(messages),
"max_tokens": max_tokens,
"temperature": temperature,
"stream": True,
**kwargs,
}
elif _is_minimax_model(model):
client = self._minimax_client
if client is None:
raise EngineConnectionError("MiniMax client not available")
temperature = max(temperature, 0.01)
temperature = min(temperature, 1.0)
create_kwargs = {
"model": model,
"messages": messages_to_dicts(messages),
"max_tokens": max_tokens,
"temperature": temperature,
"stream": True,
**kwargs,
}
elif _is_deepseek_model(model):
client = self._deepseek_client
if client is None:
raise EngineConnectionError("DeepSeek client not available")
create_kwargs = {
"model": model,
"messages": messages_to_dicts(messages),
"max_tokens": max_tokens,
"temperature": temperature,
"stream": True,
**kwargs,
}
else:
client = self._openai_client
if client is None:
raise EngineConnectionError("OpenAI client not available")
create_kwargs = {
"model": model,
"messages": messages_to_dicts(messages),
"max_completion_tokens": max_tokens,
"stream": True,
**kwargs,
}
if not _is_openai_reasoning_model(model):
create_kwargs["temperature"] = temperature
resp = _chat_completion_with_temperature_retry(client, create_kwargs)
for chunk in resp:
choice = chunk.choices[0] if chunk.choices else None
if not choice:
continue
delta = choice.delta
content = delta.content if delta else None
tool_calls = None
if delta and delta.tool_calls:
tool_calls = [
{
"index": tc.index,
"id": tc.id or "",
"function": {
"name": (tc.function.name or "") if tc.function else "",
"arguments": (
(tc.function.arguments or "") if tc.function else ""
),
},
}
for tc in delta.tool_calls
]
finish = choice.finish_reason
if content or tool_calls or finish:
yield StreamChunk(
content=content,
tool_calls=tool_calls,
finish_reason=finish,
)
async def _stream_full_anthropic(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float,
max_tokens: int,
**kwargs: Any,
) -> AsyncIterator[StreamChunk]:
"""Yield StreamChunks from an Anthropic streaming response."""
if self._anthropic_client is None:
raise EngineConnectionError("Anthropic client not available")
system_text, chat_msgs = self._prepare_anthropic_messages(messages)
create_kwargs: Dict[str, Any] = {
"model": model,
"messages": chat_msgs,
"max_tokens": max_tokens,
}
if not model.startswith(_ANTHROPIC_FIXED_SAMPLING_PREFIXES):
# Anthropic SDK v1 moved legacy sampling parameters to extra_body.
create_kwargs["extra_body"] = {"temperature": temperature}
if system_text:
create_kwargs["system"] = system_text
raw_tools = kwargs.pop("tools", None)
if raw_tools:
create_kwargs["tools"] = _convert_tools_to_anthropic(raw_tools)
kwargs.pop("tool_choice", None)
with self._anthropic_client.messages.stream(**create_kwargs) as stream:
tool_index = -1
for event in stream:
if event.type == "content_block_start":
block = event.content_block
if block.type == "tool_use":
tool_index += 1
yield StreamChunk(
tool_calls=[
{
"index": tool_index,
"id": block.id,
"function": {"name": block.name, "arguments": ""},
}
]
)
elif event.type == "content_block_delta":
delta = event.delta
if delta.type == "text_delta":
yield StreamChunk(content=delta.text)
elif delta.type == "input_json_delta":
yield StreamChunk(
tool_calls=[
{
"index": tool_index,
"function": {"arguments": delta.partial_json},
}
]
)
elif event.type == "message_delta":
stop_reason = event.delta.stop_reason
finish = "tool_calls" if stop_reason == "tool_use" else "stop"
yield StreamChunk(finish_reason=finish)
# End-of-stream parity with ``_generate_anthropic``: emit
# ``content_blocks`` (every block kind, including server_tool_use
# and thinking) and ``tool_results`` (web_search_tool_result +
# tool_result) so streaming traces see what non-streaming traces
# see.
try:
final_msg = stream.get_final_message()
except Exception as exc: # noqa: BLE001 — SDK shape varies
logger.debug("Anthropic stream.get_final_message() failed: %s", exc)
final_msg = None
if final_msg is not None and getattr(final_msg, "content", None):
content_blocks: list[Dict[str, Any]] = []
tool_results: list[Dict[str, Any]] = []
for block in final_msg.content:
btype = getattr(block, "type", None) or type(block).__name__
serialized = _serialize_anthropic_block(block)
content_blocks.append(serialized)
if btype in ("web_search_tool_result", "tool_result"):
tool_results.append(serialized)
if content_blocks or tool_results:
yield StreamChunk(
content_blocks=content_blocks or None,
tool_results=tool_results or None,
)
async def stream_full(
self,
messages: Sequence[Message],
*,
model: str,
temperature: float = 0.7,
max_tokens: int = 1024,
**kwargs: Any,
) -> AsyncIterator[StreamChunk]:
"""Yield StreamChunks with content, tool_calls, and finish_reason."""
kw = dict(
model=model,
temperature=temperature,
max_tokens=max_tokens,
**kwargs,
)
if _is_anthropic_model(model):
async for chunk in self._stream_full_anthropic(messages, **kw):
yield chunk
elif _is_google_model(model):
async for chunk in self._stream_full_google(messages, **kw):
yield chunk
else:
async for chunk in self._stream_full_openai(messages, **kw):
yield chunk
def list_models(self) -> List[str]:
models: List[str] = []
if self._openai_client is not None:
models.extend(_OPENAI_MODELS)
if self._anthropic_client is not None:
models.extend(_ANTHROPIC_MODELS)
if self._google_client is not None:
models.extend(_GOOGLE_MODELS)
if self._openrouter_client is not None:
models.extend(_OPENROUTER_POPULAR)
if self._atlascloud_client is not None:
models.extend(_ATLASCLOUD_POPULAR)
if self._minimax_client is not None:
models.extend(_MINIMAX_MODELS)
if self._deepseek_client is not None:
models.extend(_DEEPSEEK_MODELS)
if self._codex_client is not None:
models.extend(_CODEX_MODELS)
return models
def _client_for_model(self, model: str) -> Any:
"""Return the provider client ``generate``/``stream`` will dispatch to
for *model*, or ``None`` for a model this engine cannot route.
Mirrors the routing in ``generate``/``stream``, but is intentionally
*stricter* on the OpenAI fall-through: only genuine OpenAI models map to
the OpenAI client. Unrecognized names (e.g. a local Ollama model like
``qwen3.5:0.8b``) return ``None`` so ``can_serve`` declines them and the
cloud engine is not mis-selected as a fallback when the local engine is
transiently down and any (even dummy) ``OPENAI_API_KEY`` is set (#335).
``generate``/``stream`` keep their OpenAI fall-through, so an
explicitly-requested unknown cloud model still fails loudly at call time.
"""
if _is_codex_model(model):
return self._codex_client
if _is_openrouter_model(model):
return self._openrouter_client
if _is_atlascloud_model(model):
return self._atlascloud_client
if _is_minimax_model(model):
return self._minimax_client
if _is_deepseek_model(model):
return self._deepseek_client
if _is_anthropic_model(model):
return self._anthropic_client
if _is_google_model(model):
return self._google_client
if _is_openai_model(model):
return self._openai_client
return None
def can_serve(self, model: str) -> bool:
"""Return ``True`` only if the provider client for *model* exists.
``health()`` is ``True`` whenever *any* provider client is configured,
but a request for, say, a ``gpt-*`` model still needs the OpenAI
client specifically. Without this check the cloud engine gets picked
as a fallback (when the local engine is down) for a model it can't
serve, then dies at call time with "<provider> client not available"
instead of the user getting a helpful "start your local engine"
message (see #532).
"""
return self._client_for_model(model) is not None
def health(self) -> bool:
return (
self._openai_client is not None
or self._anthropic_client is not None
or self._google_client is not None
or self._openrouter_client is not None
or self._atlascloud_client is not None
or self._minimax_client is not None
or self._deepseek_client is not None
or self._codex_client is not None
)
def close(self) -> None:
if self._openai_client is not None:
if hasattr(self._openai_client, "close"):
self._openai_client.close()
self._openai_client = None
if self._anthropic_client is not None:
if hasattr(self._anthropic_client, "close"):
self._anthropic_client.close()
self._anthropic_client = None
if self._google_client is not None:
self._google_client = None
if self._openrouter_client is not None:
if hasattr(self._openrouter_client, "close"):
self._openrouter_client.close()
self._openrouter_client = None
if self._atlascloud_client is not None:
if hasattr(self._atlascloud_client, "close"):
self._atlascloud_client.close()
self._atlascloud_client = None
if self._minimax_client is not None:
if hasattr(self._minimax_client, "close"):
self._minimax_client.close()
self._minimax_client = None
# The DeepSeek client was never released here, so its httpx pool
# outlived close() (every other provider client is dropped).
if self._deepseek_client is not None:
if hasattr(self._deepseek_client, "close"):
self._deepseek_client.close()
self._deepseek_client = None
if self._codex_client is not None:
self._codex_client = None
__all__ = ["CloudEngine", "PRICING", "_annotate_anthropic_cache", "estimate_cost"]