Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

refactor(core): share OpenAI-compatible provider plumbing (#2557)

Raghav Chamadiya55a289fde666
diff --git a/packages/core/src/repowise/core/providers/llm/anthropic.py b/packages/core/src/repowise/core/providers/llm/anthropic.py
index 40f6076db..31dea487b 100644
--- a/packages/core/src/repowise/core/providers/llm/anthropic.py
+++ b/packages/core/src/repowise/core/providers/llm/anthropic.py
@@ -12,9 +12,9 @@ Recommended models (as of 2026):
from __future__ import annotations
-import contextlib
import os
from collections.abc import AsyncIterator
+from contextlib import AbstractContextManager
from typing import TYPE_CHECKING, Any
import structlog
@@ -31,15 +31,16 @@ from repowise.core.providers.llm.base import (
GeneratedResponse,
ProviderError,
ProviderModelOption,
- RateLimitError,
ensure_reasoning_supported,
fallback_model_option,
normalize_stop_reason,
- parse_retry_after,
+ parse_tool_arguments,
provider_retry_stop,
provider_retry_wait,
provider_should_retry,
+ record_generation_cost,
temperature_kwargs,
+ translate_sdk_errors,
)
from repowise.core.rate_limiter import RateLimiter
from repowise.core.reasoning import ReasoningMode
@@ -200,7 +201,7 @@ class AnthropicProvider(BaseProvider):
system_param, messages_param = _build_cached_payload(
system_prompt, user_prompt, cache_hints
)
- try:
+ with _translate_anthropic_errors():
response = await self._client.messages.create(
model=self._model,
max_tokens=max_tokens,
@@ -208,17 +209,6 @@ class AnthropicProvider(BaseProvider):
messages=messages_param,
**temperature_kwargs(self._model, temperature),
)
- except _AnthropicRateLimitError as exc:
- raise RateLimitError(
- "anthropic",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _AnthropicAPIStatusError as exc:
- raise ProviderError("anthropic", str(exc), status_code=exc.status_code) from exc
cached = getattr(response.usage, "cache_read_input_tokens", 0) or 0
stop_reason, provider_stop_reason = normalize_stop_reason(response.stop_reason)
@@ -254,22 +244,7 @@ class AnthropicProvider(BaseProvider):
request_id=request_id,
)
- if self._cost_tracker is not None:
- # Await the cost record inline rather than spawning a detached
- # task. A fire-and-forget create_task can still be flushing its
- # aiosqlite write when the event loop is torn down (e.g. the
- # asyncio.run teardown after doc generation), which surfaces as a
- # noisy "Event loop is closed" worker-thread traceback. record()
- # swallows its own persistence errors, so generation is unaffected.
- with contextlib.suppress(Exception):
- await self._cost_tracker.record(
- model=self._model,
- input_tokens=result.input_tokens,
- output_tokens=result.output_tokens,
- operation=self._cost_tracker.operation,
- file_path=None,
- )
-
+ await record_generation_cost(self._cost_tracker, model=self._model, result=result)
return result
# --- ChatProvider protocol implementation ---
@@ -284,21 +259,8 @@ class AnthropicProvider(BaseProvider):
request_id: str | None = None,
tool_executor: Any | None = None,
) -> AsyncIterator[ChatStreamEvent]:
- import json as _json
-
- # Convert OpenAI-format tools to Anthropic format
- anthropic_tools = []
- for t in tools:
- fn = t.get("function", t)
- anthropic_tools.append(
- {
- "name": fn["name"],
- "description": fn.get("description", ""),
- "input_schema": fn.get("parameters", {}),
- }
- )
-
- # Convert OpenAI-format messages to Anthropic format
+ # Convert OpenAI-format tools and messages to Anthropic format
+ anthropic_tools = [_to_anthropic_tool(t) for t in tools]
anthropic_messages = _to_anthropic_messages(messages)
kwargs: dict[str, Any] = {
@@ -311,71 +273,84 @@ class AnthropicProvider(BaseProvider):
if anthropic_tools:
kwargs["tools"] = anthropic_tools
- try:
+ with _translate_anthropic_errors():
async with self._client.messages.stream(**kwargs) as stream:
- current_tool_id: str | None = None
- current_tool_name: str | None = None
- current_tool_input_json = ""
-
- async for event in stream:
- if event.type == "content_block_start":
- block = event.content_block
- if hasattr(block, "type") and block.type == "tool_use":
- current_tool_id = block.id
- current_tool_name = block.name
- current_tool_input_json = ""
- elif event.type == "content_block_delta":
- delta = event.delta
- if hasattr(delta, "type"):
- if delta.type == "text_delta":
- yield ChatStreamEvent(type="text_delta", text=delta.text)
- elif delta.type == "input_json_delta":
- current_tool_input_json += delta.partial_json
- elif event.type == "content_block_stop":
- if current_tool_name:
- try:
- args = (
- _json.loads(current_tool_input_json)
- if current_tool_input_json
- else {}
- )
- except Exception:
- args = {}
- yield ChatStreamEvent(
- type="tool_start",
- tool_call=ChatToolCall(
- id=current_tool_id or "",
- name=current_tool_name,
- arguments=args,
- ),
- )
- current_tool_id = None
- current_tool_name = None
- current_tool_input_json = ""
- elif event.type == "message_delta":
- stop = getattr(event.delta, "stop_reason", None)
- usage = getattr(event, "usage", None)
- if usage:
- yield ChatStreamEvent(
- type="usage",
- input_tokens=getattr(usage, "input_tokens", 0) or 0,
- output_tokens=getattr(usage, "output_tokens", 0) or 0,
- )
- if stop:
- yield ChatStreamEvent(type="stop", stop_reason=stop)
- elif event.type == "message_stop":
- pass # Final cleanup; stop already yielded via message_delta
- except _AnthropicRateLimitError as exc:
- raise RateLimitError(
- "anthropic",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _AnthropicAPIStatusError as exc:
- raise ProviderError("anthropic", str(exc), status_code=exc.status_code) from exc
+ async for event in _iter_anthropic_stream_events(stream):
+ yield event
+
+
+def _translate_anthropic_errors() -> AbstractContextManager[None]:
+ return translate_sdk_errors(
+ "anthropic",
+ rate_limit_error=_AnthropicRateLimitError,
+ status_error=_AnthropicAPIStatusError,
+ )
+
+
+def _to_anthropic_tool(tool: dict[str, Any]) -> dict[str, Any]:
+ """Convert one OpenAI-format tool definition to Anthropic's shape."""
+ fn = tool.get("function", tool)
+ return {
+ "name": fn["name"],
+ "description": fn.get("description", ""),
+ "input_schema": fn.get("parameters", {}),
+ }
+
+
+async def _iter_anthropic_stream_events(stream: Any) -> AsyncIterator[ChatStreamEvent]:
+ """Map Anthropic message-stream events to ``ChatStreamEvent``s.
+
+ A ``tool_use`` block streams its input as JSON fragments between its
+ start and stop events; it is emitted whole at ``content_block_stop``.
+ """
+ tool_id: str | None = None
+ tool_name: str | None = None
+ tool_input_json = ""
+
+ async for event in stream:
+ if event.type == "content_block_start":
+ block = event.content_block
+ if getattr(block, "type", None) == "tool_use":
+ tool_id, tool_name, tool_input_json = block.id, block.name, ""
+ elif event.type == "content_block_delta":
+ delta_type = getattr(event.delta, "type", None)
+ if delta_type == "text_delta":
+ yield ChatStreamEvent(type="text_delta", text=event.delta.text)
+ elif delta_type == "input_json_delta":
+ tool_input_json += event.delta.partial_json
+ elif event.type == "content_block_stop":
+ if tool_name:
+ yield ChatStreamEvent(
+ type="tool_start",
+ tool_call=ChatToolCall(
+ id=tool_id or "",
+ name=tool_name,
+ arguments=parse_tool_arguments(tool_input_json),
+ ),
+ )
+ tool_id, tool_name, tool_input_json = None, None, ""
+ elif event.type == "message_delta":
+ for message_event in _message_delta_events(event):
+ yield message_event
+ # message_stop needs nothing: the stop was already yielded via message_delta.
+
+
+def _message_delta_events(event: Any) -> list[ChatStreamEvent]:
+ """Usage and stop events carried by a ``message_delta``, in that order."""
+ events: list[ChatStreamEvent] = []
+ usage = getattr(event, "usage", None)
+ if usage:
+ events.append(
+ ChatStreamEvent(
+ type="usage",
+ input_tokens=getattr(usage, "input_tokens", 0) or 0,
+ output_tokens=getattr(usage, "output_tokens", 0) or 0,
+ )
+ )
+ stop = getattr(event.delta, "stop_reason", None)
+ if stop:
+ events.append(ChatStreamEvent(type="stop", stop_reason=stop))
+ return events
def _build_cached_payload(
@@ -457,14 +432,9 @@ def _to_anthropic_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any
# Tool calls
for tc in msg.get("tool_calls", []):
fn = tc.get("function", {})
- import json as _json
-
args = fn.get("arguments", "{}")
if isinstance(args, str):
- try:
- args = _json.loads(args)
- except Exception:
- args = {}
+ args = parse_tool_arguments(args)
content_blocks.append(
{
"type": "tool_use",
diff --git a/packages/core/src/repowise/core/providers/llm/base.py b/packages/core/src/repowise/core/providers/llm/base.py
index 3ca9329a2..b53616e09 100644
--- a/packages/core/src/repowise/core/providers/llm/base.py
+++ b/packages/core/src/repowise/core/providers/llm/base.py
@@ -13,8 +13,10 @@ Adding a new provider:
from __future__ import annotations
+import contextlib
+import json
from abc import ABC, abstractmethod
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Iterator
from dataclasses import dataclass, field
from typing import Any, Literal, Protocol, runtime_checkable
@@ -390,6 +392,68 @@ def parse_retry_after(headers: Any) -> float | None:
return None
+def rate_limit_error_from(provider: str, exc: BaseException) -> RateLimitError:
+ """Wrap an SDK 429 exception, keeping the ``retry-after`` it carried."""
+ return RateLimitError(
+ provider,
+ str(exc),
+ status_code=429,
+ retry_after=parse_retry_after(getattr(getattr(exc, "response", None), "headers", None)),
+ )
+
+
+def translate_sdk_errors(
+ provider: str,
+ *,
+ rate_limit_error: type[BaseException],
+ status_error: type[BaseException],
+ api_error: type[BaseException] | None = None,
+) -> Iterator[None]:
+ """Re-raise a vendor SDK's errors as repowise provider errors.
+
+ 429s become ``RateLimitError`` with ``retry-after``; status errors keep
+ their code; *api_error*, when given, wraps status-less errors too.
+ """
+ # An empty tuple makes the last ``except`` match nothing.
+ wrapped_api_errors = (api_error,) if api_error is not None else ()
+ try:
+ yield
+ except rate_limit_error as exc:
+ raise rate_limit_error_from(provider, exc) from exc
+ except status_error as exc:
+ raise ProviderError(provider, str(exc), status_code=exc.status_code) from exc # type: ignore[attr-defined]
+ except wrapped_api_errors as exc:
+ raise ProviderError(
+ provider, str(exc), status_code=getattr(exc, "status_code", None)
+ ) from exc
+
+
+async def record_generation_cost(
+ tracker: Any,
+ *,
+ model: str,
+ result: GeneratedResponse,
+ operation: str | None = None,
+) -> None:
+ """Record one generation's token spend on *tracker*, if one is attached.
+
+ Awaited inline: a detached task can outlive the event loop. Failures are
+ suppressed so cost bookkeeping never fails a generation. *operation*
+ defaults to the tracker's current one.
+ """
+ if tracker is None:
+ return
+ with contextlib.suppress(Exception):
+ await tracker.record(
+ model=model,
+ input_tokens=result.input_tokens,
+ output_tokens=result.output_tokens,
+ operation=tracker.operation if operation is None else operation,
+ file_path=None,
+ )
+
+
# ---------------------------------------------------------------------------
# Shared tenacity retry policy for all HTTP providers
#
@@ -495,6 +559,14 @@ class ChatToolCall:
arguments: dict[str, Any]
+def parse_tool_arguments(raw: str) -> dict[str, Any]:
+ """Decode a tool call's JSON arguments; empty or malformed JSON reads as ``{}``."""
+ try:
+ return json.loads(raw) if raw else {}
+ except Exception:
+ return {}
+
+
@dataclass
class ChatStreamEvent:
"""A single event yielded by stream_chat().
diff --git a/packages/core/src/repowise/core/providers/llm/deepseek.py b/packages/core/src/repowise/core/providers/llm/deepseek.py
index 9b7351045..36b8f3b35 100644
--- a/packages/core/src/repowise/core/providers/llm/deepseek.py
+++ b/packages/core/src/repowise/core/providers/llm/deepseek.py
@@ -12,33 +12,12 @@ Models:
from __future__ import annotations
-import contextlib
-import os
-from collections.abc import AsyncIterator
from typing import TYPE_CHECKING, Any
-import structlog
-from openai import APIError as _OpenAIAPIError
-from openai import APIStatusError as _OpenAIAPIStatusError
-from openai import AsyncOpenAI
-from openai import RateLimitError as _OpenAIRateLimitError
-from tenacity import RetryError, retry
-
-from repowise.core.providers.llm.base import (
- BaseProvider,
- ChatStreamEvent,
- ChatToolCall,
- GeneratedResponse,
- ProviderError,
- ProviderModelOption,
- RateLimitError,
- ensure_reasoning_supported,
- fallback_model_option,
- normalize_stop_reason,
- parse_retry_after,
- provider_retry_stop,
- provider_retry_wait,
- provider_should_retry,
+from repowise.core.providers.llm.base import ProviderModelOption
+from repowise.core.providers.llm.openai_compat import (
+ OpenAICompatibleProvider,
+ listed_model_options,
)
from repowise.core.rate_limiter import RateLimiter
from repowise.core.reasoning import ReasoningMode, normalize_reasoning
@@ -46,8 +25,6 @@ from repowise.core.reasoning import ReasoningMode, normalize_reasoning
if TYPE_CHECKING:
from repowise.core.generation.cost_tracker import CostTracker
-log = structlog.get_logger(__name__)
-
_DEFAULT_BASE_URL = "https://api.deepseek.com"
_DEEPSEEK_REASONING_MODES: tuple[ReasoningMode, ...] = (
"off",
@@ -76,23 +53,6 @@ def _deepseek_supported_reasoning_modes(model: str) -> tuple[ReasoningMode, ...]
return ()
-def _resolve_deepseek_reasoning_mode(
- reasoning: ReasoningMode,
- *,
- model: str,
-) -> ReasoningMode:
- return ensure_reasoning_supported(
- "deepseek",
- model,
- normalize_reasoning(reasoning),
- _deepseek_supported_reasoning_modes(model),
- detail=(
- "DeepSeek /models lists IDs only; reasoning controls are enabled "
- "for the documented DeepSeek Flash and V4 model families."
- ),
- )
-
-
def _deepseek_reasoning_kwargs(reasoning: ReasoningMode) -> dict[str, Any]:
mode = normalize_reasoning(reasoning)
if mode == "auto":
@@ -106,59 +66,27 @@ def _deepseek_reasoning_kwargs(reasoning: ReasoningMode) -> dict[str, Any]:
}
+def _deepseek_notes(model_id: str, reasoning_modes: tuple[ReasoningMode, ...]) -> str:
+ if len(reasoning_modes) > 1:
+ return "thinking toggle and graded effort documented by DeepSeek"
+ return ""
+
+
def _deepseek_model_options(
api_key: str,
base_url: str,
fallback_model: str,
) -> tuple[ProviderModelOption, ...]:
- fallback = fallback_model_option(
+ return listed_model_options(
+ api_key,
+ base_url,
fallback_model,
- reasoning_modes=("auto", *_deepseek_supported_reasoning_modes(fallback_model)),
+ reasoning_modes_for=_deepseek_supported_reasoning_modes,
+ notes_for=_deepseek_notes,
)
- try:
- import httpx
-
- response = httpx.get(
- f"{base_url.rstrip('/')}/models",
- headers={"Authorization": f"Bearer {api_key}"},
- timeout=5.0,
- )
- response.raise_for_status()
- data = response.json().get("data", [])
- except Exception:
- return (fallback,)
-
- if not isinstance(data, list):
- return (fallback,)
- options: list[ProviderModelOption] = []
- for raw in data:
- if not isinstance(raw, dict) or not isinstance(raw.get("id"), str):
- continue
- model_id = raw["id"]
- reasoning_modes = ("auto", *_deepseek_supported_reasoning_modes(model_id))
- options.append(
- ProviderModelOption(
- model=model_id,
- label=model_id,
- reasoning_modes=reasoning_modes,
- recommended=model_id == fallback_model,
- source="api",
- notes=(
- "thinking toggle and graded effort documented by DeepSeek"
- if len(reasoning_modes) > 1
- else ""
- ),
- )
- )
- if not options:
- return (fallback,)
-
- return tuple(options)
-
-
-class DeepSeekProvider(BaseProvider):
+class DeepSeekProvider(OpenAICompatibleProvider):
"""DeepSeek provider — access DeepSeek V4 models via OpenAI-compatible API.
Args:
@@ -169,6 +97,15 @@ class DeepSeekProvider(BaseProvider):
cost_tracker: Optional CostTracker instance for usage recording.
"""
+ provider_id = "deepseek"
+ api_key_env = "DEEPSEEK_API_KEY"
+ base_url_env = "DEEPSEEK_BASE_URL"
+ default_base_url = _DEFAULT_BASE_URL
+ reasoning_detail = (
+ "DeepSeek /models lists IDs only; reasoning controls are enabled "
+ "for the documented DeepSeek Flash and V4 model families."
+ )
+
def __init__(
self,
api_key: str | None = None,
@@ -177,176 +114,25 @@ class DeepSeekProvider(BaseProvider):
rate_limiter: RateLimiter | None = None,
cost_tracker: CostTracker | None = None,
) -> None:
- resolved_key = api_key or os.environ.get("DEEPSEEK_API_KEY")
- if not resolved_key:
- raise ProviderError(
- "deepseek",
- "No API key provided. Pass api_key= or set DEEPSEEK_API_KEY.",
- )
- resolved_base_url = base_url or os.environ.get("DEEPSEEK_BASE_URL") or _DEFAULT_BASE_URL
- self._api_key = resolved_key
- self._base_url = resolved_base_url
- self._client = AsyncOpenAI(
- api_key=resolved_key,
- base_url=resolved_base_url,
- )
- self._model = model
- self._rate_limiter = rate_limiter
- self._cost_tracker = cost_tracker
-
- @property
- def provider_name(self) -> str:
- return "deepseek"
-
- @property
- def model_name(self) -> str:
- return self._model
-
- def supported_reasoning_modes(self) -> tuple[ReasoningMode, ...]:
- return ("auto", *_deepseek_supported_reasoning_modes(self._model))
+ super().__init__(api_key, model, base_url, rate_limiter, cost_tracker)
def available_model_options(self) -> tuple[ProviderModelOption, ...]:
return _deepseek_model_options(self._api_key, self._base_url, self._model)
- async def generate(
- self,
- system_prompt: str,
- user_prompt: str,
- max_tokens: int = 4096,
- temperature: float = 0.3,
- request_id: str | None = None,
- reasoning: ReasoningMode = "auto",
- cache_hints: tuple = (),
- ) -> GeneratedResponse:
- reasoning_mode = _resolve_deepseek_reasoning_mode(reasoning, model=self._model)
- if self._rate_limiter:
- await self._rate_limiter.acquire(estimated_tokens=max_tokens)
-
- log.debug(
- "deepseek.generate.start",
- model=self._model,
- max_tokens=max_tokens,
- request_id=request_id,
- )
+ def _explicit_reasoning_modes(self, model: str) -> tuple[ReasoningMode, ...]:
+ return _deepseek_supported_reasoning_modes(model)
- try:
- return await self._generate_with_retry(
- system_prompt=system_prompt,
- user_prompt=user_prompt,
- max_tokens=max_tokens,
- temperature=temperature,
- request_id=request_id,
- reasoning=reasoning_mode,
- )
- except RetryError as exc:
- raise ProviderError(
- "deepseek",
- f"All retries exhausted: {exc}",
- ) from exc
+ def _reasoning_kwargs(self, reasoning: ReasoningMode) -> dict[str, Any]:
+ return _deepseek_reasoning_kwargs(reasoning)
- @retry(
- retry=provider_should_retry,
- stop=provider_retry_stop,
- wait=provider_retry_wait,
- reraise=True,
- )
- async def _generate_with_retry(
+ def _stream_kwargs(
self,
- system_prompt: str,
- user_prompt: str,
+ messages: list[dict[str, Any]],
max_tokens: int,
temperature: float,
- request_id: str | None,
- reasoning: ReasoningMode,
- ) -> GeneratedResponse:
- try:
- kwargs: dict[str, Any] = {
- "model": self._model,
- "max_tokens": max_tokens,
- "temperature": temperature,
- "messages": [
- {"role": "system", "content": system_prompt},
- {"role": "user", "content": user_prompt},
- ],
- }
- kwargs.update(_deepseek_reasoning_kwargs(reasoning))
- response = await self._client.chat.completions.create(**kwargs)
- except _OpenAIRateLimitError as exc:
- raise RateLimitError(
- "deepseek",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _OpenAIAPIStatusError as exc:
- raise ProviderError("deepseek", str(exc), status_code=exc.status_code) from exc
- except _OpenAIAPIError as exc:
- raise ProviderError(
- "deepseek", str(exc), status_code=getattr(exc, "status_code", None)
- ) from exc
-
- usage = response.usage
- choice = response.choices[0]
- stop_reason, provider_stop_reason = normalize_stop_reason(choice.finish_reason)
- result = GeneratedResponse(
- content=choice.message.content or "",
- input_tokens=usage.prompt_tokens if usage else 0,
- output_tokens=usage.completion_tokens if usage else 0,
- cached_tokens=0,
- stop_reason=stop_reason,
- provider_stop_reason=provider_stop_reason,
- usage={
- "prompt_tokens": usage.prompt_tokens if usage else 0,
- "completion_tokens": usage.completion_tokens if usage else 0,
- "total_tokens": usage.total_tokens if usage else 0,
- },
- )
- log.debug(
- "deepseek.generate.done",
- input_tokens=result.input_tokens,
- output_tokens=result.output_tokens,
- request_id=request_id,
- )
-
- if self._cost_tracker is not None:
- # Await the cost record inline rather than spawning a detached
- # task. A fire-and-forget create_task can still be flushing its
- # aiosqlite write when the event loop is torn down (e.g. the
- # asyncio.run teardown after doc generation), which surfaces as a
- # noisy "Event loop is closed" worker-thread traceback. record()
- # swallows its own persistence errors, so generation is unaffected.
- with contextlib.suppress(Exception):
- await self._cost_tracker.record(
- model=self._model,
- input_tokens=result.input_tokens,
- output_tokens=result.output_tokens,
- operation=self._cost_tracker.operation,
- file_path=None,
- )
-
- return result
-
- async def stream_chat(
- self,
- messages: list[dict[str, Any]],
- tools: list[dict[str, Any]],
- system_prompt: str,
- max_tokens: int = 8192,
- temperature: float = 0.7,
- request_id: str | None = None,
- tool_executor: Any | None = None,
- ) -> AsyncIterator[ChatStreamEvent]:
- import json as _json
-
- full_messages = [{"role": "system", "content": system_prompt}, *messages]
- kwargs: dict[str, Any] = {
- "model": self._model,
- "max_tokens": max_tokens,
- "temperature": temperature,
- "messages": full_messages,
- "stream": True,
+ ) -> dict[str, Any]:
+ return {
+ **super()._stream_kwargs(messages, max_tokens, temperature),
# Thinking tool calls require reasoning_content from every prior
# assistant turn. Repowise's shared chat history intentionally
# does not retain hidden reasoning, so use DeepSeek's documented
@@ -354,96 +140,3 @@ class DeepSeekProvider(BaseProvider):
# invalid follow-up request after the first tool call.
"extra_body": {"thinking": {"type": "disabled"}},
}
- if tools:
- kwargs["tools"] = tools
-
- try:
- stream = await self._client.chat.completions.create(**kwargs)
- except _OpenAIRateLimitError as exc:
- raise RateLimitError(
- "deepseek",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _OpenAIAPIStatusError as exc:
- raise ProviderError("deepseek", str(exc), status_code=exc.status_code) from exc
- except _OpenAIAPIError as exc:
- raise ProviderError(
- "deepseek", str(exc), status_code=getattr(exc, "status_code", None)
- ) from exc
-
- tool_calls_acc: dict[int, dict[str, Any]] = {}
-
- try:
- async for chunk in stream:
- choice = chunk.choices[0] if chunk.choices else None
- if not choice:
- if chunk.usage:
- yield ChatStreamEvent(
- type="usage",
- input_tokens=chunk.usage.prompt_tokens or 0,
- output_tokens=chunk.usage.completion_tokens or 0,
- )
- continue
-
- delta = choice.delta
- finish = choice.finish_reason
-
- if delta and delta.content:
- yield ChatStreamEvent(type="text_delta", text=delta.content)
-
- if delta and delta.tool_calls:
- for tc_delta in delta.tool_calls:
- idx = tc_delta.index
- if idx not in tool_calls_acc:
- tool_calls_acc[idx] = {
- "id": tc_delta.id or "",
- "name": "",
- "arguments": "",
- }
- acc = tool_calls_acc[idx]
- if tc_delta.id:
- acc["id"] = tc_delta.id
- if tc_delta.function:
- if tc_delta.function.name:
- acc["name"] = tc_delta.function.name
- if tc_delta.function.arguments:
- acc["arguments"] += tc_delta.function.arguments
-
- if finish:
- for idx in sorted(tool_calls_acc.keys()):
- acc = tool_calls_acc[idx]
- try:
- args = _json.loads(acc["arguments"]) if acc["arguments"] else {}
- except Exception:
- args = {}
- yield ChatStreamEvent(
- type="tool_start",
- tool_call=ChatToolCall(
- id=acc["id"],
- name=acc["name"],
- arguments=args,
- ),
- )
- tool_calls_acc.clear()
-
- stop_reason = "tool_use" if finish == "tool_calls" else "end_turn"
- yield ChatStreamEvent(type="stop", stop_reason=stop_reason)
- except _OpenAIRateLimitError as exc:
- raise RateLimitError(
- "deepseek",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _OpenAIAPIStatusError as exc:
- raise ProviderError("deepseek", str(exc), status_code=exc.status_code) from exc
- except _OpenAIAPIError as exc:
- raise ProviderError(
- "deepseek", str(exc), status_code=getattr(exc, "status_code", None)
- ) from exc
diff --git a/packages/core/src/repowise/core/providers/llm/edenai.py b/packages/core/src/repowise/core/providers/llm/edenai.py
index e7abd0c05..60a09aa8c 100644
--- a/packages/core/src/repowise/core/providers/llm/edenai.py
+++ b/packages/core/src/repowise/core/providers/llm/edenai.py
@@ -23,31 +23,12 @@ Set the EU endpoint for data residency:
from __future__ import annotations
-import contextlib
-import os
-from collections.abc import AsyncIterator
from typing import TYPE_CHECKING, Any
-import structlog
-from openai import APIStatusError as _OpenAIAPIStatusError
-from openai import AsyncOpenAI
-from openai import RateLimitError as _OpenAIRateLimitError
-from tenacity import RetryError, retry
-
-from repowise.core.providers.llm.base import (
- BaseProvider,
- ChatStreamEvent,
- ChatToolCall,
- GeneratedResponse,
- ProviderError,
- ProviderModelOption,
- RateLimitError,
- ensure_reasoning_supported,
- fallback_model_option,
- parse_retry_after,
- provider_retry_stop,
- provider_retry_wait,
- provider_should_retry,
+from repowise.core.providers.llm.base import ProviderModelOption
+from repowise.core.providers.llm.openai_compat import (
+ OpenAICompatibleProvider,
+ listed_model_options,
)
from repowise.core.rate_limiter import RateLimiter
from repowise.core.reasoning import ReasoningMode, normalize_reasoning
@@ -55,8 +36,6 @@ from repowise.core.reasoning import ReasoningMode, normalize_reasoning
if TYPE_CHECKING:
from repowise.core.generation.cost_tracker import CostTracker
-log = structlog.get_logger(__name__)
-
_DEFAULT_BASE_URL = "https://api.edenai.run/v3"
@@ -88,20 +67,6 @@ def _edenai_supported_reasoning_modes(model: str) -> tuple[ReasoningMode, ...]:
return ("low", "medium", "high")
-def _resolve_edenai_reasoning_mode(reasoning: ReasoningMode, *, model: str) -> ReasoningMode:
- """Validate reasoning support before issuing an API call."""
- return ensure_reasoning_supported(
- "edenai",
- model,
- normalize_reasoning(reasoning),
- _edenai_supported_reasoning_modes(model),
- detail=(
- "EdenAIProvider maps explicit efforts to the OpenAI reasoning_effort "
- "parameter for OpenAI reasoning model ids routed via Eden AI."
- ),
- )
-
-
def _edenai_reasoning_kwargs(reasoning: ReasoningMode) -> dict[str, Any]:
"""Translate a validated repowise reasoning intent to Eden AI kwargs."""
mode = normalize_reasoning(reasoning)
@@ -112,60 +77,28 @@ def _edenai_reasoning_kwargs(reasoning: ReasoningMode) -> dict[str, Any]:
return {}
+def _edenai_notes(model_id: str, reasoning_modes: tuple[ReasoningMode, ...]) -> str:
+ if len(reasoning_modes) > 1:
+ return "reasoning_effort forwarded to the underlying OpenAI model"
+ return ""
+
+
def _edenai_model_options(
api_key: str,
base_url: str,
fallback_model: str,
) -> tuple[ProviderModelOption, ...]:
- fallback = fallback_model_option(
+ return listed_model_options(
+ api_key,
+ base_url,
fallback_model,
- reasoning_modes=("auto", *_edenai_supported_reasoning_modes(fallback_model)),
+ reasoning_modes_for=_edenai_supported_reasoning_modes,
+ notes_for=_edenai_notes,
+ sort=True,
)
- try:
- import httpx
-
- response = httpx.get(
- f"{base_url.rstrip('/')}/models",
- headers={"Authorization": f"Bearer {api_key}"},
- timeout=5.0,
- )
- response.raise_for_status()
- data = response.json().get("data", [])
- except Exception:
- return (fallback,)
-
- if not isinstance(data, list):
- return (fallback,)
-
- options: list[ProviderModelOption] = []
- for raw in data:
- if not isinstance(raw, dict) or not isinstance(raw.get("id"), str):
- continue
- model_id = raw["id"]
- reasoning_modes = ("auto", *_edenai_supported_reasoning_modes(model_id))
- options.append(
- ProviderModelOption(
- model=model_id,
- label=model_id,
- reasoning_modes=reasoning_modes,
- recommended=model_id == fallback_model,
- source="api",
- notes=(
- "reasoning_effort forwarded to the underlying OpenAI model"
- if len(reasoning_modes) > 1
- else ""
- ),
- )
- )
-
- if not options:
- return (fallback,)
-
- options.sort(key=lambda option: option.model)
- return tuple(options)
-
-
-class EdenAIProvider(BaseProvider):
+
+
+class EdenAIProvider(OpenAICompatibleProvider):
"""Eden AI provider, reaching many vendors via a single OpenAI-compatible key.
Args:
@@ -179,6 +112,20 @@ class EdenAIProvider(BaseProvider):
cost_tracker: Optional CostTracker instance for usage recording.
"""
+ provider_id = "edenai"
+ api_key_env = "EDENAI_API_KEY"
+ base_url_env = "EDENAI_BASE_URL"
+ default_base_url = _DEFAULT_BASE_URL
+ reasoning_detail = (
+ "EdenAIProvider maps explicit efforts to the OpenAI reasoning_effort "
+ "parameter for OpenAI reasoning model ids routed via Eden AI."
+ )
+ # Status-less SDK errors are not retried, no stop reason is reported, and
+ # spend is always recorded as doc generation.
+ wrap_connection_errors = False
+ reports_stop_reason = False
+ cost_operation = "doc_generation"
+
def __init__(
self,
api_key: str | None = None,
@@ -187,256 +134,16 @@ class EdenAIProvider(BaseProvider):
rate_limiter: RateLimiter | None = None,
cost_tracker: CostTracker | None = None,
) -> None:
- resolved_key = api_key or os.environ.get("EDENAI_API_KEY")
- if not resolved_key:
- raise ProviderError(
- "edenai",
- "No API key provided. Pass api_key= or set EDENAI_API_KEY.",
- )
- resolved_base_url = base_url or os.environ.get("EDENAI_BASE_URL") or _DEFAULT_BASE_URL
- self._api_key = resolved_key
- self._base_url = resolved_base_url.rstrip("/")
- self._client = AsyncOpenAI(
- api_key=resolved_key,
- base_url=self._base_url,
- )
- self._model = model
- self._rate_limiter = rate_limiter
- self._cost_tracker = cost_tracker
-
- @property
- def provider_name(self) -> str:
- return "edenai"
-
- @property
- def model_name(self) -> str:
- return self._model
-
- def supported_reasoning_modes(self) -> tuple[ReasoningMode, ...]:
- return ("auto", *_edenai_supported_reasoning_modes(self._model))
+ super().__init__(api_key, model, base_url, rate_limiter, cost_tracker)
def available_model_options(self) -> tuple[ProviderModelOption, ...]:
return _edenai_model_options(self._api_key, self._base_url, self._model)
- async def generate(
- self,
- system_prompt: str,
- user_prompt: str,
- max_tokens: int = 4096,
- temperature: float = 0.3,
- request_id: str | None = None,
- reasoning: ReasoningMode = "auto",
- cache_hints: tuple = (),
- ) -> GeneratedResponse:
- reasoning_mode = _resolve_edenai_reasoning_mode(reasoning, model=self._model)
- if self._rate_limiter:
- await self._rate_limiter.acquire(estimated_tokens=max_tokens)
-
- log.debug(
- "edenai.generate.start",
- model=self._model,
- max_tokens=max_tokens,
- request_id=request_id,
- )
-
- try:
- return await self._generate_with_retry(
- system_prompt=system_prompt,
- user_prompt=user_prompt,
- max_tokens=max_tokens,
- temperature=temperature,
- request_id=request_id,
- reasoning=reasoning_mode,
- )
- except RetryError as exc:
- raise ProviderError(
- "edenai",
- f"All retries exhausted: {exc}",
- ) from exc
-
- @retry(
- retry=provider_should_retry,
- stop=provider_retry_stop,
- wait=provider_retry_wait,
- reraise=True,
- )
- async def _generate_with_retry(
- self,
- system_prompt: str,
- user_prompt: str,
- max_tokens: int,
- temperature: float,
- request_id: str | None,
- reasoning: ReasoningMode,
- ) -> GeneratedResponse:
- try:
- kwargs: dict[str, Any] = {
- "model": self._model,
- "max_tokens": max_tokens,
- "temperature": temperature,
- "messages": [
- {"role": "system", "content": system_prompt},
- {"role": "user", "content": user_prompt},
- ],
- }
- kwargs.update(_edenai_reasoning_kwargs(reasoning))
- response = await self._client.chat.completions.create(**kwargs)
- except _OpenAIRateLimitError as exc:
- raise RateLimitError(
- "edenai",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _OpenAIAPIStatusError as exc:
- raise ProviderError("edenai", str(exc), status_code=exc.status_code) from exc
-
- usage = response.usage
- result = GeneratedResponse(
- content=response.choices[0].message.content or "",
- input_tokens=usage.prompt_tokens if usage else 0,
- output_tokens=usage.completion_tokens if usage else 0,
- cached_tokens=0,
- usage={
- "prompt_tokens": usage.prompt_tokens if usage else 0,
- "completion_tokens": usage.completion_tokens if usage else 0,
- "total_tokens": usage.total_tokens if usage else 0,
- },
- )
- log.debug(
- "edenai.generate.done",
- input_tokens=result.input_tokens,
- output_tokens=result.output_tokens,
- request_id=request_id,
- )
-
- if self._cost_tracker is not None:
- # Await the cost record inline rather than spawning a detached task,
- # a fire-and-forget create_task can still be flushing its aiosqlite
- # write when the event loop is torn down, surfacing as a noisy
- # "Event loop is closed" traceback. record() swallows its own
- # persistence errors, so generation is unaffected.
- with contextlib.suppress(Exception):
- await self._cost_tracker.record(
- model=self._model,
- input_tokens=result.input_tokens,
- output_tokens=result.output_tokens,
- operation="doc_generation",
- file_path=None,
- )
-
- return result
-
- # --- ChatProvider protocol implementation ---
-
- async def stream_chat(
- self,
- messages: list[dict[str, Any]],
- tools: list[dict[str, Any]],
- system_prompt: str,
- max_tokens: int = 8192,
- temperature: float = 0.7,
- request_id: str | None = None,
- tool_executor: Any | None = None,
- ) -> AsyncIterator[ChatStreamEvent]:
- import json as _json
-
- full_messages = [{"role": "system", "content": system_prompt}, *messages]
- kwargs: dict[str, Any] = {
- "model": self._model,
- "max_tokens": max_tokens,
- "temperature": temperature,
- "messages": full_messages,
- "stream": True,
- }
- if tools:
- kwargs["tools"] = tools
-
- try:
- stream = await self._client.chat.completions.create(**kwargs)
- except _OpenAIRateLimitError as exc:
- raise RateLimitError(
- "edenai",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _OpenAIAPIStatusError as exc:
- raise ProviderError("edenai", str(exc), status_code=exc.status_code) from exc
-
- # Track in-progress tool calls (OpenAI-compatible streaming)
- tool_calls_acc: dict[int, dict[str, Any]] = {}
-
- try:
- async for chunk in stream:
- choice = chunk.choices[0] if chunk.choices else None
- if not choice:
- if chunk.usage:
- yield ChatStreamEvent(
- type="usage",
- input_tokens=chunk.usage.prompt_tokens or 0,
- output_tokens=chunk.usage.completion_tokens or 0,
- )
- continue
-
- delta = choice.delta
- finish = choice.finish_reason
-
- # Text content
- if delta and delta.content:
- yield ChatStreamEvent(type="text_delta", text=delta.content)
-
- # Tool call fragments
- if delta and delta.tool_calls:
- for tc_delta in delta.tool_calls:
- idx = tc_delta.index
- if idx not in tool_calls_acc:
- tool_calls_acc[idx] = {
- "id": tc_delta.id or "",
- "name": "",
- "arguments": "",
- }
- acc = tool_calls_acc[idx]
- if tc_delta.id:
- acc["id"] = tc_delta.id
- if tc_delta.function:
- if tc_delta.function.name:
- acc["name"] = tc_delta.function.name
- if tc_delta.function.arguments:
- acc["arguments"] += tc_delta.function.arguments
-
- if finish:
- # Emit accumulated tool calls
- for idx in sorted(tool_calls_acc.keys()):
- acc = tool_calls_acc[idx]
- try:
- args = _json.loads(acc["arguments"]) if acc["arguments"] else {}
- except Exception:
- args = {}
- yield ChatStreamEvent(
- type="tool_start",
- tool_call=ChatToolCall(
- id=acc["id"],
- name=acc["name"],
- arguments=args,
- ),
- )
- tool_calls_acc.clear()
-
- stop_reason = "tool_use" if finish == "tool_calls" else "end_turn"
- yield ChatStreamEvent(type="stop", stop_reason=stop_reason)
- except _OpenAIRateLimitError as exc:
- raise RateLimitError(
- "edenai",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _OpenAIAPIStatusError as exc:
- raise ProviderError("edenai", str(exc), status_code=exc.status_code) from exc
+ def _clean_base_url(self, base_url: str) -> str:
+ return base_url.rstrip("/")
+
+ def _explicit_reasoning_modes(self, model: str) -> tuple[ReasoningMode, ...]:
+ return _edenai_supported_reasoning_modes(model)
+
+ def _reasoning_kwargs(self, reasoning: ReasoningMode) -> dict[str, Any]:
+ return _edenai_reasoning_kwargs(reasoning)
diff --git a/packages/core/src/repowise/core/providers/llm/gemini.py b/packages/core/src/repowise/core/providers/llm/gemini.py
index b588f4bfa..941419ff8 100644
--- a/packages/core/src/repowise/core/providers/llm/gemini.py
+++ b/packages/core/src/repowise/core/providers/llm/gemini.py
@@ -11,7 +11,7 @@ Recommended models:
from __future__ import annotations
import asyncio
-import contextlib
+import json
import logging
import os
from collections.abc import AsyncIterator
@@ -31,9 +31,11 @@ from repowise.core.providers.llm.base import (
ensure_reasoning_supported,
fallback_model_option,
normalize_stop_reason,
+ parse_tool_arguments,
provider_retry_stop,
provider_retry_wait,
provider_should_retry,
+ record_generation_cost,
)
from repowise.core.rate_limiter import RateLimiter
from repowise.core.reasoning import ReasoningMode, normalize_reasoning
@@ -127,65 +129,19 @@ def _gemini_model_options(
) -> tuple[ProviderModelOption, ...]:
fallback = fallback_model_option(fallback_model)
try:
- import httpx
-
- url = f"{base_url.rstrip('/')}/models" if base_url else _DEFAULT_MODELS_URL
- models: list[dict[str, Any]] = []
- page_token: str | None = None
- while True:
- params: dict[str, str | int] = {"pageSize": 1000}
- if page_token:
- params["pageToken"] = page_token
- response = httpx.get(
- url,
- headers={"x-goog-api-key": api_key},
- params=params,
- timeout=5.0,
- )
- response.raise_for_status()
- payload = response.json()
- raw_models = payload.get("models", [])
- if isinstance(raw_models, list):
- models.extend(raw for raw in raw_models if isinstance(raw, dict))
- page_token = payload.get("nextPageToken")
- if not isinstance(page_token, str) or not page_token:
- break
+ models = _list_gemini_models(api_key, base_url)
except Exception:
return (fallback,)
options: list[ProviderModelOption] = []
thinking_models: set[str] = set()
for raw in models:
- name = raw.get("name")
- if not isinstance(name, str):
- continue
- methods = raw.get("supportedGenerationMethods")
- if isinstance(methods, list) and "generateContent" not in methods:
+ option = _gemini_model_option(raw, fallback_model)
+ if option is None:
continue
- model_id = name.removeprefix("models/")
- display_name = raw.get("displayName")
- has_thinking = raw.get("thinking") is True
- if has_thinking:
- thinking_models.add(model_id)
- options.append(
- ProviderModelOption(
- model=model_id,
- label=display_name if isinstance(display_name, str) else model_id,
- reasoning_modes=(
- "auto",
- *_GEMINI_THINKING_MODES,
- )
- if has_thinking
- else ("auto",),
- recommended=model_id == fallback_model,
- source="api",
- notes=(
- "generateContent; thinking advertised by models.list"
- if has_thinking
- else "generateContent"
- ),
- )
- )
+ options.append(option)
+ if len(option.reasoning_modes) > 1:
+ thinking_models.add(option.model)
if thinking_models:
_GEMINI_THINKING_MODELS_BY_BASE[_gemini_cache_key(base_url)] = thinking_models
@@ -197,6 +153,76 @@ def _gemini_model_options(
return tuple(options)
+def _list_gemini_models(api_key: str, base_url: str | None) -> list[dict[str, Any]]:
+ """Page through ``models.list``; raises when any page cannot be fetched."""
+ import httpx
+
+ url = f"{base_url.rstrip('/')}/models" if base_url else _DEFAULT_MODELS_URL
+ models: list[dict[str, Any]] = []
+ page_token: str | None = None
+ while True:
+ params: dict[str, str | int] = {"pageSize": 1000}
+ if page_token:
+ params["pageToken"] = page_token
+ response = httpx.get(
+ url,
+ headers={"x-goog-api-key": api_key},
+ params=params,
+ timeout=5.0,
+ )
+ response.raise_for_status()
+ payload = response.json()
+ raw_models = payload.get("models", [])
+ if isinstance(raw_models, list):
+ models.extend(raw for raw in raw_models if isinstance(raw, dict))
+ page_token = payload.get("nextPageToken")
+ if not isinstance(page_token, str) or not page_token:
+ return models
+
+
+def _gemini_model_option(raw: dict[str, Any], fallback_model: str) -> ProviderModelOption | None:
+ """One ``models.list`` entry as an option, or ``None`` if it cannot generate text."""
+ name = raw.get("name")
+ if not isinstance(name, str):
+ return None
+ methods = raw.get("supportedGenerationMethods")
+ if isinstance(methods, list) and "generateContent" not in methods:
+ return None
+ model_id = name.removeprefix("models/")
+ display_name = raw.get("displayName")
+ has_thinking = raw.get("thinking") is True
+ return ProviderModelOption(
+ model=model_id,
+ label=display_name if isinstance(display_name, str) else model_id,
+ reasoning_modes=("auto", *_GEMINI_THINKING_MODES) if has_thinking else ("auto",),
+ recommended=model_id == fallback_model,
+ source="api",
+ notes=(
+ "generateContent; thinking advertised by models.list"
+ if has_thinking
+ else "generateContent"
+ ),
+ )
+
+
+def _gemini_error(exc: Exception) -> ProviderError:
+ """Classify a google-genai failure; quota errors become ``RateLimitError``.
+
+ The SDK does not type its 429s consistently, so the status attribute and
+ the message text are both consulted.
+ """
+ exc_str = str(exc)
+ status_code = getattr(exc, "status_code", None) or getattr(exc, "code", None)
+ if status_code == 429 or "429" in exc_str or "quota" in exc_str.lower():
+ return RateLimitError(
+ "gemini",
+ exc_str,
+ status_code=429,
+ retry_after=_gemini_retry_delay(exc_str),
+ )
+ return ProviderError("gemini", f"{type(exc).__name__}: {exc_str}")
+
+
class GeminiProvider(BaseProvider):
"""Native Gemini provider using the google-genai SDK.
@@ -321,31 +347,7 @@ class GeminiProvider(BaseProvider):
from google import genai # type: ignore[import-untyped]
from google.genai import types as genai_types # type: ignore[import-untyped]
- if self._client is None:
- client_kwargs: dict[str, Any] = {"api_key": api_key}
- http_opts = None
-
- if base_url:
- try:
- http_opts = genai_types.HttpOptions(base_url=base_url)
- except TypeError:
- log.warning(
- "gemini.http_options.base_url_unsupported",
- base_url=base_url,
- )
-
- if http_opts is not None:
- try:
- self._client = genai.Client(**client_kwargs, http_options=http_opts)
- except TypeError:
- log.warning(
- "gemini.client.http_options_unsupported",
- base_url=base_url,
- )
- self._client = genai.Client(**client_kwargs)
- else:
- self._client = genai.Client(**client_kwargs)
- client = self._client
+ client = self._ensure_client(genai, genai_types, api_key, base_url)
try:
thinking_config = _gemini_thinking_config(reasoning, genai_types)
config_kwargs: dict[str, Any] = {
@@ -362,16 +364,7 @@ class GeminiProvider(BaseProvider):
config=genai_types.GenerateContentConfig(**config_kwargs),
)
except Exception as exc:
- exc_str = str(exc)
- status_code = getattr(exc, "status_code", None) or getattr(exc, "code", None)
- if status_code == 429 or "429" in exc_str or "quota" in exc_str.lower():
- raise RateLimitError(
- "gemini",
- exc_str,
- status_code=429,
- retry_after=_gemini_retry_delay(exc_str),
- ) from exc
- raise ProviderError("gemini", f"{type(exc).__name__}: {exc_str}") from exc
+ raise _gemini_error(exc) from exc
usage = response.usage_metadata
candidate = response.candidates[0] if response.candidates else None
@@ -408,24 +401,33 @@ class GeminiProvider(BaseProvider):
request_id=request_id,
)
- if self._cost_tracker is not None:
- # Await the cost record inline rather than spawning a detached
- # task. A fire-and-forget create_task can still be flushing its
- # aiosqlite write when the event loop is torn down (e.g. the
- # asyncio.run teardown after doc generation), which surfaces as a
- # noisy "Event loop is closed" worker-thread traceback. record()
- # swallows its own persistence errors, so generation is unaffected.
- with contextlib.suppress(Exception):
- await self._cost_tracker.record(
- model=self._model,
- input_tokens=result.input_tokens,
- output_tokens=result.output_tokens,
- operation=self._cost_tracker.operation,
- file_path=None,
- )
-
+ await record_generation_cost(self._cost_tracker, model=self._model, result=result)
return result
+ def _ensure_client(
+ self, genai: Any, genai_types: Any, api_key: str | None, base_url: str | None
+ ) -> Any:
+ """Create the google-genai client once, tolerating SDKs without ``base_url``."""
+ if self._client is not None:
+ return self._client
+ client_kwargs: dict[str, Any] = {"api_key": api_key}
+ http_opts = None
+ if base_url:
+ try:
+ http_opts = genai_types.HttpOptions(base_url=base_url)
+ except TypeError:
+ log.warning("gemini.http_options.base_url_unsupported", base_url=base_url)
+
+ if http_opts is None:
+ self._client = genai.Client(**client_kwargs)
+ return self._client
+ try:
+ self._client = genai.Client(**client_kwargs, http_options=http_opts)
+ except TypeError:
+ log.warning("gemini.client.http_options_unsupported", base_url=base_url)
+ self._client = genai.Client(**client_kwargs)
+ return self._client
+
# --- ChatProvider protocol implementation ---
async def stream_chat(
@@ -469,35 +471,12 @@ class GeminiProvider(BaseProvider):
config=config,
)
except Exception as exc:
- exc_str = str(exc)
- status_code = getattr(exc, "status_code", None) or getattr(exc, "code", None)
- if status_code == 429 or "429" in exc_str or "quota" in exc_str.lower():
- raise RateLimitError(
- "gemini",
- exc_str,
- status_code=429,
- retry_after=_gemini_retry_delay(exc_str),
- ) from exc
- raise ProviderError("gemini", f"{type(exc).__name__}: {exc_str}") from exc
+ raise _gemini_error(exc) from exc
return response
from google.genai import types as genai_types # type: ignore[import-untyped]
- # Convert OpenAI tools to Gemini FunctionDeclarations
- gemini_tools = None
- if tools:
- declarations = []
- for t in tools:
- fn = t.get("function", t)
- params = fn.get("parameters", {})
- declarations.append(
- genai_types.FunctionDeclaration(
- name=fn["name"],
- description=fn.get("description", ""),
- parameters=params if params else None,
- )
- )
- gemini_tools = [genai_types.Tool(function_declarations=declarations)]
+ gemini_tools = _to_gemini_tools(tools, genai_types)
config = genai_types.GenerateContentConfig(
system_instruction=system_prompt,
@@ -525,24 +504,9 @@ class GeminiProvider(BaseProvider):
# The model's response content — preserved as-is for the next turn
model_content = response.candidates[0].content
- # Extract events from response parts
- function_calls_found: list[tuple[str, str, dict]] = []
- for part in model_content.parts:
- if hasattr(part, "text") and part.text:
- yield ChatStreamEvent(type="text_delta", text=part.text)
- elif hasattr(part, "function_call") and part.function_call:
- fc = part.function_call
- args = dict(fc.args) if fc.args else {}
- tc_id = f"gemini-{fc.name}-{id(part)}"
- function_calls_found.append((tc_id, fc.name, args))
- yield ChatStreamEvent(
- type="tool_start",
- tool_call=ChatToolCall(
- id=tc_id,
- name=fc.name,
- arguments=args,
- ),
- )
+ part_events, function_calls_found = _response_part_events(model_content)
+ for event in part_events:
+ yield event
if not function_calls_found:
yield ChatStreamEvent(type="stop", stop_reason="end_turn")
@@ -582,6 +546,51 @@ class GeminiProvider(BaseProvider):
yield ChatStreamEvent(type="stop", stop_reason="end_turn")
+def _to_gemini_tools(tools: list[dict[str, Any]], genai_types: Any) -> list[Any] | None:
+ """Convert OpenAI-format tool definitions to one Gemini ``Tool``."""
+ if not tools:
+ return None
+ declarations = []
+ for t in tools:
+ fn = t.get("function", t)
+ params = fn.get("parameters", {})
+ declarations.append(
+ genai_types.FunctionDeclaration(
+ name=fn["name"],
+ description=fn.get("description", ""),
+ parameters=params if params else None,
+ )
+ )
+ return [genai_types.Tool(function_declarations=declarations)]
+
+
+def _response_part_events(
+ model_content: Any,
+) -> tuple[list[ChatStreamEvent], list[tuple[str, str, dict]]]:
+ """Text and tool-call events for one model turn, in part order.
+
+ Also returns the requested calls as ``(id, name, args)`` so the caller can
+ execute them. Gemini assigns no call ids, so one is minted per part.
+ """
+ events: list[ChatStreamEvent] = []
+ function_calls: list[tuple[str, str, dict]] = []
+ for part in model_content.parts:
+ if hasattr(part, "text") and part.text:
+ events.append(ChatStreamEvent(type="text_delta", text=part.text))
+ elif hasattr(part, "function_call") and part.function_call:
+ fc = part.function_call
+ args = dict(fc.args) if fc.args else {}
+ tc_id = f"gemini-{fc.name}-{id(part)}"
+ function_calls.append((tc_id, fc.name, args))
+ events.append(
+ ChatStreamEvent(
+ type="tool_start",
+ tool_call=ChatToolCall(id=tc_id, name=fc.name, arguments=args),
+ )
+ )
+ return events, function_calls
+
+
def _to_gemini_contents(messages: list[dict[str, Any]]) -> list:
"""Convert OpenAI-format messages to Gemini Content objects.
@@ -589,8 +598,6 @@ def _to_gemini_contents(messages: list[dict[str, Any]]) -> list:
round-trips use native Gemini Content objects (preserving thought
signatures).
"""
- import json as _json
-
from google.genai import types as genai_types # type: ignore[import-untyped]
contents = []
@@ -600,48 +607,47 @@ def _to_gemini_contents(messages: list[dict[str, Any]]) -> list:
continue # Handled via system_instruction
gemini_role = "model" if role == "assistant" else "user"
- parts = []
-
if role == "tool":
- # Tool result → function_response part
- content_str = msg.get("content", "{}")
- try:
- response_data = (
- _json.loads(content_str) if isinstance(content_str, str) else content_str
- )
- except Exception:
- response_data = {"result": content_str}
- parts.append(
- genai_types.Part.from_function_response(
- name=msg.get("name", "unknown"),
- response=response_data,
- )
- )
- gemini_role = "user"
+ parts = [_tool_result_part(msg, genai_types)]
elif role == "assistant":
- text = msg.get("content")
- if text:
- parts.append(genai_types.Part.from_text(text=text))
- for tc in msg.get("tool_calls", []):
- fn = tc.get("function", {})
- args_str = fn.get("arguments", "{}")
- if isinstance(args_str, str):
- try:
- args = _json.loads(args_str)
- except Exception:
- args = {}
- else:
- args = args_str
- parts.append(
- genai_types.Part.from_function_call(
- name=fn.get("name", ""),
- args=args,
- )
- )
+ parts = _assistant_parts(msg, genai_types)
else:
- parts.append(genai_types.Part.from_text(text=msg.get("content", "")))
+ parts = [genai_types.Part.from_text(text=msg.get("content", ""))]
if parts:
contents.append(genai_types.Content(role=gemini_role, parts=parts))
return contents
+
+
+def _tool_result_part(msg: dict[str, Any], genai_types: Any) -> Any:
+ """An OpenAI ``tool`` message as a Gemini function_response part."""
+ content_str = msg.get("content", "{}")
+ try:
+ response_data = json.loads(content_str) if isinstance(content_str, str) else content_str
+ except Exception:
+ response_data = {"result": content_str}
+ return genai_types.Part.from_function_response(
+ name=msg.get("name", "unknown"),
+ response=response_data,
+ )
+
+
+def _assistant_parts(msg: dict[str, Any], genai_types: Any) -> list[Any]:
+ """An OpenAI ``assistant`` message as Gemini text and function_call parts."""
+ parts = []
+ text = msg.get("content")
+ if text:
+ parts.append(genai_types.Part.from_text(text=text))
+ for tc in msg.get("tool_calls", []):
+ fn = tc.get("function", {})
+ args = fn.get("arguments", "{}")
+ if isinstance(args, str):
+ args = parse_tool_arguments(args)
+ parts.append(
+ genai_types.Part.from_function_call(
+ name=fn.get("name", ""),
+ args=args,
+ )
+ )
+ return parts
diff --git a/packages/core/src/repowise/core/providers/llm/kimi.py b/packages/core/src/repowise/core/providers/llm/kimi.py
index 44dd3a8c6..117d6f3c4 100644
--- a/packages/core/src/repowise/core/providers/llm/kimi.py
+++ b/packages/core/src/repowise/core/providers/llm/kimi.py
@@ -15,32 +15,12 @@ Models:
from __future__ import annotations
-import contextlib
-import os
-from collections.abc import AsyncIterator
from typing import TYPE_CHECKING, Any
-import structlog
-from openai import APIStatusError as _OpenAIAPIStatusError
-from openai import AsyncOpenAI
-from openai import RateLimitError as _OpenAIRateLimitError
-from tenacity import RetryError, retry
-
-from repowise.core.providers.llm.base import (
- BaseProvider,
- ChatStreamEvent,
- ChatToolCall,
- GeneratedResponse,
- ProviderError,
- ProviderModelOption,
- RateLimitError,
- ensure_reasoning_supported,
- fallback_model_option,
- normalize_stop_reason,
- parse_retry_after,
- provider_retry_stop,
- provider_retry_wait,
- provider_should_retry,
+from repowise.core.providers.llm.base import ProviderModelOption
+from repowise.core.providers.llm.openai_compat import (
+ OpenAICompatibleProvider,
+ listed_model_options,
)
from repowise.core.rate_limiter import RateLimiter
from repowise.core.reasoning import ReasoningMode, normalize_reasoning
@@ -48,8 +28,6 @@ from repowise.core.reasoning import ReasoningMode, normalize_reasoning
if TYPE_CHECKING:
from repowise.core.generation.cost_tracker import CostTracker
-log = structlog.get_logger(__name__)
-
_DEFAULT_BASE_URL = "https://api.kimi.com/coding/v1"
_KIMI_K2_REASONING_MODES: tuple[ReasoningMode, ...] = ("off",)
_KIMI_K3_REASONING_MODES: tuple[ReasoningMode, ...] = ("low", "high", "max")
@@ -63,23 +41,6 @@ def _kimi_supported_reasoning_modes(model: str) -> tuple[ReasoningMode, ...]:
return ()
-def _resolve_kimi_reasoning_mode(
- reasoning: ReasoningMode,
- *,
- model: str,
-) -> ReasoningMode:
- return ensure_reasoning_supported(
- "kimi",
- model,
- normalize_reasoning(reasoning),
- _kimi_supported_reasoning_modes(model),
- detail=(
- "Kimi /models lists IDs only; reasoning controls are enabled "
- "from the documented K2 and K3 model-family contracts."
- ),
- )
-
-
def _kimi_reasoning_kwargs(model: str, reasoning: ReasoningMode) -> dict[str, Any]:
mode = normalize_reasoning(reasoning)
if mode == "auto":
@@ -104,61 +65,27 @@ def _kimi_temperature(
return requested
+def _kimi_notes(model_id: str, reasoning_modes: tuple[ReasoningMode, ...]) -> str:
+ if model_id.startswith("kimi-k3"):
+ return "K3 supports low, high, and max reasoning effort"
+ return "thinking can be disabled for Kimi K2" if len(reasoning_modes) > 1 else ""
+
+
def _kimi_model_options(
api_key: str,
base_url: str,
fallback_model: str,
) -> tuple[ProviderModelOption, ...]:
- fallback = fallback_model_option(
+ return listed_model_options(
+ api_key,
+ base_url,
fallback_model,
- reasoning_modes=("auto", *_kimi_supported_reasoning_modes(fallback_model)),
+ reasoning_modes_for=_kimi_supported_reasoning_modes,
+ notes_for=_kimi_notes,
)
- try:
- import httpx
-
- response = httpx.get(
- f"{base_url.rstrip('/')}/models",
- headers={"Authorization": f"Bearer {api_key}"},
- timeout=5.0,
- )
- response.raise_for_status()
- data = response.json().get("data", [])
- except Exception:
- return (fallback,)
- if not isinstance(data, list):
- return (fallback,)
- options: list[ProviderModelOption] = []
- for raw in data:
- if not isinstance(raw, dict) or not isinstance(raw.get("id"), str):
- continue
- model_id = raw["id"]
- reasoning_modes = ("auto", *_kimi_supported_reasoning_modes(model_id))
- options.append(
- ProviderModelOption(
- model=model_id,
- label=model_id,
- reasoning_modes=reasoning_modes,
- recommended=model_id == fallback_model,
- source="api",
- notes=(
- "K3 supports low, high, and max reasoning effort"
- if model_id.startswith("kimi-k3")
- else (
- "thinking can be disabled for Kimi K2" if len(reasoning_modes) > 1 else ""
- )
- ),
- )
- )
-
- if not options:
- return (fallback,)
-
- return tuple(options)
-
-
-class KimiProvider(BaseProvider):
+class KimiProvider(OpenAICompatibleProvider):
"""Kimi provider — access Kimi models via OpenAI-compatible API.
Args:
@@ -169,6 +96,17 @@ class KimiProvider(BaseProvider):
cost_tracker: Optional CostTracker instance for usage recording.
"""
+ provider_id = "kimi"
+ api_key_env = "KIMI_API_KEY"
+ base_url_env = "KIMI_BASE_URL"
+ default_base_url = _DEFAULT_BASE_URL
+ reasoning_detail = (
+ "Kimi /models lists IDs only; reasoning controls are enabled "
+ "from the documented K2 and K3 model-family contracts."
+ )
+ # Status-less SDK errors are not retried for Kimi.
+ wrap_connection_errors = False
+
def __init__(
self,
api_key: str | None = None,
@@ -177,257 +115,38 @@ class KimiProvider(BaseProvider):
rate_limiter: RateLimiter | None = None,
cost_tracker: CostTracker | None = None,
) -> None:
- resolved_key = api_key or os.environ.get("KIMI_API_KEY")
- if not resolved_key:
- raise ProviderError(
- "kimi",
- "No API key provided. Pass api_key= or set KIMI_API_KEY.",
- )
- resolved_base_url = base_url or os.environ.get("KIMI_BASE_URL") or _DEFAULT_BASE_URL
- self._api_key = resolved_key
- self._base_url = resolved_base_url
- self._client = AsyncOpenAI(
- api_key=resolved_key,
- base_url=resolved_base_url,
- )
- self._model = model
- self._rate_limiter = rate_limiter
- self._cost_tracker = cost_tracker
-
- @property
- def provider_name(self) -> str:
- return "kimi"
-
- @property
- def model_name(self) -> str:
- return self._model
-
- def supported_reasoning_modes(self) -> tuple[ReasoningMode, ...]:
- return ("auto", *_kimi_supported_reasoning_modes(self._model))
+ super().__init__(api_key, model, base_url, rate_limiter, cost_tracker)
def available_model_options(self) -> tuple[ProviderModelOption, ...]:
return _kimi_model_options(self._api_key, self._base_url, self._model)
- async def generate(
- self,
- system_prompt: str,
- user_prompt: str,
- max_tokens: int = 4096,
- temperature: float = 0.3,
- request_id: str | None = None,
- reasoning: ReasoningMode = "auto",
- cache_hints: tuple = (),
- ) -> GeneratedResponse:
- reasoning_mode = _resolve_kimi_reasoning_mode(reasoning, model=self._model)
- if self._rate_limiter:
- await self._rate_limiter.acquire(estimated_tokens=max_tokens)
+ def _explicit_reasoning_modes(self, model: str) -> tuple[ReasoningMode, ...]:
+ return _kimi_supported_reasoning_modes(model)
- log.debug(
- "kimi.generate.start",
- model=self._model,
- max_tokens=max_tokens,
- request_id=request_id,
- )
+ def _reasoning_kwargs(self, reasoning: ReasoningMode) -> dict[str, Any]:
+ return _kimi_reasoning_kwargs(self._model, reasoning)
- try:
- return await self._generate_with_retry(
- system_prompt=system_prompt,
- user_prompt=user_prompt,
- max_tokens=max_tokens,
- temperature=temperature,
- request_id=request_id,
- reasoning=reasoning_mode,
- )
- except RetryError as exc:
- raise ProviderError(
- "kimi",
- f"All retries exhausted: {exc}",
- ) from exc
-
- @retry(
- retry=provider_should_retry,
- stop=provider_retry_stop,
- wait=provider_retry_wait,
- reraise=True,
- )
- async def _generate_with_retry(
+ def _completion_kwargs(
self,
- system_prompt: str,
- user_prompt: str,
+ messages: list[dict[str, Any]],
max_tokens: int,
temperature: float,
- request_id: str | None,
reasoning: ReasoningMode,
- ) -> GeneratedResponse:
- try:
- kwargs: dict[str, Any] = {
- "model": self._model,
- "max_tokens": max_tokens,
- "temperature": _kimi_temperature(self._model, reasoning, temperature),
- "top_p": 0.95,
- "messages": [
- {"role": "system", "content": system_prompt},
- {"role": "user", "content": user_prompt},
- ],
- }
- kwargs.update(_kimi_reasoning_kwargs(self._model, reasoning))
- response = await self._client.chat.completions.create(**kwargs)
- except _OpenAIRateLimitError as exc:
- raise RateLimitError(
- "kimi",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _OpenAIAPIStatusError as exc:
- raise ProviderError("kimi", str(exc), status_code=exc.status_code) from exc
-
- usage = response.usage
- choice = response.choices[0]
- stop_reason, provider_stop_reason = normalize_stop_reason(choice.finish_reason)
- result = GeneratedResponse(
- content=choice.message.content or "",
- input_tokens=usage.prompt_tokens if usage else 0,
- output_tokens=usage.completion_tokens if usage else 0,
- cached_tokens=0,
- stop_reason=stop_reason,
- provider_stop_reason=provider_stop_reason,
- usage={
- "prompt_tokens": usage.prompt_tokens if usage else 0,
- "completion_tokens": usage.completion_tokens if usage else 0,
- "total_tokens": usage.total_tokens if usage else 0,
- },
- )
- log.debug(
- "kimi.generate.done",
- input_tokens=result.input_tokens,
- output_tokens=result.output_tokens,
- request_id=request_id,
- )
-
- if self._cost_tracker is not None:
- # Await the cost record inline rather than spawning a detached
- # task. A fire-and-forget create_task can still be flushing its
- # aiosqlite write when the event loop is torn down (e.g. the
- # asyncio.run teardown after doc generation), which surfaces as a
- # noisy "Event loop is closed" worker-thread traceback. record()
- # swallows its own persistence errors, so generation is unaffected.
- with contextlib.suppress(Exception):
- await self._cost_tracker.record(
- model=self._model,
- input_tokens=result.input_tokens,
- output_tokens=result.output_tokens,
- operation=self._cost_tracker.operation,
- file_path=None,
- )
-
- return result
+ ) -> dict[str, Any]:
+ return {
+ **super()._completion_kwargs(messages, max_tokens, temperature, reasoning),
+ "temperature": _kimi_temperature(self._model, reasoning, temperature),
+ "top_p": 0.95,
+ }
- async def stream_chat(
+ def _stream_kwargs(
self,
messages: list[dict[str, Any]],
- tools: list[dict[str, Any]],
- system_prompt: str,
- max_tokens: int = 8192,
- temperature: float = 0.7,
- request_id: str | None = None,
- tool_executor: Any | None = None,
- ) -> AsyncIterator[ChatStreamEvent]:
- import json as _json
-
- full_messages = [{"role": "system", "content": system_prompt}, *messages]
- kwargs: dict[str, Any] = {
- "model": self._model,
- "max_tokens": max_tokens,
+ max_tokens: int,
+ temperature: float,
+ ) -> dict[str, Any]:
+ return {
+ **super()._stream_kwargs(messages, max_tokens, temperature),
"temperature": _kimi_temperature(self._model, "auto", temperature),
"top_p": 0.95,
- "messages": full_messages,
- "stream": True,
}
- if tools:
- kwargs["tools"] = tools
-
- try:
- stream = await self._client.chat.completions.create(**kwargs)
- except _OpenAIRateLimitError as exc:
- raise RateLimitError(
- "kimi",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _OpenAIAPIStatusError as exc:
- raise ProviderError("kimi", str(exc), status_code=exc.status_code) from exc
-
- tool_calls_acc: dict[int, dict[str, Any]] = {}
-
- try:
- async for chunk in stream:
- choice = chunk.choices[0] if chunk.choices else None
- if not choice:
- if chunk.usage:
- yield ChatStreamEvent(
- type="usage",
- input_tokens=chunk.usage.prompt_tokens or 0,
- output_tokens=chunk.usage.completion_tokens or 0,
- )
- continue
-
- delta = choice.delta
- finish = choice.finish_reason
-
- if delta and delta.content:
- yield ChatStreamEvent(type="text_delta", text=delta.content)
-
- if delta and delta.tool_calls:
- for tc_delta in delta.tool_calls:
- idx = tc_delta.index
- if idx not in tool_calls_acc:
- tool_calls_acc[idx] = {
- "id": tc_delta.id or "",
- "name": "",
- "arguments": "",
- }
- acc = tool_calls_acc[idx]
- if tc_delta.id:
- acc["id"] = tc_delta.id
- if tc_delta.function:
- if tc_delta.function.name:
- acc["name"] = tc_delta.function.name
- if tc_delta.function.arguments:
- acc["arguments"] += tc_delta.function.arguments
-
- if finish:
- for idx in sorted(tool_calls_acc.keys()):
- acc = tool_calls_acc[idx]
- try:
- args = _json.loads(acc["arguments"]) if acc["arguments"] else {}
- except Exception:
- args = {}
- yield ChatStreamEvent(
- type="tool_start",
- tool_call=ChatToolCall(
- id=acc["id"],
- name=acc["name"],
- arguments=args,
- ),
- )
- tool_calls_acc.clear()
-
- stop_reason = "tool_use" if finish == "tool_calls" else "end_turn"
- yield ChatStreamEvent(type="stop", stop_reason=stop_reason)
- except _OpenAIRateLimitError as exc:
- raise RateLimitError(
- "kimi",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _OpenAIAPIStatusError as exc:
- raise ProviderError("kimi", str(exc), status_code=exc.status_code) from exc
diff --git a/packages/core/src/repowise/core/providers/llm/litellm.py b/packages/core/src/repowise/core/providers/llm/litellm.py
index 89362922d..76711bf70 100644
--- a/packages/core/src/repowise/core/providers/llm/litellm.py
+++ b/packages/core/src/repowise/core/providers/llm/litellm.py
@@ -19,7 +19,6 @@ Reference: https://docs.litellm.ai/docs/providers
from __future__ import annotations
-import contextlib
import os
from collections.abc import AsyncIterator, Mapping
from typing import TYPE_CHECKING, Any
@@ -30,22 +29,22 @@ from tenacity import RetryError, retry
from repowise.core.providers.llm.base import (
BaseProvider,
ChatStreamEvent,
- ChatToolCall,
GeneratedResponse,
ProviderError,
ProviderModelOption,
- RateLimitError,
ensure_reasoning_supported,
fallback_model_option,
is_temperature_rejection,
normalize_stop_reason,
- parse_retry_after,
provider_retry_stop,
provider_retry_wait,
provider_should_retry,
+ rate_limit_error_from,
+ record_generation_cost,
remember_temperature_rejection,
temperature_kwargs,
)
+from repowise.core.providers.llm.openai_compat import iter_chat_stream_events
from repowise.core.rate_limiter import RateLimiter
from repowise.core.reasoning import ReasoningMode, normalize_reasoning
@@ -401,34 +400,13 @@ class LiteLLMProvider(BaseProvider):
"max_tokens": max_tokens,
**temperature_kwargs(self._model, temperature),
}
- if self._api_key:
- call_kwargs["api_key"] = self._api_key
- if self._api_base:
- call_kwargs["api_base"] = self._api_base
+ self._add_endpoint_kwargs(call_kwargs)
call_kwargs.update(_litellm_reasoning_kwargs(reasoning))
try:
- try:
- response = await litellm.acompletion(**call_kwargs)
- except litellm.APIError as exc:
- # LiteLLM proxies arbitrary vendors, same as OpenRouter: the
- # models that reject `temperature` cannot be enumerated up
- # front, so learn from the rejection and retry once without it.
- if "temperature" not in call_kwargs or not is_temperature_rejection(exc):
- raise
- remember_temperature_rejection(self._model)
- log.debug("litellm.temperature.unsupported", model=self._model)
- call_kwargs.pop("temperature")
- response = await litellm.acompletion(**call_kwargs)
+ response = await self._acompletion(litellm, call_kwargs)
except litellm.RateLimitError as exc:
- raise RateLimitError(
- "litellm",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
+ raise rate_limit_error_from("litellm", exc) from exc
except litellm.APIError as exc:
raise ProviderError("litellm", str(exc)) from exc
except Exception as exc:
@@ -454,24 +432,33 @@ class LiteLLMProvider(BaseProvider):
request_id=request_id,
)
- if self._cost_tracker is not None:
- # Await the cost record inline rather than spawning a detached
- # task. A fire-and-forget create_task can still be flushing its
- # aiosqlite write when the event loop is torn down (e.g. the
- # asyncio.run teardown after doc generation), which surfaces as a
- # noisy "Event loop is closed" worker-thread traceback. record()
- # swallows its own persistence errors, so generation is unaffected.
- with contextlib.suppress(Exception):
- await self._cost_tracker.record(
- model=self._model,
- input_tokens=result.input_tokens,
- output_tokens=result.output_tokens,
- operation=self._cost_tracker.operation,
- file_path=None,
- )
-
+ await record_generation_cost(self._cost_tracker, model=self._model, result=result)
return result
+ def _add_endpoint_kwargs(self, call_kwargs: dict[str, Any]) -> None:
+ """Route the call with the configured key and base URL, when set."""
+ if self._api_key:
+ call_kwargs["api_key"] = self._api_key
+ if self._api_base:
+ call_kwargs["api_base"] = self._api_base
+
+ async def _acompletion(self, litellm: Any, call_kwargs: dict[str, Any]) -> Any:
+ """Complete, retrying once without a rejected ``temperature``.
+
+ LiteLLM proxies arbitrary vendors, same as OpenRouter: the models that
+ reject ``temperature`` cannot be enumerated up front, so learn from the
+ rejection and retry once without it.
+ """
+ try:
+ return await litellm.acompletion(**call_kwargs)
+ except litellm.APIError as exc:
+ if "temperature" not in call_kwargs or not is_temperature_rejection(exc):
+ raise
+ remember_temperature_rejection(self._model)
+ log.debug("litellm.temperature.unsupported", model=self._model)
+ call_kwargs.pop("temperature")
+ return await litellm.acompletion(**call_kwargs)
+
# --- ChatProvider protocol implementation ---
async def stream_chat(
@@ -484,8 +471,6 @@ class LiteLLMProvider(BaseProvider):
request_id: str | None = None,
tool_executor: Any | None = None,
) -> AsyncIterator[ChatStreamEvent]:
- import json as _json
-
import litellm # type: ignore[import-untyped]
litellm.set_verbose = False
@@ -501,80 +486,19 @@ class LiteLLMProvider(BaseProvider):
}
if tools:
call_kwargs["tools"] = tools
- if self._api_key:
- call_kwargs["api_key"] = self._api_key
- if self._api_base:
- call_kwargs["api_base"] = self._api_base
+ self._add_endpoint_kwargs(call_kwargs)
try:
stream = await litellm.acompletion(**call_kwargs)
except litellm.RateLimitError as exc:
- raise RateLimitError(
- "litellm",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
+ raise rate_limit_error_from("litellm", exc) from exc
except litellm.APIError as exc:
raise ProviderError("litellm", str(exc)) from exc
- tool_calls_acc: dict[int, dict[str, Any]] = {}
-
try:
- async for chunk in stream:
- choice = chunk.choices[0] if chunk.choices else None
- if not choice:
- continue
-
- delta = choice.delta
- finish = choice.finish_reason
-
- if delta and getattr(delta, "content", None):
- yield ChatStreamEvent(type="text_delta", text=delta.content)
-
- if delta and getattr(delta, "tool_calls", None):
- for tc_delta in delta.tool_calls:
- idx = tc_delta.index
- if idx not in tool_calls_acc:
- tool_calls_acc[idx] = {
- "id": getattr(tc_delta, "id", "") or "",
- "name": "",
- "arguments": "",
- }
- acc = tool_calls_acc[idx]
- if getattr(tc_delta, "id", None):
- acc["id"] = tc_delta.id
- fn = getattr(tc_delta, "function", None)
- if fn:
- if getattr(fn, "name", None):
- acc["name"] = fn.name
- if getattr(fn, "arguments", None):
- acc["arguments"] += fn.arguments
-
- if finish:
- for idx in sorted(tool_calls_acc.keys()):
- acc = tool_calls_acc[idx]
- try:
- args = _json.loads(acc["arguments"]) if acc["arguments"] else {}
- except Exception:
- args = {}
- yield ChatStreamEvent(
- type="tool_start",
- tool_call=ChatToolCall(id=acc["id"], name=acc["name"], arguments=args),
- )
- tool_calls_acc.clear()
- stop_reason = "tool_use" if finish == "tool_calls" else "end_turn"
- yield ChatStreamEvent(type="stop", stop_reason=stop_reason)
+ async for event in iter_chat_stream_events(stream, emit_usage=False):
+ yield event
except litellm.RateLimitError as exc:
- raise RateLimitError(
- "litellm",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
+ raise rate_limit_error_from("litellm", exc) from exc
except Exception as exc:
raise ProviderError("litellm", f"{type(exc).__name__}: {exc}") from exc
diff --git a/packages/core/src/repowise/core/providers/llm/ollama.py b/packages/core/src/repowise/core/providers/llm/ollama.py
index a0582af5a..a5e2f7047 100644
--- a/packages/core/src/repowise/core/providers/llm/ollama.py
+++ b/packages/core/src/repowise/core/providers/llm/ollama.py
@@ -20,33 +20,31 @@ Usage:
from __future__ import annotations
-import contextlib
import os
from collections.abc import AsyncIterator
from typing import Any
import structlog
-from openai import APIError as _OpenAIAPIError
-from openai import APIStatusError as _OpenAIAPIStatusError
from openai import AsyncOpenAI
-from openai import RateLimitError as _OpenAIRateLimitError
from tenacity import RetryError, retry
from repowise.core.providers.llm.base import (
BaseProvider,
ChatStreamEvent,
- ChatToolCall,
GeneratedResponse,
ProviderError,
ProviderModelOption,
- RateLimitError,
ensure_reasoning_supported,
fallback_model_option,
- normalize_stop_reason,
- parse_retry_after,
provider_retry_stop,
provider_retry_wait,
provider_should_retry,
+ record_generation_cost,
+)
+from repowise.core.providers.llm.openai_compat import (
+ completion_to_response,
+ stream_openai_chat,
+ translate_openai_errors,
)
from repowise.core.rate_limiter import RateLimiter
from repowise.core.reasoning import ReasoningMode
@@ -226,16 +224,9 @@ class OllamaProvider(BaseProvider):
# (not the @retry-wrapped inner one) so a retry can never double-count.
# The tracker is attached externally by the orchestrator, so it may be
# absent.
- tracker = getattr(self, "_cost_tracker", None)
- if tracker is not None:
- with contextlib.suppress(Exception):
- await tracker.record(
- model=f"ollama/{self._model}",
- input_tokens=result.input_tokens,
- output_tokens=result.output_tokens,
- operation=tracker.operation,
- file_path=None,
- )
+ await record_generation_cost(
+ getattr(self, "_cost_tracker", None), model=f"ollama/{self._model}", result=result
+ )
return result
@retry(
@@ -253,49 +244,20 @@ class OllamaProvider(BaseProvider):
request_id: str | None,
reasoning: ReasoningMode,
) -> GeneratedResponse:
- try:
- request_kwargs: dict[str, Any] = {
- "model": self._model,
- "max_tokens": max_tokens,
- "temperature": temperature,
- "messages": [
- {"role": "system", "content": system_prompt},
- {"role": "user", "content": user_prompt},
- ],
- }
- request_kwargs.update(_ollama_reasoning_kwargs(reasoning))
+ request_kwargs: dict[str, Any] = {
+ "model": self._model,
+ "max_tokens": max_tokens,
+ "temperature": temperature,
+ "messages": [
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": user_prompt},
+ ],
+ }
+ request_kwargs.update(_ollama_reasoning_kwargs(reasoning))
+ with translate_openai_errors("ollama"):
response = await self._client.chat.completions.create(**request_kwargs)
- except _OpenAIRateLimitError as exc:
- raise RateLimitError(
- "ollama",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _OpenAIAPIStatusError as exc:
- raise ProviderError("ollama", str(exc), status_code=exc.status_code) from exc
- except _OpenAIAPIError as exc:
- raise ProviderError(
- "ollama", str(exc), status_code=getattr(exc, "status_code", None)
- ) from exc
- usage = response.usage
- choice = response.choices[0]
- stop_reason, provider_stop_reason = normalize_stop_reason(choice.finish_reason)
- result = GeneratedResponse(
- content=choice.message.content or "",
- input_tokens=usage.prompt_tokens if usage else 0,
- output_tokens=usage.completion_tokens if usage else 0,
- cached_tokens=0,
- stop_reason=stop_reason,
- provider_stop_reason=provider_stop_reason,
- usage={
- "prompt_tokens": usage.prompt_tokens if usage else 0,
- "completion_tokens": usage.completion_tokens if usage else 0,
- },
- )
+ result = completion_to_response(response, include_total_tokens=False)
log.debug(
"ollama.generate.done",
input_tokens=result.input_tokens,
@@ -317,8 +279,6 @@ class OllamaProvider(BaseProvider):
tool_executor: Any | None = None,
) -> AsyncIterator[ChatStreamEvent]:
"""Stream chat via Ollama's OpenAI-compatible endpoint."""
- import json as _json
-
full_messages = [{"role": "system", "content": system_prompt}, *messages]
kwargs: dict[str, Any] = {
"model": self._model,
@@ -330,82 +290,5 @@ class OllamaProvider(BaseProvider):
if tools:
kwargs["tools"] = tools
- try:
- stream = await self._client.chat.completions.create(**kwargs)
- except _OpenAIRateLimitError as exc:
- raise RateLimitError(
- "ollama",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _OpenAIAPIStatusError as exc:
- raise ProviderError("ollama", str(exc), status_code=exc.status_code) from exc
- except _OpenAIAPIError as exc:
- raise ProviderError(
- "ollama", str(exc), status_code=getattr(exc, "status_code", None)
- ) from exc
-
- tool_calls_acc: dict[int, dict[str, Any]] = {}
-
- try:
- async for chunk in stream:
- choice = chunk.choices[0] if chunk.choices else None
- if not choice:
- continue
-
- delta = choice.delta
- finish = choice.finish_reason
-
- if delta and delta.content:
- yield ChatStreamEvent(type="text_delta", text=delta.content)
-
- if delta and delta.tool_calls:
- for tc_delta in delta.tool_calls:
- idx = tc_delta.index
- if idx not in tool_calls_acc:
- tool_calls_acc[idx] = {
- "id": tc_delta.id or "",
- "name": "",
- "arguments": "",
- }
- acc = tool_calls_acc[idx]
- if tc_delta.id:
- acc["id"] = tc_delta.id
- if tc_delta.function:
- if tc_delta.function.name:
- acc["name"] = tc_delta.function.name
- if tc_delta.function.arguments:
- acc["arguments"] += tc_delta.function.arguments
-
- if finish:
- for idx in sorted(tool_calls_acc.keys()):
- acc = tool_calls_acc[idx]
- try:
- args = _json.loads(acc["arguments"]) if acc["arguments"] else {}
- except Exception:
- args = {}
- yield ChatStreamEvent(
- type="tool_start",
- tool_call=ChatToolCall(id=acc["id"], name=acc["name"], arguments=args),
- )
- tool_calls_acc.clear()
- stop_reason = "tool_use" if finish == "tool_calls" else "end_turn"
- yield ChatStreamEvent(type="stop", stop_reason=stop_reason)
- except _OpenAIRateLimitError as exc:
- raise RateLimitError(
- "ollama",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _OpenAIAPIStatusError as exc:
- raise ProviderError("ollama", str(exc), status_code=exc.status_code) from exc
- except _OpenAIAPIError as exc:
- raise ProviderError(
- "ollama", str(exc), status_code=getattr(exc, "status_code", None)
- ) from exc
+ async for event in stream_openai_chat(self._client, "ollama", kwargs, emit_usage=False):
+ yield event
diff --git a/packages/core/src/repowise/core/providers/llm/openai.py b/packages/core/src/repowise/core/providers/llm/openai.py
index b59a21bf9..b9334a186 100644
--- a/packages/core/src/repowise/core/providers/llm/openai.py
+++ b/packages/core/src/repowise/core/providers/llm/openai.py
@@ -12,7 +12,6 @@ Recommended models (as of 2026):
from __future__ import annotations
-import contextlib
import ipaddress
import os
from collections.abc import AsyncIterator
@@ -20,28 +19,27 @@ from typing import TYPE_CHECKING, Any
from urllib.parse import urlparse
import structlog
-from openai import APIError as _OpenAIAPIError
-from openai import APIStatusError as _OpenAIAPIStatusError
from openai import AsyncOpenAI
-from openai import RateLimitError as _OpenAIRateLimitError
from tenacity import RetryError, retry
from repowise.core.providers.llm.base import (
BaseProvider,
CacheHint,
ChatStreamEvent,
- ChatToolCall,
GeneratedResponse,
ProviderError,
ProviderModelOption,
- RateLimitError,
ensure_reasoning_supported,
fallback_model_option,
- normalize_stop_reason,
- parse_retry_after,
provider_retry_stop,
provider_retry_wait,
provider_should_retry,
+ record_generation_cost,
+)
+from repowise.core.providers.llm.openai_compat import (
+ completion_to_response,
+ stream_openai_chat,
+ translate_openai_errors,
)
from repowise.core.rate_limiter import RateLimiter
from repowise.core.reasoning import ReasoningMode, normalize_reasoning
@@ -180,6 +178,12 @@ def _openai_chat_tool_kwargs(model: str, *, has_tools: bool) -> dict[str, Any]:
return {}
+def _cached_prompt_tokens(response: Any) -> int:
+ """Prompt tokens OpenAI served from its automatic prefix cache."""
+ details = getattr(response.usage, "prompt_tokens_details", None) if response.usage else None
+ return (getattr(details, "cached_tokens", 0) or 0) if details is not None else 0
+
+
def _is_openai_text_model(model_id: str, *, allow_namespaced: bool = False) -> bool:
"""Keep chat-capable ids, including arbitrary ids from custom gateways."""
leaf = _model_leaf(model_id)
@@ -388,81 +392,27 @@ class OpenAIProvider(BaseProvider):
request_id: str | None,
reasoning: ReasoningMode,
) -> GeneratedResponse:
- try:
- kwargs: dict[str, Any] = {
- "model": self._model,
- "max_completion_tokens": max_tokens,
- "temperature": _openai_temperature(self._model, temperature),
- "messages": [
- {"role": "system", "content": system_prompt},
- {"role": "user", "content": user_prompt},
- ],
- }
- kwargs.update(_openai_reasoning_kwargs(reasoning, model=self._model))
- response = await self._client.chat.completions.create(
- **kwargs,
- )
- except _OpenAIRateLimitError as exc:
- raise RateLimitError(
- "openai",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _OpenAIAPIStatusError as exc:
- raise ProviderError("openai", str(exc), status_code=exc.status_code) from exc
- except _OpenAIAPIError as exc:
- raise ProviderError(
- "openai", str(exc), status_code=getattr(exc, "status_code", None)
- ) from exc
+ kwargs: dict[str, Any] = {
+ "model": self._model,
+ "max_completion_tokens": max_tokens,
+ "temperature": _openai_temperature(self._model, temperature),
+ "messages": [
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": user_prompt},
+ ],
+ }
+ kwargs.update(_openai_reasoning_kwargs(reasoning, model=self._model))
+ with translate_openai_errors("openai"):
+ response = await self._client.chat.completions.create(**kwargs)
- usage = response.usage
- cached = 0
- if usage is not None:
- details = getattr(usage, "prompt_tokens_details", None)
- if details is not None:
- cached = getattr(details, "cached_tokens", 0) or 0
- choice = response.choices[0]
- stop_reason, provider_stop_reason = normalize_stop_reason(choice.finish_reason)
- result = GeneratedResponse(
- content=choice.message.content or "",
- input_tokens=usage.prompt_tokens if usage else 0,
- output_tokens=usage.completion_tokens if usage else 0,
- cached_tokens=cached,
- stop_reason=stop_reason,
- provider_stop_reason=provider_stop_reason,
- usage={
- "prompt_tokens": usage.prompt_tokens if usage else 0,
- "completion_tokens": usage.completion_tokens if usage else 0,
- "total_tokens": usage.total_tokens if usage else 0,
- "cached_tokens": cached,
- },
- )
+ result = completion_to_response(response, cached_tokens=_cached_prompt_tokens(response))
log.debug(
"openai.generate.done",
input_tokens=result.input_tokens,
output_tokens=result.output_tokens,
request_id=request_id,
)
-
- if self._cost_tracker is not None:
- # Await the cost record inline rather than spawning a detached
- # task. A fire-and-forget create_task can still be flushing its
- # aiosqlite write when the event loop is torn down (e.g. the
- # asyncio.run teardown after doc generation), which surfaces as a
- # noisy "Event loop is closed" worker-thread traceback. record()
- # swallows its own persistence errors, so generation is unaffected.
- with contextlib.suppress(Exception):
- await self._cost_tracker.record(
- model=self._model,
- input_tokens=result.input_tokens,
- output_tokens=result.output_tokens,
- operation=self._cost_tracker.operation,
- file_path=None,
- )
-
+ await record_generation_cost(self._cost_tracker, model=self._model, result=result)
return result
# --- ChatProvider protocol implementation ---
@@ -477,8 +427,6 @@ class OpenAIProvider(BaseProvider):
request_id: str | None = None,
tool_executor: Any | None = None,
) -> AsyncIterator[ChatStreamEvent]:
- import json as _json
-
full_messages = [{"role": "system", "content": system_prompt}, *messages]
kwargs: dict[str, Any] = {
"model": self._model,
@@ -491,97 +439,5 @@ class OpenAIProvider(BaseProvider):
kwargs["tools"] = tools
kwargs.update(_openai_chat_tool_kwargs(self._model, has_tools=bool(tools)))
- try:
- stream = await self._client.chat.completions.create(**kwargs)
- except _OpenAIRateLimitError as exc:
- raise RateLimitError(
- "openai",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _OpenAIAPIStatusError as exc:
- raise ProviderError("openai", str(exc), status_code=exc.status_code) from exc
- except _OpenAIAPIError as exc:
- raise ProviderError(
- "openai", str(exc), status_code=getattr(exc, "status_code", None)
- ) from exc
-
- # Track in-progress tool calls (OpenAI streams them incrementally)
- tool_calls_acc: dict[int, dict[str, Any]] = {}
-
- try:
- async for chunk in stream:
- choice = chunk.choices[0] if chunk.choices else None
- if not choice:
- if chunk.usage:
- yield ChatStreamEvent(
- type="usage",
- input_tokens=chunk.usage.prompt_tokens or 0,
- output_tokens=chunk.usage.completion_tokens or 0,
- )
- continue
-
- delta = choice.delta
- finish = choice.finish_reason
-
- # Text content
- if delta and delta.content:
- yield ChatStreamEvent(type="text_delta", text=delta.content)
-
- # Tool call fragments
- if delta and delta.tool_calls:
- for tc_delta in delta.tool_calls:
- idx = tc_delta.index
- if idx not in tool_calls_acc:
- tool_calls_acc[idx] = {
- "id": tc_delta.id or "",
- "name": "",
- "arguments": "",
- }
- acc = tool_calls_acc[idx]
- if tc_delta.id:
- acc["id"] = tc_delta.id
- if tc_delta.function:
- if tc_delta.function.name:
- acc["name"] = tc_delta.function.name
- if tc_delta.function.arguments:
- acc["arguments"] += tc_delta.function.arguments
-
- if finish:
- # Emit accumulated tool calls
- for idx in sorted(tool_calls_acc.keys()):
- acc = tool_calls_acc[idx]
- try:
- args = _json.loads(acc["arguments"]) if acc["arguments"] else {}
- except Exception:
- args = {}
- yield ChatStreamEvent(
- type="tool_start",
- tool_call=ChatToolCall(
- id=acc["id"],
- name=acc["name"],
- arguments=args,
- ),
- )
- tool_calls_acc.clear()
-
- stop_reason = "tool_use" if finish == "tool_calls" else "end_turn"
- yield ChatStreamEvent(type="stop", stop_reason=stop_reason)
- except _OpenAIRateLimitError as exc:
- raise RateLimitError(
- "openai",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _OpenAIAPIStatusError as exc:
- raise ProviderError("openai", str(exc), status_code=exc.status_code) from exc
- except _OpenAIAPIError as exc:
- raise ProviderError(
- "openai", str(exc), status_code=getattr(exc, "status_code", None)
- ) from exc
+ async for event in stream_openai_chat(self._client, "openai", kwargs):
+ yield event
diff --git a/packages/core/src/repowise/core/providers/llm/openai_compat.py b/packages/core/src/repowise/core/providers/llm/openai_compat.py
new file mode 100644
index 000000000..b1cf3b24d
--- /dev/null
+++ b/packages/core/src/repowise/core/providers/llm/openai_compat.py
@@ -0,0 +1,457 @@
+"""Shared plumbing for providers that speak the OpenAI Chat Completions shape.
+
+Request construction stays in each provider; error translation, stream parsing,
+response mapping and the ``/models`` listing live here.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import AsyncIterator, Callable
+from contextlib import AbstractContextManager
+from typing import TYPE_CHECKING, Any, ClassVar
+
+import structlog
+from openai import APIError as _OpenAIAPIError
+from openai import APIStatusError as _OpenAIAPIStatusError
+from openai import AsyncOpenAI
+from openai import RateLimitError as _OpenAIRateLimitError
+from tenacity import RetryError, retry
+
+from repowise.core.providers.llm.base import (
+ BaseProvider,
+ ChatStreamEvent,
+ ChatToolCall,
+ GeneratedResponse,
+ ProviderError,
+ ProviderModelOption,
+ ensure_reasoning_supported,
+ fallback_model_option,
+ normalize_stop_reason,
+ parse_tool_arguments,
+ provider_retry_stop,
+ provider_retry_wait,
+ provider_should_retry,
+ record_generation_cost,
+ translate_sdk_errors,
+)
+from repowise.core.reasoning import ReasoningMode, normalize_reasoning
+
+if TYPE_CHECKING:
+ from repowise.core.generation.cost_tracker import CostTracker
+ from repowise.core.rate_limiter import RateLimiter
+
+log = structlog.get_logger(__name__)
+
+
+def translate_openai_errors(
+ provider: str, *, include_api_error: bool = True
+) -> AbstractContextManager[None]:
+ """Re-raise ``openai`` SDK errors as repowise provider errors.
+
+ *include_api_error* also wraps status-less errors (connection failures,
+ timeouts), which makes them retryable.
+ """
+ return translate_sdk_errors(
+ provider,
+ rate_limit_error=_OpenAIRateLimitError,
+ status_error=_OpenAIAPIStatusError,
+ api_error=_OpenAIAPIError if include_api_error else None,
+ )
+
+
+class ToolCallAccumulator:
+ """Reassembles tool calls that Chat Completions streams in fragments.
+
+ Arguments arrive in pieces per ``index`` and are parsed once the choice finishes.
+ """
+
+ def __init__(self) -> None:
+ self._calls: dict[int, dict[str, Any]] = {}
+
+ def add(self, fragment: Any) -> None:
+ acc = self._calls.setdefault(
+ fragment.index,
+ {"id": getattr(fragment, "id", "") or "", "name": "", "arguments": ""},
+ )
+ if getattr(fragment, "id", None):
+ acc["id"] = fragment.id
+ fn = getattr(fragment, "function", None)
+ if not fn:
+ return
+ if getattr(fn, "name", None):
+ acc["name"] = fn.name
+ if getattr(fn, "arguments", None):
+ acc["arguments"] += fn.arguments
+
+ def drain(self) -> list[ChatToolCall]:
+ """Return the completed calls in index order and reset."""
+ calls = [
+ ChatToolCall(
+ id=acc["id"],
+ name=acc["name"],
+ arguments=parse_tool_arguments(acc["arguments"]),
+ )
+ for _, acc in sorted(self._calls.items())
+ ]
+ self._calls.clear()
+ return calls
+
+
+async def iter_chat_stream_events(
+ stream: AsyncIterator[Any],
+ *,
+ emit_usage: bool = True,
+) -> AsyncIterator[ChatStreamEvent]:
+ """Map streamed Chat Completions chunks to ``ChatStreamEvent``s.
+
+ *emit_usage* reports the trailing choice-less chunk as a ``usage`` event.
+ """
+ tool_calls = ToolCallAccumulator()
+ async for chunk in stream:
+ choice = chunk.choices[0] if chunk.choices else None
+ if not choice:
+ if emit_usage and chunk.usage:
+ yield ChatStreamEvent(
+ type="usage",
+ input_tokens=chunk.usage.prompt_tokens or 0,
+ output_tokens=chunk.usage.completion_tokens or 0,
+ )
+ continue
+
+ delta = choice.delta
+ if delta and getattr(delta, "content", None):
+ yield ChatStreamEvent(type="text_delta", text=delta.content)
+ if delta and getattr(delta, "tool_calls", None):
+ for fragment in delta.tool_calls:
+ tool_calls.add(fragment)
+
+ finish = choice.finish_reason
+ if finish:
+ for call in tool_calls.drain():
+ yield ChatStreamEvent(type="tool_start", tool_call=call)
+ stop_reason = "tool_use" if finish == "tool_calls" else "end_turn"
+ yield ChatStreamEvent(type="stop", stop_reason=stop_reason)
+
+
+async def stream_openai_chat(
+ client: Any,
+ provider: str,
+ kwargs: dict[str, Any],
+ *,
+ emit_usage: bool = True,
+ include_api_error: bool = True,
+) -> AsyncIterator[ChatStreamEvent]:
+ """Open a streamed completion on an ``AsyncOpenAI`` client and yield events.
+
+ Errors are translated on open and while consuming: a 429 can surface at either.
+ """
+ with translate_openai_errors(provider, include_api_error=include_api_error):
+ stream = await client.chat.completions.create(**kwargs)
+ with translate_openai_errors(provider, include_api_error=include_api_error):
+ async for event in iter_chat_stream_events(stream, emit_usage=emit_usage):
+ yield event
+
+
+def completion_to_response(
+ response: Any,
+ *,
+ cached_tokens: int | None = None,
+ include_stop_reason: bool = True,
+ include_total_tokens: bool = True,
+) -> GeneratedResponse:
+ """Build a ``GeneratedResponse`` from a non-streamed chat completion.
+
+ *cached_tokens*, when given, is also added to the usage dict. The
+ ``include_*`` flags drop fields a provider does not report.
+ """
+ usage = response.usage
+ choice = response.choices[0]
+ prompt_tokens = usage.prompt_tokens if usage else 0
+ completion_tokens = usage.completion_tokens if usage else 0
+ usage_dict: dict[str, Any] = {
+ "prompt_tokens": prompt_tokens,
+ "completion_tokens": completion_tokens,
+ }
+ if include_total_tokens:
+ usage_dict["total_tokens"] = usage.total_tokens if usage else 0
+ if cached_tokens is not None:
+ usage_dict["cached_tokens"] = cached_tokens
+ stop_reason, provider_stop_reason = (
+ normalize_stop_reason(choice.finish_reason) if include_stop_reason else (None, None)
+ )
+ return GeneratedResponse(
+ content=choice.message.content or "",
+ input_tokens=prompt_tokens,
+ output_tokens=completion_tokens,
+ cached_tokens=cached_tokens or 0,
+ stop_reason=stop_reason,
+ provider_stop_reason=provider_stop_reason,
+ usage=usage_dict,
+ )
+
+
+def fetch_model_list(api_key: str, base_url: str) -> list[Any] | None:
+ """The ``data`` list from an OpenAI-style ``GET /models``, or None on any failure."""
+ try:
+ import httpx
+
+ response = httpx.get(
+ f"{base_url.rstrip('/')}/models",
+ headers={"Authorization": f"Bearer {api_key}"},
+ timeout=5.0,
+ )
+ response.raise_for_status()
+ data = response.json().get("data", [])
+ except Exception:
+ return None
+ return data if isinstance(data, list) else None
+
+
+def listed_model_options(
+ api_key: str,
+ base_url: str,
+ fallback_model: str,
+ *,
+ reasoning_modes_for: Callable[[str], tuple[ReasoningMode, ...]],
+ notes_for: Callable[[str, tuple[ReasoningMode, ...]], str],
+ sort: bool = False,
+) -> tuple[ProviderModelOption, ...]:
+ """Model options from ``GET /models`` for an endpoint that lists ids only.
+
+ Falls back to the configured model when the listing is unavailable or empty.
+ """
+ fallback = fallback_model_option(
+ fallback_model,
+ reasoning_modes=("auto", *reasoning_modes_for(fallback_model)),
+ )
+ data = fetch_model_list(api_key, base_url)
+ if data is None:
+ return (fallback,)
+
+ options: list[ProviderModelOption] = []
+ for raw in data:
+ if not isinstance(raw, dict) or not isinstance(raw.get("id"), str):
+ continue
+ model_id = raw["id"]
+ reasoning_modes: tuple[ReasoningMode, ...] = ("auto", *reasoning_modes_for(model_id))
+ options.append(
+ ProviderModelOption(
+ model=model_id,
+ label=model_id,
+ reasoning_modes=reasoning_modes,
+ recommended=model_id == fallback_model,
+ source="api",
+ notes=notes_for(model_id, reasoning_modes),
+ )
+ )
+
+ if not options:
+ return (fallback,)
+ if sort:
+ options.sort(key=lambda option: option.model)
+ return tuple(options)
+
+
+class OpenAICompatibleProvider(BaseProvider):
+ """Base for a gateway that is the Chat Completions API at another URL.
+
+ Subclasses set the class attributes and override the ``_*_kwargs`` hooks.
+ """
+
+ provider_id: ClassVar[str]
+ api_key_env: ClassVar[str]
+ base_url_env: ClassVar[str]
+ default_base_url: ClassVar[str]
+ #: Appended to the "reasoning not supported" error.
+ reasoning_detail: ClassVar[str]
+ #: Wrap status-less SDK errors (connection failures, timeouts) so they retry.
+ wrap_connection_errors: ClassVar[bool] = True
+ #: Report the completion's stop reason on ``GeneratedResponse``.
+ reports_stop_reason: ClassVar[bool] = True
+ #: Fixed cost-tracker operation; ``None`` uses the tracker's current one.
+ cost_operation: ClassVar[str | None] = None
+
+ def __init__(
+ self,
+ api_key: str | None,
+ model: str,
+ base_url: str | None,
+ rate_limiter: RateLimiter | None,
+ cost_tracker: CostTracker | None,
+ ) -> None:
+ resolved_key = api_key or os.environ.get(self.api_key_env)
+ if not resolved_key:
+ raise ProviderError(
+ self.provider_id,
+ f"No API key provided. Pass api_key= or set {self.api_key_env}.",
+ )
+ resolved_base_url = (
+ base_url or os.environ.get(self.base_url_env) or self.default_base_url
+ )
+ self._api_key = resolved_key
+ self._base_url = self._clean_base_url(resolved_base_url)
+ self._client = AsyncOpenAI(api_key=resolved_key, base_url=self._base_url)
+ self._model = model
+ self._rate_limiter = rate_limiter
+ self._cost_tracker = cost_tracker
+
+ @property
+ def provider_name(self) -> str:
+ return self.provider_id
+
+ @property
+ def model_name(self) -> str:
+ return self._model
+
+ def supported_reasoning_modes(self) -> tuple[ReasoningMode, ...]:
+ return ("auto", *self._explicit_reasoning_modes(self._model))
+
+ # --- hooks -----------------------------------------------------------
+
+ def _clean_base_url(self, base_url: str) -> str:
+ return base_url
+
+ def _explicit_reasoning_modes(self, model: str) -> tuple[ReasoningMode, ...]:
+ """The reasoning modes *model* accepts besides ``auto``."""
+ raise NotImplementedError
+
+ def _reasoning_kwargs(self, reasoning: ReasoningMode) -> dict[str, Any]:
+ """Request kwargs that carry a validated reasoning mode."""
+ raise NotImplementedError
+
+ def _completion_kwargs(
+ self,
+ messages: list[dict[str, Any]],
+ max_tokens: int,
+ temperature: float,
+ reasoning: ReasoningMode,
+ ) -> dict[str, Any]:
+ return {
+ "model": self._model,
+ "max_tokens": max_tokens,
+ "temperature": temperature,
+ "messages": messages,
+ **self._reasoning_kwargs(reasoning),
+ }
+
+ def _stream_kwargs(
+ self,
+ messages: list[dict[str, Any]],
+ max_tokens: int,
+ temperature: float,
+ ) -> dict[str, Any]:
+ return {
+ "model": self._model,
+ "max_tokens": max_tokens,
+ "temperature": temperature,
+ "messages": messages,
+ "stream": True,
+ }
+
+ # --- generation --------------------------------------------------------
+
+ async def generate(
+ self,
+ system_prompt: str,
+ user_prompt: str,
+ max_tokens: int = 4096,
+ temperature: float = 0.3,
+ request_id: str | None = None,
+ reasoning: ReasoningMode = "auto",
+ cache_hints: tuple = (),
+ ) -> GeneratedResponse:
+ reasoning_mode = ensure_reasoning_supported(
+ self.provider_id,
+ self._model,
+ normalize_reasoning(reasoning),
+ self._explicit_reasoning_modes(self._model),
+ detail=self.reasoning_detail,
+ )
+ if self._rate_limiter:
+ await self._rate_limiter.acquire(estimated_tokens=max_tokens)
+
+ log.debug(
+ f"{self.provider_id}.generate.start",
+ model=self._model,
+ max_tokens=max_tokens,
+ request_id=request_id,
+ )
+
+ try:
+ return await self._generate_with_retry(
+ system_prompt=system_prompt,
+ user_prompt=user_prompt,
+ max_tokens=max_tokens,
+ temperature=temperature,
+ request_id=request_id,
+ reasoning=reasoning_mode,
+ )
+ except RetryError as exc:
+ raise ProviderError(
+ self.provider_id,
+ f"All retries exhausted: {exc}",
+ ) from exc
+
+ @retry(
+ retry=provider_should_retry,
+ stop=provider_retry_stop,
+ wait=provider_retry_wait,
+ reraise=True,
+ )
+ async def _generate_with_retry(
+ self,
+ system_prompt: str,
+ user_prompt: str,
+ max_tokens: int,
+ temperature: float,
+ request_id: str | None,
+ reasoning: ReasoningMode,
+ ) -> GeneratedResponse:
+ messages = [
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": user_prompt},
+ ]
+ kwargs = self._completion_kwargs(messages, max_tokens, temperature, reasoning)
+ with translate_openai_errors(
+ self.provider_id, include_api_error=self.wrap_connection_errors
+ ):
+ response = await self._client.chat.completions.create(**kwargs)
+
+ result = completion_to_response(response, include_stop_reason=self.reports_stop_reason)
+ log.debug(
+ f"{self.provider_id}.generate.done",
+ input_tokens=result.input_tokens,
+ output_tokens=result.output_tokens,
+ request_id=request_id,
+ )
+
+ await record_generation_cost(
+ self._cost_tracker, model=self._model, result=result, operation=self.cost_operation
+ )
+ return result
+
+ # --- ChatProvider protocol implementation ---
+
+ async def stream_chat(
+ self,
+ messages: list[dict[str, Any]],
+ tools: list[dict[str, Any]],
+ system_prompt: str,
+ max_tokens: int = 8192,
+ temperature: float = 0.7,
+ request_id: str | None = None,
+ tool_executor: Any | None = None,
+ ) -> AsyncIterator[ChatStreamEvent]:
+ full_messages = [{"role": "system", "content": system_prompt}, *messages]
+ kwargs = self._stream_kwargs(full_messages, max_tokens, temperature)
+ if tools:
+ kwargs["tools"] = tools
+
+ async for event in stream_openai_chat(
+ self._client,
+ self.provider_id,
+ kwargs,
+ include_api_error=self.wrap_connection_errors,
+ ):
+ yield event
diff --git a/packages/core/src/repowise/core/providers/llm/opencode.py b/packages/core/src/repowise/core/providers/llm/opencode.py
index 397a86d50..45bf2e8a6 100644
--- a/packages/core/src/repowise/core/providers/llm/opencode.py
+++ b/packages/core/src/repowise/core/providers/llm/opencode.py
@@ -20,6 +20,7 @@ import re
import shutil
import subprocess
import uuid
+from collections.abc import Iterator
from functools import lru_cache
from pathlib import Path
from typing import Any
@@ -109,40 +110,52 @@ def _parse_jsonl(stdout: str) -> tuple[str, dict[str, Any]]:
content_parts: list[str] = []
usage: dict[str, Any] = {}
- for raw_line in stdout.splitlines():
- line = raw_line.strip()
- if not line:
- continue
- try:
- event = json.loads(line)
- except json.JSONDecodeError:
- continue
-
+ for event in _iter_jsonl_events(stdout):
event_type = event.get("type")
+ part = event.get("part") or {}
if event_type == "text":
- part = event.get("part") or {}
text = part.get("text")
if isinstance(text, str) and text:
content_parts.append(text)
elif event_type == "step_finish":
- part = event.get("part") or {}
tokens = part.get("tokens")
if isinstance(tokens, dict):
- for key, value in tokens.items():
- if isinstance(value, dict):
- existing = usage.get(key, {})
- if not isinstance(existing, dict):
- existing = {}
- for sub_key, sub_value in value.items():
- if isinstance(sub_value, (int, float)):
- existing[sub_key] = existing.get(sub_key, 0) + int(sub_value)
- usage[key] = existing
- elif isinstance(value, (int, float)):
- usage[key] = usage.get(key, 0) + int(value)
+ _add_token_counts(usage, tokens)
return "\n".join(content_parts), usage
+def _iter_jsonl_events(stdout: str) -> Iterator[Any]:
+ """Decoded JSON lines, skipping blank and undecodable ones."""
+ for raw_line in stdout.splitlines():
+ line = raw_line.strip()
+ if not line:
+ continue
+ try:
+ yield json.loads(line)
+ except json.JSONDecodeError:
+ continue
+
+
+def _add_token_counts(usage: dict[str, Any], tokens: dict[str, Any]) -> None:
+ """Sum one step's token counts into *usage*, one level of nesting deep.
+
+ ``step_finish`` reports flat counts (``input``, ``output``) alongside
+ nested ones (``cache: {read, write}``); both accumulate across steps.
+ """
+ for key, value in tokens.items():
+ if isinstance(value, dict):
+ existing = usage.get(key, {})
+ if not isinstance(existing, dict):
+ existing = {}
+ for sub_key, sub_value in value.items():
+ if isinstance(sub_value, (int, float)):
+ existing[sub_key] = existing.get(sub_key, 0) + int(sub_value)
+ usage[key] = existing
+ elif isinstance(value, (int, float)):
+ usage[key] = usage.get(key, 0) + int(value)
+
+
def _tail(text: str, max_chars: int = _MAX_STDERR_CHARS) -> str:
text = text.strip()
if len(text) <= max_chars:
@@ -158,6 +171,19 @@ def _error_message(stderr: str, stdout: str, returncode: int) -> str:
continue
return candidate
+ msg = _error_event_message(stdout)
+ if msg:
+ return msg
+
+ return f"opencode run exited with {returncode}"
+
+
+def _error_event_message(stdout: str) -> str | None:
+ """Message of the first ``error`` event in *stdout*.
+
+ Stops at the first line that is not JSON: past that point the output is
+ not an event stream, so nothing later is trusted.
+ """
try:
for raw_line in stdout.splitlines():
line = raw_line.strip()
@@ -171,8 +197,7 @@ def _error_message(stderr: str, stdout: str, returncode: int) -> str:
return str(msg)
except Exception:
pass
-
- return f"opencode run exited with {returncode}"
+ return None
_MODEL_LINE_RE = re.compile(r"^\s*([a-zA-Z0-9][a-zA-Z0-9._\-]*)/([a-zA-Z0-9][a-zA-Z0-9._/\-]*)\s*$")
@@ -363,6 +388,26 @@ class OpenCodeProvider(BaseProvider):
request_id=request_id,
)
+ returncode, stdout, stderr = await self._run(cmd, prompt)
+ if returncode != 0:
+ raise ProviderError(
+ "opencode",
+ _error_message(stderr, stdout, returncode),
+ status_code=returncode,
+ )
+
+ response = self._response_from_output(stdout, stderr)
+ log.debug(
+ "opencode.generate.done",
+ input_tokens=response.input_tokens,
+ output_tokens=response.output_tokens,
+ cached_tokens=response.cached_tokens,
+ request_id=request_id,
+ )
+ return response
+
+ async def _run(self, cmd: list[str], prompt: str) -> tuple[int, str, str]:
+ """Run ``opencode run`` with *prompt* on stdin; returns (code, stdout, stderr)."""
async with self._get_semaphore():
try:
proc = await asyncio.create_subprocess_exec(
@@ -401,14 +446,10 @@ class OpenCodeProvider(BaseProvider):
stdout = stdout_bytes.decode("utf-8", errors="replace") if stdout_bytes else ""
stderr = stderr_bytes.decode("utf-8", errors="replace") if stderr_bytes else ""
+ return proc.returncode, stdout, stderr
- if proc.returncode != 0:
- raise ProviderError(
- "opencode",
- _error_message(stderr, stdout, proc.returncode),
- status_code=proc.returncode,
- )
-
+ def _response_from_output(self, stdout: str, stderr: str) -> GeneratedResponse:
+ """Build the response from a successful run's JSONL event stream."""
content, usage = _parse_jsonl(stdout)
if not content:
raise ProviderError(
@@ -416,36 +457,23 @@ class OpenCodeProvider(BaseProvider):
"opencode run completed but no text was found in JSONL output.",
)
- usage_missing = not usage
- input_tokens = int(usage.get("input", 0) or 0)
- output_tokens = int(usage.get("output", 0) or 0)
- cached_tokens = int((usage.get("cache") or {}).get("read", 0) or 0)
-
- log.debug(
- "opencode.generate.done",
- input_tokens=input_tokens,
- output_tokens=output_tokens,
- cached_tokens=cached_tokens,
- request_id=request_id,
- )
usage_payload = {
**usage,
"source": "opencode_run",
"model": self.model_name,
"stderr": _tail(stderr) if stderr.strip() else "",
}
- if usage_missing:
+ if not usage:
usage_payload["estimated"] = True
return GeneratedResponse(
content=content,
- input_tokens=input_tokens,
- output_tokens=output_tokens,
- cached_tokens=cached_tokens,
+ input_tokens=int(usage.get("input", 0) or 0),
+ output_tokens=int(usage.get("output", 0) or 0),
+ cached_tokens=int((usage.get("cache") or {}).get("read", 0) or 0),
usage=usage_payload,
)
-
async def _close_subprocess_transport(proc: asyncio.subprocess.Process) -> None:
transport = getattr(proc, "_transport", None)
close = getattr(transport, "close", None)
diff --git a/packages/core/src/repowise/core/providers/llm/openrouter.py b/packages/core/src/repowise/core/providers/llm/openrouter.py
index c97fe8ae1..744a18843 100644
--- a/packages/core/src/repowise/core/providers/llm/openrouter.py
+++ b/packages/core/src/repowise/core/providers/llm/openrouter.py
@@ -13,37 +13,37 @@ Popular models:
from __future__ import annotations
-import contextlib
import os
from collections.abc import AsyncIterator
from typing import TYPE_CHECKING, Any
import structlog
-from openai import APIError as _OpenAIAPIError
from openai import APIStatusError as _OpenAIAPIStatusError
from openai import AsyncOpenAI
-from openai import RateLimitError as _OpenAIRateLimitError
from tenacity import RetryError, retry
from repowise.core.providers.llm.base import (
BaseProvider,
ChatStreamEvent,
- ChatToolCall,
GeneratedResponse,
ProviderError,
ProviderModelOption,
- RateLimitError,
ensure_reasoning_supported,
fallback_model_option,
is_temperature_rejection,
- normalize_stop_reason,
- parse_retry_after,
provider_retry_stop,
provider_retry_wait,
provider_should_retry,
+ record_generation_cost,
remember_temperature_rejection,
temperature_kwargs,
)
+from repowise.core.providers.llm.openai_compat import (
+ completion_to_response,
+ fetch_model_list,
+ stream_openai_chat,
+ translate_openai_errors,
+)
from repowise.core.rate_limiter import RateLimiter
from repowise.core.reasoning import ReasoningMode, normalize_reasoning
@@ -129,20 +129,8 @@ def _openrouter_model_options(
*_openrouter_supported_reasoning_modes(fallback_model, base_url=base_url),
),
)
- try:
- import httpx
-
- response = httpx.get(
- f"{base_url.rstrip('/')}/models",
- headers={"Authorization": f"Bearer {api_key}"},
- timeout=5.0,
- )
- response.raise_for_status()
- data = response.json().get("data", [])
- except Exception:
- return (fallback,)
-
- if not isinstance(data, list):
+ data = fetch_model_list(api_key, base_url)
+ if data is None:
return (fallback,)
base_key = base_url.rstrip("/")
@@ -303,16 +291,9 @@ class OpenRouterProvider(BaseProvider):
# page shows $0. The tracker is attached externally by the orchestrator,
# so it may be absent (guard with getattr); record() swallows its own
# persistence errors, so generation is unaffected.
- tracker = getattr(self, "_cost_tracker", None)
- if tracker is not None:
- with contextlib.suppress(Exception):
- await tracker.record(
- model=self._model,
- input_tokens=result.input_tokens,
- output_tokens=result.output_tokens,
- operation=tracker.operation,
- file_path=None,
- )
+ await record_generation_cost(
+ getattr(self, "_cost_tracker", None), model=self._model, result=result
+ )
return result
@retry(
@@ -340,52 +321,10 @@ class OpenRouterProvider(BaseProvider):
**temperature_kwargs(self._model, temperature),
}
kwargs.update(_openrouter_reasoning_kwargs(reasoning))
- try:
- try:
- response = await self._client.chat.completions.create(**kwargs)
- except _OpenAIAPIStatusError as exc:
- # OpenRouter fronts every vendor, so the set of models that
- # reject `temperature` is not knowable ahead of time. Drop the
- # parameter and retry once; the model is remembered so the rest
- # of the run skips it.
- if "temperature" not in kwargs or not is_temperature_rejection(exc):
- raise
- remember_temperature_rejection(self._model)
- log.debug("openrouter.temperature.unsupported", model=self._model)
- kwargs.pop("temperature")
- response = await self._client.chat.completions.create(**kwargs)
- except _OpenAIRateLimitError as exc:
- raise RateLimitError(
- "openrouter",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _OpenAIAPIStatusError as exc:
- raise ProviderError("openrouter", str(exc), status_code=exc.status_code) from exc
- except _OpenAIAPIError as exc:
- raise ProviderError(
- "openrouter", str(exc), status_code=getattr(exc, "status_code", None)
- ) from exc
+ with translate_openai_errors("openrouter"):
+ response = await self._create_completion(kwargs)
- usage = response.usage
- choice = response.choices[0]
- stop_reason, provider_stop_reason = normalize_stop_reason(choice.finish_reason)
- result = GeneratedResponse(
- content=choice.message.content or "",
- input_tokens=usage.prompt_tokens if usage else 0,
- output_tokens=usage.completion_tokens if usage else 0,
- cached_tokens=0,
- stop_reason=stop_reason,
- provider_stop_reason=provider_stop_reason,
- usage={
- "prompt_tokens": usage.prompt_tokens if usage else 0,
- "completion_tokens": usage.completion_tokens if usage else 0,
- "total_tokens": usage.total_tokens if usage else 0,
- },
- )
+ result = completion_to_response(response)
log.debug(
"openrouter.generate.done",
input_tokens=result.input_tokens,
@@ -395,6 +334,23 @@ class OpenRouterProvider(BaseProvider):
return result
+ async def _create_completion(self, kwargs: dict[str, Any]) -> Any:
+ """Create a completion, retrying once without a rejected ``temperature``.
+
+ OpenRouter fronts every vendor, so the set of models that reject
+ ``temperature`` is not knowable ahead of time. Drop the parameter and
+ retry once; the model is remembered so the rest of the run skips it.
+ """
+ try:
+ return await self._client.chat.completions.create(**kwargs)
+ except _OpenAIAPIStatusError as exc:
+ if "temperature" not in kwargs or not is_temperature_rejection(exc):
+ raise
+ remember_temperature_rejection(self._model)
+ log.debug("openrouter.temperature.unsupported", model=self._model)
+ kwargs.pop("temperature")
+ return await self._client.chat.completions.create(**kwargs)
+
# --- ChatProvider protocol implementation ---
async def stream_chat(
@@ -407,8 +363,6 @@ class OpenRouterProvider(BaseProvider):
request_id: str | None = None,
tool_executor: Any | None = None,
) -> AsyncIterator[ChatStreamEvent]:
- import json as _json
-
full_messages = [{"role": "system", "content": system_prompt}, *messages]
kwargs: dict[str, Any] = {
"model": self._model,
@@ -420,97 +374,5 @@ class OpenRouterProvider(BaseProvider):
if tools:
kwargs["tools"] = tools
- try:
- stream = await self._client.chat.completions.create(**kwargs)
- except _OpenAIRateLimitError as exc:
- raise RateLimitError(
- "openrouter",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _OpenAIAPIStatusError as exc:
- raise ProviderError("openrouter", str(exc), status_code=exc.status_code) from exc
- except _OpenAIAPIError as exc:
- raise ProviderError(
- "openrouter", str(exc), status_code=getattr(exc, "status_code", None)
- ) from exc
-
- # Track in-progress tool calls (OpenAI-compatible streaming)
- tool_calls_acc: dict[int, dict[str, Any]] = {}
-
- try:
- async for chunk in stream:
- choice = chunk.choices[0] if chunk.choices else None
- if not choice:
- if chunk.usage:
- yield ChatStreamEvent(
- type="usage",
- input_tokens=chunk.usage.prompt_tokens or 0,
- output_tokens=chunk.usage.completion_tokens or 0,
- )
- continue
-
- delta = choice.delta
- finish = choice.finish_reason
-
- # Text content
- if delta and delta.content:
- yield ChatStreamEvent(type="text_delta", text=delta.content)
-
- # Tool call fragments
- if delta and delta.tool_calls:
- for tc_delta in delta.tool_calls:
- idx = tc_delta.index
- if idx not in tool_calls_acc:
- tool_calls_acc[idx] = {
- "id": tc_delta.id or "",
- "name": "",
- "arguments": "",
- }
- acc = tool_calls_acc[idx]
- if tc_delta.id:
- acc["id"] = tc_delta.id
- if tc_delta.function:
- if tc_delta.function.name:
- acc["name"] = tc_delta.function.name
- if tc_delta.function.arguments:
- acc["arguments"] += tc_delta.function.arguments
-
- if finish:
- # Emit accumulated tool calls
- for idx in sorted(tool_calls_acc.keys()):
- acc = tool_calls_acc[idx]
- try:
- args = _json.loads(acc["arguments"]) if acc["arguments"] else {}
- except Exception:
- args = {}
- yield ChatStreamEvent(
- type="tool_start",
- tool_call=ChatToolCall(
- id=acc["id"],
- name=acc["name"],
- arguments=args,
- ),
- )
- tool_calls_acc.clear()
-
- stop_reason = "tool_use" if finish == "tool_calls" else "end_turn"
- yield ChatStreamEvent(type="stop", stop_reason=stop_reason)
- except _OpenAIRateLimitError as exc:
- raise RateLimitError(
- "openrouter",
- str(exc),
- status_code=429,
- retry_after=parse_retry_after(
- getattr(getattr(exc, "response", None), "headers", None)
- ),
- ) from exc
- except _OpenAIAPIStatusError as exc:
- raise ProviderError("openrouter", str(exc), status_code=exc.status_code) from exc
- except _OpenAIAPIError as exc:
- raise ProviderError(
- "openrouter", str(exc), status_code=getattr(exc, "status_code", None)
- ) from exc
+ async for event in stream_openai_chat(self._client, "openrouter", kwargs):
+ yield event