Skip to content

Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)

Python

feat(lmstudio): route natively to Anthropic /v1/messages endpoint

Alishahryar187d8ce1196c6
diff --git a/.env.example b/.env.example
index 8965e367..cd6aa495 100644
--- a/.env.example
+++ b/.env.example
@@ -13,6 +13,9 @@ LM_STUDIO_BASE_URL="http://localhost:1234/v1"
# All Claude model requests are mapped to these models, plain model is fallback
# Format: provider_type/model/name
# Valid providers: "nvidia_nim" | "open_router" | "lmstudio"
+# Note for LM Studio: You MUST specify the exact model identifier (e.g. "lmstudio/qwen3.5-27b").
+# If specify a model that isn't currently loaded, LM Studio will route the request to whatever
+# model that happens to be loaded.
MODEL_OPUS="nvidia_nim/z-ai/glm4.7"
MODEL_SONNET="open_router/arcee-ai/trinity-large-preview:free"
MODEL_HAIKU="open_router/stepfun/step-3.5-flash:free"
diff --git a/providers/lmstudio/__init__.py b/providers/lmstudio/__init__.py
index 55287e92..dbcd0f9e 100644
--- a/providers/lmstudio/__init__.py
+++ b/providers/lmstudio/__init__.py
@@ -1,4 +1,4 @@
-"""LM Studio provider - OpenAI-compatible local API."""
+"""LM Studio provider - Anthropic-compatible local API."""
from .client import LMStudioProvider
diff --git a/providers/lmstudio/client.py b/providers/lmstudio/client.py
index e7e74f32..ce5ababc 100644
--- a/providers/lmstudio/client.py
+++ b/providers/lmstudio/client.py
@@ -1,26 +1,139 @@
"""LM Studio provider implementation."""
+import json
+from collections.abc import AsyncIterator
from typing import Any
-from providers.base import ProviderConfig
-from providers.openai_compat import OpenAICompatibleProvider
+import httpx
+from loguru import logger
-from .request import build_request_body
+from providers.base import BaseProvider, ProviderConfig
+from providers.common import get_user_facing_error_message, map_error
+from providers.rate_limit import GlobalRateLimiter
LMSTUDIO_DEFAULT_BASE_URL = "http://localhost:1234/v1"
-class LMStudioProvider(OpenAICompatibleProvider):
- """LM Studio provider using OpenAI-compatible local API."""
+class LMStudioProvider(BaseProvider):
+ """LM Studio provider using native Anthropic Messages API endpoint."""
def __init__(self, config: ProviderConfig):
- super().__init__(
- config,
- provider_name="LMSTUDIO",
- base_url=config.base_url or LMSTUDIO_DEFAULT_BASE_URL,
- api_key=config.api_key or "lm-studio",
+ super().__init__(config)
+ self._provider_name = "LMSTUDIO"
+ self._base_url = (config.base_url or LMSTUDIO_DEFAULT_BASE_URL).rstrip("/")
+
+ # We need the base URL without /v1 if the user provided it with /v1,
+ # so we can append /v1/messages safely.
+ # Actually, if they provided http://localhost:1234/v1, we can just use
+ # {base_url}/messages which becomes http://localhost:1234/v1/messages
+
+ self._global_rate_limiter = GlobalRateLimiter.get_instance(
+ rate_limit=config.rate_limit,
+ rate_window=config.rate_window,
+ max_concurrency=config.max_concurrency,
+ )
+ self._client = httpx.AsyncClient(
+ base_url=self._base_url,
+ timeout=httpx.Timeout(
+ config.http_read_timeout,
+ connect=config.http_connect_timeout,
+ read=config.http_read_timeout,
+ write=config.http_write_timeout,
+ ),
+ )
+
+ async def cleanup(self) -> None:
+ """Release HTTP client resources."""
+ await self._client.aclose()
+
+ async def stream_response(
+ self,
+ request: Any,
+ input_tokens: int = 0,
+ *,
+ request_id: str | None = None,
+ ) -> AsyncIterator[str]:
+ """Stream response natively via LM Studio's Anthropic-compatible endpoint."""
+ tag = self._provider_name
+ req_tag = f" request_id={request_id}" if request_id else ""
+
+ # Dump the Anthropic Pydantic model directly into a dict
+ body = request.model_dump(exclude_none=True)
+
+ # Remove extra_body, original_model, resolved_provider_model which are internal
+ body.pop("extra_body", None)
+ body.pop("original_model", None)
+ body.pop("resolved_provider_model", None)
+
+ # Translate internal ThinkingConfig to Anthropic API schema
+ if "thinking" in body:
+ thinking_cfg = body.pop("thinking")
+ if isinstance(thinking_cfg, dict) and thinking_cfg.get("enabled"):
+ # Anthropic API requires a budget_tokens value when enabled
+ body["thinking"] = {"type": "enabled"}
+
+ # Ensure max_tokens is present (Claude API requires it)
+ if "max_tokens" not in body:
+ body["max_tokens"] = 81920
+
+ logger.info(
+ "{}_STREAM:{} natively passing Anthropic request to LMStudio model={} msgs={} tools={}",
+ tag,
+ req_tag,
+ body.get("model"),
+ len(body.get("messages", [])),
+ len(body.get("tools", [])),
)
- def _build_request_body(self, request: Any) -> dict:
- """Internal helper for tests and shared building."""
- return build_request_body(request)
+ async with self._global_rate_limiter.concurrency_slot():
+ try:
+ # We use execute_with_retry around the streaming request context
+ # To do this safely with httpx streaming, we await the chunk stream
+
+ async def _make_request():
+ request_obj = self._client.build_request(
+ "POST",
+ "/messages",
+ json=body,
+ headers={"Content-Type": "application/json"},
+ )
+ return await self._client.send(request_obj, stream=True)
+
+ response = await self._global_rate_limiter.execute_with_retry(
+ _make_request
+ )
+
+ if response.status_code != 200:
+ text = await response.aread()
+ error_msg = f"HTTP {response.status_code}: {text.decode('utf-8', errors='replace')}"
+ logger.error("{}_ERROR:{} {}", tag, req_tag, error_msg)
+ raise RuntimeError(f"LMStudio natively returned error: {error_msg}")
+
+ async for line in response.aiter_lines():
+ if line:
+ yield f"{line}\n"
+ else:
+ yield "\n"
+
+ except Exception as e:
+ logger.error("{}_ERROR:{} {}: {}", tag, req_tag, type(e).__name__, e)
+ mapped_e = map_error(e)
+ error_message = get_user_facing_error_message(
+ mapped_e, read_timeout_s=self._config.http_read_timeout
+ )
+ if request_id:
+ error_message += f"\nRequest ID: {request_id}"
+
+ logger.info(
+ "{}_STREAM: Emitting native SSE error event for {}{}",
+ tag,
+ type(e).__name__,
+ req_tag,
+ )
+
+ # Emit an Anthropic-compatible error event
+ error_event = {
+ "type": "error",
+ "error": {"type": "api_error", "message": error_message},
+ }
+ yield f"event: error\ndata: {json.dumps(error_event)}\n\n"
diff --git a/providers/lmstudio/request.py b/providers/lmstudio/request.py
deleted file mode 100644
index 16934ebd..00000000
--- a/providers/lmstudio/request.py
+++ /dev/null
@@ -1,29 +0,0 @@
-"""Request builder for LM Studio provider."""
-
-from typing import Any
-
-from loguru import logger
-
-from providers.common.message_converter import build_base_request_body
-
-LMSTUDIO_DEFAULT_MAX_TOKENS = 81920
-
-
-def build_request_body(request_data: Any) -> dict:
- """Build OpenAI-format request body from Anthropic request for LM Studio."""
- logger.debug(
- "LMSTUDIO_REQUEST: conversion start model={} msgs={}",
- getattr(request_data, "model", "?"),
- len(getattr(request_data, "messages", [])),
- )
- body = build_base_request_body(
- request_data, default_max_tokens=LMSTUDIO_DEFAULT_MAX_TOKENS
- )
-
- logger.debug(
- "LMSTUDIO_REQUEST: conversion done model={} msgs={} tools={}",
- body.get("model"),
- len(body.get("messages", [])),
- len(body.get("tools", [])),
- )
- return body
diff --git a/tests/api/test_dependencies.py b/tests/api/test_dependencies.py
index 7d130b16..bd30b95b 100644
--- a/tests/api/test_dependencies.py
+++ b/tests/api/test_dependencies.py
@@ -118,7 +118,6 @@ async def test_get_provider_lmstudio():
assert isinstance(provider, LMStudioProvider)
assert provider._base_url == "http://localhost:1234/v1"
- assert provider._api_key == "lm-studio"
@pytest.mark.asyncio
diff --git a/tests/providers/test_lmstudio.py b/tests/providers/test_lmstudio.py
index 1a49e490..a0910b6d 100644
--- a/tests/providers/test_lmstudio.py
+++ b/tests/providers/test_lmstudio.py
@@ -1,72 +1,12 @@
-"""Tests for LM Studio provider."""
+"""Tests for LM Studio native Anthropic provider."""
-import json
from unittest.mock import AsyncMock, MagicMock, patch
+import httpx
import pytest
from providers.base import ProviderConfig
from providers.lmstudio import LMStudioProvider
-from providers.lmstudio.request import LMSTUDIO_DEFAULT_MAX_TOKENS
-
-
-class AsyncStreamMock:
- """Async iterable mock that yields chunks then optionally raises."""
-
- def __init__(self, chunks, error=None):
- self._chunks = chunks
- self._error = error
-
- def __aiter__(self):
- return self._aiter()
-
- async def _aiter(self):
- for chunk in self._chunks:
- yield chunk
- if self._error:
- raise self._error
-
-
-def _make_chunk(
- content=None, finish_reason=None, tool_calls=None, reasoning_content=None
-):
- """Create a mock streaming chunk."""
- delta = MagicMock()
- delta.content = content
- delta.tool_calls = tool_calls
- delta.reasoning_content = reasoning_content if reasoning_content else None
-
- choice = MagicMock()
- choice.delta = delta
- choice.finish_reason = finish_reason
-
- chunk = MagicMock()
- chunk.choices = [choice]
- chunk.usage = None
- return chunk
-
-
-def _make_request(model="test-model", **kwargs):
- """Create a mock request with all fields build_request_body needs."""
- req = MagicMock()
- req.model = model
- req.messages = [MagicMock(role="user", content="Hello")]
- req.system = None
- req.max_tokens = 100
- req.temperature = None
- req.top_p = None
- req.stop_sequences = None
- req.tools = None
- req.tool_choice = None
- req.thinking = MagicMock(enabled=True)
- for k, v in kwargs.items():
- setattr(req, k, v)
- return req
-
-
-async def _collect_stream(provider, request):
- """Collect all SSE events from a stream."""
- return [e async for e in provider.stream_response(request)]
class MockMessage:
@@ -91,6 +31,16 @@ class MockRequest:
for k, v in kwargs.items():
setattr(self, k, v)
+ def model_dump(self, exclude_none=True):
+ return {
+ "model": self.model,
+ "messages": [{"role": m.role, "content": m.content} for m in self.messages],
+ "max_tokens": self.max_tokens,
+ "temperature": self.temperature,
+ "extra_body": self.extra_body,
+ "thinking": {"enabled": self.thinking.enabled} if self.thinking else None,
+ }
+
@pytest.fixture
def lmstudio_config():
@@ -105,7 +55,7 @@ def lmstudio_config():
@pytest.fixture(autouse=True)
def mock_rate_limiter():
"""Mock the global rate limiter to prevent waiting."""
- with patch("providers.openai_compat.GlobalRateLimiter") as mock:
+ with patch("providers.lmstudio.client.GlobalRateLimiter") as mock:
instance = mock.get_instance.return_value
instance.wait_if_blocked = AsyncMock(return_value=False)
@@ -123,24 +73,10 @@ def lmstudio_provider(lmstudio_config):
def test_init(lmstudio_config):
"""Test provider initialization."""
- with patch("providers.openai_compat.AsyncOpenAI") as mock_openai:
+ with patch("httpx.AsyncClient"):
provider = LMStudioProvider(lmstudio_config)
- assert provider._api_key == "lm-studio"
assert provider._base_url == "http://localhost:1234/v1"
- mock_openai.assert_called_once()
-
-
-def test_init_with_empty_api_key():
- """Provider uses lm-studio placeholder when api_key is empty."""
- config = ProviderConfig(
- api_key="",
- base_url="http://localhost:1234/v1",
- rate_limit=10,
- rate_window=60,
- )
- with patch("providers.openai_compat.AsyncOpenAI"):
- provider = LMStudioProvider(config)
- assert provider._api_key == "lm-studio"
+ assert provider._provider_name == "LMSTUDIO"
def test_init_uses_configurable_timeouts():
@@ -152,499 +88,165 @@ def test_init_uses_configurable_timeouts():
http_write_timeout=15.0,
http_connect_timeout=5.0,
)
- with patch("providers.openai_compat.AsyncOpenAI") as mock_openai:
+ with patch("httpx.AsyncClient") as mock_client:
LMStudioProvider(config)
- call_kwargs = mock_openai.call_args[1]
+ call_kwargs = mock_client.call_args[1]
timeout = call_kwargs["timeout"]
assert timeout.read == 600.0
assert timeout.write == 15.0
assert timeout.connect == 5.0
-def test_build_request_body_no_extra_body(lmstudio_provider):
- """LM Studio request body does NOT include extra_body/reasoning."""
- req = MockRequest()
- body = lmstudio_provider._build_request_body(req)
-
- assert body["model"] == "lmstudio-community/qwen2.5-7b-instruct"
- assert body["temperature"] == 0.5
- assert len(body["messages"]) == 2 # System + User
- assert body["messages"][0]["role"] == "system"
- assert body["messages"][0]["content"] == "System prompt"
-
- assert "extra_body" not in body
-
-
-def test_build_request_body_base_url_and_model(lmstudio_provider):
- """Base URL and model are correct in provider config."""
- assert lmstudio_provider._base_url == "http://localhost:1234/v1"
- req = MockRequest(model="lmstudio-community/qwen2.5-7b-instruct")
- body = lmstudio_provider._build_request_body(req)
- assert body["model"] == "lmstudio-community/qwen2.5-7b-instruct"
+def test_init_base_url_strips_trailing_slash():
+ """Config with base_url trailing slash is stored without it."""
+ config = ProviderConfig(
+ api_key="lm-studio",
+ base_url="http://localhost:1234/v1/",
+ rate_limit=10,
+ rate_window=60,
+ )
+ with patch("httpx.AsyncClient"):
+ provider = LMStudioProvider(config)
+ assert provider._base_url == "http://localhost:1234/v1"
@pytest.mark.asyncio
-async def test_stream_response_text(lmstudio_provider):
- """Test streaming text response."""
+async def test_stream_response(lmstudio_provider):
+ """Test streaming native Anthropic response."""
req = MockRequest()
- mock_chunk1 = MagicMock()
- mock_chunk1.choices = [
- MagicMock(
- delta=MagicMock(content="Hello", reasoning_content=None),
- finish_reason=None,
- )
- ]
- mock_chunk1.usage = None
-
- mock_chunk2 = MagicMock()
- mock_chunk2.choices = [
- MagicMock(
- delta=MagicMock(content=" World", reasoning_content=None),
- finish_reason="stop",
- )
- ]
- mock_chunk2.usage = MagicMock(completion_tokens=10)
-
- async def mock_stream():
- yield mock_chunk1
- yield mock_chunk2
-
- with patch.object(
- lmstudio_provider._client.chat.completions, "create", new_callable=AsyncMock
- ) as mock_create:
- mock_create.return_value = mock_stream()
-
+ mock_response = MagicMock()
+ mock_response.status_code = 200
+
+ async def mock_aiter_lines():
+ yield "event: message_start"
+ yield 'data: {"type":"message_start","message":{}}'
+ yield ""
+ yield "event: content_block_delta"
+ yield 'data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"Hello World"}}'
+ yield ""
+ yield "event: message_stop"
+ yield 'data: {"type":"message_stop"}'
+ yield ""
+
+ mock_response.aiter_lines = mock_aiter_lines
+
+ with (
+ patch.object(
+ lmstudio_provider._client, "build_request", return_value=MagicMock()
+ ) as mock_build,
+ patch.object(
+ lmstudio_provider._client,
+ "send",
+ new_callable=AsyncMock,
+ return_value=mock_response,
+ ),
+ ):
events = [e async for e in lmstudio_provider.stream_response(req)]
- assert len(events) > 0
- assert "event: message_start" in events[0]
+ # Verify request construction
+ mock_build.assert_called_once()
+ args, kwargs = mock_build.call_args
+ assert args[0] == "POST"
+ assert args[1] == "/messages"
+ assert kwargs["json"]["model"] == "lmstudio-community/qwen2.5-7b-instruct"
+ # Verify internal fields are popped
+ assert "extra_body" not in kwargs["json"]
+ assert kwargs["json"]["max_tokens"] == 100
- text_content = ""
- for e in events:
- if "event: content_block_delta" in e and '"text_delta"' in e:
- for line in e.splitlines():
- if line.startswith("data: "):
- data = json.loads(line[6:])
- if "delta" in data and "text" in data["delta"]:
- text_content += data["delta"]["text"]
+ # Verify internal ThinkingConfig is mapped to Anthropic API format
+ assert kwargs["json"]["thinking"] == {"type": "enabled"}
- assert "Hello World" in text_content
+ # Verify events yielded correctly
+ assert len(events) == 9
+ assert events[0] == "event: message_start\n"
+ assert events[1] == 'data: {"type":"message_start","message":{}}\n'
@pytest.mark.asyncio
-async def test_stream_response_reasoning_content(lmstudio_provider):
- """Test streaming with reasoning_content delta (if LM Studio adds support)."""
+async def test_stream_response_adds_max_tokens_if_missing(lmstudio_provider):
+ """Fallback max_tokens to 81920 if not present."""
req = MockRequest()
-
- mock_chunk = MagicMock()
- mock_chunk.choices = [
- MagicMock(
- delta=MagicMock(content=None, reasoning_content="Thinking..."),
- finish_reason=None,
- )
- ]
- mock_chunk.usage = None
-
- async def mock_stream():
- yield mock_chunk
-
- with patch.object(
- lmstudio_provider._client.chat.completions, "create", new_callable=AsyncMock
- ) as mock_create:
- mock_create.return_value = mock_stream()
-
- events = [e async for e in lmstudio_provider.stream_response(req)]
-
- found_thinking = False
- for e in events:
- if (
- "event: content_block_delta" in e
- and '"thinking_delta"' in e
- and "Thinking..." in e
- ):
- found_thinking = True
- assert found_thinking
-
-
-# --- Stream Error Handling ---
-
-
-class TestLMStudioStreamingExceptionHandling:
- """Tests for error paths during stream_response."""
-
- @pytest.mark.asyncio
- async def test_api_error_emits_sse_error_event(self, lmstudio_provider):
- """When API raises during streaming, SSE error event is emitted."""
- request = _make_request()
-
- with patch.object(
- lmstudio_provider._client.chat.completions,
- "create",
- new_callable=AsyncMock,
- side_effect=RuntimeError("API failed"),
- ):
- events = await _collect_stream(lmstudio_provider, request)
-
- event_text = "".join(events)
- assert "message_start" in event_text
- assert "API failed" in event_text
- assert "message_stop" in event_text
-
- @pytest.mark.asyncio
- async def test_error_after_partial_content(self, lmstudio_provider):
- """Error after partial content: blocks closed, error emitted."""
- request = _make_request()
- chunk1 = _make_chunk(content="Hello ")
- stream_mock = AsyncStreamMock(
- [chunk1], error=ConnectionResetError("Connection lost")
- )
-
- with patch.object(
- lmstudio_provider._client.chat.completions,
- "create",
+ # Replace the existing method completely to avoid proxy mapping errors from type checkers
+ req.model_dump = lambda **kwargs: {"model": "test"} # type: ignore
+ mock_response = MagicMock()
+ mock_response.status_code = 200
+
+ async def empty_aiter():
+ if False:
+ yield ""
+
+ mock_response.aiter_lines = empty_aiter
+
+ with (
+ patch.object(lmstudio_provider._client, "build_request") as mock_build,
+ patch.object(
+ lmstudio_provider._client,
+ "send",
new_callable=AsyncMock,
- return_value=stream_mock,
- ):
- events = await _collect_stream(lmstudio_provider, request)
-
- event_text = "".join(events)
- assert "Hello" in event_text
- assert "Connection lost" in event_text
- assert "message_stop" in event_text
-
- @pytest.mark.asyncio
- async def test_empty_response_gets_space(self, lmstudio_provider):
- """Empty response with no text/tools gets a single space text block."""
- request = _make_request()
- empty_chunk = _make_chunk(finish_reason="stop")
- stream_mock = AsyncStreamMock([empty_chunk])
-
- with patch.object(
- lmstudio_provider._client.chat.completions,
- "create",
- new_callable=AsyncMock,
- return_value=stream_mock,
- ):
- events = await _collect_stream(lmstudio_provider, request)
-
- event_text = "".join(events)
- assert '"text_delta"' in event_text
- assert "message_stop" in event_text
-
-
-# --- Stream Chunk Edge Cases ---
+ return_value=mock_response,
+ ),
+ ):
+ # Just run the generator to completion
+ [e async for e in lmstudio_provider.stream_response(req)]
+ _, kwargs = mock_build.call_args
+ assert kwargs["json"]["max_tokens"] == 81920
-class TestLMStudioStreamChunkEdgeCases:
- """Tests for edge cases in stream chunk handling."""
- @pytest.mark.asyncio
- async def test_stream_chunk_with_empty_choices_skipped(self, lmstudio_provider):
- """Chunk with choices=[] is skipped without crashing."""
- request = _make_request()
- empty_choices_chunk = MagicMock()
- empty_choices_chunk.choices = []
- empty_choices_chunk.usage = None
- finish_chunk = _make_chunk(finish_reason="stop")
- stream_mock = AsyncStreamMock([empty_choices_chunk, finish_chunk])
+async def test_stream_error_status_code(lmstudio_provider):
+ """Non-200 status code raises an error that gets caught and yielded as an SSE API error."""
+ req = MockRequest()
- with patch.object(
- lmstudio_provider._client.chat.completions,
- "create",
+ mock_response = MagicMock()
+ mock_response.status_code = 500
+ mock_response.aread = AsyncMock(return_value=b"Internal Server Error")
+
+ with (
+ patch.object(
+ lmstudio_provider._client, "build_request", return_value=MagicMock()
+ ),
+ patch.object(
+ lmstudio_provider._client,
+ "send",
new_callable=AsyncMock,
- return_value=stream_mock,
- ):
- events = await _collect_stream(lmstudio_provider, request)
-
- event_text = "".join(events)
- assert "message_start" in event_text
- assert "message_stop" in event_text
-
- @pytest.mark.asyncio
- async def test_stream_chunk_with_none_delta_handled(self, lmstudio_provider):
- """Chunk with choice.delta=None is handled defensively."""
- request = _make_request()
- none_delta_chunk = MagicMock()
- none_delta_chunk.usage = None
- choice = MagicMock()
- choice.delta = None
- choice.finish_reason = None
- none_delta_chunk.choices = [choice]
- finish_chunk = _make_chunk(finish_reason="stop")
- stream_mock = AsyncStreamMock([none_delta_chunk, finish_chunk])
-
- with patch.object(
- lmstudio_provider._client.chat.completions,
- "create",
- new_callable=AsyncMock,
- return_value=stream_mock,
- ):
- events = await _collect_stream(lmstudio_provider, request)
-
- event_text = "".join(events)
- assert "message_start" in event_text
- assert "message_stop" in event_text
-
-
-# --- Native Tool Calls ---
-
-
-async def test_stream_response_tool_call(lmstudio_provider):
- """Test streaming tool calls."""
- request = _make_request()
- mock_tc = MagicMock()
- mock_tc.index = 0
- mock_tc.id = "call_1"
- mock_tc.function.name = "search"
- mock_tc.function.arguments = '{"q": "test"}'
-
- mock_chunk = MagicMock()
- mock_chunk.choices = [
- MagicMock(
- delta=MagicMock(content=None, reasoning_content=None, tool_calls=[mock_tc]),
- finish_reason=None,
- )
- ]
- mock_chunk.usage = None
-
- async def mock_stream():
- yield mock_chunk
-
- with patch.object(
- lmstudio_provider._client.chat.completions,
- "create",
- new_callable=AsyncMock,
- return_value=mock_stream(),
+ return_value=mock_response,
+ ),
):
- events = [e async for e in lmstudio_provider.stream_response(request)]
-
- starts = [
- e for e in events if "event: content_block_start" in e and '"tool_use"' in e
- ]
- assert len(starts) == 1
- assert "search" in starts[0]
+ events = [
+ e
+ async for e in lmstudio_provider.stream_response(req, request_id="TEST_ID")
+ ]
-
-# --- Think Tag Parsing ---
+ assert len(events) == 1
+ assert events[0].startswith("event: error\ndata: {")
+ assert "Internal Server Error" in events[0]
+ assert "TEST_ID" in events[0]
@pytest.mark.asyncio
-async def test_stream_response_think_tag_parsing(lmstudio_provider):
- """Thinking content via think tags is emitted as thinking blocks."""
- request = _make_request()
- chunk1 = _make_chunk(content="<think>reasoning</think>answer")
- chunk2 = _make_chunk(finish_reason="stop")
- stream_mock = AsyncStreamMock([chunk1, chunk2])
-
- with patch.object(
- lmstudio_provider._client.chat.completions,
- "create",
- new_callable=AsyncMock,
- return_value=stream_mock,
- ):
- events = await _collect_stream(lmstudio_provider, request)
-
- event_text = "".join(events)
- assert "thinking" in event_text
- assert "reasoning" in event_text
- assert "answer" in event_text
-
-
-# --- _process_tool_call and _flush_task_arg_buffers ---
-
-
-class TestLMStudioProcessToolCall:
- """Tests for _process_tool_call method."""
-
- def test_tool_call_with_id(self, lmstudio_provider):
- """Tool call with id starts a tool block."""
- from providers.common import SSEBuilder
-
- sse = SSEBuilder("msg_test", "test-model")
- tc = {
- "index": 0,
- "id": "call_123",
- "function": {"name": "search", "arguments": '{"q": "test"}'},
- }
- events = list(lmstudio_provider._process_tool_call(tc, sse))
- event_text = "".join(events)
- assert "tool_use" in event_text
- assert "search" in event_text
- assert "call_123" in event_text
-
- def test_tool_call_without_id_generates_uuid(self, lmstudio_provider):
- """Tool call without id generates a uuid-based id."""
- from providers.common import SSEBuilder
-
- sse = SSEBuilder("msg_test", "test-model")
- tc = {
- "index": 0,
- "id": None,
- "function": {"name": "test", "arguments": "{}"},
- }
- events = list(lmstudio_provider._process_tool_call(tc, sse))
- event_text = "".join(events)
- assert "tool_" in event_text
-
- def test_task_tool_forces_background_false(self, lmstudio_provider):
- """Task tool with run_in_background=true is forced to false."""
- from providers.common import SSEBuilder
-
- sse = SSEBuilder("msg_test", "test-model")
- args = json.dumps({"run_in_background": True, "prompt": "test"})
- tc = {
- "index": 0,
- "id": "call_task",
- "function": {"name": "Task", "arguments": args},
- }
- events = list(lmstudio_provider._process_tool_call(tc, sse))
- event_text = "".join(events)
- assert "false" in event_text.lower()
-
- def test_task_tool_chunked_args_forces_background_false(self, lmstudio_provider):
- """Chunked Task args are buffered until valid JSON, then forced to false."""
- from providers.common import SSEBuilder
-
- sse = SSEBuilder("msg_test", "test-model")
- tc1 = {
- "index": 0,
- "id": "call_task_chunked",
- "function": {"name": "Task", "arguments": '{"run_in_background": true,'},
- }
- tc2 = {
- "index": 0,
- "id": "call_task_chunked",
- "function": {"name": None, "arguments": ' "prompt": "test"}'},
- }
-
- events1 = list(lmstudio_provider._process_tool_call(tc1, sse))
- assert len(events1) > 0
- assert "false" not in "".join(events1).lower()
-
- events2 = list(lmstudio_provider._process_tool_call(tc2, sse))
- event_text = "".join(events1 + events2)
- assert "false" in event_text.lower()
+async def test_stream_network_error(lmstudio_provider):
+ """Network errors are caught and yielded as SSE API error events."""
+ req = MockRequest()
- def test_task_tool_invalid_json_logs_warning_on_flush(
- self, lmstudio_provider, caplog
+ with (
+ patch.object(
+ lmstudio_provider._client, "build_request", return_value=MagicMock()
+ ),
+ patch.object(
+ lmstudio_provider._client,
+ "send",
+ new_callable=AsyncMock,
+ side_effect=httpx.ConnectError("Connection refused"),
+ ),
):
- """Invalid JSON args for Task tool emits {} on flush and logs a warning."""
- from providers.common import SSEBuilder
-
- sse = SSEBuilder("msg_test", "test-model")
- tc = {
- "index": 0,
- "id": "call_task2",
- "function": {"name": "Task", "arguments": "not json"},
- }
- events = list(lmstudio_provider._process_tool_call(tc, sse))
- assert len(events) > 0
-
- with caplog.at_level("WARNING"):
- flushed = list(lmstudio_provider._flush_task_arg_buffers(sse))
- assert len(flushed) > 0
- assert "{}" in "".join(flushed)
- assert any("Task args invalid JSON" in r.message for r in caplog.records)
-
- def test_negative_tool_index_fallback(self, lmstudio_provider):
- """tc_index < 0 uses len(tool_indices) as fallback."""
- from providers.common import SSEBuilder
-
- sse = SSEBuilder("msg_test", "test-model")
- tc = {
- "index": -1,
- "id": "call_neg",
- "function": {"name": "test", "arguments": "{}"},
- }
- events = list(lmstudio_provider._process_tool_call(tc, sse))
- assert len(events) > 0
-
- def test_tool_args_emitted_as_delta(self, lmstudio_provider):
- """Arguments are emitted as input_json_delta events."""
- from providers.common import SSEBuilder
-
- sse = SSEBuilder("msg_test", "test-model")
- tc = {
- "index": 0,
- "id": "call_args",
- "function": {"name": "grep", "arguments": '{"pattern": "test"}'},
- }
- events = list(lmstudio_provider._process_tool_call(tc, sse))
- event_text = "".join(events)
- assert "input_json_delta" in event_text
-
- def test_stream_malformed_tool_args_chunked(self, lmstudio_provider):
- """Chunked tool args that never form valid JSON are flushed with {}."""
- from providers.common import SSEBuilder
-
- sse = SSEBuilder("msg_test", "test-model")
- tc1 = {
- "index": 0,
- "id": "call_malformed",
- "function": {"name": "Task", "arguments": '{"broken":'},
- }
- tc2 = {
- "index": 0,
- "id": "call_malformed",
- "function": {"name": None, "arguments": " never valid }"},
- }
-
- events1 = list(lmstudio_provider._process_tool_call(tc1, sse))
- events2 = list(lmstudio_provider._process_tool_call(tc2, sse))
- flushed = list(lmstudio_provider._flush_task_arg_buffers(sse))
-
- event_text = "".join(events1 + events2 + flushed)
- assert "tool_use" in event_text
- assert "{}" in event_text
-
-
-# --- Request Body Edge Cases ---
-
-
-def test_build_request_body_max_tokens_default(lmstudio_provider):
- """max_tokens=None or 0 uses LMSTUDIO_DEFAULT_MAX_TOKENS."""
- req = MockRequest(max_tokens=None)
- body = lmstudio_provider._build_request_body(req)
- assert body["max_tokens"] == LMSTUDIO_DEFAULT_MAX_TOKENS
- assert body["max_tokens"] == 81920
-
- req2 = MockRequest(max_tokens=0)
- body2 = lmstudio_provider._build_request_body(req2)
- assert body2["max_tokens"] == LMSTUDIO_DEFAULT_MAX_TOKENS
-
-
-def test_build_request_body_stop_sequences(lmstudio_provider):
- """stop_sequences non-empty adds stop key to body."""
- req = MockRequest(stop_sequences=["STOP", "END"])
- body = lmstudio_provider._build_request_body(req)
- assert body["stop"] == ["STOP", "END"]
-
-
-def test_build_request_body_tools_and_tool_choice(lmstudio_provider):
- """tools and tool_choice non-empty add to body."""
- tool = MagicMock()
- tool.name = "test_tool"
- tool.description = "A test"
- tool.input_schema = {"type": "object"}
- req = MockRequest(tools=[tool], tool_choice="auto")
- body = lmstudio_provider._build_request_body(req)
- assert "tools" in body
- assert body["tool_choice"] == "auto"
-
-
-# --- Base URL Trailing Slash ---
-
-
-def test_init_base_url_strips_trailing_slash():
- """Config with base_url trailing slash is stored without it."""
- config = ProviderConfig(
- api_key="lm-studio",
- base_url="http://localhost:1234/v1/",
- rate_limit=10,
- rate_window=60,
- )
- with patch("providers.openai_compat.AsyncOpenAI"):
- provider = LMStudioProvider(config)
- assert provider._base_url == "http://localhost:1234/v1"
+ events = [
+ e
+ async for e in lmstudio_provider.stream_response(req, request_id="TEST_ID2")
+ ]
+
+ assert len(events) == 1
+ assert events[0].startswith("event: error\ndata: {")
+ assert "Connection refused" in events[0]
+ assert "TEST_ID2" in events[0]