Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)
Python
Add queue-level concurrency limit to provider streaming
Claudeafaf50a9721f
diff --git a/.env.example b/.env.exampleindex 3f4dac84..f4a023f0 100644--- a/.env.example+++ b/.env.example@@ -2,6 +2,8 @@PROVIDER_TYPE="nvidia_nim"PROVIDER_RATE_LIMIT=40PROVIDER_RATE_WINDOW=60+# Maximum simultaneous open provider streams (unset = unlimited)+# PROVIDER_MAX_CONCURRENCY=3# HTTP client timeouts (seconds) for provider API requestsdiff --git a/api/dependencies.py b/api/dependencies.pyindex be52a0c3..e0cb8b51 100644--- a/api/dependencies.py+++ b/api/dependencies.py@@ -41,6 +41,7 @@ def get_provider() -> BaseProvider:base_url=NVIDIA_NIM_BASE_URL,rate_limit=settings.provider_rate_limit,rate_window=settings.provider_rate_window,+ max_concurrency=settings.provider_max_concurrency,http_read_timeout=settings.http_read_timeout,http_write_timeout=settings.http_write_timeout,http_connect_timeout=settings.http_connect_timeout,@@ -66,6 +67,7 @@ def get_provider() -> BaseProvider:base_url="https://openrouter.ai/api/v1",rate_limit=settings.provider_rate_limit,rate_window=settings.provider_rate_window,+ max_concurrency=settings.provider_max_concurrency,http_read_timeout=settings.http_read_timeout,http_write_timeout=settings.http_write_timeout,http_connect_timeout=settings.http_connect_timeout,@@ -80,6 +82,7 @@ def get_provider() -> BaseProvider:base_url=settings.lm_studio_base_url,rate_limit=settings.provider_rate_limit,rate_window=settings.provider_rate_window,+ max_concurrency=settings.provider_max_concurrency,http_read_timeout=settings.http_read_timeout,http_write_timeout=settings.http_write_timeout,http_connect_timeout=settings.http_connect_timeout,diff --git a/config/settings.py b/config/settings.pyindex 7142d7dc..0b2a44e7 100644--- a/config/settings.py+++ b/config/settings.py@@ -48,6 +48,9 @@ class Settings(BaseSettings):provider_rate_window: int = Field(default=60, validation_alias="PROVIDER_RATE_WINDOW")+ provider_max_concurrency: int | None = Field(+ default=None, validation_alias="PROVIDER_MAX_CONCURRENCY"+ )# ==================== HTTP Client Timeouts ====================http_read_timeout: float = Field(diff --git a/providers/base.py b/providers/base.pyindex ca4921fe..ad4a7ebc 100644--- a/providers/base.py+++ b/providers/base.py@@ -18,6 +18,7 @@ class ProviderConfig(BaseModel):base_url: str | None = Nonerate_limit: int | None = Nonerate_window: int = 60+ max_concurrency: int | None = Nonehttp_read_timeout: float = 300.0http_write_timeout: float = 10.0http_connect_timeout: float = 2.0diff --git a/providers/openai_compat.py b/providers/openai_compat.pyindex 7740788f..c308f659 100644--- a/providers/openai_compat.py+++ b/providers/openai_compat.py@@ -41,6 +41,7 @@ class OpenAICompatibleProvider(BaseProvider):self._global_rate_limiter = GlobalRateLimiter.get_instance(rate_limit=config.rate_limit,rate_window=config.rate_window,+ max_concurrency=config.max_concurrency,)self._client = AsyncOpenAI(api_key=self._api_key,@@ -155,108 +156,109 @@ class OpenAICompatibleProvider(BaseProvider):error_occurred = Falseerror_message = ""- try:- stream = await self._global_rate_limiter.execute_with_retry(- self._client.chat.completions.create, **body, stream=True- )- async for chunk in stream:- if getattr(chunk, "usage", None):- usage_info = chunk.usage-- if not chunk.choices:- continue-- choice = chunk.choices[0]- delta = choice.delta- if delta is None:- continue-- if choice.finish_reason:- finish_reason = choice.finish_reason- logger.debug("%s finish_reason: %s", tag, finish_reason)-- # Handle reasoning_content (OpenAI extended format)- reasoning = getattr(delta, "reasoning_content", None)- if reasoning:- for event in sse.ensure_thinking_block():- yield event- yield sse.emit_thinking_delta(reasoning)-- # Provider-specific extra reasoning (e.g. OpenRouter reasoning_details)- for event in self._handle_extra_reasoning(delta, sse):- yield event+ async with self._global_rate_limiter.concurrency_slot():+ try:+ stream = await self._global_rate_limiter.execute_with_retry(+ self._client.chat.completions.create, **body, stream=True+ )+ async for chunk in stream:+ if getattr(chunk, "usage", None):+ usage_info = chunk.usage++ if not chunk.choices:+ continue++ choice = chunk.choices[0]+ delta = choice.delta+ if delta is None:+ continue++ if choice.finish_reason:+ finish_reason = choice.finish_reason+ logger.debug("%s finish_reason: %s", tag, finish_reason)++ # Handle reasoning_content (OpenAI extended format)+ reasoning = getattr(delta, "reasoning_content", None)+ if reasoning:+ for event in sse.ensure_thinking_block():+ yield event+ yield sse.emit_thinking_delta(reasoning)- # Handle text content- if delta.content:- for part in think_parser.feed(delta.content):- if part.type == ContentType.THINKING:- for event in sse.ensure_thinking_block():- yield event- yield sse.emit_thinking_delta(part.content)- else:- filtered_text, detected_tools = heuristic_parser.feed(- part.content- )-- if filtered_text:- for event in sse.ensure_text_block():- yield event- yield sse.emit_text_delta(filtered_text)+ # Provider-specific extra reasoning (e.g. OpenRouter reasoning_details)+ for event in self._handle_extra_reasoning(delta, sse):+ yield event- for tool_use in detected_tools:- for event in sse.close_content_blocks():+ # Handle text content+ if delta.content:+ for part in think_parser.feed(delta.content):+ if part.type == ContentType.THINKING:+ for event in sse.ensure_thinking_block():yield event-- block_idx = sse.blocks.allocate_index()- if tool_use.get("name") == "Task" and isinstance(- tool_use.get("input"), dict- ):- tool_use["input"]["run_in_background"] = False- yield sse.content_block_start(- block_idx,- "tool_use",- id=tool_use["id"],- name=tool_use["name"],+ yield sse.emit_thinking_delta(part.content)+ else:+ filtered_text, detected_tools = heuristic_parser.feed(+ part.content)- yield sse.content_block_delta(- block_idx,- "input_json_delta",- json.dumps(tool_use["input"]),- )- yield sse.content_block_stop(block_idx)- # Handle native tool calls- if delta.tool_calls:- for event in sse.close_content_blocks():- yield event- for tc in delta.tool_calls:- tc_info = {- "index": tc.index,- "id": tc.id,- "function": {- "name": tc.function.name,- "arguments": tc.function.arguments,- },- }- for event in self._process_tool_call(tc_info, sse):+ if filtered_text:+ for event in sse.ensure_text_block():+ yield event+ yield sse.emit_text_delta(filtered_text)++ for tool_use in detected_tools:+ for event in sse.close_content_blocks():+ yield event++ block_idx = sse.blocks.allocate_index()+ if tool_use.get("name") == "Task" and isinstance(+ tool_use.get("input"), dict+ ):+ tool_use["input"]["run_in_background"] = False+ yield sse.content_block_start(+ block_idx,+ "tool_use",+ id=tool_use["id"],+ name=tool_use["name"],+ )+ yield sse.content_block_delta(+ block_idx,+ "input_json_delta",+ json.dumps(tool_use["input"]),+ )+ yield sse.content_block_stop(block_idx)++ # Handle native tool calls+ if delta.tool_calls:+ for event in sse.close_content_blocks():yield event+ for tc in delta.tool_calls:+ tc_info = {+ "index": tc.index,+ "id": tc.id,+ "function": {+ "name": tc.function.name,+ "arguments": tc.function.arguments,+ },+ }+ for event in self._process_tool_call(tc_info, sse):+ yield event- except Exception as e:- req_tag = f" request_id={request_id}" if request_id else ""- logger.error("%s_ERROR:%s %s: %s", tag, req_tag, type(e).__name__, e)- mapped_e = map_error(e)- error_occurred = True- error_message = str(mapped_e)- logger.info(- "%s_STREAM: Emitting SSE error event for %s%s",- tag,- type(e).__name__,- req_tag,- )- for event in sse.close_content_blocks():- yield event- for event in sse.emit_error(error_message):- yield event+ except Exception as e:+ req_tag = f" request_id={request_id}" if request_id else ""+ logger.error("%s_ERROR:%s %s: %s", tag, req_tag, type(e).__name__, e)+ mapped_e = map_error(e)+ error_occurred = True+ error_message = str(mapped_e)+ logger.info(+ "%s_STREAM: Emitting SSE error event for %s%s",+ tag,+ type(e).__name__,+ req_tag,+ )+ for event in sse.close_content_blocks():+ yield event+ for event in sse.emit_error(error_message):+ yield event# Flush remaining contentremaining = think_parser.flush()diff --git a/providers/rate_limit.py b/providers/rate_limit.pyindex 25b1f720..02123e25 100644--- a/providers/rate_limit.py+++ b/providers/rate_limit.py@@ -4,7 +4,8 @@ import asyncioimport randomimport timefrom collections import deque-from collections.abc import Callable+from collections.abc import AsyncIterator, Callable+from contextlib import asynccontextmanagerfrom typing import Any, TypeVarimport openai@@ -19,13 +20,22 @@ class GlobalRateLimiter:when a rate limit error is encountered (reactive) andthrottles requests (proactive) using a strict rolling window.+ Optionally enforces a max_concurrency cap: at most N provider streams+ may be open simultaneously, independent of the sliding window.+Proactive limits - throttles requests to stay within API limits.Reactive limits - pauses all requests when a 429 is hit.+ Concurrency limit - caps simultaneously open streams."""_instance: GlobalRateLimiter | None = None- def __init__(self, rate_limit: int = 40, rate_window: float = 60.0):+ def __init__(+ self,+ rate_limit: int = 40,+ rate_window: float = 60.0,+ max_concurrency: int | None = None,+ ):# Prevent double initialization in singletonif hasattr(self, "_initialized"):return@@ -34,6 +44,8 @@ class GlobalRateLimiter:raise ValueError("rate_limit must be > 0")if rate_window <= 0:raise ValueError("rate_window must be > 0")+ if max_concurrency is not None and max_concurrency <= 0:+ raise ValueError("max_concurrency must be > 0")self._rate_limit = rate_limitself._rate_window = float(rate_window)@@ -41,10 +53,18 @@ class GlobalRateLimiter:self._request_times: deque[float] = deque()self._blocked_until: float = 0self._lock = asyncio.Lock()+ self._concurrency_sem: asyncio.Semaphore | None = (+ asyncio.Semaphore(max_concurrency) if max_concurrency is not None else None+ )self._initialized = True+ concurrency_info = (+ f", max_concurrency={max_concurrency}"+ if max_concurrency is not None+ else ""+ )logger.info(- f"GlobalRateLimiter (Provider) initialized ({rate_limit} req / {rate_window}s)"+ f"GlobalRateLimiter (Provider) initialized ({rate_limit} req / {rate_window}s{concurrency_info})")@classmethod@@ -52,17 +72,20 @@ class GlobalRateLimiter:cls,rate_limit: int | None = None,rate_window: float | None = None,+ max_concurrency: int | None = None,) -> GlobalRateLimiter:"""Get or create the singleton instance.Args:rate_limit: Requests per window (only used on first creation)rate_window: Window in seconds (only used on first creation)+ max_concurrency: Max simultaneous open streams (only used on first creation)"""if cls._instance is None:cls._instance = cls(rate_limit=rate_limit or 40,rate_window=rate_window or 60.0,+ max_concurrency=max_concurrency,)return cls._instance@@ -140,6 +163,21 @@ class GlobalRateLimiter:"""Get remaining reactive wait time in seconds."""return max(0.0, self._blocked_until - time.monotonic())+ @asynccontextmanager+ async def concurrency_slot(self) -> AsyncIterator[None]:+ """Async context manager that holds one concurrency slot for a stream.++ Blocks until a slot is available when max_concurrency is set.+ Is a no-op when max_concurrency was not configured.+ """+ if self._concurrency_sem is not None:+ await self._concurrency_sem.acquire()+ try:+ yield+ finally:+ if self._concurrency_sem is not None:+ self._concurrency_sem.release()+async def execute_with_retry(self,fn: Callable[..., Any],diff --git a/tests/api/test_dependencies.py b/tests/api/test_dependencies.pyindex 095bc9d2..2b36cd58 100644--- a/tests/api/test_dependencies.py+++ b/tests/api/test_dependencies.py@@ -17,6 +17,7 @@ def _make_mock_settings(**overrides):mock.nvidia_nim_api_key = "test_key"mock.provider_rate_limit = 40mock.provider_rate_window = 60+ mock.provider_max_concurrency = Nonemock.open_router_api_key = "test_openrouter_key"mock.lm_studio_base_url = "http://localhost:1234/v1"mock.nim = NimSettings()diff --git a/tests/providers/test_provider_rate_limit.py b/tests/providers/test_provider_rate_limit.pyindex b17d3ea4..cd0e0706 100644--- a/tests/providers/test_provider_rate_limit.py+++ b/tests/providers/test_provider_rate_limit.py@@ -251,3 +251,80 @@ class TestProviderRateLimiter:)assert result == "ok"assert call_count == 2++ @pytest.mark.asyncio+ async def test_max_concurrency_zero_raises(self):+ """max_concurrency <= 0 raises ValueError."""+ GlobalRateLimiter.reset_instance()+ with pytest.raises(ValueError, match="max_concurrency must be > 0"):+ GlobalRateLimiter(rate_limit=10, rate_window=60, max_concurrency=0)++ @pytest.mark.asyncio+ async def test_concurrency_slot_noop_when_not_configured(self):+ """concurrency_slot() is a no-op when max_concurrency is None."""+ GlobalRateLimiter.reset_instance()+ limiter = GlobalRateLimiter.get_instance(rate_limit=100, rate_window=60)+ assert limiter._concurrency_sem is None++ # Should not block and complete immediately+ entered = False+ async with limiter.concurrency_slot():+ entered = True+ assert entered++ @pytest.mark.asyncio+ async def test_concurrency_slot_limits_simultaneous_streams(self):+ """At most max_concurrency streams can hold a slot simultaneously."""+ GlobalRateLimiter.reset_instance()+ max_concurrency = 2+ limiter = GlobalRateLimiter.get_instance(+ rate_limit=100, rate_window=60, max_concurrency=max_concurrency+ )++ peak_concurrent = 0+ current_concurrent = 0+ lock = asyncio.Lock()++ async def stream_task(hold_time: float) -> None:+ nonlocal peak_concurrent, current_concurrent+ async with limiter.concurrency_slot():+ async with lock:+ current_concurrent += 1+ if current_concurrent > peak_concurrent:+ peak_concurrent = current_concurrent+ await asyncio.sleep(hold_time)+ async with lock:+ current_concurrent -= 1++ # Launch 5 tasks that each hold the slot; only 2 can be active at once+ await asyncio.gather(*(stream_task(0.05) for _ in range(5)))++ assert peak_concurrent <= max_concurrency, (+ f"Concurrency exceeded: peak={peak_concurrent}, max={max_concurrency}"+ )++ @pytest.mark.asyncio+ async def test_concurrency_slot_releases_on_exception(self):+ """Slot is released even when the body raises an exception."""+ GlobalRateLimiter.reset_instance()+ limiter = GlobalRateLimiter.get_instance(+ rate_limit=100, rate_window=60, max_concurrency=1+ )+ assert limiter._concurrency_sem is not None++ with pytest.raises(RuntimeError):+ async with limiter.concurrency_slot():+ raise RuntimeError("boom")++ # Semaphore value should be restored (1 available again)+ assert limiter._concurrency_sem._value == 1++ @pytest.mark.asyncio+ async def test_get_instance_passes_max_concurrency(self):+ """get_instance forwards max_concurrency to the singleton."""+ GlobalRateLimiter.reset_instance()+ limiter = GlobalRateLimiter.get_instance(+ rate_limit=10, rate_window=60, max_concurrency=3+ )+ assert limiter._concurrency_sem is not None+ assert limiter._concurrency_sem._value == 3