Skip to content

Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)

Python

Add queue-level concurrency limit to provider streaming

Claudeafaf50a9721f
diff --git a/.env.example b/.env.example
index 3f4dac84..f4a023f0 100644
--- a/.env.example
+++ b/.env.example
@@ -2,6 +2,8 @@
PROVIDER_TYPE="nvidia_nim"
PROVIDER_RATE_LIMIT=40
PROVIDER_RATE_WINDOW=60
+# Maximum simultaneous open provider streams (unset = unlimited)
+# PROVIDER_MAX_CONCURRENCY=3
# HTTP client timeouts (seconds) for provider API requests
diff --git a/api/dependencies.py b/api/dependencies.py
index be52a0c3..e0cb8b51 100644
--- a/api/dependencies.py
+++ b/api/dependencies.py
@@ -41,6 +41,7 @@ def get_provider() -> BaseProvider:
base_url=NVIDIA_NIM_BASE_URL,
rate_limit=settings.provider_rate_limit,
rate_window=settings.provider_rate_window,
+ max_concurrency=settings.provider_max_concurrency,
http_read_timeout=settings.http_read_timeout,
http_write_timeout=settings.http_write_timeout,
http_connect_timeout=settings.http_connect_timeout,
@@ -66,6 +67,7 @@ def get_provider() -> BaseProvider:
base_url="https://openrouter.ai/api/v1",
rate_limit=settings.provider_rate_limit,
rate_window=settings.provider_rate_window,
+ max_concurrency=settings.provider_max_concurrency,
http_read_timeout=settings.http_read_timeout,
http_write_timeout=settings.http_write_timeout,
http_connect_timeout=settings.http_connect_timeout,
@@ -80,6 +82,7 @@ def get_provider() -> BaseProvider:
base_url=settings.lm_studio_base_url,
rate_limit=settings.provider_rate_limit,
rate_window=settings.provider_rate_window,
+ max_concurrency=settings.provider_max_concurrency,
http_read_timeout=settings.http_read_timeout,
http_write_timeout=settings.http_write_timeout,
http_connect_timeout=settings.http_connect_timeout,
diff --git a/config/settings.py b/config/settings.py
index 7142d7dc..0b2a44e7 100644
--- a/config/settings.py
+++ b/config/settings.py
@@ -48,6 +48,9 @@ class Settings(BaseSettings):
provider_rate_window: int = Field(
default=60, validation_alias="PROVIDER_RATE_WINDOW"
)
+ provider_max_concurrency: int | None = Field(
+ default=None, validation_alias="PROVIDER_MAX_CONCURRENCY"
+ )
# ==================== HTTP Client Timeouts ====================
http_read_timeout: float = Field(
diff --git a/providers/base.py b/providers/base.py
index ca4921fe..ad4a7ebc 100644
--- a/providers/base.py
+++ b/providers/base.py
@@ -18,6 +18,7 @@ class ProviderConfig(BaseModel):
base_url: str | None = None
rate_limit: int | None = None
rate_window: int = 60
+ max_concurrency: int | None = None
http_read_timeout: float = 300.0
http_write_timeout: float = 10.0
http_connect_timeout: float = 2.0
diff --git a/providers/openai_compat.py b/providers/openai_compat.py
index 7740788f..c308f659 100644
--- a/providers/openai_compat.py
+++ b/providers/openai_compat.py
@@ -41,6 +41,7 @@ class OpenAICompatibleProvider(BaseProvider):
self._global_rate_limiter = GlobalRateLimiter.get_instance(
rate_limit=config.rate_limit,
rate_window=config.rate_window,
+ max_concurrency=config.max_concurrency,
)
self._client = AsyncOpenAI(
api_key=self._api_key,
@@ -155,108 +156,109 @@ class OpenAICompatibleProvider(BaseProvider):
error_occurred = False
error_message = ""
- try:
- stream = await self._global_rate_limiter.execute_with_retry(
- self._client.chat.completions.create, **body, stream=True
- )
- async for chunk in stream:
- if getattr(chunk, "usage", None):
- usage_info = chunk.usage
-
- if not chunk.choices:
- continue
-
- choice = chunk.choices[0]
- delta = choice.delta
- if delta is None:
- continue
-
- if choice.finish_reason:
- finish_reason = choice.finish_reason
- logger.debug("%s finish_reason: %s", tag, finish_reason)
-
- # Handle reasoning_content (OpenAI extended format)
- reasoning = getattr(delta, "reasoning_content", None)
- if reasoning:
- for event in sse.ensure_thinking_block():
- yield event
- yield sse.emit_thinking_delta(reasoning)
-
- # Provider-specific extra reasoning (e.g. OpenRouter reasoning_details)
- for event in self._handle_extra_reasoning(delta, sse):
- yield event
+ async with self._global_rate_limiter.concurrency_slot():
+ try:
+ stream = await self._global_rate_limiter.execute_with_retry(
+ self._client.chat.completions.create, **body, stream=True
+ )
+ async for chunk in stream:
+ if getattr(chunk, "usage", None):
+ usage_info = chunk.usage
+
+ if not chunk.choices:
+ continue
+
+ choice = chunk.choices[0]
+ delta = choice.delta
+ if delta is None:
+ continue
+
+ if choice.finish_reason:
+ finish_reason = choice.finish_reason
+ logger.debug("%s finish_reason: %s", tag, finish_reason)
+
+ # Handle reasoning_content (OpenAI extended format)
+ reasoning = getattr(delta, "reasoning_content", None)
+ if reasoning:
+ for event in sse.ensure_thinking_block():
+ yield event
+ yield sse.emit_thinking_delta(reasoning)
- # Handle text content
- if delta.content:
- for part in think_parser.feed(delta.content):
- if part.type == ContentType.THINKING:
- for event in sse.ensure_thinking_block():
- yield event
- yield sse.emit_thinking_delta(part.content)
- else:
- filtered_text, detected_tools = heuristic_parser.feed(
- part.content
- )
-
- if filtered_text:
- for event in sse.ensure_text_block():
- yield event
- yield sse.emit_text_delta(filtered_text)
+ # Provider-specific extra reasoning (e.g. OpenRouter reasoning_details)
+ for event in self._handle_extra_reasoning(delta, sse):
+ yield event
- for tool_use in detected_tools:
- for event in sse.close_content_blocks():
+ # Handle text content
+ if delta.content:
+ for part in think_parser.feed(delta.content):
+ if part.type == ContentType.THINKING:
+ for event in sse.ensure_thinking_block():
yield event
-
- block_idx = sse.blocks.allocate_index()
- if tool_use.get("name") == "Task" and isinstance(
- tool_use.get("input"), dict
- ):
- tool_use["input"]["run_in_background"] = False
- yield sse.content_block_start(
- block_idx,
- "tool_use",
- id=tool_use["id"],
- name=tool_use["name"],
+ yield sse.emit_thinking_delta(part.content)
+ else:
+ filtered_text, detected_tools = heuristic_parser.feed(
+ part.content
)
- yield sse.content_block_delta(
- block_idx,
- "input_json_delta",
- json.dumps(tool_use["input"]),
- )
- yield sse.content_block_stop(block_idx)
- # Handle native tool calls
- if delta.tool_calls:
- for event in sse.close_content_blocks():
- yield event
- for tc in delta.tool_calls:
- tc_info = {
- "index": tc.index,
- "id": tc.id,
- "function": {
- "name": tc.function.name,
- "arguments": tc.function.arguments,
- },
- }
- for event in self._process_tool_call(tc_info, sse):
+ if filtered_text:
+ for event in sse.ensure_text_block():
+ yield event
+ yield sse.emit_text_delta(filtered_text)
+
+ for tool_use in detected_tools:
+ for event in sse.close_content_blocks():
+ yield event
+
+ block_idx = sse.blocks.allocate_index()
+ if tool_use.get("name") == "Task" and isinstance(
+ tool_use.get("input"), dict
+ ):
+ tool_use["input"]["run_in_background"] = False
+ yield sse.content_block_start(
+ block_idx,
+ "tool_use",
+ id=tool_use["id"],
+ name=tool_use["name"],
+ )
+ yield sse.content_block_delta(
+ block_idx,
+ "input_json_delta",
+ json.dumps(tool_use["input"]),
+ )
+ yield sse.content_block_stop(block_idx)
+
+ # Handle native tool calls
+ if delta.tool_calls:
+ for event in sse.close_content_blocks():
yield event
+ for tc in delta.tool_calls:
+ tc_info = {
+ "index": tc.index,
+ "id": tc.id,
+ "function": {
+ "name": tc.function.name,
+ "arguments": tc.function.arguments,
+ },
+ }
+ for event in self._process_tool_call(tc_info, sse):
+ yield event
- except Exception as e:
- req_tag = f" request_id={request_id}" if request_id else ""
- logger.error("%s_ERROR:%s %s: %s", tag, req_tag, type(e).__name__, e)
- mapped_e = map_error(e)
- error_occurred = True
- error_message = str(mapped_e)
- logger.info(
- "%s_STREAM: Emitting SSE error event for %s%s",
- tag,
- type(e).__name__,
- req_tag,
- )
- for event in sse.close_content_blocks():
- yield event
- for event in sse.emit_error(error_message):
- yield event
+ except Exception as e:
+ req_tag = f" request_id={request_id}" if request_id else ""
+ logger.error("%s_ERROR:%s %s: %s", tag, req_tag, type(e).__name__, e)
+ mapped_e = map_error(e)
+ error_occurred = True
+ error_message = str(mapped_e)
+ logger.info(
+ "%s_STREAM: Emitting SSE error event for %s%s",
+ tag,
+ type(e).__name__,
+ req_tag,
+ )
+ for event in sse.close_content_blocks():
+ yield event
+ for event in sse.emit_error(error_message):
+ yield event
# Flush remaining content
remaining = think_parser.flush()
diff --git a/providers/rate_limit.py b/providers/rate_limit.py
index 25b1f720..02123e25 100644
--- a/providers/rate_limit.py
+++ b/providers/rate_limit.py
@@ -4,7 +4,8 @@ import asyncio
import random
import time
from collections import deque
-from collections.abc import Callable
+from collections.abc import AsyncIterator, Callable
+from contextlib import asynccontextmanager
from typing import Any, TypeVar
import openai
@@ -19,13 +20,22 @@ class GlobalRateLimiter:
when a rate limit error is encountered (reactive) and
throttles requests (proactive) using a strict rolling window.
+ Optionally enforces a max_concurrency cap: at most N provider streams
+ may be open simultaneously, independent of the sliding window.
+
Proactive limits - throttles requests to stay within API limits.
Reactive limits - pauses all requests when a 429 is hit.
+ Concurrency limit - caps simultaneously open streams.
"""
_instance: GlobalRateLimiter | None = None
- def __init__(self, rate_limit: int = 40, rate_window: float = 60.0):
+ def __init__(
+ self,
+ rate_limit: int = 40,
+ rate_window: float = 60.0,
+ max_concurrency: int | None = None,
+ ):
# Prevent double initialization in singleton
if hasattr(self, "_initialized"):
return
@@ -34,6 +44,8 @@ class GlobalRateLimiter:
raise ValueError("rate_limit must be > 0")
if rate_window <= 0:
raise ValueError("rate_window must be > 0")
+ if max_concurrency is not None and max_concurrency <= 0:
+ raise ValueError("max_concurrency must be > 0")
self._rate_limit = rate_limit
self._rate_window = float(rate_window)
@@ -41,10 +53,18 @@ class GlobalRateLimiter:
self._request_times: deque[float] = deque()
self._blocked_until: float = 0
self._lock = asyncio.Lock()
+ self._concurrency_sem: asyncio.Semaphore | None = (
+ asyncio.Semaphore(max_concurrency) if max_concurrency is not None else None
+ )
self._initialized = True
+ concurrency_info = (
+ f", max_concurrency={max_concurrency}"
+ if max_concurrency is not None
+ else ""
+ )
logger.info(
- f"GlobalRateLimiter (Provider) initialized ({rate_limit} req / {rate_window}s)"
+ f"GlobalRateLimiter (Provider) initialized ({rate_limit} req / {rate_window}s{concurrency_info})"
)
@classmethod
@@ -52,17 +72,20 @@ class GlobalRateLimiter:
cls,
rate_limit: int | None = None,
rate_window: float | None = None,
+ max_concurrency: int | None = None,
) -> GlobalRateLimiter:
"""Get or create the singleton instance.
Args:
rate_limit: Requests per window (only used on first creation)
rate_window: Window in seconds (only used on first creation)
+ max_concurrency: Max simultaneous open streams (only used on first creation)
"""
if cls._instance is None:
cls._instance = cls(
rate_limit=rate_limit or 40,
rate_window=rate_window or 60.0,
+ max_concurrency=max_concurrency,
)
return cls._instance
@@ -140,6 +163,21 @@ class GlobalRateLimiter:
"""Get remaining reactive wait time in seconds."""
return max(0.0, self._blocked_until - time.monotonic())
+ @asynccontextmanager
+ async def concurrency_slot(self) -> AsyncIterator[None]:
+ """Async context manager that holds one concurrency slot for a stream.
+
+ Blocks until a slot is available when max_concurrency is set.
+ Is a no-op when max_concurrency was not configured.
+ """
+ if self._concurrency_sem is not None:
+ await self._concurrency_sem.acquire()
+ try:
+ yield
+ finally:
+ if self._concurrency_sem is not None:
+ self._concurrency_sem.release()
+
async def execute_with_retry(
self,
fn: Callable[..., Any],
diff --git a/tests/api/test_dependencies.py b/tests/api/test_dependencies.py
index 095bc9d2..2b36cd58 100644
--- a/tests/api/test_dependencies.py
+++ b/tests/api/test_dependencies.py
@@ -17,6 +17,7 @@ def _make_mock_settings(**overrides):
mock.nvidia_nim_api_key = "test_key"
mock.provider_rate_limit = 40
mock.provider_rate_window = 60
+ mock.provider_max_concurrency = None
mock.open_router_api_key = "test_openrouter_key"
mock.lm_studio_base_url = "http://localhost:1234/v1"
mock.nim = NimSettings()
diff --git a/tests/providers/test_provider_rate_limit.py b/tests/providers/test_provider_rate_limit.py
index b17d3ea4..cd0e0706 100644
--- a/tests/providers/test_provider_rate_limit.py
+++ b/tests/providers/test_provider_rate_limit.py
@@ -251,3 +251,80 @@ class TestProviderRateLimiter:
)
assert result == "ok"
assert call_count == 2
+
+ @pytest.mark.asyncio
+ async def test_max_concurrency_zero_raises(self):
+ """max_concurrency <= 0 raises ValueError."""
+ GlobalRateLimiter.reset_instance()
+ with pytest.raises(ValueError, match="max_concurrency must be > 0"):
+ GlobalRateLimiter(rate_limit=10, rate_window=60, max_concurrency=0)
+
+ @pytest.mark.asyncio
+ async def test_concurrency_slot_noop_when_not_configured(self):
+ """concurrency_slot() is a no-op when max_concurrency is None."""
+ GlobalRateLimiter.reset_instance()
+ limiter = GlobalRateLimiter.get_instance(rate_limit=100, rate_window=60)
+ assert limiter._concurrency_sem is None
+
+ # Should not block and complete immediately
+ entered = False
+ async with limiter.concurrency_slot():
+ entered = True
+ assert entered
+
+ @pytest.mark.asyncio
+ async def test_concurrency_slot_limits_simultaneous_streams(self):
+ """At most max_concurrency streams can hold a slot simultaneously."""
+ GlobalRateLimiter.reset_instance()
+ max_concurrency = 2
+ limiter = GlobalRateLimiter.get_instance(
+ rate_limit=100, rate_window=60, max_concurrency=max_concurrency
+ )
+
+ peak_concurrent = 0
+ current_concurrent = 0
+ lock = asyncio.Lock()
+
+ async def stream_task(hold_time: float) -> None:
+ nonlocal peak_concurrent, current_concurrent
+ async with limiter.concurrency_slot():
+ async with lock:
+ current_concurrent += 1
+ if current_concurrent > peak_concurrent:
+ peak_concurrent = current_concurrent
+ await asyncio.sleep(hold_time)
+ async with lock:
+ current_concurrent -= 1
+
+ # Launch 5 tasks that each hold the slot; only 2 can be active at once
+ await asyncio.gather(*(stream_task(0.05) for _ in range(5)))
+
+ assert peak_concurrent <= max_concurrency, (
+ f"Concurrency exceeded: peak={peak_concurrent}, max={max_concurrency}"
+ )
+
+ @pytest.mark.asyncio
+ async def test_concurrency_slot_releases_on_exception(self):
+ """Slot is released even when the body raises an exception."""
+ GlobalRateLimiter.reset_instance()
+ limiter = GlobalRateLimiter.get_instance(
+ rate_limit=100, rate_window=60, max_concurrency=1
+ )
+ assert limiter._concurrency_sem is not None
+
+ with pytest.raises(RuntimeError):
+ async with limiter.concurrency_slot():
+ raise RuntimeError("boom")
+
+ # Semaphore value should be restored (1 available again)
+ assert limiter._concurrency_sem._value == 1
+
+ @pytest.mark.asyncio
+ async def test_get_instance_passes_max_concurrency(self):
+ """get_instance forwards max_concurrency to the singleton."""
+ GlobalRateLimiter.reset_instance()
+ limiter = GlobalRateLimiter.get_instance(
+ rate_limit=10, rate_window=60, max_concurrency=3
+ )
+ assert limiter._concurrency_sem is not None
+ assert limiter._concurrency_sem._value == 3