Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)
Python
Make PROVIDER_MAX_CONCURRENCY required with default of 5
Claude45b7e4cafdb4
diff --git a/.env.example b/.env.exampleindex 6bc114c3..bf2569b8 100644--- a/.env.example+++ b/.env.example@@ -2,8 +2,8 @@PROVIDER_TYPE="nvidia_nim"PROVIDER_RATE_LIMIT=40PROVIDER_RATE_WINDOW=60-# Maximum simultaneous open provider streams (unset = unlimited)-# PROVIDER_MAX_CONCURRENCY=3+# Maximum simultaneous open provider streams (default: 5)+PROVIDER_MAX_CONCURRENCY=5# HTTP client timeouts (seconds) for provider API requestsdiff --git a/README.md b/README.mdindex 4adf3ab8..3035a41d 100644--- a/README.md+++ b/README.md@@ -320,7 +320,7 @@ Browse: [model.lmstudio.ai](https://model.lmstudio.ai)| `LM_STUDIO_BASE_URL` | LM Studio server URL | `http://localhost:1234/v1` || `PROVIDER_RATE_LIMIT` | LLM API requests per window | `40` || `PROVIDER_RATE_WINDOW` | Rate limit window (seconds) | `60` |-| `PROVIDER_MAX_CONCURRENCY` | Max simultaneous open provider streams (unset = unlimited) | — |+| `PROVIDER_MAX_CONCURRENCY` | Max simultaneous open provider streams | `5` || `HTTP_READ_TIMEOUT` | Read timeout for provider API requests (seconds) | `300` || `HTTP_WRITE_TIMEOUT` | Write timeout for provider API requests (seconds) | `10` || `HTTP_CONNECT_TIMEOUT` | Connect timeout for provider API requests (seconds) | `2` |diff --git a/config/settings.py b/config/settings.pyindex cde9b9dd..4f367bd7 100644--- a/config/settings.py+++ b/config/settings.py@@ -48,8 +48,8 @@ class Settings(BaseSettings):provider_rate_window: int = Field(default=60, validation_alias="PROVIDER_RATE_WINDOW")- provider_max_concurrency: int | None = Field(- default=None, validation_alias="PROVIDER_MAX_CONCURRENCY"+ provider_max_concurrency: int = Field(+ default=5, validation_alias="PROVIDER_MAX_CONCURRENCY")# ==================== HTTP Client Timeouts ====================diff --git a/providers/base.py b/providers/base.pyindex ad4a7ebc..83c655af 100644--- a/providers/base.py+++ b/providers/base.py@@ -18,7 +18,7 @@ class ProviderConfig(BaseModel):base_url: str | None = Nonerate_limit: int | None = Nonerate_window: int = 60- max_concurrency: int | None = None+ max_concurrency: int = 5http_read_timeout: float = 300.0http_write_timeout: float = 10.0http_connect_timeout: float = 2.0diff --git a/providers/rate_limit.py b/providers/rate_limit.pyindex 02123e25..110307a1 100644--- a/providers/rate_limit.py+++ b/providers/rate_limit.py@@ -34,7 +34,7 @@ class GlobalRateLimiter:self,rate_limit: int = 40,rate_window: float = 60.0,- max_concurrency: int | None = None,+ max_concurrency: int = 5,):# Prevent double initialization in singletonif hasattr(self, "_initialized"):@@ -44,7 +44,7 @@ class GlobalRateLimiter:raise ValueError("rate_limit must be > 0")if rate_window <= 0:raise ValueError("rate_window must be > 0")- if max_concurrency is not None and max_concurrency <= 0:+ if max_concurrency <= 0:raise ValueError("max_concurrency must be > 0")self._rate_limit = rate_limit@@ -53,18 +53,11 @@ class GlobalRateLimiter:self._request_times: deque[float] = deque()self._blocked_until: float = 0self._lock = asyncio.Lock()- self._concurrency_sem: asyncio.Semaphore | None = (- asyncio.Semaphore(max_concurrency) if max_concurrency is not None else None- )+ self._concurrency_sem = asyncio.Semaphore(max_concurrency)self._initialized = True- concurrency_info = (- f", max_concurrency={max_concurrency}"- if max_concurrency is not None- else ""- )logger.info(- f"GlobalRateLimiter (Provider) initialized ({rate_limit} req / {rate_window}s{concurrency_info})"+ f"GlobalRateLimiter (Provider) initialized ({rate_limit} req / {rate_window}s, max_concurrency={max_concurrency})")@classmethod@@ -72,7 +65,7 @@ class GlobalRateLimiter:cls,rate_limit: int | None = None,rate_window: float | None = None,- max_concurrency: int | None = None,+ max_concurrency: int = 5,) -> GlobalRateLimiter:"""Get or create the singleton instance.@@ -167,16 +160,13 @@ class GlobalRateLimiter:async def concurrency_slot(self) -> AsyncIterator[None]:"""Async context manager that holds one concurrency slot for a stream.- Blocks until a slot is available when max_concurrency is set.- Is a no-op when max_concurrency was not configured.+ Blocks until a slot is available (controlled by max_concurrency)."""- if self._concurrency_sem is not None:- await self._concurrency_sem.acquire()+ await self._concurrency_sem.acquire()try:yieldfinally:- if self._concurrency_sem is not None:- self._concurrency_sem.release()+ self._concurrency_sem.release()async def execute_with_retry(self,diff --git a/tests/api/test_dependencies.py b/tests/api/test_dependencies.pyindex 2b36cd58..e50dffad 100644--- a/tests/api/test_dependencies.py+++ b/tests/api/test_dependencies.py@@ -17,7 +17,7 @@ def _make_mock_settings(**overrides):mock.nvidia_nim_api_key = "test_key"mock.provider_rate_limit = 40mock.provider_rate_window = 60- mock.provider_max_concurrency = None+ mock.provider_max_concurrency = 5mock.open_router_api_key = "test_openrouter_key"mock.lm_studio_base_url = "http://localhost:1234/v1"mock.nim = NimSettings()diff --git a/tests/providers/test_provider_rate_limit.py b/tests/providers/test_provider_rate_limit.pyindex cd0e0706..c8d2061f 100644--- a/tests/providers/test_provider_rate_limit.py+++ b/tests/providers/test_provider_rate_limit.py@@ -259,19 +259,6 @@ class TestProviderRateLimiter:with pytest.raises(ValueError, match="max_concurrency must be > 0"):GlobalRateLimiter(rate_limit=10, rate_window=60, max_concurrency=0)- @pytest.mark.asyncio- async def test_concurrency_slot_noop_when_not_configured(self):- """concurrency_slot() is a no-op when max_concurrency is None."""- GlobalRateLimiter.reset_instance()- limiter = GlobalRateLimiter.get_instance(rate_limit=100, rate_window=60)- assert limiter._concurrency_sem is None-- # Should not block and complete immediately- entered = False- async with limiter.concurrency_slot():- entered = True- assert entered-@pytest.mark.asyncioasync def test_concurrency_slot_limits_simultaneous_streams(self):"""At most max_concurrency streams can hold a slot simultaneously."""