Skip to content

Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)

Python

Make PROVIDER_MAX_CONCURRENCY required with default of 5

Claude45b7e4cafdb4
diff --git a/.env.example b/.env.example
index 6bc114c3..bf2569b8 100644
--- a/.env.example
+++ b/.env.example
@@ -2,8 +2,8 @@
PROVIDER_TYPE="nvidia_nim"
PROVIDER_RATE_LIMIT=40
PROVIDER_RATE_WINDOW=60
-# Maximum simultaneous open provider streams (unset = unlimited)
-# PROVIDER_MAX_CONCURRENCY=3
+# Maximum simultaneous open provider streams (default: 5)
+PROVIDER_MAX_CONCURRENCY=5
# HTTP client timeouts (seconds) for provider API requests
diff --git a/README.md b/README.md
index 4adf3ab8..3035a41d 100644
--- a/README.md
+++ b/README.md
@@ -320,7 +320,7 @@ Browse: [model.lmstudio.ai](https://model.lmstudio.ai)
| `LM_STUDIO_BASE_URL` | LM Studio server URL | `http://localhost:1234/v1` |
| `PROVIDER_RATE_LIMIT` | LLM API requests per window | `40` |
| `PROVIDER_RATE_WINDOW` | Rate limit window (seconds) | `60` |
-| `PROVIDER_MAX_CONCURRENCY` | Max simultaneous open provider streams (unset = unlimited) | — |
+| `PROVIDER_MAX_CONCURRENCY` | Max simultaneous open provider streams | `5` |
| `HTTP_READ_TIMEOUT` | Read timeout for provider API requests (seconds) | `300` |
| `HTTP_WRITE_TIMEOUT` | Write timeout for provider API requests (seconds) | `10` |
| `HTTP_CONNECT_TIMEOUT` | Connect timeout for provider API requests (seconds) | `2` |
diff --git a/config/settings.py b/config/settings.py
index cde9b9dd..4f367bd7 100644
--- a/config/settings.py
+++ b/config/settings.py
@@ -48,8 +48,8 @@ class Settings(BaseSettings):
provider_rate_window: int = Field(
default=60, validation_alias="PROVIDER_RATE_WINDOW"
)
- provider_max_concurrency: int | None = Field(
- default=None, validation_alias="PROVIDER_MAX_CONCURRENCY"
+ provider_max_concurrency: int = Field(
+ default=5, validation_alias="PROVIDER_MAX_CONCURRENCY"
)
# ==================== HTTP Client Timeouts ====================
diff --git a/providers/base.py b/providers/base.py
index ad4a7ebc..83c655af 100644
--- a/providers/base.py
+++ b/providers/base.py
@@ -18,7 +18,7 @@ class ProviderConfig(BaseModel):
base_url: str | None = None
rate_limit: int | None = None
rate_window: int = 60
- max_concurrency: int | None = None
+ max_concurrency: int = 5
http_read_timeout: float = 300.0
http_write_timeout: float = 10.0
http_connect_timeout: float = 2.0
diff --git a/providers/rate_limit.py b/providers/rate_limit.py
index 02123e25..110307a1 100644
--- a/providers/rate_limit.py
+++ b/providers/rate_limit.py
@@ -34,7 +34,7 @@ class GlobalRateLimiter:
self,
rate_limit: int = 40,
rate_window: float = 60.0,
- max_concurrency: int | None = None,
+ max_concurrency: int = 5,
):
# Prevent double initialization in singleton
if hasattr(self, "_initialized"):
@@ -44,7 +44,7 @@ class GlobalRateLimiter:
raise ValueError("rate_limit must be > 0")
if rate_window <= 0:
raise ValueError("rate_window must be > 0")
- if max_concurrency is not None and max_concurrency <= 0:
+ if max_concurrency <= 0:
raise ValueError("max_concurrency must be > 0")
self._rate_limit = rate_limit
@@ -53,18 +53,11 @@ class GlobalRateLimiter:
self._request_times: deque[float] = deque()
self._blocked_until: float = 0
self._lock = asyncio.Lock()
- self._concurrency_sem: asyncio.Semaphore | None = (
- asyncio.Semaphore(max_concurrency) if max_concurrency is not None else None
- )
+ self._concurrency_sem = asyncio.Semaphore(max_concurrency)
self._initialized = True
- concurrency_info = (
- f", max_concurrency={max_concurrency}"
- if max_concurrency is not None
- else ""
- )
logger.info(
- f"GlobalRateLimiter (Provider) initialized ({rate_limit} req / {rate_window}s{concurrency_info})"
+ f"GlobalRateLimiter (Provider) initialized ({rate_limit} req / {rate_window}s, max_concurrency={max_concurrency})"
)
@classmethod
@@ -72,7 +65,7 @@ class GlobalRateLimiter:
cls,
rate_limit: int | None = None,
rate_window: float | None = None,
- max_concurrency: int | None = None,
+ max_concurrency: int = 5,
) -> GlobalRateLimiter:
"""Get or create the singleton instance.
@@ -167,16 +160,13 @@ class GlobalRateLimiter:
async def concurrency_slot(self) -> AsyncIterator[None]:
"""Async context manager that holds one concurrency slot for a stream.
- Blocks until a slot is available when max_concurrency is set.
- Is a no-op when max_concurrency was not configured.
+ Blocks until a slot is available (controlled by max_concurrency).
"""
- if self._concurrency_sem is not None:
- await self._concurrency_sem.acquire()
+ await self._concurrency_sem.acquire()
try:
yield
finally:
- if self._concurrency_sem is not None:
- self._concurrency_sem.release()
+ self._concurrency_sem.release()
async def execute_with_retry(
self,
diff --git a/tests/api/test_dependencies.py b/tests/api/test_dependencies.py
index 2b36cd58..e50dffad 100644
--- a/tests/api/test_dependencies.py
+++ b/tests/api/test_dependencies.py
@@ -17,7 +17,7 @@ def _make_mock_settings(**overrides):
mock.nvidia_nim_api_key = "test_key"
mock.provider_rate_limit = 40
mock.provider_rate_window = 60
- mock.provider_max_concurrency = None
+ mock.provider_max_concurrency = 5
mock.open_router_api_key = "test_openrouter_key"
mock.lm_studio_base_url = "http://localhost:1234/v1"
mock.nim = NimSettings()
diff --git a/tests/providers/test_provider_rate_limit.py b/tests/providers/test_provider_rate_limit.py
index cd0e0706..c8d2061f 100644
--- a/tests/providers/test_provider_rate_limit.py
+++ b/tests/providers/test_provider_rate_limit.py
@@ -259,19 +259,6 @@ class TestProviderRateLimiter:
with pytest.raises(ValueError, match="max_concurrency must be > 0"):
GlobalRateLimiter(rate_limit=10, rate_window=60, max_concurrency=0)
- @pytest.mark.asyncio
- async def test_concurrency_slot_noop_when_not_configured(self):
- """concurrency_slot() is a no-op when max_concurrency is None."""
- GlobalRateLimiter.reset_instance()
- limiter = GlobalRateLimiter.get_instance(rate_limit=100, rate_window=60)
- assert limiter._concurrency_sem is None
-
- # Should not block and complete immediately
- entered = False
- async with limiter.concurrency_slot():
- entered = True
- assert entered
-
@pytest.mark.asyncio
async def test_concurrency_slot_limits_simultaneous_streams(self):
"""At most max_concurrency streams can hold a slot simultaneously."""