Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

studio: reuse ssl setup for local gguf requests (#12495)

Maheswar Kumar6316710532a7
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 1c6d9fb8ff9..7ada48eef08 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -7453,8 +7453,8 @@ def _extra_args_have_tensor_split(
@functools.lru_cache(maxsize = 1)
-def _count_ssl_context() -> ssl.SSLContext:
- # httpx loads the CA bundle per Client (~15 ms); admission counts once per chat request.
+def _local_ssl_context() -> ssl.SSLContext:
+ # httpx loads CA certificates even for loopback HTTP. Share that setup, not streaming sockets.
return ssl.create_default_context()
@@ -34999,6 +34999,7 @@ class LlamaCppBackend:
timeout = stream_timeout,
limits = httpx.Limits(max_keepalive_connections = 0),
trust_env = False,
+ verify = _local_ssl_context(),
) as client:
first_token_deadline = time.monotonic() + _DEFAULT_FIRST_TOKEN_TIMEOUT_S
with self._stream_with_retry(
@@ -40207,7 +40208,7 @@ class LlamaCppBackend:
timeout = 10,
headers = self._auth_headers,
trust_env = False,
- verify = _count_ssl_context(),
+ verify = _local_ssl_context(),
) as client:
def _tokenize(text: str) -> int:
@@ -40543,6 +40544,7 @@ class LlamaCppBackend:
timeout = httpx.Timeout(read_timeout, connect = 10),
headers = self._auth_headers,
trust_env = False,
+ verify = _local_ssl_context(),
) as client:
finished = threading.Event()
watcher: Optional[threading.Thread] = None
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index fc0aa5666bc..5526d54d93a 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -139,7 +139,11 @@ from core.inference.llama_admission import (
peek_llama_admission_snapshot,
)
from core.inference.tool_stream_exec import TOOL_APPROVAL_FLUSH_DELAY_S
-from core.inference.llama_cpp import _llama_chunk_has_generated_output, requested_video_fps
+from core.inference.llama_cpp import (
+ _llama_chunk_has_generated_output,
+ _local_ssl_context,
+ requested_video_fps,
+)
from core.inference.llama_video_input import shrink_video_for_llama
@@ -5521,6 +5525,7 @@ def _cancelable_nonstreaming_client() -> httpx.AsyncClient:
return httpx.AsyncClient(
limits = httpx.Limits(max_connections = 1, max_keepalive_connections = 0),
trust_env = False,
+ verify = _local_ssl_context(),
)
@@ -32604,6 +32609,7 @@ async def openai_completions(request: Request, current_subject: str = Depends(ge
client = httpx.AsyncClient(
timeout = _llama_streaming_generation_timeout(),
trust_env = False,
+ verify = _local_ssl_context(),
)
resp = None
bytes_iter = None
@@ -35010,6 +35016,7 @@ async def _responses_stream(
client = httpx.AsyncClient(
timeout = _llama_streaming_generation_timeout(),
trust_env = False,
+ verify = _local_ssl_context(),
)
resp = None
lines_iter = None
@@ -39245,6 +39252,7 @@ async def _anthropic_passthrough_stream(
timeout = _llama_streaming_generation_timeout(),
limits = httpx.Limits(max_keepalive_connections = 0),
trust_env = False,
+ verify = _local_ssl_context(),
)
resp = None
lines_iter = None
@@ -40518,6 +40526,7 @@ async def _openai_passthrough_stream_admitted(
timeout = _llama_streaming_generation_timeout(),
limits = httpx.Limits(max_keepalive_connections = 0),
trust_env = False,
+ verify = _local_ssl_context(),
)
_truncate_policy = _overflow_truncation_policy(payload)
_truncate_budget = _OVERFLOW_TRUNCATE_MAX_RETRIES if _truncate_policy else 0
diff --git a/studio/backend/tests/test_llama_cpp_stream_cancel.py b/studio/backend/tests/test_llama_cpp_stream_cancel.py
index f87ce864507..8a8da3f806e 100644
--- a/studio/backend/tests/test_llama_cpp_stream_cancel.py
+++ b/studio/backend/tests/test_llama_cpp_stream_cancel.py
@@ -14,6 +14,7 @@ import pytest
_backend = os.path.join(os.path.dirname(__file__), "..")
sys.path.insert(0, _backend)
+from core.inference import llama_cpp
from core.inference.llama_cpp import LlamaCppBackend, _LlamaStreamCancelled
@@ -160,33 +161,62 @@ class _StallUpstream:
return
-def test_cancel_interrupts_a_read_blocked_on_a_mid_stream_stall():
+def test_streams_reuse_ssl_setup_without_sharing_connections(monkeypatch):
+ backend = LlamaCppBackend(manages_processes = False)
+
+ def repeated_ssl_setup(*_args, **_kwargs):
+ pytest.fail("A later loopback request must not reload CA certificates")
+
+ with _StallUpstream() as first_server, _StallUpstream() as second_server:
+ with backend._open_stream(first_server.url, {}, None) as (first, _):
+ assert next(first.iter_text()) == "data: hello\n\n"
+ monkeypatch.setattr(llama_cpp.ssl, "create_default_context", repeated_ssl_setup)
+ with backend._open_stream(second_server.url, {}, None) as (second, _):
+ assert next(second.iter_text()) == "data: hello\n\n"
+ assert first.extensions["network_stream"] is not second.extensions["network_stream"]
+ assert second.is_closed
+ assert not first.is_closed
+ assert first.is_closed
+
+
+def test_route_upstream_clients_reuse_ssl_setup(monkeypatch):
+ import asyncio
+
+ from routes.inference import _cancelable_nonstreaming_client
+
+ llama_cpp._local_ssl_context()
+
+ def repeated_ssl_setup(*_args, **_kwargs):
+ pytest.fail("A passthrough request must not reload CA certificates")
+
+ monkeypatch.setattr(llama_cpp.ssl, "create_default_context", repeated_ssl_setup)
+ first, second = _cancelable_nonstreaming_client(), _cancelable_nonstreaming_client()
+ assert first is not second
+ asyncio.run(first.aclose())
+ asyncio.run(second.aclose())
+
+
+def test_cancel_interrupts_a_read_blocked_on_a_mid_stream_stall(monkeypatch):
# Mid-stream stall: the reader is parked in recv() on a long bound read timeout,
# so response.close() alone can't wake it; the watcher must shut the socket down.
# Assert cancel lands in seconds, not at the far-off deadline (pre-fix: hung ~30s).
+ monkeypatch.setattr(llama_cpp, "_DEFAULT_FIRST_TOKEN_TIMEOUT_S", 30)
with _StallUpstream() as server:
+ backend = LlamaCppBackend(manages_processes = False)
cancel_event = threading.Event()
def _cancel_soon():
time.sleep(0.3)
cancel_event.set()
- threading.Thread(target = _cancel_soon, daemon = True).start()
-
started = time.monotonic()
- with httpx.Client(
- limits = httpx.Limits(max_keepalive_connections = 0), trust_env = False
- ) as client:
- with pytest.raises(_LlamaStreamCancelled):
- with LlamaCppBackend._stream_with_retry(
- client,
- server.url,
- {},
- cancel_event,
- first_token_deadline = started + 30,
- ) as response:
- for _chunk in response.iter_text():
- pass # first chunk arrives, then the read blocks silently
+ with pytest.raises(_LlamaStreamCancelled):
+ with backend._open_stream(server.url, {}, cancel_event) as (response, _):
+ chunks = response.iter_text()
+ assert next(chunks) == "data: hello\n\n"
+ threading.Thread(target = _cancel_soon, daemon = True).start()
+ for _chunk in chunks:
+ pass # the next read blocks silently until cancellation
elapsed = time.monotonic() - started
assert elapsed < 10, f"cancel took {elapsed:.1f}s; the blocked read was not interrupted"