Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
studio: reuse ssl setup for local gguf requests (#12495)
Maheswar Kumar6316710532a7
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.pyindex 1c6d9fb8ff9..7ada48eef08 100644--- a/studio/backend/core/inference/llama_cpp.py+++ b/studio/backend/core/inference/llama_cpp.py@@ -7453,8 +7453,8 @@ def _extra_args_have_tensor_split(@functools.lru_cache(maxsize = 1)-def _count_ssl_context() -> ssl.SSLContext:- # httpx loads the CA bundle per Client (~15 ms); admission counts once per chat request.+def _local_ssl_context() -> ssl.SSLContext:+ # httpx loads CA certificates even for loopback HTTP. Share that setup, not streaming sockets.return ssl.create_default_context()@@ -34999,6 +34999,7 @@ class LlamaCppBackend:timeout = stream_timeout,limits = httpx.Limits(max_keepalive_connections = 0),trust_env = False,+ verify = _local_ssl_context(),) as client:first_token_deadline = time.monotonic() + _DEFAULT_FIRST_TOKEN_TIMEOUT_Swith self._stream_with_retry(@@ -40207,7 +40208,7 @@ class LlamaCppBackend:timeout = 10,headers = self._auth_headers,trust_env = False,- verify = _count_ssl_context(),+ verify = _local_ssl_context(),) as client:def _tokenize(text: str) -> int:@@ -40543,6 +40544,7 @@ class LlamaCppBackend:timeout = httpx.Timeout(read_timeout, connect = 10),headers = self._auth_headers,trust_env = False,+ verify = _local_ssl_context(),) as client:finished = threading.Event()watcher: Optional[threading.Thread] = Nonediff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex fc0aa5666bc..5526d54d93a 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -139,7 +139,11 @@ from core.inference.llama_admission import (peek_llama_admission_snapshot,)from core.inference.tool_stream_exec import TOOL_APPROVAL_FLUSH_DELAY_S-from core.inference.llama_cpp import _llama_chunk_has_generated_output, requested_video_fps+from core.inference.llama_cpp import (+ _llama_chunk_has_generated_output,+ _local_ssl_context,+ requested_video_fps,+)from core.inference.llama_video_input import shrink_video_for_llama@@ -5521,6 +5525,7 @@ def _cancelable_nonstreaming_client() -> httpx.AsyncClient:return httpx.AsyncClient(limits = httpx.Limits(max_connections = 1, max_keepalive_connections = 0),trust_env = False,+ verify = _local_ssl_context(),)@@ -32604,6 +32609,7 @@ async def openai_completions(request: Request, current_subject: str = Depends(geclient = httpx.AsyncClient(timeout = _llama_streaming_generation_timeout(),trust_env = False,+ verify = _local_ssl_context(),)resp = Nonebytes_iter = None@@ -35010,6 +35016,7 @@ async def _responses_stream(client = httpx.AsyncClient(timeout = _llama_streaming_generation_timeout(),trust_env = False,+ verify = _local_ssl_context(),)resp = Nonelines_iter = None@@ -39245,6 +39252,7 @@ async def _anthropic_passthrough_stream(timeout = _llama_streaming_generation_timeout(),limits = httpx.Limits(max_keepalive_connections = 0),trust_env = False,+ verify = _local_ssl_context(),)resp = Nonelines_iter = None@@ -40518,6 +40526,7 @@ async def _openai_passthrough_stream_admitted(timeout = _llama_streaming_generation_timeout(),limits = httpx.Limits(max_keepalive_connections = 0),trust_env = False,+ verify = _local_ssl_context(),)_truncate_policy = _overflow_truncation_policy(payload)_truncate_budget = _OVERFLOW_TRUNCATE_MAX_RETRIES if _truncate_policy else 0diff --git a/studio/backend/tests/test_llama_cpp_stream_cancel.py b/studio/backend/tests/test_llama_cpp_stream_cancel.pyindex f87ce864507..8a8da3f806e 100644--- a/studio/backend/tests/test_llama_cpp_stream_cancel.py+++ b/studio/backend/tests/test_llama_cpp_stream_cancel.py@@ -14,6 +14,7 @@ import pytest_backend = os.path.join(os.path.dirname(__file__), "..")sys.path.insert(0, _backend)+from core.inference import llama_cppfrom core.inference.llama_cpp import LlamaCppBackend, _LlamaStreamCancelled@@ -160,33 +161,62 @@ class _StallUpstream:return-def test_cancel_interrupts_a_read_blocked_on_a_mid_stream_stall():+def test_streams_reuse_ssl_setup_without_sharing_connections(monkeypatch):+ backend = LlamaCppBackend(manages_processes = False)++ def repeated_ssl_setup(*_args, **_kwargs):+ pytest.fail("A later loopback request must not reload CA certificates")++ with _StallUpstream() as first_server, _StallUpstream() as second_server:+ with backend._open_stream(first_server.url, {}, None) as (first, _):+ assert next(first.iter_text()) == "data: hello\n\n"+ monkeypatch.setattr(llama_cpp.ssl, "create_default_context", repeated_ssl_setup)+ with backend._open_stream(second_server.url, {}, None) as (second, _):+ assert next(second.iter_text()) == "data: hello\n\n"+ assert first.extensions["network_stream"] is not second.extensions["network_stream"]+ assert second.is_closed+ assert not first.is_closed+ assert first.is_closed+++def test_route_upstream_clients_reuse_ssl_setup(monkeypatch):+ import asyncio++ from routes.inference import _cancelable_nonstreaming_client++ llama_cpp._local_ssl_context()++ def repeated_ssl_setup(*_args, **_kwargs):+ pytest.fail("A passthrough request must not reload CA certificates")++ monkeypatch.setattr(llama_cpp.ssl, "create_default_context", repeated_ssl_setup)+ first, second = _cancelable_nonstreaming_client(), _cancelable_nonstreaming_client()+ assert first is not second+ asyncio.run(first.aclose())+ asyncio.run(second.aclose())+++def test_cancel_interrupts_a_read_blocked_on_a_mid_stream_stall(monkeypatch):# Mid-stream stall: the reader is parked in recv() on a long bound read timeout,# so response.close() alone can't wake it; the watcher must shut the socket down.# Assert cancel lands in seconds, not at the far-off deadline (pre-fix: hung ~30s).+ monkeypatch.setattr(llama_cpp, "_DEFAULT_FIRST_TOKEN_TIMEOUT_S", 30)with _StallUpstream() as server:+ backend = LlamaCppBackend(manages_processes = False)cancel_event = threading.Event()def _cancel_soon():time.sleep(0.3)cancel_event.set()- threading.Thread(target = _cancel_soon, daemon = True).start()-started = time.monotonic()- with httpx.Client(- limits = httpx.Limits(max_keepalive_connections = 0), trust_env = False- ) as client:- with pytest.raises(_LlamaStreamCancelled):- with LlamaCppBackend._stream_with_retry(- client,- server.url,- {},- cancel_event,- first_token_deadline = started + 30,- ) as response:- for _chunk in response.iter_text():- pass # first chunk arrives, then the read blocks silently+ with pytest.raises(_LlamaStreamCancelled):+ with backend._open_stream(server.url, {}, cancel_event) as (response, _):+ chunks = response.iter_text()+ assert next(chunks) == "data: hello\n\n"+ threading.Thread(target = _cancel_soon, daemon = True).start()+ for _chunk in chunks:+ pass # the next read blocks silently until cancellationelapsed = time.monotonic() - startedassert elapsed < 10, f"cancel took {elapsed:.1f}s; the blocked read was not interrupted"