Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop tool calls from getting stuck in Running forever (#12234)

Nilay2b76e7d70658
diff --git a/studio/backend/storage/chat_generation_runs_db.py b/studio/backend/storage/chat_generation_runs_db.py
index 818cfc7d398..caf940e56af 100644
--- a/studio/backend/storage/chat_generation_runs_db.py
+++ b/studio/backend/storage/chat_generation_runs_db.py
@@ -7,6 +7,7 @@ from __future__ import annotations
import hashlib
import json
+import os
import secrets
import sqlite3
import threading
@@ -51,7 +52,7 @@ class _PoolEntry:
self.generation = generation
# Runs when the last reference goes, which for a short-lived thread is when its thread-local
# storage is torn down.
- self._closer = weakref.finalize(self, _close_quietly, conn)
+ self._closer = weakref.finalize(self, _close_quietly, conn, os.getpid())
def release(self) -> None:
"""Close now, and disarm the finalizer so it cannot close it a second time."""
@@ -121,7 +122,14 @@ _pool_generation = 0
_pool_lock = threading.Lock()
-def _close_quietly(conn: sqlite3.Connection) -> None:
+# Closing or freeing a handle in a forked child can block on a lock held when it forked.
+_inherited_by_fork: list[sqlite3.Connection] = []
+
+
+def _close_quietly(conn: sqlite3.Connection, owner_pid: int) -> None:
+ if os.getpid() != owner_pid:
+ _inherited_by_fork.append(conn)
+ return
try:
conn.close()
except Exception:
diff --git a/studio/backend/tests/test_chat_generation_runs_pool.py b/studio/backend/tests/test_chat_generation_runs_pool.py
index 9e467a674be..9345223834b 100644
--- a/studio/backend/tests/test_chat_generation_runs_pool.py
+++ b/studio/backend/tests/test_chat_generation_runs_pool.py
@@ -9,8 +9,14 @@ database. These are the properties that make that safe to do; each fails if the
"""
import gc
+import os
+import signal
import sqlite3
+import subprocess
+import sys
+import textwrap
import threading
+from pathlib import Path
import pytest
@@ -254,6 +260,46 @@ def test_a_short_lived_threads_connection_is_released_when_it_exits():
assert alive == [], "a dead thread must leave nothing behind in the registry"
[email protected](sys.platform == "win32", reason = "preexec_fn needs fork")
+def test_spawning_a_tool_while_another_thread_is_inside_sqlite_does_not_hang(tmp_path):
+ script = textwrap.dedent(
+ """
+ import subprocess, sys, threading
+ from storage import chat_generation_runs_db as runs_db
+
+ inside = threading.Event()
+ release = threading.Event()
+
+ def block():
+ inside.set()
+ release.wait()
+ return 1
+
+ def query():
+ conn = runs_db._connect()
+ conn.create_function("block", 0, block)
+ conn.execute("SELECT block()").fetchone()
+ conn.close()
+
+ worker = threading.Thread(target = query)
+ worker.start()
+ inside.wait()
+ subprocess.Popen([sys.executable, "-c", "pass"], preexec_fn = lambda: None).wait()
+ release.set()
+ worker.join()
+ """
+ )
+ backend = Path(runs_db.__file__).resolve().parents[1]
+ env = {**os.environ, "UNSLOTH_STUDIO_HOME": str(tmp_path), "PYTHONPATH": str(backend)}
+ proc = subprocess.Popen([sys.executable, "-c", script], env = env, start_new_session = True)
+ try:
+ assert proc.wait(timeout = 60) == 0
+ except subprocess.TimeoutExpired:
+ os.killpg(proc.pid, signal.SIGKILL)
+ proc.wait()
+ pytest.fail("the forked child hung on a SQLite lock another thread held at the fork")
+
+
def test_closing_the_keeper_drops_the_pool_even_when_there_was_no_keeper():
"""journal_mode=WAL declines on filesystems without shared memory, so those installs never have
a keeper. Retirement still calls close_wal_keeper_for and still needs the handle released."""
diff --git a/studio/backend/tests/test_hub_settings.py b/studio/backend/tests/test_hub_settings.py
index 6fb3ce2bd19..a4303f45fd1 100644
--- a/studio/backend/tests/test_hub_settings.py
+++ b/studio/backend/tests/test_hub_settings.py
@@ -19,6 +19,9 @@ sys.modules.setdefault("loggers", _loggers_stub)
import json
import os
+import signal
+import subprocess
+import textwrap
import time
import pytest
@@ -751,3 +754,41 @@ def test_a_saved_endpoint_stays_out_of_the_csp_the_operators_does_not(store):
assert csp_connect_sources() == (OPERATOR_DS,) and csp_asset_sources() == ()
hub_settings.set_hub_settings("", False)
assert csp_connect_sources() == ("https://huggingface.co", OPERATOR_DS)
+
+
[email protected](sys.platform == "win32", reason = "preexec_fn needs fork")
+def test_spawning_a_tool_while_another_thread_holds_the_hub_client_does_not_hang(tmp_path):
+ script = textwrap.dedent(
+ """
+ import subprocess, sys, threading
+ import utils.hub_settings
+ from huggingface_hub.utils import _http
+
+ client = _http.get_session()
+ pool_lock = client._transport._pool._optional_thread_lock._lock
+ inside = threading.Event()
+ release = threading.Event()
+
+ def hold_pool_lock():
+ with pool_lock:
+ inside.set()
+ release.wait()
+
+ worker = threading.Thread(target = hold_pool_lock)
+ worker.start()
+ inside.wait()
+ subprocess.Popen([sys.executable, "-c", "pass"], preexec_fn = lambda: None).wait()
+ release.set()
+ worker.join()
+ _http.close_session()
+ assert client.is_closed
+ """
+ )
+ env = {**os.environ, "UNSLOTH_STUDIO_HOME": str(tmp_path), "PYTHONPATH": _BACKEND_DIR}
+ proc = subprocess.Popen([sys.executable, "-c", script], env = env, start_new_session = True)
+ try:
+ assert proc.wait(timeout = 60) == 0
+ except subprocess.TimeoutExpired:
+ os.killpg(proc.pid, signal.SIGKILL)
+ proc.wait()
+ pytest.fail("the forked child hung closing the hub client another thread was using")
diff --git a/studio/backend/utils/hub_settings.py b/studio/backend/utils/hub_settings.py
index 5a945e715e4..156c8709c61 100644
--- a/studio/backend/utils/hub_settings.py
+++ b/studio/backend/utils/hub_settings.py
@@ -19,8 +19,10 @@ import os
import sqlite3
import sys
import threading
+import weakref
from contextlib import closing
from dataclasses import dataclass
+from functools import partial
from urllib.parse import urlsplit
from loggers import get_logger
@@ -268,6 +270,34 @@ def _bypass_proxy_for(url: str) -> None:
http._GLOBAL_CLIENT = None
+# huggingface_hub closes its shared client in every forked child, so any preexec_fn child (tools,
+# sidecar servers) can block before exec on httpx locks another thread held at the fork. Child
+# hooks run in registration order, which import order decides, so the parent disarms the client.
+_inherited_hub_clients: list = []
+
+
+def _close_if_owner(client_ref: weakref.ref, owner_pid: int) -> None:
+ client = client_ref()
+ if client is None:
+ return
+ if os.getpid() == owner_pid:
+ type(client).close(client)
+ else:
+ _inherited_hub_clients.append(client)
+
+
+def _keep_hub_client_open_in_forks() -> None:
+ http = sys.modules.get("huggingface_hub.utils._http")
+ client = getattr(http, "_GLOBAL_CLIENT", None)
+ if client is not None and "close" not in vars(client):
+ # A weak reference, so the client is still freed by refcount once dropped.
+ client.close = partial(_close_if_owner, weakref.ref(client), os.getpid())
+
+
+if hasattr(os, "register_at_fork"):
+ os.register_at_fork(before = _keep_hub_client_open_in_forks)
+
+
def _refresh_imported_hub_libraries() -> None:
constants = sys.modules.get("huggingface_hub.constants")
if constants is None or getattr(constants, "_staging_mode", False):