Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: retry Lemonade on a new port when its free port is taken before it binds (#12498)

Daniel Han7e81057d0a01
diff --git a/studio/backend/core/inference/lemonade_server.py b/studio/backend/core/inference/lemonade_server.py
index a74870f580f..f06fc584bb1 100644
--- a/studio/backend/core/inference/lemonade_server.py
+++ b/studio/backend/core/inference/lemonade_server.py
@@ -12,6 +12,7 @@ from __future__ import annotations
import json
import logging
+import re
import secrets
import socket
import subprocess
@@ -52,6 +53,18 @@ _CONFIG_OVERRIDES: dict[str, Any] = {
}
+# _find_free_port releases the port before lemond binds it, so another process can take it in
+# between. Only an exit that names the collision is retried, on a fresh port; any other early exit
+# is a real failure and is reported at once.
+_START_ATTEMPTS = 3
+_PORT_TAKEN = re.compile(
+ r"address already in use|eaddrinuse|only one usage of each socket address|errno 98\b|errno 48\b"
+ # A localized Windows message still carries the code; never a bare 10048, which can be a port.
+ r"|(?:winerror|errno|error)\s*:?\s*10048\b",
+ re.IGNORECASE,
+)
+
+
class LemonadeUnavailable(RuntimeError):
"""lemond is not running or stopped answering."""
@@ -135,61 +148,82 @@ class LemonadeServer:
self.cache_dir.mkdir(parents = True, exist_ok = True)
self.flm_model_dir.mkdir(parents = True, exist_ok = True)
self._write_config()
- port = self._find_free_port()
- api_key = secrets.token_urlsafe(32)
- env = child_env_without_native_path_secret()
- env["LEMONADE_API_KEY"] = api_key
- env["FLM_MODEL_PATH"] = str(self.flm_model_dir)
- env["FLM_DISABLE_UPDATE_CHECK"] = "1"
- cmd = [
- str(self.binary),
- str(self.cache_dir),
- str(self.config_dir),
- "--port",
- str(port),
- "--host",
- "127.0.0.1",
- "--no-broadcast",
- "--log-file",
- "disabled",
- ]
- logger.info("Starting Lemonade: %s", " ".join(cmd))
- self._tail.clear()
-
- def _spawn() -> subprocess.Popen:
- return subprocess.Popen(
- cmd,
- cwd = str(self.binary.parent),
- stdout = subprocess.PIPE,
- stderr = subprocess.STDOUT,
- stdin = subprocess.DEVNULL,
- text = True,
- encoding = "utf-8",
- errors = "replace",
- env = env,
- **windows_hidden_subprocess_kwargs(),
- **child_popen_kwargs(_kill_with_the_parent),
- )
-
- proc = spawn_on_lifetime_thread(_spawn)
- adopt_pid(proc.pid)
- self._process = proc
- self.port = port
- self.api_key = api_key
- # Shutdown may have swept children before this process was adopted.
- if is_process_shutting_down():
- self._kill_locked()
- raise LemonadeUnavailable("Unsloth is shutting down; not starting Lemonade.")
- self._drain_thread = threading.Thread(
- target = self._drain, args = (proc,), daemon = True, name = "lemond-drain"
- )
- self._drain_thread.start()
- if not self._wait_ready(timeout):
- # After the kill, which joins the drain thread, so the tail has the last lines.
- self._kill_locked()
+ for attempt in range(1, _START_ATTEMPTS + 1):
+ if self._start_once(timeout):
+ return
tail = self.log_tail()
+ if (
+ attempt < _START_ATTEMPTS
+ and not self._stop_requested.is_set()
+ and _PORT_TAKEN.search(tail)
+ ):
+ logger.warning(
+ "Lemonade lost its port to another process; retrying on a new one."
+ )
+ if is_process_shutting_down():
+ raise LemonadeUnavailable(
+ "Unsloth is shutting down; not starting Lemonade."
+ )
+ continue
raise LemonadeUnavailable(f"Lemonade did not start. Last output:\n{tail}")
+ def _start_once(self, timeout: float) -> bool:
+ """Spawn lemond on a fresh port and wait for it; False (with the child gone) if it never got ready."""
+ port = self._find_free_port()
+ api_key = secrets.token_urlsafe(32)
+ env = child_env_without_native_path_secret()
+ env["LEMONADE_API_KEY"] = api_key
+ env["FLM_MODEL_PATH"] = str(self.flm_model_dir)
+ env["FLM_DISABLE_UPDATE_CHECK"] = "1"
+ cmd = [
+ str(self.binary),
+ str(self.cache_dir),
+ str(self.config_dir),
+ "--port",
+ str(port),
+ "--host",
+ "127.0.0.1",
+ "--no-broadcast",
+ "--log-file",
+ "disabled",
+ ]
+ logger.info("Starting Lemonade: %s", " ".join(cmd))
+ self._tail.clear()
+
+ def _spawn() -> subprocess.Popen:
+ return subprocess.Popen(
+ cmd,
+ cwd = str(self.binary.parent),
+ stdout = subprocess.PIPE,
+ stderr = subprocess.STDOUT,
+ stdin = subprocess.DEVNULL,
+ text = True,
+ encoding = "utf-8",
+ errors = "replace",
+ env = env,
+ **windows_hidden_subprocess_kwargs(),
+ **child_popen_kwargs(_kill_with_the_parent),
+ )
+
+ proc = spawn_on_lifetime_thread(_spawn)
+ adopt_pid(proc.pid)
+ self._process = proc
+ self.port = port
+ self.api_key = api_key
+ # Shutdown may have swept children before this process was adopted.
+ if is_process_shutting_down():
+ self._kill_locked()
+ raise LemonadeUnavailable("Unsloth is shutting down; not starting Lemonade.")
+ self._drain_thread = threading.Thread(
+ target = self._drain, args = (proc,), daemon = True, name = "lemond-drain"
+ )
+ self._drain_thread.start()
+ if not self._wait_ready(timeout):
+ # After the kill, which joins the drain thread, so the tail has the last lines.
+ self._kill_locked()
+ return False
+ return True
+
def _drain(self, proc: subprocess.Popen) -> None:
try:
assert proc.stdout is not None
diff --git a/studio/backend/tests/test_npu_backend.py b/studio/backend/tests/test_npu_backend.py
index 5d75a5ea2d9..7a0bdd07188 100644
--- a/studio/backend/tests/test_npu_backend.py
+++ b/studio/backend/tests/test_npu_backend.py
@@ -537,6 +537,99 @@ def test_a_server_that_never_answers_is_reported(tmp_path):
assert not server.is_alive()
+def _held_port():
+ """A port another process could have taken between _find_free_port and lemond's bind."""
+ import socket
+
+ holder = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ holder.bind(("127.0.0.1", 0))
+ holder.listen(1)
+ return holder, holder.getsockname()[1]
+
+
+def _server(tmp_path, binary):
+ return LemonadeServer(
+ binary,
+ cache_dir = tmp_path / "cache",
+ config_dir = tmp_path / "config",
+ flm_model_dir = tmp_path / "flm",
+ )
+
+
+def test_a_port_taken_before_lemond_binds_it_is_retried_on_a_new_one(tmp_path, monkeypatch):
+ """The release-then-bind window lost a CI start to another xdist worker (#12493's run)."""
+ holder, taken = _held_port()
+ original = LemonadeServer._find_free_port
+ handed = []
+
+ def _find_free_port():
+ port = taken if not handed else original()
+ handed.append(port)
+ return port
+
+ monkeypatch.setattr(LemonadeServer, "_find_free_port", staticmethod(_find_free_port))
+ server = _server(tmp_path, _binary(tmp_path))
+ try:
+ server.start(timeout = 30)
+ assert server.is_alive()
+ assert handed[0] == taken and server.port == handed[-1] != taken
+ assert len(handed) == 2
+ finally:
+ holder.close()
+ server.close()
+
+
+def test_a_port_that_stays_taken_gives_up_after_the_attempts(tmp_path, monkeypatch):
+ holder, taken = _held_port()
+ handed = []
+ monkeypatch.setattr(
+ LemonadeServer,
+ "_find_free_port",
+ staticmethod(lambda: handed.append(taken) or taken),
+ )
+ server = _server(tmp_path, _binary(tmp_path))
+ try:
+ with pytest.raises(LemonadeUnavailable, match = "Address already in use"):
+ server.start(timeout = 30)
+ assert len(handed) == 3
+ assert not server.is_alive()
+ finally:
+ holder.close()
+ server.close()
+
+
+ "tail, collision",
+ [
+ ("OSError: [Errno 98] Address already in use", True),
+ ("bind: EADDRINUSE", True),
+ ("OSError: [WinError 10048] Normalerweise darf jede Socketadresse nur einmal", True),
+ ("socket error 10048", True),
+ ("Only one usage of each socket address is normally permitted", True),
+ ("listening on port 10048", False),
+ ("bind failed", False),
+ ("flm validate: memlock too low", False),
+ ],
+)
+def test_the_collision_pattern(tail, collision):
+ from core.inference.lemonade_server import _PORT_TAKEN
+ assert bool(_PORT_TAKEN.search(tail)) is collision
+
+
+def test_an_exit_that_is_not_a_port_collision_is_not_retried(tmp_path):
+ spawns = tmp_path / "spawns"
+ binary = tmp_path / "lemond"
+ binary.write_text(
+ f"#!/bin/sh\necho x >> '{spawns}'\necho 'bind failed'\nexit 3\n", encoding = "utf-8"
+ )
+ binary.chmod(0o755)
+ server = _server(tmp_path, binary)
+ with pytest.raises(LemonadeUnavailable, match = "bind failed"):
+ server.start(timeout = 10)
+ assert spawns.read_text(encoding = "utf-8").count("x") == 1
+ server.close()
+
+
def test_stop_interrupts_a_start_that_never_becomes_ready(tmp_path):
import threading