Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Wait for the model load before reading the unsloth run banner (#11679)
Daniel Hanaede992c207c
diff --git a/.github/scripts/serve-unsloth-run.sh b/.github/scripts/serve-unsloth-run.shindex 2079aee2ed1..067b2fcb6e6 100755--- a/.github/scripts/serve-unsloth-run.sh+++ b/.github/scripts/serve-unsloth-run.sh@@ -9,7 +9,7 @@# Usage:# serve-unsloth-run.sh --model REPO --gguf-variant VAR --port PORT \# [--gguf-file PATH] [--extra "--seed 3407 --temp 0"] \-# [--log-dir logs] [--health-timeout 300]+# [--log-dir logs] [--health-timeout 300] [--banner-timeout 300]# Why a helper and not inline YAML# * Every `unsloth run` invocation here is the *Unsloth server* under test.# A failure to come up healthy is class (a) "server/API regression" and@@ -39,6 +39,7 @@ PORT=""EXTRA=""LOG_DIR="logs"HEALTH_TIMEOUT="300"+BANNER_TIMEOUT="300"while [ "$#" -gt 0 ]; docase "$1" in@@ -49,6 +50,7 @@ while [ "$#" -gt 0 ]; do--extra) EXTRA="$2"; shift 2 ;;--log-dir) LOG_DIR="$2"; shift 2 ;;--health-timeout) HEALTH_TIMEOUT="$2"; shift 2 ;;+ --banner-timeout) BANNER_TIMEOUT="$2"; shift 2 ;;*) echo "serve-unsloth-run.sh: unknown arg '$1'" >&2; exit 2 ;;esacdone@@ -134,12 +136,24 @@ echo "[serve] /api/health healthy"# ── parse the API key from the banner ────────────────────────────────────# Match both the non-silent " API Key: <key>" and silent "API Key: <key>"# forms. We do NOT trust a fixed column count; we take the sk-unsloth-* token.+# The banner is printed once the model has LOADED, which /api/health does not+# wait for. That usually takes about 10s, but a 4.8 GB GGUF on a busy CPU+# runner has taken over 30s, so wait on the load itself: until the key is+# printed, the server exits, or --banner-timeout runs out.API_KEY=""-for _ in $(seq 1 30); do+for _ in $(seq 1 "$BANNER_TIMEOUT"); doAPI_KEY="$(grep -aoE 'sk-unsloth-[A-Za-z0-9_-]+' "$SERVER_LOG" 2>/dev/null | head -1 || true)"[ -n "$API_KEY" ] && break+ # The label is out but carries no sk-unsloth- token: the fallback below reads it.+ grep -aq 'API Key:' "$SERVER_LOG" 2>/dev/null && break+ if ! kill -0 "$SERVER_PID" 2>/dev/null; then+ server_fail "process exited before printing its banner (the model load failed)"+ fisleep 1done+if [ -z "$API_KEY" ] && ! grep -aq 'API Key:' "$SERVER_LOG" 2>/dev/null; then+ server_fail "no banner within ${BANNER_TIMEOUT}s of /api/health: the model is still loading"+fiif [ -z "$API_KEY" ]; then# Fallback: take whatever follows an "API Key:" label, in case the key# prefix scheme changes. Still a parse-fragility guard, not silent.diff --git a/tests/studio/test_serve_unsloth_run_waits_for_the_load.py b/tests/studio/test_serve_unsloth_run_waits_for_the_load.pynew file mode 100644index 00000000000..9b67d788aa8--- /dev/null+++ b/tests/studio/test_serve_unsloth_run_waits_for_the_load.py@@ -0,0 +1,173 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0+"""serve-unsloth-run.sh waits for the model load, not a fixed 30 seconds.++`unsloth run` answers /api/health before it loads the model and prints its banner, API key+included, only once the load is done. The script used to give the banner 30 one-second polls+after health. That is about 10s of work normally, but a 4.8 GB GGUF on a busy CPU runner took+longer, and the `connection (opencode, stable)` leg failed with "could not parse an API key+from the banner" while the model was still loading.++The script runs here against a stand-in `unsloth` on PATH, and a stand-in `sleep` that returns+at once and counts its calls, so the script's seconds are counted polls. The stand-in prints+the banner 40 polls after it first answers health, past the old window, in well under a second+of real time.+"""++from __future__ import annotations++import os+import shutil+import socket+import subprocess+import sys+import textwrap+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parents[2]+SCRIPT = ROOT / ".github" / "scripts" / "serve-unsloth-run.sh"+KEY = "sk-unsloth-standin0123456789"++pytestmark = pytest.mark.skipif(+ os.name == "nt" or shutil.which("setsid") is None or shutil.which("jq") is None,+ reason = "the script needs bash, setsid and jq, as on the Linux runner",+)++# `unsloth run -H HOST -p PORT ...`: healthy at once, banner BANNER_AFTER sleeps after the first+# health answer, or exit that many sleeps after it without a banner when BANNER_AFTER is negative.+STANDIN_UNSLOTH = textwrap.dedent(+ f"""\+ #!{sys.executable}+ import json, os, sys, threading, time+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer++ port = int(sys.argv[sys.argv.index("-p") + 1])+ after = int(os.environ["BANNER_AFTER"])+ counter = os.environ["SLEEP_COUNTER"]++ healthy_at = [] # the poll count when /api/health first answered++ def polls():+ try:+ return len(open(counter).read())+ except FileNotFoundError:+ return 0++ class Handler(BaseHTTPRequestHandler):+ def do_GET(self):+ body = {{"status": "healthy"}}+ if self.path.startswith("/api/health") and not healthy_at:+ healthy_at.append(polls())+ if self.path.startswith("/v1/models"):+ if self.headers.get("Authorization") != "Bearer {KEY}":+ self.send_response(401); self.end_headers(); return+ body = {{"data": [{{"id": "standin-model"}}]}}+ data = json.dumps(body).encode()+ self.send_response(200)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(data)))+ self.end_headers()+ self.wfile.write(data)++ def log_message(self, *args):+ pass++ # A slow start: health is not answered until the script has polled it this many times.+ while polls() < int(os.environ["START_AFTER"]):+ time.sleep(0.005)+ server = ThreadingHTTPServer(("127.0.0.1", port), Handler)+ threading.Thread(target = server.serve_forever, daemon = True).start()++ # Counted from health, so a slow start cannot spend the banner's polls before the wait begins.+ while not healthy_at:+ time.sleep(0.005)+ with open(os.environ["HEALTHY_AT"], "w") as out:+ out.write(str(healthy_at[0]))+ if after < 0:+ while polls() < healthy_at[0] - after:+ time.sleep(0.005)+ sys.exit(1)+ while polls() < healthy_at[0] + after:+ time.sleep(0.005)+ print(" API Key: {KEY}", flush = True)+ time.sleep(600)+ """+)++# Counts a call and returns: the script's `sleep 1` becomes one poll.+STANDIN_SLEEP = '#!/bin/sh\nprintf x >> "$SLEEP_COUNTER"\nexec /bin/sleep 0.01\n'+++def _free_port() -> int:+ with socket.socket() as sock:+ sock.bind(("127.0.0.1", 0))+ return sock.getsockname()[1]+++def _serve(+ tmp_path: Path,+ banner_after: int,+ start_after: int = 0,+) -> tuple[subprocess.CompletedProcess, int]:+ bin_dir = tmp_path / "bin"+ bin_dir.mkdir()+ for name, body in (("unsloth", STANDIN_UNSLOTH), ("sleep", STANDIN_SLEEP)):+ path = bin_dir / name+ path.write_text(body, encoding = "utf-8")+ path.chmod(0o755)+ counter = tmp_path / "sleeps"+ env = {key: value for key, value in os.environ.items() if not key.startswith("GITHUB_")}+ env.update(+ PATH = f"{bin_dir}{os.pathsep}{env['PATH']}",+ BANNER_AFTER = str(banner_after),+ START_AFTER = str(start_after),+ SLEEP_COUNTER = str(counter),+ HEALTHY_AT = str(tmp_path / "healthy_at"),+ STUDIO_HOME = str(tmp_path / "studio"),+ )+ port = _free_port()+ try:+ result = subprocess.run(+ [+ "bash",+ str(SCRIPT),+ "--gguf-file",+ "/standin/model.gguf",+ "--port",+ str(port),+ "--log-dir",+ str(tmp_path / "logs"),+ ],+ env = env,+ capture_output = True,+ text = True,+ timeout = 60,+ )+ finally:+ # The script leaves the server running for the steps after it, as CI wants.+ subprocess.run(["pkill", "-f", f"{bin_dir}/unsloth run "], capture_output = True)+ polls = len(counter.read_text()) if counter.exists() else 0+ healthy_at = tmp_path / "healthy_at"+ # Polls spent waiting for the banner, not starting the server.+ if healthy_at.exists():+ polls -= int(healthy_at.read_text())+ return result, polls+++def test_a_banner_after_the_old_30_second_window_is_still_read(tmp_path):+ # 15 polls to come up healthy, then 40 more to load: the load alone outlasts the old window.+ result, polls = _serve(tmp_path, banner_after = 40, start_after = 15)+ assert result.returncode == 0, result.stdout + result.stderr+ assert f"UNSLOTH_API_KEY={KEY}" in result.stdout+ assert "UNSLOTH_MODEL_ID=standin-model" in result.stdout+ assert polls >= 40, f"the banner came after {polls} polls, so the stand-in did not delay it"+++def test_a_server_that_dies_while_loading_fails_at_once_and_says_so(tmp_path):+ result, polls = _serve(tmp_path, banner_after = -5)+ assert result.returncode == 1+ assert "process exited before printing its banner" in result.stderr+ # Not the whole banner budget: the script noticed the exit within a few polls.+ assert polls < 30, f"the script kept polling a dead server for {polls} polls"