Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

26.5 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Exercise a running Studio and time every request."""

from __future__ import annotations

import argparse
import base64
import hashlib
import hmac
import json
import os
import secrets
import sqlite3
import statistics
import sys
import threading
import time
import urllib.error
import urllib.request
from pathlib import Path
from typing import Any, Optional


DEFAULT_PORTS = list(range(8888, 8896))


def split_model_ref(ref: str) -> tuple[str, Optional[str]]:
    """`repo:variant` into the two fields /api/inference/load takes."""
    repo, sep, variant = ref.rpartition(":")
    if not sep or not repo or "/" in variant or "\\" in variant:
        return ref, None
    return repo, variant


class Timed:
    """Every HTTP call this script makes, with how long it took."""

    def __init__(self) -> None:
        self.calls: list[dict[str, Any]] = []
        self._lock = threading.Lock()

    def record(
        self,
        method: str,
        path: str,
        ms: float,
        status: Any,
        note: str = "",
    ) -> None:
        with self._lock:
            self.calls.append(
                {
                    "at": time.strftime("%Y-%m-%dT%H:%M:%S"),
                    "method": method,
                    "path": path,
                    "ms": round(ms, 1),
                    "status": status,
                    "note": note,
                }
            )

    def slowest(self, n: int = 10) -> list[dict[str, Any]]:
        with self._lock:
            return sorted(self.calls, key = lambda c: -c["ms"])[:n]


TIMED = Timed()


class Credentials:
    """The session login returned, renewed when the access token expires."""

    def __init__(
        self,
        access: str,
        refresh: Optional[str] = None,
    ) -> None:
        self.access = access
        self.refresh = refresh
        self._lock = threading.Lock()

    def renew(self, base_url: str, stale: str) -> bool:
        """Replace `stale`; True when a fresh access token is now held."""
        with self._lock:
            if self.access != stale:
                return True
            if not self.refresh:
                return False
            status, body = _request_once(
                base_url, "POST", "/api/auth/refresh", {"refresh_token": self.refresh}, None, 30
            )
            if status != 200 or not isinstance(body, dict) or not body.get("access_token"):
                # Only a refusal spends the token.
                if status in (400, 401, 403):
                    self.refresh = None
                return False
            self.access = body["access_token"]
            self.refresh = body.get("refresh_token")
            return True


def _bearer(token: Any) -> Optional[str]:
    return token.access if isinstance(token, Credentials) else token


def _request(
    base_url: str,
    method: str,
    path: str,
    payload: Optional[dict] = None,
    token: Any = None,
    timeout: int = 900,
) -> tuple[int, Any]:
    """One call; an expired session is renewed and the call made once more."""
    bearer = _bearer(token)
    status, body = _request_once(base_url, method, path, payload, bearer, timeout)
    if status == 401 and isinstance(token, Credentials) and token.renew(base_url, bearer):
        status, body = _request_once(base_url, method, path, payload, token.access, timeout)
    return status, body


def _request_once(
    base_url: str,
    method: str,
    path: str,
    payload: Optional[dict],
    token: Optional[str],
    timeout: float,
) -> tuple[int, Any]:
    headers = {"Content-Type": "application/json"}
    if token:
        headers["Authorization"] = f"Bearer {token}"
    data = json.dumps(payload).encode() if payload is not None else None
    req = urllib.request.Request(base_url + path, data = data, headers = headers, method = method)
    start = time.monotonic()
    try:
        with urllib.request.urlopen(req, timeout = timeout) as response:
            body = response.read()
            ms = (time.monotonic() - start) * 1000.0
            TIMED.record(method, path, ms, response.status)
            try:
                return response.status, json.loads(body)
            except ValueError:
                return response.status, body.decode("utf-8", "replace")
    except urllib.error.HTTPError as exc:
        ms = (time.monotonic() - start) * 1000.0
        body = exc.read().decode("utf-8", "replace")
        TIMED.record(method, path, ms, exc.code, body[:400])
        return exc.code, body
    except Exception as exc:  # noqa: BLE001 - a transport failure is a result here
        ms = (time.monotonic() - start) * 1000.0
        TIMED.record(method, path, ms, "error", str(exc)[:400])
        return 0, str(exc)


# /load and /unload commit a 200 after ~15s and pad the body so a proxy cannot time out.
_DEFERRED_ERROR_KEY = "_deferred_error"


def padded_route_failure(status: int, body: Any) -> Optional[str]:
    """Why a /load or /unload reply is not a success, or None when it is."""
    if status != 200:
        return str(body)[:800]
    if not isinstance(body, dict) or not body:
        return "the connection closed before the server's reply arrived (empty or truncated body)"
    deferred = body.get(_DEFERRED_ERROR_KEY)
    if isinstance(deferred, dict):
        return f"deferred {deferred.get('status_code')}: {str(deferred.get('detail'))[:700]}"
    return None


def _stream_events(
    base_url: str,
    path: str,
    payload: dict,
    token: Any,
    timeout: int = 900,
) -> tuple[int, list[dict], Optional[str]]:
    """POST a streaming request and return every parsed `data:` object."""
    bearer = _bearer(token)
    result = _stream_events_once(base_url, path, payload, bearer, timeout)
    if result[0] == 401 and isinstance(token, Credentials) and token.renew(base_url, bearer):
        result = _stream_events_once(base_url, path, payload, token.access, timeout)
    return result


def _stream_events_once(
    base_url: str, path: str, payload: dict, token: Optional[str], timeout: int
) -> tuple[int, list[dict], Optional[str]]:
    headers = {
        "Content-Type": "application/json",
        "Accept": "text/event-stream",
        # Without this header, tool_start/tool_end are suppressed for external clients.
        "X-Unsloth-Events": "1",
    }
    if token:
        headers["Authorization"] = f"Bearer {token}"
    req = urllib.request.Request(
        base_url + path, data = json.dumps(payload).encode(), headers = headers, method = "POST"
    )
    start = time.monotonic()
    events: list[dict] = []
    error: Optional[str] = None
    done = False
    try:
        with urllib.request.urlopen(req, timeout = timeout) as response:
            for raw in response:
                line = raw.decode("utf-8", "replace").strip()
                if not line.startswith("data:"):
                    continue
                data = line[5:].strip()
                if data == "[DONE]":
                    done = True
                    break
                try:
                    parsed = json.loads(data)
                except ValueError:
                    continue
                if not isinstance(parsed, dict):
                    continue
                events.append(parsed)
                if "error" in parsed and error is None:
                    detail = parsed["error"]
                    message = detail.get("message") if isinstance(detail, dict) else detail
                    error = f"stream error: {str(message)[:300]}"
            if error is None and not done:
                error = "stream ended without [DONE]: the turn did not complete"
            TIMED.record("POST", path, (time.monotonic() - start) * 1000.0, response.status)
            return response.status, events, error
    except urllib.error.HTTPError as exc:
        body = exc.read().decode("utf-8", "replace")
        TIMED.record("POST", path, (time.monotonic() - start) * 1000.0, exc.code, body[:400])
        return exc.code, events, body[:400]
    except Exception as exc:  # noqa: BLE001 - a transport failure is a result here
        TIMED.record("POST", path, (time.monotonic() - start) * 1000.0, "error", str(exc)[:400])
        return 0, events, str(exc)[:400]


def discover_port(explicit: Optional[int]) -> int:
    ports = [explicit] if explicit else DEFAULT_PORTS
    for port in ports:
        status, body = _request(f"http://127.0.0.1:{port}", "GET", "/api/liveness", timeout = 5)
        if status == 200 and isinstance(body, dict) and body.get("service") == "Unsloth UI Backend":
            return port
    raise SystemExit(
        "no Unsloth Studio backend answered /api/liveness on "
        + ", ".join(str(p) for p in ports)
        + ". Start Unsloth Studio first, or pass --port."
    )


def default_studio_home() -> str:
    """Studio's own precedence (utils/paths/storage_roots.studio_root)"""
    override = (os.environ.get("UNSLOTH_STUDIO_HOME") or "").strip()
    if not override:
        override = (os.environ.get("STUDIO_HOME") or "").strip()
    if override:
        return override
    master = (os.environ.get("UNSLOTH_HOME") or "").strip()
    if master:
        return str(Path(master).expanduser() / "studio")
    return str(Path.home() / ".unsloth" / "studio")


def resolve_studio_home(value: str) -> Path:
    """The same normalization Studio applies"""
    home = Path(value.strip()).expanduser()
    try:
        return home.resolve()
    except (OSError, ValueError):
        return home


class _NoRedirect(urllib.request.HTTPRedirectHandler):
    def redirect_request(self, req, fp, code, msg, headers, newurl):
        raise urllib.error.HTTPError(
            req.full_url, code, f"refusing redirect to {newurl}", headers, fp
        )


def studio_identity_error(
    base_url: str,
    home: Path,
    timeout: float = 10,
) -> Optional[str]:
    """None when `base_url` signs our nonce with this home's secret (storage.compute_identity_proof), else why not.
    /api/liveness is public, so anything holding the port can claim to be Studio."""
    nonce = secrets.token_bytes(32)
    url = f"{base_url}/api/auth/identity?nonce={base64.urlsafe_b64encode(nonce).decode()}"
    start = time.monotonic()
    try:
        # No redirects: one could relay a real Studio's proof.
        with urllib.request.build_opener(_NoRedirect).open(url, timeout = timeout) as response:
            proof = json.loads(response.read(65536)).get("proof")
        TIMED.record("GET", "/api/auth/identity", (time.monotonic() - start) * 1000.0, 200)
    except Exception as exc:  # noqa: BLE001 - any failure means unverified
        TIMED.record(
            "GET",
            "/api/auth/identity",
            (time.monotonic() - start) * 1000.0,
            "error",
            str(exc)[:400],
        )
        return f"/api/auth/identity did not answer ({str(exc)[:200]})"
    # Read after the challenge: Studio creates the secret on the first one it answers.
    db = home / "auth" / "auth.db"
    try:
        conn = sqlite3.connect(f"{db.resolve().as_uri()}?mode=rw", uri = True, timeout = 10)
        try:
            row = conn.execute(
                "SELECT value FROM app_secrets WHERE key = 'studio_identity_secret'"
            ).fetchone()
        finally:
            conn.close()
        secret = bytes.fromhex(row[0])
    except Exception as exc:  # noqa: BLE001
        return f"could not read the identity secret from {db} ({str(exc)[:200]}); pass --home"
    host, _, port = base_url.rpartition("//")[2].rpartition(":")
    expected = hmac.new(
        secret, b"|".join([nonce, host.encode(), port.encode()]), hashlib.sha256
    ).hexdigest()
    if not isinstance(proof, str) or not hmac.compare_digest(proof, expected):
        return "its identity proof does not match this installation"
    return None


def authenticate(base_url: str, home: Path, password: Optional[str]) -> Credentials:
    """Log in, rotating the bootstrap credential when this home has never been used."""
    boot_file = home / "auth" / ".bootstrap_password"
    if not password:
        raise SystemExit(
            "no password given. Pass --password (or set UNSLOTH_STUDIO_PASSWORD): on a "
            "Unsloth Studio that has never been opened it becomes the account password, on one "
            "that has it must be the password you sign in with."
        )
    # Before any password is sent: the port may be held by something other than Studio.
    reason = studio_identity_error(base_url, home)
    if reason:
        raise SystemExit(
            f"refusing to send a password to {base_url}: {reason}. It may not be this "
            f"machine's Unsloth Studio (home {home})."
        )
    # An EMPTY file is a rotated installation
    bootstrap = boot_file.read_text(encoding = "utf-8").strip() if boot_file.exists() else ""
    if bootstrap:
        secret = bootstrap
        rotate = True
    else:
        secret = password
        rotate = False

    status, body = _request(
        base_url, "POST", "/api/auth/login", {"username": "unsloth", "password": secret}
    )
    if status != 200:
        raise SystemExit(f"login failed ({status}): {str(body)[:300]}")
    token = body["access_token"]
    refresh = body.get("refresh_token")

    if rotate:
        status, body = _request(
            base_url,
            "POST",
            "/api/auth/change-password",
            {"current_password": secret, "new_password": password},
            token = token,
        )
        if status != 200:
            raise SystemExit(f"password rotation failed ({status}): {str(body)[:300]}")
        token = body["access_token"]
        refresh = body.get("refresh_token")
        print("  rotated the bootstrap password to the one given with --password")
    return Credentials(token, refresh)


STATUS_INTERVAL_S = 5.0
STATUS_READ_TIMEOUT_S = 10.0


class StatusPoller(threading.Thread):
    """Poll /api/inference/status the way the frontend does, and time it."""

    def __init__(
        self,
        base_url: str,
        token: Any,
        interval: float = STATUS_INTERVAL_S,
        read_timeout: float = STATUS_READ_TIMEOUT_S,
    ) -> None:
        super().__init__(daemon = True)
        self.base_url = base_url
        self.token = token
        self.interval = interval
        self.read_timeout = read_timeout
        self.polls: list[tuple[float, float, bool]] = []
        self._stop_event = threading.Event()
        self.in_flight_since: Optional[float] = None

    @property
    def durations(self) -> list[float]:
        return [ms for _, ms, _ in self.polls]

    def run(self) -> None:
        while not self._stop_event.is_set():
            start = time.monotonic()
            self.in_flight_since = start
            status, body = _request(
                self.base_url,
                "GET",
                "/api/inference/status",
                token = self.token,
                timeout = self.read_timeout,
            )
            self.in_flight_since = None
            elapsed = time.monotonic() - start
            # Windows timer granularity can end a timed-out read a few ms early.
            timed_out = status == 0 and (elapsed >= self.read_timeout or "timed out" in str(body))
            self.polls.append((start, elapsed * 1000.0, timed_out))
            self._stop_event.wait(max(0.0, self.interval - elapsed))

    def stalls_ms(self) -> list[float]:
        """Length of each run of consecutive abandoned reads."""
        runs: list[float] = []
        run_start: Optional[float] = None
        run_end = 0.0
        for start, ms, timed_out in self.polls:
            if timed_out:
                run_start = start if run_start is None else run_start
                run_end = start + ms / 1000.0
            elif run_start is not None:
                runs.append((start + ms / 1000.0 - run_start) * 1000.0)
                run_start = None
        if run_start is not None:
            runs.append((run_end - run_start) * 1000.0)
        return runs

    def stop(self) -> None:
        self._stop_event.set()

    def in_flight_ms(self) -> Optional[float]:
        """How long the current poll has been waiting, if one is."""
        since = self.in_flight_since
        return None if since is None else (time.monotonic() - since) * 1000.0


TOOL_REJECTED_MESSAGE = "The user declined to run this tool call."
TOOL_NOT_RUN_PREFIXES = ("Unsloth did not ", "Unsloth stopped this tool call")
TOOL_ERROR_PREFIXES = (
    "Error:",
    "Error ",
    "Search failed",
    "Execution error",
    "Blocked:",
    "Exit code",
    "Failed to fetch",
    "Failed to resolve",
    "No query provided",
)


def tool_end_failure(result: Any) -> Optional[str]:
    """Why a tool_end did not come from an executed tool, or None if it did."""
    if not isinstance(result, str) or not result.strip():
        return "empty result"
    text = result.strip()
    if text == TOOL_REJECTED_MESSAGE:
        return "declined before running"
    if text.startswith(TOOL_ERROR_PREFIXES) or text.startswith(TOOL_NOT_RUN_PREFIXES):
        return text[:160]
    return None


def chat(
    base_url: str,
    token: Any,
    model: str,
    prompt: str,
    *,
    tools: bool = False,
    enabled_tools: Optional[list[str]] = None,
) -> dict[str, Any]:
    payload: dict[str, Any] = {
        "model": model,
        "messages": [{"role": "user", "content": prompt}],
        "max_tokens": 256,
        "stream": False,
    }
    if tools:
        # Streamed: only the stream's tool_start/tool_end events show a tool ran.
        names = enabled_tools or ["web_search"]
        payload["stream"] = True
        payload["enable_tools"] = True
        payload["enabled_tools"] = names
        payload["tool_choice"] = {"type": "function", "function": {"name": names[0]}}
        payload["permission_mode"] = "off"
        status, events, error = _stream_events(
            base_url, "/v1/chat/completions", payload, token = token
        )
        text = ""
        started: list[str] = []
        finished: list[str] = []
        failed: list[dict[str, str]] = []
        for event in events:
            kind = event.get("type")
            if kind == "tool_start":
                started.append(str(event.get("tool_name") or ""))
            elif kind == "tool_end":
                name = str(event.get("tool_name") or "")
                why = tool_end_failure(event.get("result"))
                if why is None:
                    finished.append(name)
                else:
                    failed.append({"tool": name, "why": why})
            for choice in event.get("choices", []) or []:
                text += (choice.get("delta") or {}).get("content") or ""
        if error:
            reason = error
        elif finished:
            reason = None
        elif failed:
            reason = "tool ended without executing: " + "; ".join(f["why"] for f in failed)[:300]
        else:
            reason = "no tool_end event: the turn executed no tool"
        return {
            "status": status,
            "ok": status == 200 and bool(finished) and error is None,
            "chars": len(text),
            "tool_calls": len(finished),
            "tools_started": started,
            "tools_run": finished,
            "tools_failed": failed,
            "text": text[:600],
            "error": reason,
        }
    status, body = _request(base_url, "POST", "/v1/chat/completions", payload, token = token)
    text = ""
    if status == 200 and isinstance(body, dict):
        for choice in body.get("choices", []) or []:
            message = choice.get("message") or {}
            text += message.get("content") or ""
    return {
        "status": status,
        "ok": status == 200 and bool(text.strip()),
        "chars": len(text),
        "tool_calls": 0,
        "text": text[:600],
        "error": None if status == 200 else str(body)[:400],
    }


def main() -> int:
    parser = argparse.ArgumentParser(description = __doc__)
    parser.add_argument("--model", default = "unsloth/Qwen3.5-2B-MTP-GGUF:UD-Q4_K_XL")
    parser.add_argument("--out", default = ".", help = "directory for scenario-results.json")
    parser.add_argument("--port", type = int, default = None)
    parser.add_argument("--password", default = os.environ.get("SAC_PROBE_STUDIO_PASSWORD"))
    parser.add_argument(
        "--home",
        default = default_studio_home(),
        help = "Unsloth Studio home (where auth/ lives); follows UNSLOTH_STUDIO_HOME like Unsloth Studio does",
    )
    parser.add_argument(
        "--poll-seconds",
        type = float,
        default = 5.0,
        help = "status poll cadence; 5s matches the frontend",
    )
    args = parser.parse_args()

    out_dir = Path(args.out)
    out_dir.mkdir(parents = True, exist_ok = True)

    port = discover_port(args.port)
    base_url = f"http://127.0.0.1:{port}"
    print(f"Unsloth Studio on {base_url}")

    token = authenticate(base_url, resolve_studio_home(args.home), args.password)
    print("authenticated")

    status, body = _request(base_url, "GET", "/api/settings/llama-cpp-path", token = token)
    runtime = (
        body
        if status == 200 and isinstance(body, dict)
        else {"error": f"{status}: {str(body)[:200]}"}
    )
    (out_dir / "runtime-selection.json").write_text(json.dumps(runtime, indent = 2), encoding = "utf-8")
    print(
        f"runtime: {runtime.get('resolved_binary') or runtime.get('error') or 'unresolved'} ({runtime.get('source')})"
    )

    poller = StatusPoller(base_url, token, args.poll_seconds)
    poller.start()

    repo, variant = split_model_ref(args.model)
    results: dict[str, Any] = {
        "model": args.model,
        "model_path": repo,
        "gguf_variant": variant,
        "port": port,
        "runtime": runtime,
        "steps": {},
    }
    load_payload: dict[str, Any] = {"model_path": repo}
    if variant:
        load_payload["gguf_variant"] = variant
    try:
        # A resident model makes /load a no-op, so no PE would load inside the window.
        status, body = _request(
            base_url,
            "POST",
            "/api/inference/unload",
            {"model_path": repo},
            token = token,
            timeout = 300,
        )
        results["evicted_before_load"] = {"status": status, "body": str(body)[:200]}

        print(f"loading {args.model} ...")
        status, body = _request(
            base_url,
            "POST",
            "/api/inference/load",
            load_payload,
            token = token,
            timeout = 1800,
        )
        load_error = padded_route_failure(status, body)
        if load_error is None and isinstance(body, dict) and body.get("status") == "already_loaded":
            load_error = (
                "the model was already resident, so this load started no llama-server and "
                "loaded no PE; the evidence window contains nothing for it"
            )
        elif load_error is None and isinstance(body, dict) and body.get("is_gguf") is not True:
            load_error = (
                "the load answered is_gguf false, so no llama-server started and no llama.cpp PE "
                "was loaded; pass a GGUF model with -Model"
            )
        results["steps"]["load"] = {
            "status": status,
            "ok": load_error is None,
            "error": load_error,
        }
        if load_error is not None:
            print(f"  load FAILED ({status}): {load_error[:400]}")
        else:
            print("  loaded")

            print("inference ...")
            results["steps"]["inference"] = chat(
                base_url,
                token,
                repo,
                "Reply with exactly one short sentence about the Antarctic.",
            )
            print(f"  {results['steps']['inference']['chars']} chars")

            print("web search ...")
            results["steps"]["web_search"] = chat(
                base_url,
                token,
                repo,
                "Search the web for today's date in Reykjavik and tell me what you found.",
                tools = True,
                enabled_tools = ["web_search"],
            )
            print(f"  tool calls: {results['steps']['web_search']['tool_calls']}")

            print("tool calls ...")
            results["steps"]["tool_calls"] = chat(
                base_url,
                token,
                repo,
                "Use your tools to look up what the tallest building in the world is.",
                tools = True,
                enabled_tools = ["web_search"],
            )
            print(f"  tool calls: {results['steps']['tool_calls']['tool_calls']}")

            status, body = _request(
                base_url,
                "POST",
                "/api/inference/unload",
                {"model_path": repo},
                token = token,
                timeout = 300,
            )
            unload_error = padded_route_failure(status, body)
            results["steps"]["unload"] = {
                "status": status,
                "ok": unload_error is None,
                "error": unload_error,
            }
            print(
                "  unloaded"
                if unload_error is None
                else f"  unload FAILED ({status}): {unload_error[:200]}"
            )
    finally:
        poller.stop()
        poller.join(timeout = STATUS_READ_TIMEOUT_S + 15)

    durations = poller.durations
    in_flight = poller.in_flight_ms() if poller.is_alive() else None
    stalls = poller.stalls_ms()
    results["status_poll"] = {
        "count": len(durations),
        "in_flight_ms": None if in_flight is None else round(in_flight, 1),
        "max_ms": round(max(durations), 1) if durations else None,
        "median_ms": round(statistics.median(durations), 1) if durations else None,
        "abandoned_reads": sum(1 for _, _, t in poller.polls if t),
        "stalls_ms": [round(x, 1) for x in stalls],
        "over_10s": sum(1 for d in durations if d >= 10_000),
        "over_75s": sum(1 for x in stalls if x >= 75_000),
    }
    results["slowest_calls"] = TIMED.slowest(15)
    results["all_calls"] = TIMED.calls

    path = out_dir / "scenario-results.json"
    path.write_text(json.dumps(results, indent = 2), encoding = "utf-8")

    print()
    print("=" * 60)
    for name, step in results["steps"].items():
        print(f"{name:12s} {'ok' if step.get('ok') else 'FAILED'}  ({step.get('status')})")
    poll = results["status_poll"]
    print(
        f"status polls  {poll['count']}, median {poll['median_ms']} ms, max {poll['max_ms']} ms, "
        f"{poll['abandoned_reads']} abandoned at {STATUS_READ_TIMEOUT_S:.0f}s, "
        f"{poll['over_75s']} stall(s) over 75s"
    )
    print(f"written to {path}")

    return 0 if all(s.get("ok") for s in results["steps"].values()) else 1


if __name__ == "__main__":
    sys.exit(main())