Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

1.2 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.

"""Launches the pinned vLLM server, run by the isolated engine Python, with the API key guarding '/'
(every route): vLLM's middleware only covers /v1, /v2, /inference (and /cohere in 0.30), not
/invocations or /tokenize; key comes from VLLM_API_KEY, never on the command line."""

import importlib
import runpy
import sys

# vLLM 0.26 keeps the prefix in serve/utils/server_utils, 0.30 in serve/middleware/authenticate.
for _name in (
    "vllm.entrypoints.serve.middleware.authenticate",
    "vllm.entrypoints.serve.utils.server_utils",
):
    try:
        guard = importlib.import_module(_name)
    except ImportError:
        continue
    if isinstance(getattr(guard, "GUARDED_PREFIX", None), tuple):
        guard.GUARDED_PREFIX = ("/",)
        break
else:
    # A vLLM that moved this again would silently reopen routes; refuse to start instead.
    raise SystemExit("Unsloth: vLLM's API-key middleware changed; refusing to start unguarded.")

if __name__ == "__main__":
    module = sys.argv.pop(1)
    runpy.run_module(module, run_name = "__main__", alter_sys = True)