Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

AsyncGRPO: require a per-run secret on the local vLLM server; validate BF16 AdamW state on load (#17390)

Daniel Han41bee13710b5
diff --git a/tests/test_async_grpo_fast_inference.py b/tests/test_async_grpo_fast_inference.py
index 3b8dfd78515..d8ef334c7f7 100644
--- a/tests/test_async_grpo_fast_inference.py
+++ b/tests/test_async_grpo_fast_inference.py
@@ -10,6 +10,7 @@ import threading
import time
import types
import urllib.error
+import urllib.parse
import urllib.request
from pathlib import Path
@@ -242,6 +243,43 @@ def test_pause_holds_new_requests_until_resume(served):
assert box["out"]["choices"][0]["token_ids"] == [4, 5]
+def _raw_post(
+ url,
+ payload,
+ content_type = "application/json",
+):
+ request = urllib.request.Request(
+ url, data = json.dumps(payload).encode(), headers = {"Content-Type": content_type}
+ )
+ try:
+ with urllib.request.urlopen(request, timeout = 10) as response:
+ return response.status
+ except urllib.error.HTTPError as error:
+ return error.code
+
+
+def test_control_endpoints_need_the_url_secret(served):
+ server, llm = served
+ origin = "http://127.0.0.1:{}".format(urllib.parse.urlsplit(server.url).port)
+ adapter = {"lora_name": "other", "lora_path": "/tmp/other"}
+ for base in (origin, origin + "/wrong-token"):
+ assert _raw_post(base + "/v1/load_lora_adapter", adapter) == 404
+ assert _raw_post(base + "/pause", {}) == 404
+ with pytest.raises(urllib.error.HTTPError) as error:
+ urllib.request.urlopen(base + "/v1/models", timeout = 10)
+ assert error.value.code == 404
+ assert llm.llm_engine.added_loras == [] and server.engine.paused is False
+ assert _get(server, "/health") == (200, {"status": "ok"})
+
+
+def test_text_plain_posts_are_refused(served):
+ server, llm = served
+ adapter = {"lora_name": "other", "lora_path": "/tmp/other"}
+ assert _raw_post(server.url + "/v1/load_lora_adapter", adapter, "text/plain") == 415
+ assert _raw_post(server.url + "/pause", {}, "text/plain;charset=UTF-8") == 415
+ assert llm.llm_engine.added_loras == [] and server.engine.paused is False
+
+
def test_adapters_load_eagerly_serve_newest_and_unload_after_drain(served):
server, llm = served
engine = llm.llm_engine
diff --git a/tests/test_bf16_adamw.py b/tests/test_bf16_adamw.py
index 6f242401919..f31ef1b252d 100644
--- a/tests/test_bf16_adamw.py
+++ b/tests/test_bf16_adamw.py
@@ -355,3 +355,34 @@ def test_batched_updates_match_separate_groups_and_new_gradients(api):
continue
for key in ("step", "exp_avg", "exp_avg_sq"):
assert torch.equal(optimizer.state[p][key], control.state[q][key])
+
+
[email protected]("bad", ["short_moment", "wide_step", "empty_seed"])
+def test_load_state_dict_rejects_state_the_kernel_would_overrun(api, bad):
+ # The CUDA kernel masks moment reads / writes by the parameter's size and loads step / seed as scalars.
+ p = torch.nn.Parameter(torch.ones(8, 4, dtype = torch.bfloat16))
+ opt = api.AdamWBF16([p], lr = 1e-3)
+ p.grad = torch.ones_like(p)
+ opt.step()
+ state = opt.state_dict()
+ entry = state["state"][0]
+ if bad == "short_moment":
+ entry["exp_avg"] = torch.zeros(3)
+ elif bad == "wide_step":
+ entry["step"] = torch.zeros(4, dtype = torch.int64)
+ else:
+ entry["sr_seed"] = torch.zeros(0, dtype = torch.int64)
+ fresh = torch.nn.Parameter(torch.ones(8, 4, dtype = torch.bfloat16))
+ with pytest.raises(ValueError):
+ api.AdamWBF16([fresh], lr = 1e-3).load_state_dict(state)
+
+
+def test_load_state_dict_round_trip_still_loads(api):
+ p = torch.nn.Parameter(torch.ones(8, 4, dtype = torch.bfloat16))
+ opt = api.AdamWBF16([p], lr = 1e-3)
+ p.grad = torch.ones_like(p)
+ opt.step()
+ fresh = torch.nn.Parameter(torch.ones(8, 4, dtype = torch.bfloat16))
+ other = api.AdamWBF16([fresh], lr = 1e-3)
+ other.load_state_dict(opt.state_dict())
+ assert torch.equal(other.state[fresh]["exp_avg"], opt.state[p]["exp_avg"])
diff --git a/unsloth/models/_async_grpo_vllm.py b/unsloth/models/_async_grpo_vllm.py
index 4abf609faea..c4e5ac0a2f7 100644
--- a/unsloth/models/_async_grpo_vllm.py
+++ b/unsloth/models/_async_grpo_vllm.py
@@ -9,9 +9,11 @@ __all__ = [
"UnslothAsyncVLLMServer",
]
+import hmac
import itertools
import json
import logging
+import secrets
import threading
import time
import uuid
@@ -274,17 +276,36 @@ class _Handler(BaseHTTPRequestHandler):
# The rollout process exits with requests in flight when training ends.
self.close_connection = True
+ def _route(self):
+ # Any local process can reach 127.0.0.1, so the URL carries a per-run secret: "/<token>/<endpoint>".
+ _, _, rest = urlparse(self.path).path.partition("/")
+ token, slash, path = rest.partition("/")
+ if not slash or not hmac.compare_digest(token.encode(), self.server.owner._token.encode()):
+ return None
+ return "/" + path
+
def do_GET(self):
- self._send(*self.server.owner.get(urlparse(self.path).path))
+ path = self._route()
+ if path is None:
+ return self._send(404, {"error": "not found"})
+ self._send(*self.server.owner.get(path))
def do_POST(self):
owner = self.server.owner
length = int(self.headers.get("Content-Length") or 0)
+ path = self._route()
+ if path is None:
+ self.close_connection = True
+ return self._send(404, {"error": "not found"})
+ # TRL sends JSON or no body; a browser can POST text/plain cross-origin without a preflight.
+ content_type = self.headers.get("Content-Type")
+ if content_type and content_type.split(";")[0].strip().lower() != "application/json":
+ self.close_connection = True
+ return self._send(415, {"error": "Content-Type must be application/json"})
try:
body = json.loads(self.rfile.read(length) or b"{}") if length else {}
except ValueError as error:
return self._send(400, {"error": f"bad JSON: {error}"})
- path = urlparse(self.path).path
if path == "/v1/completions":
try:
return self._send(200, owner.finish(owner.start(body).result()))
@@ -323,7 +344,8 @@ class UnslothAsyncVLLMServer:
_keep_awake(llm)
self.engine = _Engine(llm, load_lora, sampling_params)
self.stopped = False
- self.url = f"http://127.0.0.1:{self._serve()}"
+ self._token = secrets.token_urlsafe(32)
+ self.url = f"http://127.0.0.1:{self._serve()}/{self._token}"
def _serve(self):
httpd = _Server(("127.0.0.1", 0), _Handler)
diff --git a/unsloth/optimizers/bf16_adamw.py b/unsloth/optimizers/bf16_adamw.py
index f8b896fdfb2..17f7fe24307 100644
--- a/unsloth/optimizers/bf16_adamw.py
+++ b/unsloth/optimizers/bf16_adamw.py
@@ -397,8 +397,21 @@ class AdamWBF16(torch.optim.Optimizer):
if original is None:
continue
state = self.state[p]
+ # The CUDA kernel indexes moments by the parameter's element count and reads step / seed as scalars,
+ # so a mismatched checkpoint must fail here, not read and write out of bounds on the next step.
for name in ("exp_avg", "exp_avg_sq"):
- state[name] = original[name].to(device = p.device, dtype = torch.float32)
+ if tuple(original[name].shape) != tuple(p.shape):
+ raise ValueError(
+ f"AdamWBF16 state {name} has shape {tuple(original[name].shape)}, "
+ f"but its parameter has shape {tuple(p.shape)}"
+ )
+ for name in ("step", "sr_seed"):
+ if original[name].numel() != 1:
+ raise ValueError(f"AdamWBF16 state {name} must be a single value")
+ for name in ("exp_avg", "exp_avg_sq"):
+ state[name] = (
+ original[name].to(device = p.device, dtype = torch.float32).contiguous()
+ )
state["step"] = original["step"].to(device = p.device, dtype = torch.int64)
state["sr_seed"] = original["sr_seed"].to(device = p.device, dtype = torch.int64)
self._initialize()