Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
AsyncGRPO vLLM server: raise the listen backlog past socketserver's 5 (#13381)
Daniel Han489d63ee9e3f
diff --git a/tests/test_async_grpo_fast_inference.py b/tests/test_async_grpo_fast_inference.pyindex b24f1fe21d5..c78abb3a75e 100644--- a/tests/test_async_grpo_fast_inference.py+++ b/tests/test_async_grpo_fast_inference.py@@ -207,18 +207,27 @@ def test_unset_fields_take_the_generation_config_defaults_and_set_fields_win(serdef test_concurrent_requests_all_complete(served):+ # 256 is well past socketserver's default listen backlog of 5: with that+ # backlog a burst this size has connections reset every run, while 32 only+ # tripped it now and then on a loaded runner.server, _ = served- results = [None] * 32+ count = 256+ results = [None] * count+ errors = []def worker(i):- results[i] = _complete(server, [i])["choices"][0]["token_ids"]+ try:+ results[i] = _complete(server, [i])["choices"][0]["token_ids"]+ except Exception as error:+ errors.append(f"{i}: {type(error).__name__}: {error}")- threads = [threading.Thread(target = worker, args = (i,)) for i in range(32)]+ threads = [threading.Thread(target = worker, args = (i,)) for i in range(count)]for thread in threads:thread.start()for thread in threads:- thread.join(10)- assert results == [[i + 1, i + 2] for i in range(32)]+ thread.join(30)+ assert not errors, errors[:5]+ assert results == [[i + 1, i + 2] for i in range(count)]def test_pause_holds_new_requests_until_resume(served):diff --git a/unsloth/models/_async_grpo_vllm.py b/unsloth/models/_async_grpo_vllm.pyindex eb671f78c3c..35c70108dbd 100644--- a/unsloth/models/_async_grpo_vllm.py+++ b/unsloth/models/_async_grpo_vllm.py@@ -293,6 +293,14 @@ class _Handler(BaseHTTPRequestHandler):self._send(*owner.post(path, body))+class _Server(ThreadingHTTPServer):+ # socketserver listens with a backlog of 5. Each in-flight completion is its+ # own connection, so a burst of them can overflow the accept queue and have+ # connections reset before a handler thread picks them up.+ request_queue_size = 1024+ daemon_threads = True++class UnslothAsyncVLLMServer:"""Serves `model.vllm_engine` to TRL's AsyncGRPOTrainer on 127.0.0.1."""@@ -320,8 +328,7 @@ class UnslothAsyncVLLMServer:self.url = f"http://127.0.0.1:{self._serve()}"def _serve(self):- httpd = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)- httpd.daemon_threads = True+ httpd = _Server(("127.0.0.1", 0), _Handler)httpd.owner = selfself._thread = threading.Thread(target = httpd.serve_forever, name = "unsloth-async-grpo-http", daemon = True