Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

AsyncGRPO vLLM server: raise the listen backlog past socketserver's 5 (#13381)

Daniel Han489d63ee9e3f
diff --git a/tests/test_async_grpo_fast_inference.py b/tests/test_async_grpo_fast_inference.py
index b24f1fe21d5..c78abb3a75e 100644
--- a/tests/test_async_grpo_fast_inference.py
+++ b/tests/test_async_grpo_fast_inference.py
@@ -207,18 +207,27 @@ def test_unset_fields_take_the_generation_config_defaults_and_set_fields_win(ser
def test_concurrent_requests_all_complete(served):
+ # 256 is well past socketserver's default listen backlog of 5: with that
+ # backlog a burst this size has connections reset every run, while 32 only
+ # tripped it now and then on a loaded runner.
server, _ = served
- results = [None] * 32
+ count = 256
+ results = [None] * count
+ errors = []
def worker(i):
- results[i] = _complete(server, [i])["choices"][0]["token_ids"]
+ try:
+ results[i] = _complete(server, [i])["choices"][0]["token_ids"]
+ except Exception as error:
+ errors.append(f"{i}: {type(error).__name__}: {error}")
- threads = [threading.Thread(target = worker, args = (i,)) for i in range(32)]
+ threads = [threading.Thread(target = worker, args = (i,)) for i in range(count)]
for thread in threads:
thread.start()
for thread in threads:
- thread.join(10)
- assert results == [[i + 1, i + 2] for i in range(32)]
+ thread.join(30)
+ assert not errors, errors[:5]
+ assert results == [[i + 1, i + 2] for i in range(count)]
def test_pause_holds_new_requests_until_resume(served):
diff --git a/unsloth/models/_async_grpo_vllm.py b/unsloth/models/_async_grpo_vllm.py
index eb671f78c3c..35c70108dbd 100644
--- a/unsloth/models/_async_grpo_vllm.py
+++ b/unsloth/models/_async_grpo_vllm.py
@@ -293,6 +293,14 @@ class _Handler(BaseHTTPRequestHandler):
self._send(*owner.post(path, body))
+class _Server(ThreadingHTTPServer):
+ # socketserver listens with a backlog of 5. Each in-flight completion is its
+ # own connection, so a burst of them can overflow the accept queue and have
+ # connections reset before a handler thread picks them up.
+ request_queue_size = 1024
+ daemon_threads = True
+
+
class UnslothAsyncVLLMServer:
"""Serves `model.vllm_engine` to TRL's AsyncGRPOTrainer on 127.0.0.1."""
@@ -320,8 +328,7 @@ class UnslothAsyncVLLMServer:
self.url = f"http://127.0.0.1:{self._serve()}"
def _serve(self):
- httpd = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
- httpd.daemon_threads = True
+ httpd = _Server(("127.0.0.1", 0), _Handler)
httpd.owner = self
self._thread = threading.Thread(
target = httpd.serve_forever, name = "unsloth-async-grpo-http", daemon = True