FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(engine): upload cuda graphs at capture so a stream wait cannot block the first launch (#697)
Xiaoze Fan9b68e0076769
diff --git a/python/freetoken/engine/graph.py b/python/freetoken/engine/graph.pyindex eb9684f6..a995316d 100644--- a/python/freetoken/engine/graph.py+++ b/python/freetoken/engine/graph.py@@ -1,6 +1,9 @@from __future__ import annotations+import ctypes+import functoolsimport gc+import sysfrom dataclasses import dataclassfrom typing import TYPE_CHECKING, Dict, List@@ -102,6 +105,30 @@ def get_free_memory(device: torch.device) -> int:return torch.cuda.mem_get_info(device)[0]+def _cu_graph_upload():+ if sys.platform != "linux" or torch.version.hip is not None:+ return None+ fn = ctypes.CDLL("libcuda.so.1").cuGraphUpload+ fn.argtypes = [ctypes.c_void_p, ctypes.c_void_p]+ fn.restype = ctypes.c_int+ return fn+++def _upload_graph(graph: torch.cuda.CUDAGraph, stream: torch.cuda.Stream) -> None:+ """Upload the graph at capture time.++ Otherwise the first replay pushes every node behind the graph's own stream WAIT, and a large graph then blocks cudaGraphLaunch until the WAIT is released. PLE wait-sync releases it only after the launch returns, so that deadlocks.+ """+ upload = _cu_graph_upload()+ if upload is None:+ return+ rc = upload(graph.raw_cuda_graph_exec(), stream.cuda_stream)+ if rc != 0:+ raise RuntimeError(f"cuGraphUpload failed with CUresult {rc}")+ stream.synchronize()++class GraphRunner:def __init__(self,@@ -192,6 +219,7 @@ class GraphRunner:# CUDA graph capture to replay cold-cache expert copies.with torch.cuda.graph(graph, pool=pool, stream=self.stream):self.buffer.logits[:bs] = model.forward()+ _upload_graph(graph, self.stream)self._reset_moe_offload_cache()if pool is None:pool = graph.pool() # reuse cuda graph handle to reduce memory