Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(engine): upload cuda graphs at capture so a stream wait cannot block the first launch (#697)

Xiaoze Fan9b68e0076769
diff --git a/python/freetoken/engine/graph.py b/python/freetoken/engine/graph.py
index eb9684f6..a995316d 100644
--- a/python/freetoken/engine/graph.py
+++ b/python/freetoken/engine/graph.py
@@ -1,6 +1,9 @@
from __future__ import annotations
+import ctypes
+import functools
import gc
+import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Dict, List
@@ -102,6 +105,30 @@ def get_free_memory(device: torch.device) -> int:
return torch.cuda.mem_get_info(device)[0]
+def _cu_graph_upload():
+ if sys.platform != "linux" or torch.version.hip is not None:
+ return None
+ fn = ctypes.CDLL("libcuda.so.1").cuGraphUpload
+ fn.argtypes = [ctypes.c_void_p, ctypes.c_void_p]
+ fn.restype = ctypes.c_int
+ return fn
+
+
+def _upload_graph(graph: torch.cuda.CUDAGraph, stream: torch.cuda.Stream) -> None:
+ """Upload the graph at capture time.
+
+ Otherwise the first replay pushes every node behind the graph's own stream WAIT, and a large graph then blocks cudaGraphLaunch until the WAIT is released. PLE wait-sync releases it only after the launch returns, so that deadlocks.
+ """
+ upload = _cu_graph_upload()
+ if upload is None:
+ return
+ rc = upload(graph.raw_cuda_graph_exec(), stream.cuda_stream)
+ if rc != 0:
+ raise RuntimeError(f"cuGraphUpload failed with CUresult {rc}")
+ stream.synchronize()
+
+
class GraphRunner:
def __init__(
self,
@@ -192,6 +219,7 @@ class GraphRunner:
# CUDA graph capture to replay cold-cache expert copies.
with torch.cuda.graph(graph, pool=pool, stream=self.stream):
self.buffer.logits[:bs] = model.forward()
+ _upload_graph(graph, self.stream)
self._reset_moe_offload_cache()
if pool is None:
pool = graph.pool() # reuse cuda graph handle to reduce memory