FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(server)!: add --gpu to choose the GPU on multi-GPU machines (#117)
Xiaoze Fan2757bb5f9115
diff --git a/benchmarks/bench_decode_moe.py b/benchmarks/bench_decode_moe.pyindex 723c2be5..56621792 100644--- a/benchmarks/bench_decode_moe.py+++ b/benchmarks/bench_decode_moe.py@@ -100,6 +100,8 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:help="hybrid: max PCIe fetches/layer; -1 = auto (benched pcie/cpu bandwidth fraction)",)p.add_argument("--mem-ratio", type=float, default=0.9, help="target VRAM utilization")+ p.add_argument("--gpu", default=None,+ help="GPU for the serve: a UUID or nvidia-smi index (as ft serve --gpu)")p.add_argument("--no-graph", action="store_true", help="eager decode instead of CUDA graph")p.add_argument("--greedy",@@ -183,6 +185,8 @@ def serve_cmd(args: argparse.Namespace, backend: str, port: int) -> list[str]:"--cuda-graph-max-bs", "0" if args.no_graph else "1","--moe-hybrid-max-fetch", str(args.hybrid_fetch),]+ if args.gpu:+ cmd += ["--gpu", args.gpu]if args.cache > 0:cmd += ["--moe-cache-size", str(args.cache)]elif args.cache_rate is not None:diff --git a/benchmarks/bench_load_weight_generic.py b/benchmarks/bench_load_weight_generic.pyindex 0ef4bd4c..0e1905a0 100644--- a/benchmarks/bench_load_weight_generic.py+++ b/benchmarks/bench_load_weight_generic.py@@ -140,8 +140,10 @@ def _model_config(model_path: str):if try_get_tp_info() is None:set_tp_info(rank=0, size=1)- torch.cuda.set_device(0)- torch.zeros(1, device="cuda") # init CUDA context (pinning / nvfp4 backend pick)+ from freetoken.gpu_select import bind_assigned_gpu++ dev = bind_assigned_gpu()+ torch.zeros(1, device=dev) # init CUDA context (pinning / nvfp4 backend pick)cfg = EngineConfig(model_path=model_path, tp_info=DistributedInfo(0, 1),dtype=torch.bfloat16, moe_backend="offload")return cfg.model_config@@ -176,7 +178,7 @@ def _bench_load(mode: str, model_path: str, *, parallel: bool, workers: int, chus.start()t = time.perf_counter()try:- banks = load_expert_banks(model_path, mc, device=torch.device("cuda:0"),+ banks = load_expert_banks(model_path, mc, device=torch.device("cuda", torch.cuda.current_device()),dtype=torch.bfloat16, parallel=parallel,workers=workers, chunk=chunk)except NotImplementedError as e:@@ -220,7 +222,8 @@ def worker_build(ns):s = MemSampler()s.start()t = time.perf_counter()- idx = convert_checkpoint(ns.model, ns.ftw_dir, moe_backend="offload", shard_limit=shard_limit)+ dev = f"cuda:{torch.cuda.current_device()}" if ns.gpu else None+ idx = convert_checkpoint(ns.model, ns.ftw_dir, moe_backend="offload", shard_limit=shard_limit, device=dev)build_s = time.perf_counter() - ts.stop()print("@@RESULT@@" + json.dumps({@@ -237,6 +240,8 @@ def _spawn(worker: str, ns):cmd = [sys.executable, os.path.abspath(__file__), "--_worker", worker, "--model", ns.model,"--workers", str(ns.workers), "--chunk-mib", str(ns.chunk_mib),"--shard-gib", str(ns.shard_gib), "--ftw-dir", ns.ftw_dir]+ if ns.gpu:+ cmd += ["--gpu", ns.gpu]if ns.no_drop_cache:cmd.append("--no-drop-cache")# Capture ONLY stdout (the @@RESULT@@ line); let stderr inherit the terminal so the@@ -260,6 +265,10 @@ def main():p.add_argument("--no-drop-cache", action="store_true",help="don't evict page cache before each read (warm comparison)")p.add_argument("--keep-ftw", action="store_true", help="keep + reuse the FTW dir across runs")+ from freetoken.gpu_select import single_gpu_arg++ p.add_argument("--gpu", type=single_gpu_arg, default=None,+ help="GPU UUID or nvidia-smi index (default: the first visible GPU)")p.add_argument("--_worker", default="")ns = p.parse_args()@@ -268,7 +277,17 @@ def main():if not ns.ftw_dir:ns.ftw_dir = _default_ftw_dir(ns.model)+ from freetoken.gpu_select import assign_gpu++ try:+ assign_gpu(ns.gpu)+ except ValueError as e:+ p.error(str(e))+if ns._worker:+ from freetoken.gpu_select import bind_assigned_gpu++ bind_assigned_gpu()return _WORKERS[ns._worker](ns)from freetoken.checkpoint.ftw import is_ftw_checkpointdiff --git a/benchmarks/bench_offload_cache_copy.py b/benchmarks/bench_offload_cache_copy.pyindex a2434a66..8374510f 100644--- a/benchmarks/bench_offload_cache_copy.py+++ b/benchmarks/bench_offload_cache_copy.py@@ -16,6 +16,7 @@ from dataclasses import dataclassimport torch+from freetoken.gpu_select import assign_gpu, bind_assigned_gpu, single_gpu_argfrom freetoken.moe.offload_cache import _BANK_SCHEMAS, OffloadMoeCache@@ -80,7 +81,8 @@ def expert_bytes(profile: ModelProfile) -> int:def parse_args() -> argparse.Namespace:parser = argparse.ArgumentParser()- parser.add_argument("--device", type=int, default=0)+ parser.add_argument("--gpu", type=single_gpu_arg, default=None,+ help="GPU UUID or nvidia-smi index (default: the first visible GPU)")parser.add_argument("--repeat", type=int, default=25)parser.add_argument("--models", type=str, nargs="+", default=list(MODELS), choices=list(MODELS))parser.add_argument(@@ -228,8 +230,11 @@ def print_table(def main() -> None:args = parse_args()assert torch.cuda.is_available(), "CUDA is required"- torch.cuda.set_device(args.device)- device = torch.device("cuda")+ try:+ assign_gpu(args.gpu)+ device = bind_assigned_gpu()+ except (ValueError, RuntimeError) as e:+ raise SystemExit(f"error: {e}") from eprint("gpu", torch.cuda.get_device_name(device), flush=True)for name in args.models:diff --git a/docs/cli.md b/docs/cli.mdindex cf4b27a2..ff4af382 100644--- a/docs/cli.md+++ b/docs/cli.md@@ -40,6 +40,7 @@ parsers all resolve automatically from the checkpoint and the GPU.|---|---|---|| `--host` | 127.0.0.1 | Bind address || `--port` | 1919 | Bind port |+| `--gpu` | GPU 0 | GPU to run on: a UUID from `nvidia-smi -L` or an `nvidia-smi` index; see [below](#choosing-a-gpu) || `--max-running-requests` | 4 | Max concurrently running requests || `--max-output-tokens` | 32768 | Default output budget for requests that omit one || `--max-seq-len-override` | from checkpoint | Max sequence length |@@ -47,6 +48,21 @@ parsers all resolve automatically from the checkpoint and the GPU.| `--cuda-graph-max-bs`, `--graph` | = max running requests | Max batch size captured as CUDA graphs || `--decode-log-interval` | 40 | Scheduler status line every N decode steps |+### Choosing a GPU++For example, a machine with an RTX 5090 and an RTX 3060 Ti:++```console+$ nvidia-smi -L+GPU 0: NVIDIA GeForce RTX 3060 Ti (UUID: GPU-2f3a9b1c-8d7e-4a05-b6c1-0e5f9a3d7b42)+GPU 1: NVIDIA GeForce RTX 5090 (UUID: GPU-9e8d7c6b-5a49-4f13-8207-c1b0a4e6d3f5)+```++```bash+ft serve --model ... --gpu 1 # by nvidia-smi index -- the 5090+ft serve --model ... --gpu GPU-9e8d7c6b # the same card by UUID (a unique prefix is enough)+```+### KV cache & memory| Flag | Default | Meaning |@@ -130,7 +146,7 @@ environment so the agent cannot silently fall back to a paid endpoint.## ft checkpoint```bash-ft checkpoint --model <hf_dir> --out <ftw_dir> [--dtype bfloat16] [--moe-backend offload] [--shard-gib 8] [--device cuda:0]+ft checkpoint --model <hf_dir> --out <ftw_dir> [--dtype bfloat16] [--moe-backend offload] [--shard-gib 8] [--gpu <uuid-or-index>]```Converts an HF safetensors checkpoint to FTW, FreeToken's self-contained@@ -142,15 +158,18 @@ keeps them dense for resident serving. See the FTW caveats in## ft bench bw```bash-ft bench bw # once per machine+ft bench bw # once per GPUft bench bw --dtype nvfp4,bf16 # only the formats you serve+ft bench bw --gpu 1 # a specific GPU (UUID or nvidia-smi index, as for ft serve)```-Measures host-RAM vs PCIe bandwidth with the real cpu/offload MoE kernels and-writes a profile (`~/.cache/freetoken/benchbw.json`) that `ft serve---moe-backend auto` and `--moe-hybrid-max-fetch -1` read. Profiles are keyed on-expert format + GPU name, so a profile from different hardware is ignored-rather than misapplied. Selection flags: `--dtype`, `--model`, `--formats`,-`--isa`; decision rule: `--threshold` (default 2.0 — recommend hybrid when CPU-bandwidth > 2× PCIe).+Measures host-RAM vs PCIe bandwidth with the real cpu/offload MoE kernels and writes a+profile that `ft serve --moe-backend auto` and `--moe-hybrid-max-fetch -1` then read.++- One profile per GPU, at `~/.cache/freetoken/benchbw/<gpu-uuid>.json`.+- Keyed on expert format + GPU, so a profile from other hardware is ignored rather than+ misapplied. An older single `benchbw.json` still counts if its GPU name matches.+- What to measure: `--dtype`, `--model`, `--formats`, `--isa`.+- `--threshold` (default 2.0) sets the call: recommend hybrid when CPU bandwidth beats PCIe+ by that factor.diff --git a/python/freetoken/checkpoint/__main__.py b/python/freetoken/checkpoint/__main__.pyindex cd1f90a5..2786173f 100644--- a/python/freetoken/checkpoint/__main__.py+++ b/python/freetoken/checkpoint/__main__.py@@ -1,7 +1,7 @@"""CLI: convert an HF safetensors checkpoint to a FreeToken Weight (FTW) checkpoint.ft checkpoint --model <hf_dir> --out <ftw_dir> \- [--dtype bfloat16] [--moe-backend offload] [--shard-gib 8]+ [--dtype bfloat16] [--moe-backend offload] [--shard-gib 8] [--gpu <uuid-or-index>]The output dir is self-contained: point the server's ``--model`` at it to load via the FTWfast path (auto-detected).@@ -14,6 +14,8 @@ import timeimport torch+from freetoken.gpu_select import assign_gpu, bind_assigned_gpu, single_gpu_arg+from .convert import convert_checkpoint_DTYPES = {"bfloat16": torch.bfloat16, "float16": torch.float16, "float32": torch.float32}@@ -27,15 +29,24 @@ def main(argv: list[str] | None = None, prog: str = "freetoken.checkpoint") -> ip.add_argument("--moe-backend", default="offload",help="offload (experts -> banks) or e.g. triton (experts stay dense)")p.add_argument("--shard-gib", type=float, default=8.0, help="max shard size in GiB")- p.add_argument("--device", default=None, help="CUDA device for repack (default cuda:0)")+ p.add_argument("--gpu", type=single_gpu_arg, default=None,+ help="GPU for the repack: a GPU UUID (GPU-xxxx..., as nvidia-smi -L prints) or "+ "an nvidia-smi index (default: the first visible GPU)")ns = p.parse_args(argv)+ # same as ft serve --gpu: resolve, then bind by UUID at CUDA init+ try:+ assign_gpu(ns.gpu)+ device = f"cuda:{bind_assigned_gpu().index}"+ except (ValueError, RuntimeError) as e:+ p.error(str(e))+shard_limit = int(ns.shard_gib * (1 << 30))shard_limit -= shard_limit % 4096 # keep alignedt = time.perf_counter()index = convert_checkpoint(ns.model, ns.out, dtype=_DTYPES[ns.dtype],- moe_backend=ns.moe_backend, shard_limit=shard_limit, device=ns.device,+ moe_backend=ns.moe_backend, shard_limit=shard_limit, device=device,)dt = time.perf_counter() - tc = index["counts"]diff --git a/python/freetoken/daemon/app.py b/python/freetoken/daemon/app.pyindex e230ccf3..d7a0a53c 100644--- a/python/freetoken/daemon/app.py+++ b/python/freetoken/daemon/app.py@@ -58,16 +58,41 @@ class BenchBody(BaseModel):args: list[str] = []-def _bench_profile_path() -> str:- from freetoken.moe.bench_profile import default_profile_path # torch-free+def _bench_profile_path(gpu_uuid: str | None) -> str | None:+ # per-GPU profiles and no torch here: the serve's own card when its --gpu names one, else the newest file+ from freetoken.moe.bench_profile import default_profile_path, latest_profile_path # torch-free- return default_profile_path()+ if gpu_uuid:+ path = default_profile_path(gpu_uuid)+ if os.path.isfile(path):+ return path+ return latest_profile_path()-def _read_bench_profile() -> dict | None:- """The engine host's cached benchbw.json (this is where the serve reads it too), or None."""+def _serve_gpu_uuid(args: list[str]) -> str | None:+ """The full UUID a serve's `--gpu` pins, or None when there is none or it cannot be resolved."""+ for i, a in enumerate(args):+ val = a[len("--gpu="):] if a.startswith("--gpu=") else (args[i + 1] if a == "--gpu" and i + 1 < len(args) else None)+ if not val:+ continue+ from freetoken.gpu_select import resolve_gpu_uuids++ try:+ resolved = resolve_gpu_uuids([val])+ except ValueError:+ return None+ if resolved:+ return resolved[0]+ # no NVML: a UUID value still keys the profile file (canonical prefix), an index cannot+ return "GPU-" + val[len("GPU-"):] if val.upper().startswith("GPU-") else None+ return None+++def _read_bench_profile(path: str | None) -> dict | None:+ if path is None:+ return Nonetry:- with open(_bench_profile_path()) as f:+ with open(path) as f:return json.load(f)except (OSError, ValueError):return None@@ -331,19 +356,23 @@ def build_app(yield _bench_sse("error", {"message": f"failed to spawn bench: {exc}"})returntail: collections.deque = collections.deque(maxlen=8) # last non-progress lines (errors)+ out_path: str | None = Noneassert proc.stdout is not Noneasync for raw in proc.stdout:line = raw.decode(errors="replace").rstrip()prog = _parse_ftbench(line)if prog is not None:yield _bench_sse("progress", prog)+ elif line.startswith("FTBENCH_OUT "):+ out_path = line[len("FTBENCH_OUT "):]elif line:tail.append(line)rc = await proc.wait()if rc != 0:yield _bench_sse("error", {"message": "\n".join(tail) or f"bench exited {rc}"})return- prof = _read_bench_profile()+ # the file this run wrote (an older engine prints no FTBENCH_OUT: newest file, as before)+ prof = _read_bench_profile(out_path or _bench_profile_path(None))if prof is None:yield _bench_sse("error", {"message": "bench finished but no profile was written"})else:@@ -353,7 +382,23 @@ def build_app(@app.get("/bench/profile", dependencies=auth)async def bench_profile():- return await run(proxy_pool, _read_bench_profile)+ def read() -> dict | None:+ return _read_bench_profile(_bench_profile_path(serve_gpu_uuid()))++ def serve_gpu_uuid() -> str | None:+ # the running serve reports the full UUID of its card (/v1/stats gpus); a --gpu given as+ # a UUID prefix would not match the profile file name+ st = manager.status()+ if st.get("running"):+ try:+ gpus = probe.stats(st.get("port") or default_serve_port).get("gpus") or []+ if gpus and gpus[0].get("uuid"):+ return gpus[0]["uuid"]+ except Exception: # noqa: BLE001 -- the arg below is the fallback+ pass+ return _serve_gpu_uuid(manager.serve_args())++ return await run(proxy_pool, read)return appdiff --git a/python/freetoken/daemon/serve_manager.py b/python/freetoken/daemon/serve_manager.pyindex 78cea11f..681bdef4 100644--- a/python/freetoken/daemon/serve_manager.py+++ b/python/freetoken/daemon/serve_manager.py@@ -862,6 +862,11 @@ class ServeManager:with self._cond:return self._child.pid if self._child is not None else None+ def serve_args(self) -> list[str]:+ """Engine args of the running (or last started) serve."""+ with self._cond:+ return list(self._args)+def status(self) -> dict:with self._cond:child = self._childdiff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.pyindex 22c4a6c7..cd6505d2 100644--- a/python/freetoken/engine/engine.py+++ b/python/freetoken/engine/engine.py@@ -10,6 +10,7 @@ import torchfrom freetoken.attention import AttnType, attention_backend_info, create_attention_backendfrom freetoken.core import Batch, Context, Req, set_global_ctxfrom freetoken.distributed import destroy_distributed, enable_pynccl_distributed, set_tp_info+from freetoken.gpu_select import gpu_identityfrom freetoken.layers import set_rope_devicefrom freetoken.models import create_model, load_weightfrom freetoken.moe import create_moe_backend, is_offload_moe_backend@@ -294,10 +295,11 @@ class Engine:assert not torch.cuda.is_initialized()set_tp_info(rank=config.tp_info.rank, size=config.tp_info.size)_ensure_expandable_segments() # before the first CUDA allocation below- _adjust_config(config)- self.device = torch.device(f"cuda:{config.tp_info.rank}")- torch.cuda.set_device(self.device)+ from freetoken.gpu_select import bind_assigned_gpu++ self.device = bind_assigned_gpu(config.tp_info.rank)+ _adjust_config(config)torch.manual_seed(42)self.stream = torch.cuda.Stream()torch.cuda.set_stream(self.stream)@@ -649,8 +651,10 @@ class Engine:return # explicit fixed capfrom freetoken.moe.bench_profile import load_hybrid_fetch_fraction- gpu_name = torch.cuda.get_device_name(self.device) if torch.cuda.is_available() else None- fraction = load_hybrid_fetch_fraction(cache.quant_format, gpu_name=gpu_name)+ gpu_name, gpu_uuid = _profile_gpu(self.device.index)+ fraction = load_hybrid_fetch_fraction(+ cache.quant_format, gpu_name=gpu_name, gpu_uuid=gpu_uuid+ )if fraction is None:cache.hybrid_max_fetch = 1logger.warning_rank0(@@ -993,6 +997,14 @@ class Engine:destroy_distributed()+def _profile_gpu(index: "int | None" = None) -> Tuple[str | None, str | None]:+ """(name, uuid) of visible device ``index`` (default: the current, i.e. bound, device); (None, None) without CUDA."""+ if not torch.cuda.is_available():+ return None, None+ ident = gpu_identity(torch.cuda.current_device() if index is None else index)+ return ident["name"], ident["uuid"]++def _ensure_expandable_segments() -> None:"""Default the CUDA allocator to expandable segments.@@ -1358,8 +1370,8 @@ def _adjust_config(config: EngineConfig):bench_fmt = expert_quant if expert_quant != "none" else (moe_wfmt or "bf16")from freetoken.moe.bench_profile import load_backend_recommendation- gpu_name = torch.cuda.get_device_name(0) if torch.cuda.is_available() else None- if load_backend_recommendation(bench_fmt, gpu_name=gpu_name) == "hybrid":+ gpu_name, gpu_uuid = _profile_gpu()+ if load_backend_recommendation(bench_fmt, gpu_name=gpu_name, gpu_uuid=gpu_uuid) == "hybrid":from freetoken.moe.cpu_executor import compiled_extension_supports_act = getattr(model_config, "hidden_act", "silu")diff --git a/python/freetoken/gpu_select.py b/python/freetoken/gpu_select.pynew file mode 100644index 00000000..2c7d012b--- /dev/null+++ b/python/freetoken/gpu_select.py@@ -0,0 +1,287 @@+"""--gpu for ft serve / bench bw / checkpoint: resolve entries to GPU UUIDs, bind by UUID at CUDA init.++Three device-id namespaces, converted explicitly:+- logical: position in the --gpu list == TP rank; each worker takes its own entry by rank and the id ends there.+- physical: NVML / nvidia-smi order, carried as a GPU UUID (_assigned_physical); not affected by CUDA_VISIBLE_DEVICES.+- visible: CUDA ordinal in this process (_assigned_visible), what torch.device("cuda", n) means.++The parent resolves --gpu entries to full UUIDs via NVML (resolve_gpu_uuids) and fails fast on a typo.+Each worker publishes its own entry (set_assigned_gpu / assign_gpu) and binds it when CUDA comes up (bind_assigned_gpu) by matching the UUID against CUDA's visible devices.+One process runs on one GPU. Binding is unconditional: a process that publishes nothing binds a default ordinal and records it, so assigned_visible_gpu() names that card in every case.+No process mutates CUDA_VISIBLE_DEVICES, and the UUID match holds under any CUDA_DEVICE_ORDER.++Stdlib only (torch is imported lazily); not under freetoken.utils, which imports transformers.+"""++from __future__ import annotations++import argparse+import os+from typing import Sequence++UUID_PREFIX = "GPU-"+++def is_gpu_uuid(spec: str) -> bool:+ return spec[: len(UUID_PREFIX)].upper() == UUID_PREFIX+++def is_gpu_index(spec: str) -> bool:+ # not str.isdigit(): that also accepts superscripts and other Unicode digits+ return spec.isascii() and spec.isdecimal()+++def _canonical(entry: str) -> str:+ """A UUID in the exact form the driver matches (upper-case GPU- prefix), an index as-is."""+ if not (is_gpu_uuid(entry) or is_gpu_index(entry)):+ raise ValueError(+ f"{entry!r} is neither a GPU UUID (GPU-xxxx..., as `nvidia-smi -L` prints) "+ f"nor an nvidia-smi index"+ )+ return UUID_PREFIX + entry[len(UUID_PREFIX):] if is_gpu_uuid(entry) else entry+++def parse_gpu_spec(value: str) -> tuple[str, ...]:+ """Split a --gpu value; ValueError on a bad entry, an empty value, or a mix of UUIDs and indices."""+ entries = tuple(_canonical(e.strip()) for e in value.split(",") if e.strip())+ if not entries:+ raise ValueError("--gpu needs at least one GPU")+ if len({is_gpu_uuid(e) for e in entries}) > 1:+ # the driver parses CUDA_VISIBLE_DEVICES as all-UUID or all-index+ raise ValueError("--gpu entries must be all UUIDs or all indices")+ return entries+++def gpu_arg(value: str) -> tuple[str, ...]:+ """argparse type for a --gpu list."""+ try:+ return parse_gpu_spec(value)+ except ValueError as exc:+ raise argparse.ArgumentTypeError(str(exc)) from exc+++def single_gpu_arg(value: str) -> str:+ """argparse type for a single-GPU --gpu."""+ entries = gpu_arg(value)+ if len(entries) != 1:+ raise argparse.ArgumentTypeError("takes exactly one GPU")+ return entries[0]+++def _nvml_uuids() -> "list[str] | None":+ """Full GPU UUIDs in physical (nvidia-smi) order, or None when NVML is unavailable.++ Own ctypes loader instead of torch's _raw_device_uuid_nvml: that helper only knows the Linux library name, raises (not None) when the library is missing, and is private API.+ NVML exports are cdecl on every platform, so CDLL is right on Windows too (same as nvidia-ml-py).+ None on any failure -- no library, a stub library without the _v2 symbols, WSL, a dead device -- and callers fall back.+ """+ import ctypes++ if os.name == "nt":+ candidates = [+ "nvml.dll",+ os.path.join(os.environ.get("SystemRoot", r"C:\\Windows"), "System32", "nvml.dll"),+ os.path.join(os.environ.get("ProgramFiles", r"C:\\Program Files"), "NVIDIA Corporation", "NVSMI", "nvml.dll"),+ ]+ else:+ candidates = ["libnvidia-ml.so.1"]+ try:+ for name in candidates:+ try:+ lib = ctypes.CDLL(name)+ break+ except OSError:+ continue+ else:+ return None+ if lib.nvmlInit() != 0:+ return None+ try:+ count = ctypes.c_int()+ if lib.nvmlDeviceGetCount_v2(ctypes.byref(count)) != 0:+ return None+ uuids = []+ for i in range(count.value):+ handle = ctypes.c_void_p()+ if lib.nvmlDeviceGetHandleByIndex_v2(i, ctypes.byref(handle)) != 0:+ return None+ buf = ctypes.create_string_buffer(96)+ if lib.nvmlDeviceGetUUID(handle, buf, 96) != 0:+ return None+ uuids.append(buf.value.decode("ascii", "replace"))+ return uuids+ finally:+ lib.nvmlShutdown()+ except (OSError, AttributeError):+ return None+++def _match_uuid(spec: str, uuids: "list[str]", where: str) -> str:+ """The unique full UUID that ``spec`` prefixes, else ValueError."""+ hits = [u for u in uuids if u.upper().startswith(spec.upper())]+ if len(hits) != 1:+ raise ValueError(f"--gpu {spec}: not found or not a unique prefix {where}; run `nvidia-smi -L` to list GPUs")+ return hits[0]+++def resolve_gpu_uuids(specs: Sequence[str]) -> "tuple[str, ...] | None":+ """--gpu entries -> full GPU UUIDs, one per TP rank; raises ValueError on a bad entry.++ A preset CUDA_VISIBLE_DEVICES is a quota to stay inside: an index counts within that list, a UUID must name one of its entries.+ Returns None when NVML is unavailable -- the worker then interprets the raw entries against CUDA's own enumeration (see bind_assigned_gpu).+ """+ specs = parse_gpu_spec(",".join(specs))+ if len({s.upper() for s in specs}) != len(specs):+ raise ValueError(f"--gpu {','.join(specs)}: the same GPU appears twice")+ uuids = _nvml_uuids()+ if uuids is None:+ return None+ preset_raw = os.environ.get("CUDA_VISIBLE_DEVICES")+ preset = None if preset_raw is None else [e.strip() for e in preset_raw.split(",") if e.strip()]++ resolved: list[str] = []+ for spec in specs:+ if preset is None:+ if is_gpu_uuid(spec):+ resolved.append(_match_uuid(spec, uuids, "on this machine"))+ elif int(spec) < len(uuids):+ resolved.append(uuids[int(spec)])+ else:+ raise ValueError(f"--gpu {spec}: only {len(uuids)} GPU(s) on this machine; run `nvidia-smi -L` to list GPUs")+ else:+ entry = _preset_entry(spec, preset, preset_raw)+ # an integer entry is read in physical order, as under CUDA_DEVICE_ORDER=PCI_BUS_ID; a negative or MIG-form entry cannot name a whole GPU+ if is_gpu_uuid(entry):+ resolved.append(_match_uuid(entry, uuids, f"(from CUDA_VISIBLE_DEVICES={preset_raw!r})"))+ elif is_gpu_index(entry) and int(entry) < len(uuids):+ resolved.append(uuids[int(entry)])+ else:+ raise ValueError(+ f"--gpu {spec}: cannot resolve CUDA_VISIBLE_DEVICES entry {entry!r} "+ f"({len(uuids)} GPU(s) on this machine)"+ )+ if len(set(resolved)) != len(resolved):+ raise ValueError(f"--gpu {','.join(specs)}: the same GPU appears twice")+ return tuple(resolved)+++def _preset_entry(spec: str, preset: "list[str]", preset_raw: str) -> str:+ """The CUDA_VISIBLE_DEVICES entry ``spec`` selects, else ValueError."""+ if not is_gpu_uuid(spec):+ idx = int(spec)+ if idx >= len(preset):+ raise ValueError(+ f"--gpu {spec}: only {len(preset)} GPU(s) are visible through "+ f"CUDA_VISIBLE_DEVICES={preset_raw!r} (indices count within that list)"+ )+ return preset[idx]+ if not all(is_gpu_uuid(p) for p in preset):+ raise ValueError(+ f"--gpu {spec}: CUDA_VISIBLE_DEVICES={preset_raw!r} lists GPUs by index; "+ f"give --gpu as an index into that list"+ )+ hits = [p for p in preset if p.upper().startswith(spec.upper()) or spec.upper().startswith(p.upper())]+ if len(hits) != 1:+ raise ValueError(+ f"--gpu {spec}: not one of the GPUs visible through CUDA_VISIBLE_DEVICES={preset_raw!r}"+ )+ return hits[0]+++# The GPU this process was assigned, in whichever namespace it arrived in; bind_assigned_gpu fills in the visible one.+# Process-global on purpose: publishing is torch-free so a worker can do it before heavy imports, and kernel-compat checks (e4m3_native) need the device this process will use.+_assigned_physical: "str | None" = None+_assigned_visible: "int | None" = None+++def set_assigned_gpu(target: str) -> None:+ """Publish this process's GPU before CUDA init; second call must agree.++ A UUID names a physical GPU and is converted at bind time; a bare index is already a visible ordinal (a preset CUDA_VISIBLE_DEVICES has narrowed to it).+ """+ global _assigned_physical, _assigned_visible+ physical = target if is_gpu_uuid(target) else None+ visible = None if physical is not None else int(target)+ current = (_assigned_physical, _assigned_visible)+ if current not in ((None, None), (physical, visible)):+ raise RuntimeError(f"set_assigned_gpu called twice: {current} then {target!r}")+ _assigned_physical, _assigned_visible = physical, visible+++def assign_gpu(spec: "str | None") -> None:+ """Resolve one --gpu value and publish it for bind_assigned_gpu; no-op when the flag was not given."""+ if spec is None:+ return+ resolved = resolve_gpu_uuids([spec])+ set_assigned_gpu(resolved[0] if resolved else parse_gpu_spec(spec)[0])+++def _visible_of_physical(uuid: str) -> int:+ """CUDA ordinal of the physical GPU ``uuid`` (or unique prefix) among this process's visible devices."""+ import torch++ seen: list[str] = []+ hits: list[int] = []+ for v in range(torch.cuda.device_count()):+ u = format_gpu_uuid(getattr(torch.cuda.get_device_properties(v), "uuid", None))+ seen.append(u or "?")+ if u is not None and u.upper().startswith(uuid.upper()):+ hits.append(v)+ if len(hits) == 1:+ return hits[0]+ if hits:+ raise RuntimeError(f"--gpu {uuid}: not a unique prefix (visible: {', '.join(seen)})")+ raise RuntimeError(+ f"GPU {uuid} is not visible to CUDA in this process "+ f"(CUDA_VISIBLE_DEVICES={os.environ.get('CUDA_VISIBLE_DEVICES')!r}, "+ f"visible: {', '.join(seen) or 'none'})"+ )+++def bind_assigned_gpu(default: int = 0):+ """torch.cuda.set_device this process's GPU and return the device.++ ``default`` is a visible ordinal, used and recorded when nothing was published, so the process always knows which card it runs on.+ A published UUID (or prefix) is matched against CUDA's own device list, so the result is right under any CUDA_DEVICE_ORDER.+ """+ global _assigned_visible+ import torch++ if _assigned_visible is None:+ _assigned_visible = default if _assigned_physical is None else _visible_of_physical(_assigned_physical)+ if not 0 <= _assigned_visible < torch.cuda.device_count():+ raise RuntimeError(+ f"cannot use CUDA device {_assigned_visible}: only {torch.cuda.device_count()} device(s) visible "+ f"(CUDA_VISIBLE_DEVICES={os.environ.get('CUDA_VISIBLE_DEVICES')!r})"+ )+ device = torch.device("cuda", _assigned_visible)+ torch.cuda.set_device(device)+ return device+++def assigned_visible_gpu() -> "int | None":+ """Visible ordinal this process is pinned to, or None before it publishes or binds a GPU (= the current device).++ Published-but-not-yet-bound still counts: compat checks in the window between publish and bind must judge the assigned card, not whatever the calling thread happens to sit on.+ """+ if _assigned_visible is not None:+ return _assigned_visible+ return None if _assigned_physical is None else _visible_of_physical(_assigned_physical)+++def format_gpu_uuid(raw) -> str | None:+ """nvidia-smi form GPU-<uuid> from a uuid.UUID."""+ return None if raw is None else f"{UUID_PREFIX}{raw}"+++def gpu_identity(index: int) -> dict:+ """{index, name, uuid, total_bytes} of visible device ``index``."""+ import torch++ props = torch.cuda.get_device_properties(index)+ return {+ "index": index,+ "name": props.name,+ "uuid": format_gpu_uuid(getattr(props, "uuid", None)),+ "total_bytes": int(props.total_memory),+ }diff --git a/python/freetoken/kernel/fla/utils.py b/python/freetoken/kernel/fla/utils.pyindex 28a95858..e9e0b697 100644--- a/python/freetoken/kernel/fla/utils.py+++ b/python/freetoken/kernel/fla/utils.py@@ -271,13 +271,13 @@ is_intel = device_platform == "intel"is_nvidia = device_platform == "nvidia"is_intel_alchemist = is_intel and "Intel(R) Arc(TM) A" in torch.xpu.get_device_name(0)is_nvidia_hopper = is_nvidia and (- "NVIDIA H" in torch.cuda.get_device_name(0)+ "NVIDIA H" in torch.cuda.get_device_name(torch.cuda.current_device())or torch.cuda.get_device_capability()[0] >= 9)use_cuda_graph = is_nvidia and os.environ.get("FLA_USE_CUDA_GRAPH", "0") == "1"# Nvidia Ampere or newer, haven't check AMD and intel yet.-is_tf32_supported = is_nvidia and torch.cuda.get_device_capability(0)[0] >= 8+is_tf32_supported = is_nvidia and torch.cuda.get_device_capability()[0] >= 8is_gather_supported = hasattr(triton.language, "gather")@@ -309,8 +309,10 @@ class Backend(Enum):@lru_cache(maxsize=None)-def check_shared_mem(arch: str = "none", tensor_idx: int = 0) -> bool:+def check_shared_mem(arch: str = "none", tensor_idx: "int | None" = None) -> bool:try:+ if tensor_idx is None:+ tensor_idx = device_torch_lib.current_device()device_shared_mem_list = get_all_max_shared_mem()max_shared_memory = device_shared_mem_list[tensor_idx]return max_shared_memory >= Backend.get_shared_memory(arch)diff --git a/python/freetoken/kernel/triton/e4m3_compat.py b/python/freetoken/kernel/triton/e4m3_compat.pyindex 61d3a0e7..1d9f744c 100644--- a/python/freetoken/kernel/triton/e4m3_compat.py+++ b/python/freetoken/kernel/triton/e4m3_compat.py@@ -60,14 +60,10 @@ def e4m3_native() -> bool:if FORCE_EMU:_native = Falseelse:- native = {torch.cuda.get_device_capability(i) >= (8, 9)- for i in range(torch.cuda.device_count())}- if len(native) > 1:- raise NotImplementedError(- "GPUs on both sides of the sm_89 fp8 boundary in one process: "- "the host-side e4m3 convention is process-global"- )- _native = native.pop() if native else torch.cuda.get_device_capability() >= (8, 9)+ from freetoken.gpu_select import assigned_visible_gpu++ # one process runs on one GPU, so its convention is that GPU's; None (-> the current device) only before the process binds+ _native = torch.cuda.get_device_capability(assigned_visible_gpu()) >= (8, 9)return _nativediff --git a/python/freetoken/kernel/triton/sampling.py b/python/freetoken/kernel/triton/sampling.pyindex 9d1bd3da..7345d65f 100644--- a/python/freetoken/kernel/triton/sampling.py+++ b/python/freetoken/kernel/triton/sampling.py@@ -28,7 +28,7 @@ import triton.language as tlfrom freetoken.kernel.triton.autotune_cache import autotune_cache_kwargs-_NUM_SM = torch.cuda.get_device_properties(0).multi_processor_count+_NUM_SM = torch.cuda.get_device_properties(torch.cuda.current_device()).multi_processor_count_MIN_CHUNK = 4096 # do not split a row finer than thisdiff --git a/python/freetoken/moe/bench_profile.py b/python/freetoken/moe/bench_profile.pyindex 46b11d44..d7b72073 100644--- a/python/freetoken/moe/bench_profile.py+++ b/python/freetoken/moe/bench_profile.py@@ -1,4 +1,4 @@-"""Torch-free reader for the ``ft bench bw`` hardware profile (``benchbw.json``).+"""Torch-free reader for the ``ft bench bw`` hardware profile (``benchbw/<gpu-uuid>.json``).The engine consults this at MoE-backend *auto* resolution (``engine.py``) to make theoffload-vs-hybrid choice hardware-adaptive without importing the (torch-heavy) benchmark@@ -31,10 +31,42 @@ _QUANT_TO_BENCH_FORMAT = {}-def default_profile_path() -> str:- """``$XDG_CACHE_HOME/freetoken/benchbw.json`` (mirrors ``benchbw.default_out_path``)."""+def _cache_dir() -> str:cache = os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache")- return os.path.join(cache, "freetoken", "benchbw.json")+ return os.path.join(cache, "freetoken")+++def default_profile_path(gpu_uuid: str | None = None) -> str:+ """``$XDG_CACHE_HOME/freetoken/benchbw/<gpu-uuid>.json``, or the legacy ``benchbw.json`` without a uuid.++ One file per GPU: bandwidth differs between slots.+ """+ if gpu_uuid:+ return os.path.join(_cache_dir(), "benchbw", f"{gpu_uuid}.json")+ return os.path.join(_cache_dir(), "benchbw.json")+++def latest_profile_path() -> str | None:+ """Newest ``benchbw/*.json``, else the legacy ``benchbw.json``, else None."""+ per_gpu = os.path.join(_cache_dir(), "benchbw")+ newest: tuple[float, str] | None = None+ try:+ for name in os.listdir(per_gpu):+ if not name.endswith(".json"):+ continue+ path = os.path.join(per_gpu, name)+ try:+ mtime = os.path.getmtime(path)+ except OSError:+ continue+ if newest is None or mtime > newest[0]:+ newest = (mtime, path)+ except OSError:+ pass+ if newest is not None:+ return newest[1]+ legacy = default_profile_path()+ return legacy if os.path.isfile(legacy) else Nonedef _load(path: str) -> dict | None:@@ -45,14 +77,28 @@ def _load(path: str) -> dict | None:return None-def _usable_profile(gpu_name: str | None, path: str | None) -> dict | None:+def _usable_profile(+ gpu_name: str | None, path: str | None, gpu_uuid: str | None = None+) -> dict | None:"""The cached profile, or ``None`` when there is no file / it was benched on another GPU(bandwidths are hardware-specific, so a mismatch is ignored rather than trusted).- ``path`` overrides the profile location (else ``FREETOKEN_BENCHBW_PATH`` then the default).+ Lookup: explicit ``path`` (else ``FREETOKEN_BENCHBW_PATH``) -> ``benchbw/<gpu_uuid>.json`` -> legacy ``benchbw.json``."""- src = path or os.environ.get("FREETOKEN_BENCHBW_PATH") or default_profile_path()- prof = _load(src)+ explicit = path or os.environ.get("FREETOKEN_BENCHBW_PATH")+ if explicit:+ candidates = [explicit]+ else:+ candidates = [default_profile_path(gpu_uuid)] if gpu_uuid else []+ candidates.append(default_profile_path())+ prof = None+ for src in candidates:+ prof = _load(src)+ if isinstance(prof, dict):+ break+ if os.path.exists(src):+ # unreadable profile for this card: stay on the safe default, do not borrow the legacy file+ return Noneif not isinstance(prof, dict):return Noneprof_gpu = (prof.get("gpu") or {}).get("name")@@ -66,7 +112,10 @@ def _usable_profile(gpu_name: str | None, path: str | None) -> dict | None:def load_backend_recommendation(- quant_format: str, gpu_name: str | None = None, path: str | None = None+ quant_format: str,+ gpu_name: str | None = None,+ path: str | None = None,+ gpu_uuid: str | None = None,) -> str | None:"""Bench-recommended offload-family backend for ``quant_format`` on this GPU, or ``None``.@@ -77,7 +126,7 @@ def load_backend_recommendation(default (offload) on ``None``."""fmt = _QUANT_TO_BENCH_FORMAT.get(quant_format, quant_format)- prof = _usable_profile(gpu_name, path)+ prof = _usable_profile(gpu_name, path, gpu_uuid)if prof is None:return None@@ -105,7 +154,10 @@ def load_backend_recommendation(def load_hybrid_fetch_fraction(- quant_format: str, gpu_name: str | None = None, path: str | None = None+ quant_format: str,+ gpu_name: str | None = None,+ path: str | None = None,+ gpu_uuid: str | None = None,) -> float | None:"""Benched hybrid fetch fraction for ``quant_format``, or ``None``.@@ -119,7 +171,7 @@ def load_hybrid_fetch_fraction(``None`` = no usable profile; clamped to [0, 1]."""fmt = _QUANT_TO_BENCH_FORMAT.get(quant_format, quant_format)- prof = _usable_profile(gpu_name, path)+ prof = _usable_profile(gpu_name, path, gpu_uuid)if prof is None:return Noneentries = [(prof.get("dtype_kernels") or {}).get(fmt)] + [diff --git a/python/freetoken/moe/benchbw.py b/python/freetoken/moe/benchbw.pyindex bab06ba0..f3e5359a 100644--- a/python/freetoken/moe/benchbw.py+++ b/python/freetoken/moe/benchbw.py@@ -21,13 +21,15 @@ the CPU, so they always resolve to ``offload``.The hybrid-vs-offload choice is dtype-dominated, so the default is a **per-dtype tuning bench**(``--dtype``): one bench per expert format against a canonical geometry -- the minimal set theruntime backend pick matches on. ``--model`` additionally benches specific model geometries for-per-model detail. Results are written to a JSON file (default ``$XDG_CACHE_HOME/freetoken/-benchbw.json``) so the choice is reproducible.+per-model detail. Results are written to a JSON file per GPU (default ``$XDG_CACHE_HOME/+freetoken/benchbw/<gpu-uuid>.json``) so the choice is reproducible and a multi-GPU box keeps+one profile per card.ft bench bw # per-dtype tuning bench (default: all formats)ft bench bw --dtype nvfp4,bf16 # only these formatsft bench bw --model qwen3.6-moe # per-model detail insteadft bench bw --dtype all --model all # both+ ft bench bw --gpu GPU-2f3a... # bench a specific GPU (UUID or nvidia-smi index)"""from __future__ import annotations@@ -47,6 +49,12 @@ from types import SimpleNamespaceimport torch+from freetoken.gpu_select import (+ assign_gpu,+ bind_assigned_gpu,+ gpu_identity,+ single_gpu_arg,+)from freetoken.kernel.pinned import alloc_pinned_tensorfrom freetoken.moe.cpu_executor import physical_core_cpus, resolve_threads_and_affinityfrom freetoken.utils import init_logger@@ -134,11 +142,11 @@ DTYPE_WORKLOADS: dict[str, Workload] = {}-def default_out_path() -> str:+def default_out_path(gpu_uuid: str | None = None) -> str:# Single source of truth with the (torch-free) reader the engine consults.from freetoken.moe.bench_profile import default_profile_path- return default_profile_path()+ return default_profile_path(gpu_uuid)def _cgroup_mem_headroom() -> int | None:@@ -696,12 +704,17 @@ def run_benchbw("benchbw needs a CUDA device to measure PCIe bandwidth (both offload and ""hybrid serve experts to the GPU).")+ if torch.cuda.device_count() == 0:+ raise RuntimeError(+ f"no CUDA device visible (CUDA_VISIBLE_DEVICES={os.environ.get('CUDA_VISIBLE_DEVICES')!r})."+ )if not 0 <= device_index < torch.cuda.device_count():raise RuntimeError(- f"--device {device_index} out of range (found {torch.cuda.device_count()} CUDA devices)."+ f"device_index {device_index} out of range (found {torch.cuda.device_count()} CUDA devices).")device = torch.device("cuda", device_index)torch.cuda.set_device(device)+ gpu = gpu_identity(device_index)# Machine-parseable progress on stdout (opt-in), so the daemon/Desktop can stream feedback# while the bench runs -- mirrors ft checkpoint's FTCONVERT lines. `done`/`total` count the@@ -766,7 +779,8 @@ def run_benchbw("timestamp": datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds"),"epoch": int(time.time()),"host": socket.gethostname(),- "gpu": {"index": device_index, "name": torch.cuda.get_device_name(device)},+ # index is the CUDA ordinal the bench ran on; uuid keys the profile file+ "gpu": {"index": device_index, "name": gpu["name"], "uuid": gpu["uuid"]},"cpu": {"physical_cores": len(physical_core_cpus()), "threads_used": cpu["threads"]},"threshold": threshold,"ceilings": {@@ -780,9 +794,11 @@ def run_benchbw("workloads": workloads_out,}- out_path = os.path.expanduser(out_path or default_out_path())+ out_path = os.path.expanduser(out_path or default_out_path(gpu["uuid"]))_atomic_write_json(out_path, result)result["out_path"] = out_path+ if prog_on:+ print(f"FTBENCH_OUT {out_path}", flush=True)return result@@ -927,10 +943,12 @@ def main(argv: list[str] | None = None, prog: str = "ft bench bw") -> int:help=f"CPU MoE ISA: 'auto' (default, best), 'all', or a subset of "f"{list(_ISA_TIERS)} to sweep (kernel caps down to hw support)")p.add_argument("-o", "--out", default=None,- help=f"JSON output path (default {default_out_path()})")+ help=f"JSON output path (default {default_out_path('<gpu-uuid>')})")p.add_argument("--threshold", type=_positive_float, default=2.0,help="recommend hybrid when CPU BW > threshold x PCIe BW (default 2.0)")- p.add_argument("--device", type=_nonneg_int, default=0, help="CUDA device index (default 0)")+ p.add_argument("--gpu", type=single_gpu_arg, default=None,+ help="GPU to bench: a GPU UUID (GPU-xxxx..., as nvidia-smi -L prints) or an "+ "nvidia-smi index (default: the first visible GPU)")p.add_argument("--cpu-threads", type=_nonneg_int, default=0,help="CPU worker threads (0 = one per physical core)")p.add_argument("--cpu-iters", type=_positive_int, default=8, help="STREAM read passes to time")@@ -942,6 +960,13 @@ def main(argv: list[str] | None = None, prog: str = "ft bench bw") -> int:help="fast_index_copy gather passes to time")ns = p.parse_args(argv)+ # same as ft serve --gpu: resolve, then bind by UUID at CUDA init+ try:+ assign_gpu(ns.gpu)+ device_index = bind_assigned_gpu().index+ except (ValueError, RuntimeError) as e:+ p.error(str(e))+models = ns.model or ()dtypes = ns.dtype# Default (no selection): the full per-dtype tuning bench -- the minimal set the runtime@@ -951,7 +976,7 @@ def main(argv: list[str] | None = None, prog: str = "ft bench bw") -> int:try:result = run_benchbw(- out_path=ns.out, threshold=ns.threshold, device_index=ns.device, models=models,+ out_path=ns.out, threshold=ns.threshold, device_index=device_index, models=models,dtypes=dtypes, formats=ns.formats, isas=ns.isas, cpu_threads=ns.cpu_threads,cpu_iters=ns.cpu_iters, pcie_bytes=ns.pcie_mib << 20, pcie_iters=ns.pcie_iters,kernel_cpu_iters=ns.kernel_cpu_iters, kernel_pcie_iters=ns.kernel_pcie_iters,diff --git a/python/freetoken/moe/host_banks.py b/python/freetoken/moe/host_banks.pyindex 5923c823..d7af348a 100644--- a/python/freetoken/moe/host_banks.py+++ b/python/freetoken/moe/host_banks.py@@ -292,10 +292,14 @@ class PinPipeline:def __init__(self) -> None:self._q: queue.SimpleQueue = queue.SimpleQueue()self._exc: BaseException | None = None+ # the current device is thread-local: a fresh thread sits on device 0 and cudaHostRegister would build its context there -- carry the creator's (bound) device into the worker+ self._device = torch.cuda.current_device() if torch.cuda.is_available() else Noneself._thread = threading.Thread(target=self._run, daemon=True)self._thread.start()def _run(self) -> None:+ if self._device is not None:+ torch.cuda.set_device(self._device)while True:item = self._q.get()if item is None:diff --git a/python/freetoken/scheduler/scheduler.py b/python/freetoken/scheduler/scheduler.pyindex 35541161..48923e3b 100644--- a/python/freetoken/scheduler/scheduler.py+++ b/python/freetoken/scheduler/scheduler.py@@ -6,6 +6,7 @@ import torchfrom freetoken.attention.linear import build_fla_metadatafrom freetoken.core import Batch, Reqfrom freetoken.env import ENV+from freetoken.gpu_select import gpu_identityfrom freetoken.message import (AbortBackendMsg,BaseBackendMsg,@@ -68,6 +69,8 @@ class Scheduler(SchedulerIOMixin):self.stream = torch.cuda.Stream(device=self.device)self.engine_stream_ctx = torch.cuda.stream(self.engine.stream)torch.cuda.set_stream(self.stream)+ # sent on the readiness ack for /v1/stats gpus; a list so TP can add one entry per rank+ self.gpus = [gpu_identity(self.device.index)] if self.device.type == "cuda" else []# initialize other managersself.table_manager = TableManager(config.max_running_req, self.engine.page_table)diff --git a/python/freetoken/server/api_server.py b/python/freetoken/server/api_server.pyindex 80dbf6ab..3e2acc85 100644--- a/python/freetoken/server/api_server.py+++ b/python/freetoken/server/api_server.py@@ -174,6 +174,8 @@ class FrontendManager:# "num_mamba_slots"}, from the same ack. Seeds geometry before the first generation reply# (the running snapshot channel) has anything. None until meta arrives.cache_pools: Dict[str, int] | None = None+ # one {index, name, uuid, total_bytes} per TP rank, from the same ack; /v1/stats gpus+ gpus: List[Dict[str, Any]] = field(default_factory=list)# Backend worker Process handles (TP schedulers + tokenizer/detokenizer), captured from the# BackendHandle after start_backend(). The orderly-shutdown path (lifespan / shell signal# handler) tears these down itself, AFTER setting _SHUTTING_DOWN, so the supervisor observes@@ -1010,6 +1012,7 @@ def run_api_server(config: ServerArgs, start_backend: Callable[[], "Any"], run_s_GLOBAL_STATE.cache_pools = meta.pop("pools", None)_GLOBAL_STATE.swa_full_tokens_ratio = float(meta.pop("swa_full_tokens_ratio", 0.0) or 0.0)_GLOBAL_STATE.cache_budget_bytes = int(meta.pop("cache_budget_bytes", 0) or 0)+ _GLOBAL_STATE.gpus = list(meta.pop("gpus", None) or [])_GLOBAL_STATE.unit_bytes = meta# Early-bind: supervise the backend on a daemon thread so uvicorn can binddiff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.pyindex 4954c5f5..a71b6819 100644--- a/python/freetoken/server/args.py+++ b/python/freetoken/server/args.py@@ -39,6 +39,10 @@ class ServerArgs(SchedulerConfig):# Comma-separated CORS allow-list for browser/webview clients (e.g. the desktop# app). Empty string disables CORS headers entirely; "*" allows any origin.cors_origins: str = "tauri://localhost,http://tauri.localhost,http://localhost:1420"+ # --gpu entries in TP-rank order, empty = not given+ gpu: tuple[str, ...] = ()+ # full UUIDs resolved from --gpu, entry i = TP rank i; None = NVML unavailable, each worker then resolves its raw entry against CUDA's own enumeration+ gpu_assigned: "tuple[str, ...] | None" = None@propertydef share_tokenizer(self) -> bool:@@ -109,6 +113,11 @@ def parse_args(raise argparse.ArgumentTypeError("must be >= 1")return n+ def _lazy_gpu_arg(value: str) -> tuple[str, ...]:+ from freetoken.gpu_select import gpu_arg++ return gpu_arg(value)+def _infer_tool_call_parser(model_path: str) -> str:try:from freetoken.utils import cached_load_hf_config@@ -221,6 +230,16 @@ def parse_args(help="The tensor parallelism size.",)+ parser.add_argument(+ "--gpu",+ type=_lazy_gpu_arg,+ default=ServerArgs.gpu,+ help=(+ "GPU(s) to run on, comma-separated; entry i is TP rank i. Each entry is a GPU "+ "UUID (GPU-xxxx..., as nvidia-smi -L prints) or an nvidia-smi index"+ ),+ )+parser.add_argument("--max-running-requests",type=int,@@ -611,6 +630,15 @@ def parse_args(# Parse argumentskwargs = parser.parse_args(args).__dict__.copy()+ # reject a too-long list here with a clear reason, not as a dead rank later+ if len(kwargs["gpu"]) not in (0, kwargs["tensor_parallel_size"]):+ if kwargs["tensor_parallel_size"] == 1 and len(kwargs["gpu"]) > 1:+ parser.error("tensor parallelism is not supported yet: --gpu takes one entry")+ parser.error(+ f"--gpu has {len(kwargs['gpu'])} entries but --tensor-parallel-size is "+ f"{kwargs['tensor_parallel_size']}; give one entry per TP rank"+ )+# resolve some argumentsrun_shell |= kwargs.pop("shell_mode")kwargs["shell_mode"] = run_shelldiff --git a/python/freetoken/server/launch.py b/python/freetoken/server/launch.pyindex a5f84388..acef5517 100644--- a/python/freetoken/server/launch.py+++ b/python/freetoken/server/launch.py@@ -59,6 +59,13 @@ def _run_scheduler(args: ServerArgs, ack_queue: mp.Queue[str]) -> None:if args.shell_mode:_detach_process_group()+ # published (not bound) here: the engine binds it after the allocator setup+ from freetoken.gpu_select import set_assigned_gpu++ # resolved UUIDs when we have them, the raw --gpu entries when NVML could not resolve them, else one CUDA ordinal per rank+ targets = args.gpu_assigned or args.gpu or tuple(str(r) for r in range(args.tp_info.size))+ set_assigned_gpu(targets[args.tp_info.rank])+import torchfrom freetoken.scheduler import Scheduler@@ -90,7 +97,10 @@ def _run_scheduler(args: ServerArgs, ack_queue: mp.Queue[str]) -> None:try:from freetoken.kvcache.cache_status import compute_cache_status_meta- ack_queue.put(("meta", compute_cache_status_meta(scheduler.engine)))+ meta = compute_cache_status_meta(scheduler.engine)+ # the parent must not touch CUDA to learn this+ meta["gpus"] = scheduler.gpus+ ack_queue.put(("meta", meta))except Exception: # noqa: BLE001 -- metadata is a nicety; readiness is notpassack_queue.put("Scheduler is ready")@@ -127,6 +137,19 @@ def launch_server()logger = init_logger(__name__, "initializer")+ if server_args.gpu:+ # resolve here so a typo is one clear error before any worker spawns+ from freetoken.gpu_select import resolve_gpu_uuids++ try:+ server_args = replace(server_args, gpu_assigned=resolve_gpu_uuids(server_args.gpu))+ except ValueError as exc:+ raise SystemExit(f"{prog or 'ft serve'}: error: {exc}") from exc+ logger.info(+ f"--gpu {','.join(server_args.gpu)} -> "+ f"{', '.join(server_args.gpu_assigned) if server_args.gpu_assigned else 'resolved at CUDA init (no NVML)'}"+ )+def start_subprocess() -> "BackendHandle":import multiprocessing as mpdiff --git a/python/freetoken/server/stats.py b/python/freetoken/server/stats.pyindex e89d23a4..76c6c308 100644--- a/python/freetoken/server/stats.py+++ b/python/freetoken/server/stats.py@@ -130,7 +130,9 @@ def _swa_page_size(config: Any) -> int:def build_stats(state: Any, p95_ms: int, ttft_mean_ms: int) -> dict:"""Full /v1/stats doc. throughput is 0 when idle; kv/mamba/swa are nullwhen their total is 0 (owned-KV / non-hybrid / non-SWA). kv and swa share one shape:- pages + the pool's own page_size (tokens = pages x page_size)."""+ pages + the pool's own page_size (tokens = pages x page_size). gpus: the engine's GPU as+ [{index, name, uuid, total_bytes}] (the primary rank's; a list so TP can extend it), []+ until the readiness meta arrives."""tr: StatsTracker = state.statsconfig = state.configready_at = getattr(state, "ready_at", None)@@ -158,6 +160,7 @@ def build_stats(state: Any, p95_ms: int, ttft_mean_ms: int) -> dict:"mamba": mamba,"swa": swa,"vram_bytes": tr.vram_bytes,+ "gpus": list(getattr(state, "gpus", None) or []),"throughput": {"decode_tps": round(tr.decode_tps(), 1),"prefill_tps": round(tr.prefill_tps(), 1),diff --git a/tests/moe/test_hybrid_fetch.py b/tests/moe/test_hybrid_fetch.pyindex e080e165..169f8e2b 100644--- a/tests/moe/test_hybrid_fetch.py+++ b/tests/moe/test_hybrid_fetch.py@@ -6,11 +6,12 @@ per-step integer split (GPU kernel vs CPU reference mirror, and the balance rule"""import json+import osimport pytestimport torch-from freetoken.moe.bench_profile import load_hybrid_fetch_fraction+from freetoken.moe.bench_profile import default_profile_path, load_backend_recommendation, load_hybrid_fetch_fractionfrom freetoken.moe.offload_cache import OffloadMoeCacheQ = 1 << 16@@ -65,6 +66,25 @@ def test_load_hybrid_fetch_fraction(tmp_path):assert load_hybrid_fetch_fraction("bf16", gpu_name="OTHER", path=str(path)) is None+def test_profile_lookup_prefers_the_gpu_uuid_file(tmp_path, monkeypatch):+ monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path))+ monkeypatch.delenv("FREETOKEN_BENCHBW_PATH", raising=False)+ uuid = "GPU-2f3a9b1c-0000-1111-2222-333344445555"++ def write(path, name, verdict):+ os.makedirs(os.path.dirname(path), exist_ok=True)+ with open(path, "w") as f:+ json.dump({"gpu": {"name": name}, "dtypes": {"bf16": verdict}}, f)++ # legacy single file only: used when the name matches, ignored otherwise+ write(default_profile_path(), "FAKE GPU", "hybrid")+ assert load_backend_recommendation("bf16", gpu_name="FAKE GPU", gpu_uuid=uuid) == "hybrid"+ assert load_backend_recommendation("bf16", gpu_name="OTHER", gpu_uuid=uuid) is None+ # this card's own file wins over the legacy one+ write(default_profile_path(uuid), "FAKE GPU", "offload")+ assert load_backend_recommendation("bf16", gpu_name="FAKE GPU", gpu_uuid=uuid) == "offload"++@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")def test_hybrid_fraction_gpu_matches_cpu_reference():torch.manual_seed(0)