FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""CLI: convert an HF safetensors checkpoint to a FreeToken Weight (FTW) checkpoint.
ft checkpoint --model <hf_dir> --out <ftw_dir> \
[--dtype bfloat16] [--moe-backend offload] [--quant-backend moe.nvfp4=b12x] [--shard-gib 8] [--gpu <uuid-or-index>]
The output dir is self-contained: point the server's ``--model`` at it to load via the FTW
fast path (auto-detected).
"""
from __future__ import annotations
import argparse
import time
import torch
from freetoken.gpu_select import assign_gpu, bind_assigned_gpu, single_gpu_arg
from .convert import convert_checkpoint
_DTYPES = {"bfloat16": torch.bfloat16, "float16": torch.float16, "float32": torch.float32}
def _parse_quant_backend(value: str) -> str:
from freetoken.layers.quantization import QuantBackend
try:
QuantBackend.parse(value)
except ValueError as exc:
raise argparse.ArgumentTypeError(str(exc)) from None
return value
def main(argv: list[str] | None = None, prog: str = "freetoken.checkpoint") -> int:
p = argparse.ArgumentParser(prog=prog, description=__doc__)
p.add_argument("--model", required=True, help="source HF safetensors checkpoint dir")
p.add_argument("--out", required=True, help="output FTW checkpoint dir")
p.add_argument("--dtype", choices=sorted(_DTYPES), default="bfloat16")
p.add_argument("--moe-backend", default="offload",
help="kept for compatibility: MoE experts always go to banks, which every strategy reads")
p.add_argument("--quant-backend", type=_parse_quant_backend, default=None,
help="kernel per quantized layer type, as for ft serve; the expert banks are packed for the "
"chosen MoE kernel and the server picks that kernel back up from the FTW")
p.add_argument("--shard-gib", type=float, default=8.0, help="max shard size in GiB")
p.add_argument("--gpu", type=single_gpu_arg, default=None,
help="GPU for the repack: a GPU UUID (GPU-xxxx..., as nvidia-smi -L prints) or "
"an nvidia-smi index (default: the first visible GPU)")
ns = p.parse_args(argv)
# same as ft serve --gpu: resolve, then bind by UUID at CUDA init
try:
assign_gpu(ns.gpu)
device = f"cuda:{bind_assigned_gpu().index}"
except (ValueError, RuntimeError) as e:
p.error(str(e))
shard_limit = int(ns.shard_gib * (1 << 30))
shard_limit -= shard_limit % 4096 # keep aligned
t = time.perf_counter()
index = convert_checkpoint(
ns.model, ns.out, dtype=_DTYPES[ns.dtype],
moe_backend=ns.moe_backend, quant_backend=ns.quant_backend, shard_limit=shard_limit, device=device,
)
dt = time.perf_counter() - t
c = index["counts"]
gib = index["total_bytes"] / (1 << 30)
print(f"\nwrote FTW checkpoint -> {ns.out}")
print(f" tensors: {c['weight']} weight + {c['experts_bank']} experts_bank")
print(f" FTW: {gib:.2f} GiB across {len(index['shards'])} shard(s) "
f"(<= {ns.shard_gib} GiB each)")
print(f" quant_format: {index['quant_format']} fingerprint={index['fingerprint']}")
print(f" converted in {dt:.1f}s")
return 0
if __name__ == "__main__":
raise SystemExit(main())