Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

2.0 KB
Raw Blame
from __future__ import annotations

from typing import Any, Callable, Dict, Tuple

from freetoken.utils import init_logger

logger = init_logger(__name__)


def perf_cuda(
    f: Callable[[], Any],
    *,
    init_stream: bool = True,
    repetitions: int = 10,
    cuda_graph_repetitions: int | None = 10,
) -> float:
    import torch

    assert repetitions > 0
    tic = torch.cuda.Event(enable_timing=True)
    toc = torch.cuda.Event(enable_timing=True)
    stream = torch.cuda.Stream()
    torch.cuda.synchronize()
    if init_stream:
        stream = torch.cuda.Stream()
    else:
        stream = torch.cuda.current_stream()

    with torch.cuda.stream(stream):
        f()
        if N := cuda_graph_repetitions:
            g = torch.cuda.CUDAGraph()
            with torch.cuda.graph(g):
                for _ in range(N):
                    f()
            replay = g.replay
            del g
        else:
            replay = f
            N = 1

        torch.cuda.synchronize()

        replay()
        tic.record()
        for _ in range(repetitions):
            replay()
        toc.record()
        toc.synchronize()
        dur = tic.elapsed_time(toc)
        return dur / (N * repetitions)


def compare_memory_kernel_perf(
    *,
    baseline: Callable[[], Any],
    our_impl: Callable[[], Any],
    memory_footprint: int,  # in bytes
    description: str = " ",
    extra_kwargs: Dict[str, Any] | None = None,
    need_latency: bool = True,
) -> Tuple[float, float]:
    extra_kwargs = extra_kwargs or {}

    dur = perf_cuda(baseline, **extra_kwargs)
    bandwidth_0 = memory_footprint / (dur * 1e6)  # GB/s
    latency_msg = f"{dur:8.3f} ms | " if need_latency else ""
    message_0 = f"Baseline: {latency_msg}{bandwidth_0:8.3f} GB/s"

    dur = perf_cuda(our_impl, **extra_kwargs)
    bandwidth_1 = memory_footprint / (dur * 1e6)  # GB/s
    latency_msg = f"{dur:8.3f} ms | " if need_latency else ""
    logger.info(f"{description}{message_0} | Our Impl: {latency_msg}{bandwidth_1:8.3f} GB/s")
    return bandwidth_0, bandwidth_1