Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

4.0 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Prefill/decode timing; each stamp waits for the device or prefill reads too fast; leaf module:
torch and transformers only, no unsloth or peft imports"""

import time

import torch


def _wait_for_device(device):
    """Drain queued work so a wall-clock stamp reflects finished compute, not dispatch."""
    if device is None or device.type == "cpu":
        return
    synchronize = getattr(getattr(torch, device.type, None), "synchronize", None)
    if synchronize is None:
        return
    try:
        try:
            synchronize(device)
        except TypeError:  # torch.mps.synchronize takes no device argument
            synchronize()
    except Exception:
        # An async device fault belongs to generate()'s caller; a timing stamp must not pre-empt it.
        pass


class GenerationTimer:
    """Monotonic prefill/decode split around one ``model.generate()`` call."""

    def __init__(self):
        self.started_at = None
        self.prefill_ended_at = None
        self.ended_at = None
        self._device = None

    def start(self):
        self.started_at = time.monotonic()

    def mark_prefill_end(self, device = None):
        """Stamp the end of prefill; later decode steps must not move the boundary."""
        if self.started_at is None or self.prefill_ended_at is not None:
            return
        _wait_for_device(device)
        # latched for finish(), which has no tensor of its own to read a device off
        self._device = device
        self.prefill_ended_at = time.monotonic()

    def finish(self):
        if self.started_at is None or self.ended_at is not None:
            return
        _wait_for_device(self._device)
        self.ended_at = time.monotonic()

    @property
    def prompt_ms(self):
        """Prefill wall time, or None when generation never reached its first logits."""
        if self.started_at is None or self.prefill_ended_at is None:
            return None
        return max(0.0, (self.prefill_ended_at - self.started_at) * 1000.0)

    @property
    def predicted_ms(self):
        """Decode wall time, or None when the prefill boundary or the end is unknown."""
        if self.prefill_ended_at is None or self.ended_at is None:
            return None
        return max(0.0, (self.ended_at - self.prefill_ended_at) * 1000.0)


def with_prefill_boundary_processor(logits_processor, timer):
    """Runs first in the custom list, so the presence-penalty processor is not charged to prefill."""
    from transformers import LogitsProcessor, LogitsProcessorList

    class _PrefillBoundaryLogitsProcessor(LogitsProcessor):
        def __call__(self, input_ids, scores):
            # scores is the prefill output, so its device is the one to wait on
            timer.mark_prefill_end(scores.device)
            return scores

    processors = LogitsProcessorList([_PrefillBoundaryLogitsProcessor()])
    if logits_processor:
        processors.extend(logits_processor)
    return processors


def build_generation_timings(
    *,
    prompt_n,
    predicted_n,
    prompt_ms,
    predicted_ms,
    cached_n = 0,
):
    """A rate is omitted, not zero, when its window or token count is empty, so no speed is invented."""
    if prompt_ms is None or predicted_ms is None:
        return None
    prompt_n = int(prompt_n or 0)
    predicted_n = int(predicted_n or 0)
    prompt_ms = float(prompt_ms)
    predicted_ms = float(predicted_ms)
    timings = {
        "prompt_n": prompt_n,
        "prompt_ms": prompt_ms,
        "predicted_n": predicted_n,
        "predicted_ms": predicted_ms,
        "cache_n": int(cached_n or 0),
    }
    if prompt_n > 0 and prompt_ms > 0:
        timings["prompt_per_token_ms"] = prompt_ms / prompt_n
        timings["prompt_per_second"] = prompt_n / (prompt_ms / 1000.0)
    if predicted_n > 0 and predicted_ms > 0:
        timings["predicted_per_token_ms"] = predicted_ms / predicted_n
        timings["predicted_per_second"] = predicted_n / (predicted_ms / 1000.0)
    return timings