Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

10.0 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Attribute throughput to the window the work ran in, not the tick a counter moved on."""

from __future__ import annotations

import json

import core.inference.llama_stats as ls
from core.inference.llama_stats import LlamaServerStatsLogger

_TICK_S = 10.0


class _Resp:
    status = 200

    def __init__(self, body):
        self._body = body

    def read(self):
        return self._body

    def __enter__(self):
        return self

    def __exit__(self, *a):
        return False


class _Capture:
    def __init__(self):
        self.events = []

    def info(self, event, **kw):
        self.events.append((event, dict(kw)))

    def debug(self, *a, **k):
        pass

    def warning(self, *a, **k):
        pass


def _drive(
    snaps,
    monkeypatch,
    tick_s = _TICK_S,
):
    """Run _run() over `snaps` on a clock that advances tick_s per scrape."""
    clock = {"t": 1000.0}
    monkeypatch.setattr(ls.time, "monotonic", lambda: clock["t"])

    cap = _Capture()
    lg = LlamaServerStatsLogger("http://127.0.0.1:0", cap)
    lg._interval = 0.001  # the real sleep between ticks, not the faked elapsed time
    state = {"i": 0}

    def fake_scrape():
        i = state["i"]
        state["i"] += 1
        if i >= len(snaps):
            lg.stop()
            return None
        clock["t"] += tick_s
        return snaps[i]

    lg._scrape = fake_scrape
    lg._run()
    return [kw for ev, kw in cap.events if ev == "engine_stats"]


def _busy(
    predicted = 0.0,
    predicted_s = 0.0,
    prompt = 0.0,
    prompt_s = 0.0,
    decode = 0.0,
    running = 1.0,
    waiting = 0.0,
    gen_gauge = None,
):
    snap = {
        "tokens_predicted_total": predicted,
        "tokens_predicted_seconds_total": predicted_s,
        "prompt_tokens_total": prompt,
        "prompt_seconds_total": prompt_s,
        "n_decode_total": decode,
        "requests_processing": running,
        "requests_deferred": waiting,
    }
    if gen_gauge is not None:
        snap["predicted_tokens_seconds"] = gen_gauge
    return snap


def test_a_prefill_is_priced_by_the_seconds_it_reports_with_it(monkeypatch):
    snaps = [_busy(decode = float(i)) for i in range(8)] + [
        _busy(prompt = 1837.0, prompt_s = 80.0, decode = 8.0)
    ]
    stats = _drive(snaps, monkeypatch)

    assert max(s["prompt_tok_s"] for s in stats) == 23.0
    assert 1837.0 / _TICK_S == 183.7


def test_an_idle_gap_is_not_charged_to_the_prefill_after_it(monkeypatch):
    idle = [_busy(running = 0.0) for _ in range(6)]
    working = [_busy(decode = 1.0), _busy(prompt = 100.0, prompt_s = 20.0, decode = 2.0)]
    stats = _drive(idle + working, monkeypatch)

    assert max(s["prompt_tok_s"] for s in stats) == 5.0


def test_deferred_requests_do_not_stretch_the_denominator(monkeypatch):
    """A queued request is not running, so charging its wait understates instead."""
    snaps = [
        _busy(decode = 1.0),
        *[_busy(running = 0.0, waiting = 1.0, decode = 1.0) for _ in range(6)],
        _busy(prompt = 100.0, prompt_s = 20.0, decode = 2.0),
    ]
    stats = _drive(snaps, monkeypatch)

    assert max(s["prompt_tok_s"] for s in stats) == 5.0
    assert round(100.0 / 80.0, 1) == 1.2


def test_work_already_running_at_the_first_poll_is_priced_whole(monkeypatch):
    """Starting mid-prefill, elapsed poll time omits up to an interval; the engine's does not."""
    snaps = [
        _busy(prompt = 200.0, prompt_s = 40.0, decode = 4.0),
        _busy(prompt = 1837.0, prompt_s = 80.0, decode = 8.0),
    ]
    stats = _drive(snaps, monkeypatch)

    # 1637 tokens in the 40 seconds between the two readings
    assert max(s["prompt_tok_s"] for s in stats) == 40.9
    assert 1637.0 / _TICK_S == 163.7


def test_a_long_prefill_is_not_attributed_to_the_tick_it_flushed_on(monkeypatch):
    """The prompt counter and its seconds flush together, on a decode with output."""
    snaps = (
        [_busy(prompt = 0.0)]
        + [_busy() for _ in range(64)]
        + [_busy(prompt = 130000.0, prompt_s = 650.0)]
    )
    stats = _drive(snaps, monkeypatch)

    assert max(s["prompt_tok_s"] for s in stats) == 200.0
    assert 130000.0 / _TICK_S == 13000.0


def test_the_decode_counter_reports_while_the_token_counters_are_still(monkeypatch):
    """Why the line read 0 for 98.8% of the time: only n_decode_total moves here."""
    snaps = [_busy(decode = float(i * 20), gen_gauge = 0.0) for i in range(4)]
    stats = _drive(snaps, monkeypatch)

    assert [s.get("gen_tok_s") for s in stats] == [0.0, None, None, None]
    assert [s.get("decode_calls_s") for s in stats] == [None, 2.0, 2.0, 2.0]


def test_a_build_without_the_decode_counter_omits_the_field(monkeypatch):
    """0.0 would state the engine was measured making no calls."""
    snaps = [
        {"tokens_predicted_total": 0.0, "prompt_tokens_total": 0.0, "requests_processing": 1.0},
        {"tokens_predicted_total": 50.0, "prompt_tokens_total": 0.0, "requests_processing": 1.0},
    ]
    stats = _drive(snaps, monkeypatch)

    assert stats and all("decode_calls_s" not in s for s in stats)


def test_a_zero_gauge_is_a_reading_and_not_a_missing_one(monkeypatch):
    """One-token completion: the free prompt-batch token is no decode step, so 0."""
    snaps = [
        _busy(gen_gauge = 0.0),
        _busy(predicted = 1.0, predicted_s = 0.0001, gen_gauge = 0.0),
    ]
    stats = _drive(snaps, monkeypatch)

    assert stats and all(s["gen_tok_s"] == 0.0 for s in stats)
    assert 1.0 / 0.0001 == 10000.0


def test_a_zero_gauge_beside_moved_counters_is_still_a_reading(monkeypatch):
    """Another client's scrape empties the bucket, and understating beats inventing."""
    snaps = [
        _busy(predicted = 100.0, predicted_s = 5.0, gen_gauge = 20.0),
        _busy(predicted = 300.0, predicted_s = 15.0, gen_gauge = 0.0),
        _busy(predicted = 400.0, predicted_s = 20.0, gen_gauge = 20.0),
    ]
    stats = _drive(snaps, monkeypatch)

    assert [s["gen_tok_s"] for s in stats] == [20.0, 0.0, 20.0]
    assert 200.0 / 10.0 == 20.0


def test_tokens_with_no_seconds_yet_are_kept_for_the_tick_that_brings_them(monkeypatch):
    """Six significant digits can move one total a scrape before the other, so the
    baseline is held until both have moved rather than charged to the next seconds."""
    snaps = [
        _busy(prompt = 100.0, prompt_s = 1000.0),
        _busy(prompt = 200.0, prompt_s = 1000.0),
        _busy(prompt = 300.0, prompt_s = 1020.0),
    ]
    stats = _drive(snaps, monkeypatch)

    assert [s["prompt_tok_s"] for s in stats] == [0.0, 0.0, 10.0]


def test_seconds_that_resolve_before_their_tokens_are_kept_too(monkeypatch):
    snaps = [
        _busy(prompt = 100.0, prompt_s = 1000.0),
        _busy(prompt = 100.0, prompt_s = 1020.0),
        _busy(prompt = 200.0, prompt_s = 1020.0),
        _busy(prompt = 300.0, prompt_s = 1040.0),
    ]
    stats = _drive(snaps, monkeypatch)

    assert [s["prompt_tok_s"] for s in stats] == [0.0, 0.0, 5.0, 5.0]
    assert 100.0 / 20.0 == 5.0


def test_a_build_without_the_generation_gauge_omits_the_field(monkeypatch):
    """The seconds time n_gen - 1 steps against n_gen tokens, so the ratio is not a rate."""
    snaps = [
        _busy(predicted = 0.0),
        _busy(predicted = 1.0, predicted_s = 0.0001),
    ]
    stats = _drive(snaps, monkeypatch)

    assert stats and all("gen_tok_s" not in s for s in stats)
    assert 1.0 / 0.0001 == 10000.0


def test_a_build_with_no_prompt_metric_omits_the_field(monkeypatch):
    """The third unmeasurable case: no prompt gauge and no prompt counters."""
    snaps = [
        {"tokens_predicted_total": 0.0, "requests_processing": 1.0},
        {"tokens_predicted_total": 50.0, "requests_processing": 1.0},
    ]
    stats = _drive(snaps, monkeypatch)

    assert stats and all("prompt_tok_s" not in s for s in stats)
    assert all(s["running"] == 1 for s in stats)


def test_a_non_finite_metric_value_never_reaches_the_line(monkeypatch):
    """float() turns a long digit string into inf; _env_float refuses the same text."""

    body = (
        "llamacpp:prompt_tokens_seconds " + "9" * 400 + "\n"
        "llamacpp:prompt_tokens_total 10\n"
        "llamacpp:prompt_seconds_total 1\n"
    ).encode()
    monkeypatch.setattr(ls.urllib.request.OpenerDirector, "open", lambda *a, **k: _Resp(body))

    m = LlamaServerStatsLogger("http://127.0.0.1:0", _Capture())._scrape()

    assert "prompt_tokens_seconds" not in m
    assert m["prompt_tokens_total"] == 10.0


def test_the_llama_cpp_gauge_still_wins_when_it_reports(monkeypatch):
    snaps = [
        _busy(gen_gauge = 24.6),
        _busy(predicted = 1837.0, predicted_s = 80.0, gen_gauge = 24.6),
    ]
    stats = _drive(snaps, monkeypatch)

    assert stats and all(s["gen_tok_s"] == 24.6 for s in stats)


def _slots(monkeypatch, *ticks):
    bodies = iter(
        json.dumps(
            [
                {
                    "id": slot,
                    "is_processing": True,
                    "id_task": task,
                    "next_token": [{"n_decoded": n}],
                }
                for slot, task, n in tick
            ]
        ).encode()
        for tick in ticks
    )
    monkeypatch.setattr(
        ls.urllib.request.OpenerDirector, "open", lambda *a, **k: _Resp(next(bodies))
    )


def test_a_long_reply_reports_its_rate_while_it_is_still_generating(monkeypatch):
    _slots(monkeypatch, [(0, 7, 250)], [(0, 7, 500)], [(0, 7, 750)])
    snaps = [_busy(running = 0.0)] + [
        _busy(decode = float(i * 250), gen_gauge = 0.0) for i in range(1, 4)
    ]
    stats = _drive(snaps, monkeypatch)

    assert [s["gen_tok_s"] for s in stats] == [25.0, 25.0, 25.0]


def test_a_new_task_on_a_slot_is_counted_from_zero(monkeypatch):
    _slots(monkeypatch, [(0, 7, 900)], [(0, 8, 100)])
    snaps = [
        _busy(running = 0.0),
        _busy(decode = 900.0, gen_gauge = 0.0),
        _busy(decode = 1000.0, gen_gauge = 0.0),
    ]
    stats = _drive(snaps, monkeypatch)

    assert [s["gen_tok_s"] for s in stats] == [90.0, 10.0]