Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Safetensors timings: each stamp syncs the accelerator so it times compute, not kernel dispatch."""
import pytest
import torch
from core.inference.generation_timing import (
GenerationTimer,
build_generation_timings,
with_prefill_boundary_processor,
)
from core.inference.presence_penalty import _make_presence_penalty_processor
try:
# core.inference.inference imports unsloth, absent from the dependency-light CI job.
from core.inference.inference import InferenceBackend
except ImportError:
InferenceBackend = None
def test_windows_are_none_until_generation_is_measured():
timer = GenerationTimer()
assert timer.prompt_ms is None
assert timer.predicted_ms is None
timer.start()
assert timer.prompt_ms is None
timer.mark_prefill_end()
assert timer.prompt_ms is not None
assert timer.predicted_ms is None
timer.finish()
assert timer.predicted_ms is not None
def test_prefill_boundary_stamps_once_so_decode_steps_do_not_move_it():
timer = GenerationTimer()
timer.start()
timer.mark_prefill_end()
boundary = timer.prefill_ended_at
for _ in range(5):
timer.mark_prefill_end()
assert timer.prefill_ended_at == boundary
def test_both_stamps_wait_for_the_accelerator(monkeypatch):
"""Stamps must sync the accelerator, or a still-queued forward pass times dispatch, not prefill."""
synced = []
monkeypatch.setattr(torch.cuda, "synchronize", lambda device = None: synced.append(device))
cuda = torch.device("cuda", 0)
timer = GenerationTimer()
timer.start()
timer.mark_prefill_end(cuda)
assert synced == [cuda]
timer.finish() # reuses the latched device: no tensor in scope
assert synced == [cuda, cuda]
def test_a_faulting_device_wait_still_produces_timings(monkeypatch):
"""An async CUDA fault raised in the stamp must not strand the streamer or kill the worker thread."""
def _boom(device = None):
raise RuntimeError("CUDA error: device-side assert triggered")
monkeypatch.setattr(torch.cuda, "synchronize", _boom)
cuda = torch.device("cuda", 0)
timer = GenerationTimer()
timer.start()
timer.mark_prefill_end(cuda)
timer.finish()
assert timer.prompt_ms is not None
assert timer.predicted_ms is not None
def test_a_cpu_run_has_nothing_to_wait_for(monkeypatch):
monkeypatch.setattr(
torch.cuda, "synchronize", lambda device = None: pytest.fail("cpu run synchronized")
)
timer = GenerationTimer()
timer.start()
timer.mark_prefill_end(torch.device("cpu"))
timer.finish()
assert timer.prompt_ms is not None
def test_a_run_that_never_reached_prefill_reports_no_prompt_window():
timer = GenerationTimer()
timer.start()
timer.finish()
assert timer.prompt_ms is None
assert timer.predicted_ms is None
def test_timings_carry_llama_server_field_names_and_rates():
timings = build_generation_timings(
prompt_n = 400,
predicted_n = 50,
prompt_ms = 200.0,
predicted_ms = 2000.0,
)
assert timings["prompt_n"] == 400
assert timings["prompt_ms"] == pytest.approx(200.0)
assert timings["prompt_per_second"] == pytest.approx(2000.0)
assert timings["prompt_per_token_ms"] == pytest.approx(0.5)
assert timings["predicted_n"] == 50
assert timings["predicted_ms"] == pytest.approx(2000.0)
assert timings["predicted_per_second"] == pytest.approx(25.0)
assert timings["predicted_per_token_ms"] == pytest.approx(40.0)
assert timings["cache_n"] == 0
def test_unmeasured_split_reports_no_timings_at_all():
assert (
build_generation_timings(
prompt_n = 10,
predicted_n = 5,
prompt_ms = None,
predicted_ms = 12.0,
)
is None
)
@pytest.mark.parametrize(
"prompt_n, prompt_ms",
[
(0, 30.0),
(10, 0.0),
],
)
def test_unratable_prompt_window_omits_the_rate_instead_of_reporting_zero(prompt_n, prompt_ms):
timings = build_generation_timings(
prompt_n = prompt_n,
predicted_n = 5,
prompt_ms = prompt_ms,
predicted_ms = 100.0,
)
assert "prompt_per_second" not in timings
assert "prompt_per_token_ms" not in timings
assert timings["predicted_per_second"] == pytest.approx(50.0)
def test_processor_stamps_the_boundary_and_keeps_the_penalty_processor():
timer = GenerationTimer()
timer.start()
penalty = _make_presence_penalty_processor(1.0, prompt_len = 2)
processors = with_prefill_boundary_processor(penalty, timer)
assert len(processors) == 2
input_ids = torch.tensor([[0, 1, 3]])
scores = processors(input_ids, torch.zeros(1, 5))
assert timer.prompt_ms is not None
assert scores[0, 3].item() == pytest.approx(-1.0)
def test_boundary_lands_after_the_prompt_forward_pass_in_a_real_generate():
"""The split assumes transformers calls the processor once per step, first on the unextended prompt."""
from transformers import GPT2Config, GPT2LMHeadModel
torch.manual_seed(0)
model = GPT2LMHeadModel(
GPT2Config(vocab_size = 64, n_positions = 128, n_embd = 32, n_layer = 2, n_head = 2)
).eval()
seen_lengths = []
class _RecordingProcessor:
def __call__(self, input_ids, scores):
seen_lengths.append(int(input_ids.shape[1]))
return scores
prompt_len = 20
timer = GenerationTimer()
processors = with_prefill_boundary_processor(None, timer)
processors.append(_RecordingProcessor())
timer.start()
outputs = model.generate(
input_ids = torch.randint(0, 64, (1, prompt_len)),
max_new_tokens = 8,
do_sample = False,
logits_processor = processors,
pad_token_id = 0,
)
timer.finish()
assert seen_lengths[0] == prompt_len
assert seen_lengths == list(range(prompt_len, prompt_len + 8))
timings = build_generation_timings(
prompt_n = prompt_len,
predicted_n = int(outputs.shape[1]) - prompt_len,
prompt_ms = timer.prompt_ms,
predicted_ms = timer.predicted_ms,
)
assert timings["predicted_n"] == 8
assert timings["prompt_per_second"] > 0
assert timings["predicted_per_second"] > 0
def test_processor_wraps_a_zero_penalty_run_that_has_no_processor_of_its_own():
timer = GenerationTimer()
timer.start()
processors = with_prefill_boundary_processor(None, timer)
scores = torch.zeros(1, 5)
out = processors(torch.tensor([[0, 1, 3]]), scores)
assert timer.prompt_ms is not None
assert torch.equal(out, scores)
@pytest.mark.skipif(InferenceBackend is None, reason = "unsloth stack not installed")
def test_recorded_stats_carry_timings_only_when_a_run_was_timed():
timer = GenerationTimer()
timer.start()
timer.mark_prefill_end()
timer.finish()
backend = InferenceBackend.__new__(InferenceBackend)
InferenceBackend._record_generation_stats(
backend,
prompt_tokens = 64,
completion_tokens = 16,
max_new_tokens = 256,
timer = timer,
)
stats = backend.last_generation_stats
assert stats["usage"] == {"prompt_tokens": 64, "completion_tokens": 16, "total_tokens": 80}
assert stats["timings"]["prompt_n"] == 64
assert stats["timings"]["predicted_n"] == 16
InferenceBackend._record_generation_stats(
backend,
prompt_tokens = 64,
completion_tokens = 16,
max_new_tokens = 256,
)
assert "timings" not in backend.last_generation_stats