Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

5.6 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Hermetic tests for the standalone diffusion benchmark report."""

from __future__ import annotations

import importlib.util
import json
import math
import sys
from pathlib import Path
from types import SimpleNamespace

import pytest


_REPO_ROOT = Path(__file__).resolve().parents[3]
_SPEC = importlib.util.spec_from_file_location(
    "diffusion_bench", _REPO_ROOT / "scripts" / "diffusion_bench.py"
)
assert _SPEC is not None and _SPEC.loader is not None
_DIFFUSION_BENCH = importlib.util.module_from_spec(_SPEC)
_SPEC.loader.exec_module(_DIFFUSION_BENCH)


@pytest.mark.parametrize(
    ("psnr", "expected_exit"),
    ((math.inf, 0), (-math.inf, 1), (math.nan, 1)),
)
def test_compare_report_serializes_non_finite_psnr_as_strict_json(
    tmp_path, monkeypatch, psnr, expected_exit
):
    reference = tmp_path / "reference.png"
    reference.write_bytes(b"reference")
    baseline = {
        "env": {
            "gpu_name": "test-gpu",
            "status": {"device": "cuda", "dtype": "bfloat16"},
        },
        "generate": {
            "median_latency_s": 1.0,
            "peak_vram_bytes": 100,
            "host_rss": {"post_warmup_growth_bytes": 0},
        },
        "accuracy": {"reference_png": str(reference)},
    }
    baseline_path = tmp_path / "baseline.json"
    baseline_path.write_text(json.dumps(baseline))
    current = {
        "env": {"status": {"device": "cuda", "dtype": "bfloat16"}},
        "generate": {
            "median_latency_s": 1.0,
            "peak_vram_bytes": 100,
            "host_rss": {"post_warmup_growth_bytes": 0},
        },
    }
    monkeypatch.setattr(_DIFFUSION_BENCH, "_gpu_name", lambda: "test-gpu")
    monkeypatch.setattr(_DIFFUSION_BENCH, "_run", lambda args: current)
    monkeypatch.setattr(_DIFFUSION_BENCH, "_psnr", lambda ref, candidate: psnr)
    out_dir = tmp_path / "out"
    args = SimpleNamespace(
        compare = str(baseline_path),
        out_dir = str(out_dir),
        force_compare = False,
        max_latency_regression = 0.1,
        max_vram_regression = 0.1,
        max_host_rss_growth_mib = 1.0,
        min_psnr = 30.0,
    )

    assert _DIFFUSION_BENCH._compare(args) == expected_exit
    report_text = (out_dir / "compare.json").read_text()
    report = json.loads(
        report_text,
        parse_constant = lambda token: pytest.fail(f"non-standard JSON constant: {token}"),
    )
    assert report["comparison"]["psnr_db"] is None


def test_process_rss_is_plausible_or_absent():
    value = _DIFFUSION_BENCH._process_rss_bytes()
    assert value is None or (isinstance(value, int) and 2**20 < value < 2**44)


@pytest.mark.skipif(not sys.platform.startswith("linux"), reason = "/proc fallback is Linux only")
def test_process_rss_falls_back_to_proc_without_psutil(monkeypatch):
    """The /proc branch is unreachable wherever psutil is installed, which is everywhere the
    benchmark normally runs, so drive it directly rather than leaving it unexercised."""
    real_import = __import__

    def without_psutil(name, *args, **kwargs):
        if name == "psutil":
            raise ImportError("psutil unavailable")
        return real_import(name, *args, **kwargs)

    monkeypatch.setattr("builtins.__import__", without_psutil)
    value = _DIFFUSION_BENCH._process_rss_bytes()
    assert isinstance(value, int) and value > 0


@pytest.mark.parametrize("name", ["compare.json", "compare.png"])
def test_compare_refuses_to_overwrite_its_own_baseline(tmp_path, monkeypatch, name):
    """Compare must refuse to overwrite its own --write-baseline file, before generating, not after."""
    out_dir = tmp_path / "out"
    out_dir.mkdir()
    reference = tmp_path / "reference.png"
    reference.write_bytes(b"reference")
    baseline_path = out_dir / name
    baseline_path.write_text(
        json.dumps(
            {
                "env": {"gpu_name": "test-gpu", "status": {"device": "cuda", "dtype": "bfloat16"}},
                "generate": {
                    "median_latency_s": 1.0,
                    "peak_vram_bytes": 100,
                    "host_rss": {"post_warmup_growth_bytes": 0},
                },
                "accuracy": {"reference_png": str(reference)},
            }
        )
    )
    before = baseline_path.read_text()

    monkeypatch.setattr(_DIFFUSION_BENCH, "_gpu_name", lambda: "test-gpu")
    monkeypatch.setattr(
        _DIFFUSION_BENCH,
        "_run",
        lambda args: {
            "env": {"status": {"device": "cuda", "dtype": "bfloat16"}},
            "generate": {
                "median_latency_s": 1.0,
                "peak_vram_bytes": 100,
                "host_rss": {"post_warmup_growth_bytes": 0},
            },
        },
    )
    monkeypatch.setattr(_DIFFUSION_BENCH, "_psnr", lambda ref, candidate: 99.0)

    args = SimpleNamespace(
        compare = str(baseline_path),
        out_dir = str(out_dir),
        force_compare = False,
        max_latency_regression = 0.1,
        max_vram_regression = 0.1,
        max_host_rss_growth_mib = None,
        min_psnr = 30.0,
    )
    assert _DIFFUSION_BENCH._compare(args) == 2
    assert baseline_path.read_text() == before, "the baseline was overwritten"


def test_host_rss_guard_is_off_unless_requested():
    """Back-compat for existing scripted callers: the new threshold only engages when asked
    for, so an invocation written before this flag existed behaves exactly as it did."""
    args = _DIFFUSION_BENCH._build_parser().parse_args(["--write-baseline", "out.json"])
    assert args.max_host_rss_growth_mib is None