Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

9.8 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""CPU tests for flow_shift resolution, sigma transforms, shifted sampling and bell loss weights."""

from __future__ import annotations

import math

import pytest

from core.training.diffusion_dit_trainer import (
    _bell_loss_weights,
    _gather_sigmas,
    _sample_timesteps,
    _training_sigma_table,
)
from core.training.diffusion_train_common import DiffusionLoraConfig

QWEN_SHIFT_TERMINAL = 0.02


def _qwen_scheduler():
    # The Qwen/Qwen-Image scheduler config: shift=1.0 is SKIPPED at init (use_dynamic_shifting), base_shift = max_shift = log 3, exponential time shift, terminal stretch to 0.02.
    diffusers = pytest.importorskip("diffusers")
    FlowMatchEulerDiscreteScheduler = diffusers.FlowMatchEulerDiscreteScheduler
    return FlowMatchEulerDiscreteScheduler(
        num_train_timesteps = 1000,
        shift = 1.0,
        use_dynamic_shifting = True,
        base_shift = math.log(3.0),
        max_shift = math.log(3.0),
        shift_terminal = QWEN_SHIFT_TERMINAL,
        time_shift_type = "exponential",
    )


def _flux_static_scheduler():
    # A static-shift scheduler (shift baked into sigmas at init, no dynamic shifting).
    diffusers = pytest.importorskip("diffusers")
    return diffusers.FlowMatchEulerDiscreteScheduler(num_train_timesteps = 1000, shift = 3.0)


def test_flow_shift_defaults_per_family():
    qwen = DiffusionLoraConfig(
        base_model = "Qwen/Qwen-Image", data_dir = "d", output_dir = "o"
    ).normalized()
    assert qwen.resolved_family == "qwen-image"
    assert qwen.flow_shift == "auto"
    flux = DiffusionLoraConfig(
        base_model = "black-forest-labs/FLUX.1-dev", data_dir = "d", output_dir = "o"
    ).normalized()
    assert flux.flow_shift is None
    zimg = DiffusionLoraConfig(
        base_model = "Tongyi-MAI/Z-Image-Turbo", data_dir = "d", output_dir = "o"
    ).normalized()
    assert zimg.flow_shift is None


def test_flow_shift_explicit_values_and_validation():
    cfg = DiffusionLoraConfig(
        base_model = "Qwen/Qwen-Image", data_dir = "d", output_dir = "o", flow_shift = 2.2
    ).normalized()
    assert cfg.flow_shift == 2.2
    assert (
        DiffusionLoraConfig(base_model = "b", data_dir = "d", output_dir = "o", flow_shift = "3.0")
        .normalized()
        .flow_shift
        == 3.0
    )
    assert (
        DiffusionLoraConfig(base_model = "b", data_dir = "d", output_dir = "o", flow_shift = "AUTO")
        .normalized()
        .flow_shift
        == "auto"
    )
    with pytest.raises(ValueError, match = "flow_shift"):
        DiffusionLoraConfig(
            base_model = "b", data_dir = "d", output_dir = "o", flow_shift = 0.0
        ).normalized()
    with pytest.raises(ValueError, match = "flow_shift"):
        DiffusionLoraConfig(
            base_model = "b", data_dir = "d", output_dir = "o", flow_shift = "bogus"
        ).normalized()
    # Non-finite must be rejected too: JSON accepts 1e309, which floats to inf, and a positivity-only guard passed it to the sigma table as NaN.
    for bad in (float("inf"), float("-inf"), float("nan"), 1e309):
        with pytest.raises(ValueError, match = "flow_shift"):
            DiffusionLoraConfig(
                base_model = "b", data_dir = "d", output_dir = "o", flow_shift = bad
            ).normalized()


def test_cfg_dropout_and_weighting_scheme_validation():
    cfg = DiffusionLoraConfig(base_model = "b", data_dir = "d", output_dir = "o").normalized()
    assert cfg.cfg_dropout == 0.0
    assert cfg.weighting_scheme == "none"
    on = DiffusionLoraConfig(
        base_model = "b",
        data_dir = "d",
        output_dir = "o",
        cfg_dropout = 0.1,
        weighting_scheme = "bell",
    ).normalized()
    assert on.cfg_dropout == 0.1
    assert on.weighting_scheme == "bell"
    with pytest.raises(ValueError, match = "cfg_dropout"):
        DiffusionLoraConfig(
            base_model = "b", data_dir = "d", output_dir = "o", cfg_dropout = 1.5
        ).normalized()
    with pytest.raises(ValueError, match = "weighting_scheme"):
        DiffusionLoraConfig(
            base_model = "b", data_dir = "d", output_dir = "o", weighting_scheme = "sigma_sqrt"
        ).normalized()


def test_config_from_dict_plumbs_the_new_fields():
    from core.training.diffusion_train_common import _config_from_dict

    cfg = _config_from_dict(
        {
            "base_model": "Qwen/Qwen-Image",
            "data_dir": "d",
            "output_dir": "o",
            "flow_shift": "auto",
            "cfg_dropout": 0.05,
            "weighting_scheme": "bell",
        }
    )
    assert cfg.flow_shift == "auto"
    assert cfg.cfg_dropout == 0.05
    assert cfg.weighting_scheme == "bell"


def test_auto_table_matches_the_exact_qwen_transform():
    import torch

    sched = _qwen_scheduler()
    table = _training_sigma_table(sched, "auto")
    base = sched.sigmas
    # Exponential shift at mu = log 3 with sigma exponent 1 is exp(mu)/(exp(mu) + 1/u - 1) = 3u/(1 + 2u), then the terminal stretch maps the last sigma to 0.02.
    shifted = 3.0 * base / (1.0 + 2.0 * base)
    scale = (1.0 - shifted[-1]) / (1.0 - QWEN_SHIFT_TERMINAL)
    expected = 1.0 - (1.0 - shifted) / scale
    assert torch.allclose(table, expected, atol = 1e-6)
    assert abs(float(table[0]) - 1.0) < 1e-6
    assert abs(float(table[-1]) - QWEN_SHIFT_TERMINAL) < 1e-6
    assert abs(float(table[499]) - 0.75427) < 1e-3
    assert bool((table[:-1] > table[1:]).all())


def test_numeric_table_applies_the_linear_shift():
    import torch

    sched = _qwen_scheduler()
    table = _training_sigma_table(sched, 2.2)
    base = sched.sigmas
    assert torch.allclose(table, 2.2 * base / (1.0 + 1.2 * base), atol = 1e-6)
    # u = 0.5 under shift s maps to s/(s+1).
    assert abs(float(table[499]) - 2.2 / 3.2) < 1e-3


def test_identity_and_static_families_are_untouched():
    # None returns the scheduler's own table; "auto" on a static-shift scheduler is a no-op.
    sched = _qwen_scheduler()
    assert _training_sigma_table(sched, None) is sched.sigmas
    assert _training_sigma_table(sched, 1.0) is sched.sigmas
    static = _flux_static_scheduler()
    assert _training_sigma_table(static, None) is static.sigmas
    assert _training_sigma_table(static, "auto") is static.sigmas


def test_a_numeric_shift_is_the_effective_shift_on_a_static_scheduler():
    import torch

    # Z-Image-Turbo bakes in shift 3: a requested 3 is that schedule (not 9); 1.0 is unshifted.
    static = _flux_static_scheduler()
    assert torch.allclose(_training_sigma_table(static, 3.0), static.sigmas, atol = 1e-6)
    u = _training_sigma_table(static, 1.0)
    assert torch.allclose(u, torch.linspace(1.0, 0.001, 1000), atol = 1e-6)
    assert torch.allclose(_training_sigma_table(static, 2.0), 2.0 * u / (1.0 + u), atol = 1e-6)


def test_the_resume_identity_keeps_the_default_token_and_drops_min_snr_for_flow_families():
    from core.training import diffusion_checkpoint as dc

    flux = DiffusionLoraConfig(
        base_model = "black-forest-labs/FLUX.1-dev", data_dir = "d", output_dir = "o"
    ).normalized()
    ident = dc.identity_for_config(flux)
    # Earlier builds recorded the model's own schedule as "1.0", so a default run still resumes their checkpoints.
    assert ident.flow_shift == "1.0"
    assert ident.snr_gamma is None
    old = dc.CheckpointIdentity.from_dict({**ident.as_dict(), "snr_gamma": "5.0"})
    assert old.mismatch_reason(ident) is None
    # FLUX.1 shifts dynamically, so older checkpoints still resume.
    flux_shifted = dc.identity_for_config(
        DiffusionLoraConfig(
            base_model = "black-forest-labs/FLUX.1-dev", data_dir = "d", output_dir = "o", flow_shift = 3.0
        ).normalized()
    )
    assert flux_shifted.flow_shift == "3.0"
    legacy = dc.CheckpointIdentity.from_dict({**flux_shifted.as_dict(), "flow_shift": "3.0"})
    assert legacy.mismatch_reason(flux_shifted) is None
    # Z-Image bakes a static shift in: a number is now effective, not on top of it, so an older run is refused.
    zimage_shifted = dc.identity_for_config(
        DiffusionLoraConfig(
            base_model = "Tongyi-MAI/Z-Image-Turbo", data_dir = "d", output_dir = "o", flow_shift = 3.0
        ).normalized()
    )
    assert zimage_shifted.flow_shift == "effective:3.0"
    legacy = dc.CheckpointIdentity.from_dict({**zimage_shifted.as_dict(), "flow_shift": "3.0"})
    assert "timestep shift" in (legacy.mismatch_reason(zimage_shifted) or "")
    sdxl = DiffusionLoraConfig(
        base_model = "stabilityai/stable-diffusion-xl-base-1.0", data_dir = "d", output_dir = "o"
    ).normalized()
    assert dc.identity_for_config(sdxl).snr_gamma == "5.0"


def test_sampled_sigma_distribution_shifts_under_auto():
    import torch

    torch.manual_seed(0)
    sched = _qwen_scheduler()
    auto_table = _training_sigma_table(sched, "auto")
    _, idx = _sample_timesteps(sched, 4096, "cpu")
    base = _gather_sigmas(sched.sigmas, idx, "cpu", torch.float32, 1)
    shifted = _gather_sigmas(auto_table, idx, "cpu", torch.float32, 1)
    # Unshifted logit-normal draws center at 0.5; the mu = log 3 shift + terminal stretch push the mass to high noise (mean ~0.72) and raise EVERY sample.
    assert abs(float(base.mean()) - 0.5) < 0.03
    assert float(shifted.mean()) > 0.68
    assert bool((shifted >= base - 1e-6).all())


def test_gather_sigmas_broadcasts_to_ndim():
    import torch

    sched = _qwen_scheduler()
    sig = _gather_sigmas(sched.sigmas, torch.tensor([0, 499, 999]), "cpu", torch.float32, 4)
    assert sig.shape == (3, 1, 1, 1)
    assert abs(float(sig[0].flatten()) - 1.0) < 1e-6


def test_bell_weights_shape_peak_and_normalization():
    w = _bell_loss_weights(1000)
    assert w.shape == (1000,)
    assert float(w.min()) >= 0.0
    assert int(w.argmax()) == 500
    assert abs(float(w.mean()) - 1.0) < 1e-5
    assert float(w[500]) > float(w[0])
    assert float(w[500]) > float(w[999])