Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
from __future__ import annotations
import os
import types
import pytest
from core.inference import diffusion_nvfp4_linear as nl
from core.inference import diffusion_nvfp4_ops as ops
from core.inference import diffusion_nvfp4_protect as pr
REAL_SHAPES = ((3072, 3072), (18432, 3072), (15360, 256), (5120, 3072))
class _CapableLayer:
"""A stand-in for a converted flashinfer Linear: weak-referenceable is all the registry asks."""
def _capable(ctl):
"""Keep the result alive: the registry is weak."""
layer = _CapableLayer()
ctl.register_layer(layer)
return layer
def _cuda_or_skip():
torch = pytest.importorskip("torch")
if os.environ.get("CUDA_VISIBLE_DEVICES", None) == "":
pytest.skip("CUDA devices are masked off for this process")
if not getattr(torch, "cuda", None) or not torch.cuda.is_available():
pytest.skip("needs CUDA")
capability = tuple(torch.cuda.get_device_capability(0))
if capability not in ops.NVFP4_FLASHINFER_CAPS:
pytest.skip("sm_%d%d has no flashinfer NVFP4 kernels" % capability)
pytest.importorskip("flashinfer")
pytest.importorskip("torchao")
return torch
@pytest.mark.parametrize(
"spec,total,want",
[
("0,1,2,3,-1", 50, (0, 1, 2, 3, 49)),
("0,-1", 50, (0, 49)),
("0,1,2,3,4,5,6,7,-1", 50, (0, 1, 2, 3, 4, 5, 6, 7, 49)),
("auto", 50, (0, 1, 2, 3, 49)),
("auto", 8, (0, 7)),
("auto", 100, tuple(range(8)) + (99,)),
("auto", 1, (0,)),
(" 0 , 0 , -1 , 49 ", 50, (0, 49)),
("", 50, ()),
("off", 50, ()),
("none", 50, ()),
(None, 50, ()),
("0,1,2,3,-1", 4, (0, 1, 2, 3)),
("0,1,2,3,-1", 2, (0, 1)),
("60", 50, ()),
("0", 0, ()),
],
)
def test_the_schedule_parses(spec, total, want):
assert pr.parse_protect_steps(spec, total) == want
def test_a_typo_in_the_schedule_raises_rather_than_protecting_nothing():
with pytest.raises(ValueError) as excinfo:
pr.parse_protect_steps("0,fisrt,-1", 50)
assert "not a step index" in str(excinfo.value)
assert pr.PROTECT_STEPS_ENV in str(excinfo.value)
def test_the_env_reader_treats_blank_and_off_as_off(monkeypatch):
monkeypatch.delenv(pr.PROTECT_STEPS_ENV, raising = False)
assert pr.protect_steps_env() == ""
monkeypatch.setenv(pr.PROTECT_STEPS_ENV, " ")
assert pr.protect_steps_env() == ""
monkeypatch.setenv(pr.PROTECT_STEPS_ENV, "OFF")
assert pr.protect_steps_env() == ""
monkeypatch.setenv(pr.PROTECT_STEPS_ENV, " 0,1,2,3,-1 ")
assert pr.protect_steps_env() == "0,1,2,3,-1"
monkeypatch.setenv(pr.PROTECT_STEPS_ENV, "AUTO")
assert pr.protect_steps_env() == "auto"
def test_an_unarmed_controller_never_protects():
ctl = pr.NVFP4StepController("")
assert ctl.armed is False
assert ctl.begin(50) == ()
for _ in range(50):
assert ctl.protected is False
ctl.advance()
assert ctl.protected_steps_seen == 0
def test_the_controller_protects_exactly_the_named_steps():
ctl = pr.NVFP4StepController("0,1,2,3,-1")
assert ctl.armed is True
assert ctl.begin(50) == (0, 1, 2, 3, 49)
fired = []
for step in range(50):
assert ctl.index == step
if ctl.protected:
fired.append(step)
ctl.advance()
assert fired == [0, 1, 2, 3, 49]
assert ctl.protected_steps_seen == 5
assert ctl.protected is False
def test_reset_leaves_a_controller_protecting_nothing():
ctl = pr.NVFP4StepController("auto")
ctl.begin(50)
assert ctl.protected is True
ctl.reset()
assert (ctl.protected, ctl.index, ctl.steps, ctl.total) == (False, 0, (), 0)
assert ctl.armed is True
def test_begin_clears_the_previous_generation():
ctl = pr.NVFP4StepController("-1")
ctl.begin(4)
for _ in range(3):
ctl.advance()
assert (ctl.index, ctl.protected) == (3, True)
assert ctl.begin(50) == (49,)
assert (ctl.index, ctl.protected, ctl.generations) == (0, False, 2)
def test_a_bad_schedule_is_logged_and_costs_no_render():
class _Logger:
def __init__(self):
self.text = ""
def warning(self, message, *args):
self.text += (message % args if args else message) + "\n"
def info(self, message, *args):
self.text += (message % args if args else message) + "\n"
logger = _Logger()
ctl = pr.NVFP4StepController("0,oops")
assert ctl.begin(50, logger = logger) == ()
assert "protect schedule ignored" in logger.text
assert ctl.protected is False
def test_configure_rearms_and_disarms():
ctl = pr.NVFP4StepController("")
assert ctl.armed is False
ctl.configure("0,-1")
assert (ctl.armed, ctl.spec) == (True, "0,-1")
ctl.configure("off")
assert (ctl.armed, ctl.spec) == (False, "")
class _FakeScheduler:
def __init__(self) -> None:
self.calls = 0
def step(self, *args, **kwargs):
self.calls += 1
return "stepped"
class _FakePipe:
def __init__(self) -> None:
self.scheduler = _FakeScheduler()
def run(self, steps: int, observe) -> None:
for _ in range(steps):
observe()
self.scheduler.step()
def test_the_scheduler_wrapper_gives_the_forward_the_right_step_index():
ctl = pr.NVFP4StepController("0,1,2,3,-1")
layer = _capable(ctl)
pipe = _FakePipe()
seen: list = []
with pr.protect_generation(pipe, 50, controller = ctl):
pipe.run(50, lambda: seen.append((ctl.index, ctl.protected)))
assert [index for index, _ in seen] == list(range(50))
assert [index for index, protected in seen if protected] == [0, 1, 2, 3, 49]
assert pipe.scheduler.calls == 50
def test_the_wrapper_is_removed_and_the_controller_reset_afterwards():
ctl = pr.NVFP4StepController("0")
layer = _capable(ctl)
pipe = _FakePipe()
assert "step" not in pipe.scheduler.__dict__
with pr.protect_generation(pipe, 8, controller = ctl):
assert "step" in pipe.scheduler.__dict__
assert ctl.protected is True
assert "step" not in pipe.scheduler.__dict__
assert (ctl.protected, ctl.steps, ctl.index) == (False, (), 0)
def test_the_wrapper_is_removed_when_the_denoise_loop_raises():
ctl = pr.NVFP4StepController("0,-1")
layer = _capable(ctl)
pipe = _FakePipe()
with pytest.raises(RuntimeError):
with pr.protect_generation(pipe, 8, controller = ctl):
pipe.scheduler.step()
raise RuntimeError("cancelled")
assert "step" not in pipe.scheduler.__dict__
assert ctl.protected is False
def test_an_unarmed_controller_wraps_nothing_at_all():
ctl = pr.NVFP4StepController("")
pipe = _FakePipe()
with pr.protect_generation(pipe, 50, controller = ctl):
assert "step" not in pipe.scheduler.__dict__
pipe.run(50, lambda: None)
assert ctl.protected is False
def test_a_pipeline_with_no_scheduler_protects_nothing():
class _NoScheduler:
scheduler = None
class _Logger:
text = ""
def warning(self, message, *args):
type(self).text += message % args if args else message
ctl = pr.NVFP4StepController("0,-1")
layer = _capable(ctl)
logger = _Logger()
with pr.protect_generation(_NoScheduler(), 50, controller = ctl, logger = logger):
assert ctl.protected is False
assert "no scheduler.step" in _Logger.text
def test_a_second_wrapper_over_the_first_still_counts_once():
ctl = pr.NVFP4StepController("0,-1")
layer = _capable(ctl)
pipe = _FakePipe()
with pr.protect_generation(pipe, 10, controller = ctl):
inner = pipe.scheduler.step
ticks: list = []
progress = pipe.scheduler.step
def outer(*a, **kw):
ticks.append(1)
return progress(*a, **kw)
pipe.scheduler.step = outer
seen: list = []
pipe.run(10, lambda: seen.append(ctl.index))
pipe.scheduler.step = inner
assert seen == list(range(10))
assert len(ticks) == 10
assert "step" not in pipe.scheduler.__dict__
def test_the_graph_key_is_empty_when_the_lever_is_off(monkeypatch):
monkeypatch.delenv(pr.PROTECT_STEPS_ENV, raising = False)
pr.reset_protect_controller()
assert pr.protect_graph_key() == ()
assert pr.protect_graph_key(True) == ()
def test_the_graph_key_separates_the_two_branches(monkeypatch):
monkeypatch.setenv(pr.PROTECT_STEPS_ENV, "0,-1")
ctl = pr.reset_protect_controller()
try:
ctl.begin(50)
assert pr.protect_graph_key() == (("nvfp4_protect", True),)
ctl.advance()
assert pr.protect_graph_key() == (("nvfp4_protect", False),)
assert pr.protect_graph_key(True) != pr.protect_graph_key(False)
finally:
monkeypatch.delenv(pr.PROTECT_STEPS_ENV, raising = False)
pr.reset_protect_controller()
def test_the_wrapper_does_not_key_a_model_with_no_nvfp4_layers(monkeypatch):
"""An fp8 load in a process that names protect steps must not double its graph count."""
torch = pytest.importorskip("torch")
import torch.nn as nn
from core.inference import diffusion_cuda_graph as cg
monkeypatch.setenv(pr.PROTECT_STEPS_ENV, "0,-1")
pr.reset_protect_controller()
try:
assert cg._protect_keyed(nn.Sequential(nn.Linear(8, 8))) is False
finally:
monkeypatch.delenv(pr.PROTECT_STEPS_ENV, raising = False)
pr.reset_protect_controller()
assert torch is not None
def _cpu_layer(
torch,
*,
in_features = 64,
out_features = 32,
):
return nl.nvfp4_linear_class()(
in_features,
out_features,
wq = torch.zeros(out_features, in_features // 2, dtype = torch.uint8),
w_sf = torch.ones(128, 4, dtype = torch.uint8),
alpha = torch.tensor([0.25]),
a_gsf = torch.tensor([1344.0]),
bias = None,
)
def test_a_layer_takes_the_process_controller_and_derives_its_weight_scale():
torch = pytest.importorskip("torch")
layer = _cpu_layer(torch)
assert layer.protect is pr.protect_controller()
assert float(layer.w_scale) == pytest.approx(0.25 * 1344.0)
assert "protect=" not in layer.extra_repr()
def test_the_layer_repr_names_the_schedule_when_armed(monkeypatch):
torch = pytest.importorskip("torch")
monkeypatch.setenv(pr.PROTECT_STEPS_ENV, "0,1,2,3,-1")
pr.reset_protect_controller()
try:
assert "protect=0,1,2,3,-1" in _cpu_layer(torch).extra_repr()
finally:
monkeypatch.delenv(pr.PROTECT_STEPS_ENV, raising = False)
pr.reset_protect_controller()
def test_attach_controller_reaches_every_nvfp4_layer():
torch = pytest.importorskip("torch")
import torch.nn as nn
tree = nn.Sequential(_cpu_layer(torch), nn.SiLU(), _cpu_layer(torch), nn.Linear(4, 4))
ctl = pr.NVFP4StepController("0")
assert pr.attach_controller(tree, ctl) == 2
assert tree[0].protect is ctl and tree[2].protect is ctl
assert [name for name, _ in pr.protect_layers(tree)] == ["0", "2"]
def test_an_image_and_a_video_render_in_flight_keep_their_own_schedules(monkeypatch):
torch = pytest.importorskip("torch")
import torch.nn as nn
monkeypatch.setenv(pr.PROTECT_STEPS_ENV, "0")
pr.reset_protect_controller()
try:
image, video = _FakePipe(), _FakePipe()
image.transformer = nn.Sequential(_cpu_layer(torch))
video.transformer = nn.Sequential(_cpu_layer(torch))
a = pr.attach_own_controller(image.transformer)
b = pr.attach_own_controller(video.transformer)
assert a is not b and a.armed and b.armed
assert pr.pipeline_controllers(image) == [a]
assert pr.module_controller(video.transformer) is b
assert pr.protect_controller().capable_layers() == 0
with pr.protect_generation(video, 10):
assert b.protected
with pr.protect_generation(image, 10):
image.run(3, lambda: None)
assert a.index == 3 and not a.protected
assert b.index == 0 and b.protected and b.total == 10
finally:
monkeypatch.delenv(pr.PROTECT_STEPS_ENV, raising = False)
pr.reset_protect_controller()
@pytest.mark.parametrize("out_features,in_features", REAL_SHAPES)
def test_the_dequantiser_matches_torchao_bit_for_bit(out_features, in_features):
torch = _cuda_or_skip()
from torchao.prototype.mx_formats.nvfp4_tensor import NVFP4Tensor
torch.manual_seed(7)
with torch.cuda.device(0):
w = torch.randn(out_features, in_features, device = "cuda", dtype = torch.bfloat16) * 0.02
amax = w.float().abs().amax().clamp(min = 1e-8)
per_tensor_scale = (amax / (6.0 * 448.0)).reshape(1)
tensor = NVFP4Tensor.to_nvfp4(w, per_tensor_scale = per_tensor_scale, is_swizzled_scales = True)
wq = tensor.qdata.view(torch.uint8).contiguous()
cols = in_features // 16
w_sf = (
tensor.scale.reshape(-1)
.view(torch.uint8)
.reshape(ops.sf_matrix_shape(out_features, cols))
.contiguous()
)
want = tensor.dequantize(torch.bfloat16)
got = ops.dequantize_nvfp4_weight(wq, w_sf, per_tensor_scale, dtype = torch.bfloat16)
assert tuple(got.shape) == (out_features, in_features)
assert got.dtype == torch.bfloat16
assert torch.equal(got, want), (
f"{int((got != want).sum())} of {got.numel()} elements differ from "
f"NVFP4Tensor.dequantize"
)
with torch.cuda.device(0):
assert torch.equal(
ops.dequantize_nvfp4_weight(wq, w_sf, per_tensor_scale, dtype = torch.float32),
tensor.dequantize(torch.float32),
)
def test_the_dequantiser_matches_flashinfers_own_packing():
torch = _cuda_or_skip()
import flashinfer
out_features, in_features = 3072, 3072
torch.manual_seed(11)
with torch.cuda.device(0):
w = torch.randn(out_features, in_features, device = "cuda", dtype = torch.bfloat16) * 0.02
amax = w.float().abs().amax().clamp(min = 1e-8)
w_gsf = (6.0 * 448.0 / amax).reshape(1)
wq, w_sf = flashinfer.nvfp4_quantize(w, w_gsf, do_shuffle = False)
cols = in_features // 16
deq = ops.dequantize_nvfp4_weight(
wq.view(torch.uint8),
w_sf.reshape(-1).view(torch.uint8).reshape(ops.sf_matrix_shape(out_features, cols)),
(1.0 / w_gsf),
dtype = torch.float32,
)
rel = float((deq - w.float()).norm() / w.float().norm())
assert rel < 0.12, rel
assert bool(torch.isfinite(deq).all())
def _torchao_linear(
torch,
out_features,
in_features,
*,
bias = True,
seed = 0,
):
import torch.nn as nn
from torchao.prototype.mx_formats import NVFP4DynamicActivationNVFP4WeightConfig
from torchao.quantization import quantize_
torch.manual_seed(seed)
linear = nn.Linear(in_features, out_features, bias = bias).to("cuda", torch.bfloat16)
quantize_(linear, NVFP4DynamicActivationNVFP4WeightConfig(use_triton_kernel = False))
return linear
@pytest.mark.parametrize("m", [1, 512, 4096])
def test_a_protected_forward_is_the_dense_bf16_gemm(m):
torch = _cuda_or_skip()
import torch.nn.functional as F
with torch.cuda.device(0):
linear = _torchao_linear(torch, 3072, 3072)
x = torch.randn(m, 3072, device = "cuda", dtype = torch.bfloat16) * 0.05
converted = nl.nvfp4_linear_from_torchao(linear, ops.global_scale(x))
ctl = pr.NVFP4StepController("0")
converted.protect = ctl
ctl.begin(2)
with torch.inference_mode():
got = converted(x)
weight = ops.dequantize_nvfp4_weight(
converted.wq, converted.w_sf, converted.w_scale, dtype = torch.bfloat16
)
want = F.linear(x, weight)
want = want.to(x.dtype)
want.add_(converted.bias)
ctl.advance()
assert ctl.protected is False
w4a4 = converted(x)
assert torch.equal(got, want)
assert bool(torch.isfinite(w4a4).all())
rel = float((got.float() - w4a4.float()).norm() / w4a4.float().norm())
assert 0.0 < rel < 0.2, rel
def test_the_protected_branch_leaves_no_resident_weight_behind():
torch = _cuda_or_skip()
with torch.cuda.device(0):
linear = _torchao_linear(torch, 18432, 3072, seed = 3)
x = torch.randn(1024, 3072, device = "cuda", dtype = torch.bfloat16) * 0.05
converted = nl.nvfp4_linear_from_torchao(linear, ops.global_scale(x))
ctl = pr.NVFP4StepController("1")
converted.protect = ctl
ctl.begin(4)
resident = lambda: sum(b.numel() * b.element_size() for b in converted.buffers())
before_bytes = resident()
with torch.inference_mode():
converted(x)
torch.cuda.synchronize()
baseline = torch.cuda.memory_allocated()
ctl.advance()
assert ctl.protected is True
with torch.inference_mode():
out = converted(x)
del out
torch.cuda.synchronize()
after = torch.cuda.memory_allocated()
assert resident() == before_bytes
assert after == baseline, f"{(after - baseline) / 2 ** 20:.1f} MiB left resident"
def test_the_switch_costs_two_compiled_variants_over_a_fifty_step_render():
torch = _cuda_or_skip()
import torch._dynamo as dynamo
with torch.cuda.device(0):
linear = _torchao_linear(torch, 3072, 3072, seed = 5)
x = torch.randn(512, 3072, device = "cuda", dtype = torch.bfloat16) * 0.05
converted = nl.nvfp4_linear_from_torchao(linear, ops.global_scale(x))
ctl = pr.NVFP4StepController("0,1,2,3,-1")
converted.protect = ctl
dynamo.reset()
dynamo.utils.counters.clear()
compiled = torch.compile(converted, fullgraph = True, dynamic = False)
ctl.begin(50)
with torch.inference_mode():
for _ in range(50):
compiled(x)
ctl.advance()
frames = dynamo.utils.counters["stats"]["unique_graphs"]
assert frames == 2, f"{frames} compiled variants for one bool; expected 2"
assert ctl.protected_steps_seen == 5
def test_a_captured_block_gets_one_graph_per_branch(monkeypatch):
torch = _cuda_or_skip()
import torch.nn as nn
from core.inference import diffusion_cuda_graph as cg
monkeypatch.setenv(pr.PROTECT_STEPS_ENV, "0")
ctl = pr.reset_protect_controller()
try:
with torch.cuda.device(0):
linear = _torchao_linear(torch, 3072, 3072, seed = 9)
x = torch.randn(512, 3072, device = "cuda", dtype = torch.bfloat16) * 0.05
converted = nl.nvfp4_linear_from_torchao(linear, ops.global_scale(x))
class Block(nn.Module):
def __init__(self, inner):
super().__init__()
self.inner = inner
def forward(
self,
hidden_states,
return_dict = True,
):
out = self.inner(hidden_states)
return types.SimpleNamespace(sample = out) if return_dict else (out,)
block = Block(converted).eval()
handle = cg.GraphedForward(block).install().enable()
ctl.begin(4)
call = lambda: block(hidden_states = x, return_dict = False)[0].clone()
with torch.inference_mode():
protected = call()
ctl.advance()
assert ctl.protected is False
plain = call()
ctl.advance()
again = call()
captures = handle.stats["captures"]
replays = handle.stats["replays"]
cg.uninstall_all([handle])
finally:
monkeypatch.delenv(pr.PROTECT_STEPS_ENV, raising = False)
pr.reset_protect_controller()
assert captures == 2, handle.stats
assert replays == 3, handle.stats
assert torch.equal(plain, again)
assert not torch.equal(protected, plain)
def test_an_armed_controller_with_no_protect_capable_layer_refuses_and_says_so():
"""Only ``NVFP4FlashInferLinear`` reads the controller, so a torchao load must not arm."""
class _Logger:
def __init__(self):
self.text = ""
def warning(self, message, *args):
self.text += (message % args if args else message) + "\n"
def info(self, message, *args):
self.text += (message % args if args else message) + "\n"
ctl = pr.NVFP4StepController("0,-1")
assert ctl.capable_layers() == 0
pipe = _FakePipe()
logger = _Logger()
with pr.protect_generation(pipe, 10, controller = ctl, logger = logger):
assert "step" not in pipe.scheduler.__dict__
pipe.run(10, lambda: None)
assert "no protect-capable NVFP4 layer" in logger.text
assert (ctl.steps, ctl.protected, ctl.protected_steps_seen) == ((), False, 0)
def test_a_registered_layer_lets_the_same_controller_arm():
ctl = pr.NVFP4StepController("0,-1")
layer = _capable(ctl)
assert ctl.capable_layers() == 1
pipe = _FakePipe()
seen: list = []
with pr.protect_generation(pipe, 10, controller = ctl):
pipe.run(10, lambda: seen.append(ctl.protected))
assert [i for i, protected in enumerate(seen) if protected] == [0, 9]
assert layer is not None
def test_the_layer_registry_is_weak_so_an_unloaded_model_stops_counting():
import gc
ctl = pr.NVFP4StepController("0,-1")
layer = _capable(ctl)
assert ctl.capable_layers() == 1
del layer
gc.collect()
assert ctl.capable_layers() == 0
def _cpu_protected_layer():
torch = pytest.importorskip("torch")
from core.inference.diffusion_nvfp4_protect import NVFP4StepController
torch.manual_seed(7)
q = torch.randint(0, 256, (32, 32), dtype = torch.uint8)
scales = torch.full((32, 4), 0.5, dtype = torch.float8_e4m3fn)
packed = ops.swizzle_sf(scales, 32, 64).reshape(ops.sf_matrix_shape(32, 4))
layer = nl.nvfp4_linear_class()(
64,
32,
wq = q,
w_sf = packed,
alpha = torch.ones(1),
a_gsf = torch.ones(1),
w_scale = torch.tensor([0.125]),
)
layer.protect = NVFP4StepController("auto")
layer.protect.begin(9)
assert layer.protect.armed and layer.protect.protected
return torch, layer
def test_a_cold_lut_compiles_the_protected_branch_once():
torch, layer = _cpu_protected_layer()
from torch._dynamo.testing import CompileCounter
ops.reset_lut_cache()
torch._dynamo.reset()
counter = CompileCounter()
compiled = torch.compile(layer, backend = counter, fullgraph = True, dynamic = True)
x = torch.randn(4, 64, dtype = torch.bfloat16)
with torch.inference_mode():
ref = layer(x)
ops.reset_lut_cache() # the eager reference memoised it; the compiled calls start cold
outs = [compiled(x) for _ in range(3)]
ops.e2m1_lut(x.device) # an eager dequantise between renders memoises the table
outs.append(compiled(x))
assert counter.frame_count == 1
assert all(torch.equal(out, ref) for out in outs)
torch._dynamo.reset()
@pytest.mark.parametrize("dtype_name", ["float16", "bfloat16", "float32"])
def test_the_protected_branch_keeps_the_input_dtype(dtype_name):
torch, layer = _cpu_protected_layer()
dtype = getattr(torch, dtype_name)
x = torch.randn(4, 64, dtype = dtype)
compute = torch.bfloat16 if dtype is torch.float32 else dtype
weight = ops.dequantize_nvfp4_weight(layer.wq, layer.w_sf, layer.w_scale, dtype = compute)
with torch.inference_mode():
out = layer(x)
assert out.dtype == dtype
assert torch.equal(out, torch.nn.functional.linear(x.to(compute), weight).to(dtype))