Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio upscaler: normalise seams and finalise uint8 in place on the host result (#17083)

Daniel Han0c6edeeacc6e
diff --git a/studio/backend/core/inference/diffusion_upscalers.py b/studio/backend/core/inference/diffusion_upscalers.py
index 51fc82df041..8489b7d6c33 100644
--- a/studio/backend/core/inference/diffusion_upscalers.py
+++ b/studio/backend/core/inference/diffusion_upscalers.py
@@ -35,6 +35,7 @@ DEFAULT_OVERLAP = 32
MIN_TILE = 64
# ComfyUI ImageUpscaleWithModel halves the tile on an OOM and gives up below 128 px.
OOM_MIN_TILE = 128
+INPLACE_ENV = "UNSLOTH_UPSCALER_INPLACE"
DOWNLOAD_TIMEOUT_S = 600
_SOCKET_TIMEOUT_S = 60
@@ -278,6 +279,10 @@ def resolve_model_path(upscaler_id: str, *, cancel: Any = None) -> Path:
return dest
+def inplace_enabled() -> bool:
+ return (os.environ.get(INPLACE_ENV) or "").strip().lower() not in ("0", "off", "false", "no")
+
+
def tile_starts(length: int, tile: int, overlap: int) -> list[int]:
"""Start offsets covering ``[0, length)`` with windows of ``tile`` that overlap by at least ``overlap``; the last
window ends exactly at ``length``. One window when ``length <= tile``."""
@@ -360,7 +365,9 @@ def tiled_apply(
# Stop may land while the last tile runs: never hand back a result for an accepted cancel.
if cancel is not None and cancel.is_set():
_raise_cancelled()
- return out / weight.clamp_min(1e-8)
+ if not inplace_enabled():
+ return out / weight.clamp_min(1e-8)
+ return out.div_(weight.clamp_min_(1e-8))
_model_lock = threading.Lock()
@@ -411,6 +418,20 @@ def _load(path: Path, device: Any) -> tuple[Any, Any]:
return descriptor, dtype
+def _to_uint8_hwc(y: Any, source: Any) -> Any:
+ """In place unless ``y`` shares ``source``'s storage; inference mode because ``y`` is an inference tensor."""
+ import torch
+
+ if (
+ not inplace_enabled()
+ or y.untyped_storage().data_ptr() == source.untyped_storage().data_ptr()
+ ):
+ return (y.clamp(0, 1) * 255.0).round().to(torch.uint8)[0].permute(1, 2, 0).cpu().numpy()
+ with torch.inference_mode():
+ y.clamp_(0, 1).mul_(255.0).round_()
+ return y.to(torch.uint8)[0].permute(1, 2, 0).cpu().numpy()
+
+
@dataclass
class UpscaleOutcome:
image: Any
@@ -501,7 +522,7 @@ def upscale_with_model(
if (y.shape[3], y.shape[2]) != (tw, th):
# Resized where it accumulated (host), as ComfyUI's ImageScale after ImageUpscaleWithModel.
y = torch.nn.functional.interpolate(y, size = (th, tw), mode = "bicubic", antialias = True)
- out = (y.clamp(0, 1) * 255.0).round().to(torch.uint8)[0].permute(1, 2, 0).cpu().numpy()
+ out = _to_uint8_hwc(y, x)
result = Image.fromarray(out, mode = "RGB")
del x, y
released = not keep_resident
diff --git a/studio/backend/tests/test_diffusion_hires.py b/studio/backend/tests/test_diffusion_hires.py
index 3304c5c7bb3..4870d3fe535 100644
--- a/studio/backend/tests/test_diffusion_hires.py
+++ b/studio/backend/tests/test_diffusion_hires.py
@@ -500,6 +500,119 @@ def test_tiled_esrgan_matches_untiled_on_a_tiny_random_model(tmp_path, monkeypat
assert seen_devices == ["cpu", "cpu"] # the full-scale result accumulates in host memory
+class _LiveBytes:
+ """Peak bytes of the storages ops create inside the block while any tensor on them is alive."""
+
+ def __enter__(self):
+ import weakref
+
+ import torch.utils._pytree as pytree
+ from torch.utils._python_dispatch import TorchDispatchMode
+
+ state = self
+ self.size, self.refs, self.peak = {}, {}, 0
+
+ def drop(key):
+ state.refs[key] -= 1
+ if not state.refs[key]:
+ del state.refs[key], state.size[key]
+
+ class _Mode(TorchDispatchMode):
+ def __torch_dispatch__(
+ self,
+ func,
+ types,
+ args = (),
+ kwargs = None,
+ ):
+ known = {
+ t.untyped_storage().data_ptr()
+ for t in pytree.tree_leaves((args, kwargs))
+ if hasattr(t, "untyped_storage")
+ }
+ out = func(*args, **(kwargs or {}))
+ for t in pytree.tree_leaves(out):
+ if not hasattr(t, "untyped_storage"):
+ continue
+ key = t.untyped_storage().data_ptr()
+ if not key or (key in known and key not in state.size):
+ continue # a view or in-place result on a storage made outside the block
+ if key not in state.size:
+ state.size[key], state.refs[key] = t.untyped_storage().nbytes(), 0
+ state.refs[key] += 1
+ weakref.finalize(t, drop, key)
+ state.peak = max(state.peak, sum(state.size.values()))
+ return out
+
+ self.mode = _Mode()
+ self.mode.__enter__()
+ return self
+
+ def __exit__(self, *exc):
+ self.mode.__exit__(*exc)
+
+
[email protected]("tile", [64, 512])
+def test_inplace_finalise_is_bit_identical_and_drops_full_size_planes(tile, monkeypatch):
+ torch = pytest.importorskip("torch")
+
+ def up4(x):
+ return torch.nn.functional.interpolate(x, scale_factor = 4, mode = "bicubic").clamp(0, 1)
+
+ x = torch.rand(1, 3, 192, 160)
+ results, blend, final = {}, {}, {}
+ for flag in ("0", "1"):
+ monkeypatch.setenv(ups.INPLACE_ENV, flag)
+ with torch.inference_mode():
+ with _LiveBytes() as m:
+ y = ups.tiled_apply(x, up4, 4, tile = tile, overlap = 8, output_device = "cpu")
+ blend[flag] = m.peak
+ m.peak = 0
+ results[flag] = ups._to_uint8_hwc(y, x)
+ final[flag] = m.peak
+ del y
+ assert (results["0"] == results["1"]).all()
+ plane = 3 * (192 * 4) * (160 * 4) * 4 # one full-size float32 RGB result
+ if tile == 64:
+ assert blend["1"] <= blend["0"] - plane
+ assert final["1"] <= final["0"] - plane
+
+
+def test_inplace_finalise_never_writes_the_source():
+ torch = pytest.importorskip("torch")
+ x = torch.rand(1, 3, 16, 16) * 2 - 0.5
+ before = x.clone()
+ out = ups._to_uint8_hwc(x, x) # a result sharing the source's storage
+ assert torch.equal(x, before)
+ assert out.dtype.name == "uint8" and out.shape == (16, 16, 3)
+
+
+def test_tiny_esrgan_pixels_are_unchanged_by_the_inplace_finalise(tmp_path, monkeypatch):
+ torch = pytest.importorskip("torch")
+ pytest.importorskip("spandrel")
+ import numpy as np
+ from PIL import Image
+ from spandrel.architectures.ESRGAN import ESRGAN
+
+ torch.manual_seed(0)
+ net = ESRGAN(in_nc = 3, out_nc = 3, num_filters = 8, num_blocks = 1, scale = 4)
+ path = tmp_path / "tiny_esrgan.pth"
+ torch.save(net.state_dict(), path)
+ monkeypatch.setattr(ups, "models_dir", lambda: tmp_path)
+ img = Image.fromarray((torch.rand(96, 80, 3) * 255).to(torch.uint8).numpy())
+ out = {}
+ for flag in ("0", "1"):
+ monkeypatch.setenv(ups.INPLACE_ENV, flag)
+ for tile, target in ((512, (320, 384)), (48, (320, 384)), (48, (300, 352))):
+ r = ups.upscale_with_model(
+ img, "local:tiny_esrgan", target, device = "cpu", tile = tile, overlap = 16
+ )
+ out[(flag, tile, target)] = np.asarray(r.image)
+ for (flag, tile, target), pixels in out.items():
+ if flag == "1":
+ assert np.array_equal(pixels, out[("0", tile, target)])
+
+
def test_native_request_carries_the_hires_block():
from core.inference.sd_cpp_args import build_img_gen_request