Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

qwen-image: qwenimage_ref.py, the staged f32 reference for Qwen-Image-2.1

JustVugg7e6a913e121b
diff --git a/c/tools/qwenimage_ref.py b/c/tools/qwenimage_ref.py
new file mode 100644
index 00000000..3bd1b507
--- /dev/null
+++ b/c/tools/qwenimage_ref.py
@@ -0,0 +1,884 @@
+#!/usr/bin/env python3
+"""Qwen-Image-2.1 text-to-image reference, one component at a time.
+
+The C engine (c/qwenimage) computes in float32, so this oracle does too by
+default: the bf16 weights are upcast and every op runs in f32. The pipeline is
+never loaded whole. The text encoder is loaded, runs and is freed, then the
+transformer, then the VAE, so the real model peaks at about 31 GB (the f32
+language model) instead of the ~75 GB the three f32 components take together.
+The vision tower and the LM head are dropped before the upcast: text-to-image
+reads neither.
+
+Every stage calls the diffusers code itself (QwenImage21Pipeline.encode_prompt,
+QwenImage21Pipeline.__call__ with output_type="latent", then the pipeline's own
+VAE tail), and forward hooks record what goes through. The initial latents are
+explicit and saved, so no RNG has to be matched.
+
+ # primary oracle, f32, all dumps and the step-to-step redundancy table
+ python3 tools/qwenimage_ref.py --model /path/qwen-image-2.1 --out ref_fox \\
+ --prompt "a red fox sitting in the snow, photograph" --size 512x512 \\
+ --steps 8 --seed 42 --dump --redundancy
+
+ # steps sweep in bf16 compute, reusing the oracle's prompt embeddings and latents
+ python3 tools/qwenimage_ref.py --model ... --out sweep --size 512x512 \\
+ --steps 8,16,30 --dtype bf16 --embeds ref_fox/ref.safetensors \\
+ --latents ref_fox/ref.safetensors
+
+Output (per run directory): image.png, ref.json (config values a C
+implementation needs, ids, schedule, timings, peak RSS, versions) and, with
+--dump, ref.safetensors. make_qwenimage_tiny.py writes the same set for the
+tiny random model. Tensor layouts in ref.safetensors (batch axis dropped):
+ input_ids [T] i32 full ids fed to the text encoder
+ te_hidden_NN [T][hidden] text encoder hidden states (00 = embeddings,
+ last = final layer output before the norm)
+ prompt_embeds [T-drop][hidden] what the transformer reads
+ latents_init [N][C] packed initial latents, N = (H/16)*(W/16)
+ sigmas [steps+1], timesteps [steps], timestep_in [steps] (t/1000 as fed)
+ noise_pred_NNN [N][C], latents_NNN [N][C] (after step NNN), latents_final
+ step0_* step 0 internals (see ref.json "tensors")
+ vae_in [C][h][w] latents after de-normalization
+ vae_out_raw [4][H][W] decoder output before the clamp
+ vae_out [4][H][W] after the clamp to [-1, 1]
+ rgba [H][W][4] u8 the pipeline's postprocess, top row first
+
+Reference stack (the ds venv this was written and run against):
+ diffusers 0.41.0.dev0, git commit 80c7ed262aeffbeb43ef13ae04baeb9b84515a69
+ (pip install "git+https://github.com/huggingface/diffusers@80c7ed262aeffbeb43ef13ae04baeb9b84515a69")
+ transformers 5.17.0, torch 2.14.0+cpu, tokenizers 0.23.2, safetensors 0.8.0,
+ numpy 2.5.3, pillow 12.3.0, python 3.14.
+"""
+import argparse
+import ctypes
+import gc
+import json
+import math
+import os
+import platform
+import resource
+import sys
+import time
+from pathlib import Path
+
+import numpy as np
+import torch
+
+VAE_SCALE = 16 # QwenImage21Pipeline.vae_scale_factor: one latent token per 16x16 pixels
+RATIO_EDGES = (0.01, 0.05, 0.10)
+
+
+# ---------------------------------------------------------------- memory and timing
+
+def mem_available_gb() -> float:
+ try:
+ with open("/proc/meminfo") as f:
+ for line in f:
+ if line.startswith("MemAvailable:"):
+ return int(line.split()[1]) / 1048576.0
+ except OSError:
+ pass
+ return float("inf")
+
+
+def rss_gb(kind: str = "RssAnon") -> float:
+ """Anonymous RSS by default: safetensors can be memory-mapped, and clean file pages the kernel can drop
+ at will must not count against the cap."""
+ try:
+ with open("/proc/self/status") as f:
+ for line in f:
+ if line.startswith(kind + ":"):
+ return int(line.split()[1]) / 1048576.0
+ except OSError:
+ pass
+ return 0.0
+
+
+def peak_rss_gb() -> float:
+ return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1048576.0
+
+
+def release_memory():
+ gc.collect()
+ try: # glibc keeps freed arenas otherwise, and the next component would stack on top
+ ctypes.CDLL("libc.so.6").malloc_trim(0)
+ except OSError:
+ pass
+
+
+def log(msg: str):
+ print(f"[{time.strftime('%H:%M:%S')}] {msg} (anon {rss_gb():.1f} GB, file {rss_gb('RssFile'):.1f} GB, "
+ f"avail {mem_available_gb():.1f} GB)", flush=True)
+
+
+class MemoryGuard:
+ """Waits until MemAvailable leaves room for a component, and aborts an upcast that overshoots.
+
+ ds is shared: other jobs come and go, and swapping would slow them all down. A component is only
+ loaded once MemAvailable exceeds what it needs plus a margin; the upcast loops call check() so a
+ wrong estimate stops the run instead of pushing the machine into swap."""
+
+ def __init__(self, max_rss_gb: float, wait_minutes: float, margin_gb: float = 4.0):
+ self.max_rss_gb = max_rss_gb
+ self.wait_s = wait_minutes * 60.0
+ self.margin_gb = margin_gb
+
+ def wait_for(self, need_gb: float, what: str) -> bool:
+ deadline = time.time() + self.wait_s
+ while True:
+ avail = mem_available_gb()
+ if avail >= need_gb + self.margin_gb:
+ return True
+ if time.time() >= deadline:
+ log(f"{what}: needs {need_gb:.1f} GB + {self.margin_gb:.0f} GB margin, only {avail:.1f} GB available")
+ return False
+ log(f"{what}: waiting for memory (need {need_gb:.1f} + {self.margin_gb:.0f} GB)")
+ time.sleep(30)
+
+ def check(self, what: str):
+ rss = rss_gb()
+ if rss > self.max_rss_gb:
+ raise MemoryError(f"{what}: anonymous RSS {rss:.1f} GB is over the {self.max_rss_gb:.0f} GB cap")
+
+
+def dir_size_gb(path: Path, pattern: str = "*.safetensors") -> float:
+ return sum(p.stat().st_size for p in Path(path).glob(pattern)) / 1073741824.0
+
+
+# ---------------------------------------------------------------- component loading
+
+def load_text_encoder(model_dir: Path, dtype: torch.dtype, guard: MemoryGuard | None = None):
+ from transformers import Qwen3VLForConditionalGeneration
+
+ te = Qwen3VLForConditionalGeneration.from_pretrained(str(model_dir / "text_encoder"), dtype=torch.bfloat16)
+ te.eval()
+ # Text-to-image never runs the vision tower or the LM head (only the hidden states are read). Dropping them
+ # before the upcast saves ~5 GB of f32 on the real model. The forward still calls lm_head, so it becomes
+ # an identity instead of disappearing.
+ te.model.visual = None
+ te.lm_head = torch.nn.Identity()
+ release_memory()
+ if dtype == torch.float32:
+ lm = te.model.language_model
+ lm.embed_tokens.float()
+ for i, layer in enumerate(lm.layers):
+ layer.float()
+ if guard is not None and i % 4 == 3:
+ release_memory()
+ guard.check(f"text encoder upcast, layer {i}")
+ lm.norm.float()
+ rot = lm.rotary_emb
+ if rot.rope_type != "default":
+ raise SystemExit(f"unexpected rope_type {rot.rope_type!r} in the text encoder")
+ # inv_freq is a non-persistent buffer: recompute it in f32 rather than trust whatever dtype the load left
+ inv_freq, _ = rot.compute_default_rope_parameters(rot.config)
+ rot.inv_freq = inv_freq.float()
+ rot.original_inv_freq = inv_freq.float().clone()
+ release_memory()
+ return te
+
+
+def load_transformer(model_dir: Path, dtype: torch.dtype, guard: MemoryGuard | None = None):
+ from diffusers import QwenImage21Transformer2DModel
+ from diffusers.models.transformers.transformer_qwenimage21 import QwenImage21TemporalTimesteps
+
+ tr = QwenImage21Transformer2DModel.from_pretrained(str(model_dir / "transformer"), torch_dtype=torch.bfloat16)
+ tr.eval()
+ if dtype == torch.float32:
+ # block by block, so each bf16 block is released as its f32 copy appears
+ for name, child in tr.named_children():
+ if name == "transformer_blocks":
+ for i, block in enumerate(child):
+ block.float()
+ if guard is not None and i % 4 == 3:
+ release_memory()
+ guard.check(f"transformer upcast, block {i}")
+ else:
+ child.float()
+ # the sinusoid frequencies are a non-persistent buffer: rebuild them in f32 exactly as __init__ does
+ tp = tr.time_text_embed.time_proj
+ fresh = QwenImage21TemporalTimesteps(tp.timestep_dim, time_factor=tp.time_factor)
+ tp.freqs = fresh.freqs.float()
+ release_memory()
+ return tr
+
+
+def load_vae(model_dir: Path):
+ from diffusers import AutoencoderKLQwenImage21
+
+ vae = AutoencoderKLQwenImage21.from_pretrained(str(model_dir / "vae"), torch_dtype=torch.float32)
+ vae.eval()
+ return vae
+
+
+PROCESSOR_NOTE = {"how": "Qwen3VLProcessor.from_pretrained"}
+
+
+def load_processor(model_dir: Path):
+ """Qwen3VLProcessor as the pipeline loads it, or, when torchvision is missing (the video processor and the
+ fast image processor need it), the same class assembled from the tokenizer, the PIL image processor and
+ the chat template with no video processor. Text-to-image calls neither image nor video processor, so the
+ ids, the attention mask and the chat template rendering are the same; make_qwenimage_tiny.py checks the
+ ids against the raw tokenizers library either way."""
+ from transformers import Qwen3VLProcessor
+
+ d = model_dir / "processor"
+ try:
+ proc = Qwen3VLProcessor.from_pretrained(str(d))
+ PROCESSOR_NOTE["how"] = "Qwen3VLProcessor.from_pretrained"
+ return proc
+ except ImportError as e:
+ if "orchvision" not in str(e):
+ raise
+ from transformers import AutoTokenizer
+ from transformers.models.qwen2_vl.image_processing_pil_qwen2_vl import Qwen2VLImageProcessorPil
+ from transformers.processing_utils import ProcessorMixin
+
+ tokenizer = AutoTokenizer.from_pretrained(str(d))
+ image_processor = Qwen2VLImageProcessorPil.from_pretrained(str(d))
+ strict = ProcessorMixin.check_argument_for_proper_class
+
+ def allow_missing_video(self, name, arg):
+ if arg is None and name == "video_processor":
+ return None
+ return strict(self, name, arg)
+
+ ProcessorMixin.check_argument_for_proper_class = allow_missing_video
+ try:
+ proc = Qwen3VLProcessor(image_processor=image_processor, tokenizer=tokenizer, video_processor=None,
+ chat_template=(d / "chat_template.jinja").read_text(encoding="utf-8"))
+ finally:
+ ProcessorMixin.check_argument_for_proper_class = strict
+ PROCESSOR_NOTE["how"] = ("Qwen3VLProcessor(tokenizer, Qwen2VLImageProcessorPil, video_processor=None, "
+ "chat_template.jinja): torchvision is not installed, t2i never calls the image or "
+ "video processor")
+ return proc
+
+
+def load_scheduler(model_dir: Path):
+ from diffusers import FlowMatchEulerDiscreteScheduler
+
+ return FlowMatchEulerDiscreteScheduler.from_pretrained(str(model_dir / "scheduler"))
+
+
+def make_pipeline(model_dir: Path, processor, text_encoder=None, transformer=None, vae=None):
+ from diffusers import QwenImage21Pipeline
+
+ pipe = QwenImage21Pipeline(scheduler=load_scheduler(model_dir), vae=vae, text_encoder=text_encoder,
+ processor=processor, transformer=transformer)
+ pipe.set_progress_bar_config(disable=True)
+ return pipe
+
+
+# ---------------------------------------------------------------- stages
+
+def f32(t: torch.Tensor) -> torch.Tensor:
+ return t.detach().to(torch.float32).contiguous().clone()
+
+
+def encode_prompts(model_dir: Path, prompts: list[str], dtype: torch.dtype, dump: bool,
+ guard: MemoryGuard | None = None) -> list[dict]:
+ """Runs QwenImage21Pipeline.encode_prompt for each prompt and records ids and hidden states."""
+ processor = load_processor(model_dir)
+ t0 = time.time()
+ te = load_text_encoder(model_dir, dtype, guard)
+ load_s = time.time() - t0
+ log(f"text encoder loaded ({dtype}) in {load_s:.1f} s")
+ pipe = make_pipeline(model_dir, processor, text_encoder=te)
+
+ captured = {}
+ lm = te.model.language_model
+
+ def pre_hook(module, args, kwargs):
+ captured["input_ids"] = kwargs["input_ids"].detach().clone()
+ captured["attention_mask"] = kwargs["attention_mask"].detach().clone()
+
+ def post_hook(module, args, kwargs, output):
+ if dump:
+ captured["hidden"] = [f32(h[0]) for h in output.hidden_states]
+
+ def last_layer_hook(module, args, output):
+ out = output[0] if isinstance(output, tuple) else output
+ captured["last_layer_out"] = f32(out[0])
+
+ handles = [te.register_forward_pre_hook(pre_hook, with_kwargs=True),
+ te.register_forward_hook(post_hook, with_kwargs=True),
+ lm.layers[-1].register_forward_hook(last_layer_hook)]
+ results = []
+ try:
+ for prompt in prompts:
+ captured.clear()
+ t1 = time.time()
+ with torch.no_grad():
+ prompt_embeds, mask, image_pad_mask = pipe.encode_prompt(prompt=prompt, device=torch.device("cpu"))
+ enc_s = time.time() - t1
+ if mask is not None and not bool(mask.all()):
+ raise SystemExit("unexpected padding mask for a single prompt")
+ if bool(image_pad_mask.any()):
+ raise SystemExit("unexpected image tokens in a text-to-image prompt")
+ drop = pipe._drop_idx
+ ids = captured["input_ids"][0]
+ # The pipeline feeds the transformer the last decoder layer's output BEFORE the final norm; check
+ # that is really what came out, since transformers 5 changed what hidden_states[-1] means.
+ pre_norm = torch.equal(f32(prompt_embeds[0]), captured["last_layer_out"][drop:])
+ if not pre_norm:
+ raise SystemExit("prompt_embeds is not the pre-norm last layer output: the norm hook did not apply")
+ rec = {
+ "prompt": prompt,
+ "template_text": pipe.prompt_template_t2i.format(prompt if prompt else " "),
+ "system_prompt": pipe.sys_prompt,
+ "input_ids": [int(v) for v in ids],
+ "drop_idx": int(drop),
+ "prompt_embeds": f32(prompt_embeds[0]),
+ "prompt_embeds_is_prenorm_last_layer": pre_norm,
+ "encode_s": enc_s,
+ }
+ if dump:
+ rec["te_hidden"] = captured["hidden"]
+ results.append(rec)
+ log(f"encoded {len(ids)} tokens (drop {drop}) in {enc_s:.2f} s: {prompt!r}")
+ finally:
+ for h in handles:
+ h.remove()
+ info = {"load_s": load_s, "dtype": str(dtype).replace("torch.", ""),
+ "config": text_encoder_config(te), "peak_rss_gb": peak_rss_gb()}
+ del pipe, te, lm, handles
+ release_memory()
+ log("text encoder freed")
+ return results, info
+
+
+def text_encoder_config(te) -> dict:
+ c = te.config.text_config
+ rp = dict(c.rope_parameters)
+ return {
+ "hidden_size": c.hidden_size, "num_hidden_layers": c.num_hidden_layers,
+ "num_attention_heads": c.num_attention_heads, "num_key_value_heads": c.num_key_value_heads,
+ "head_dim": c.head_dim, "intermediate_size": c.intermediate_size, "vocab_size": c.vocab_size,
+ "rms_norm_eps": c.rms_norm_eps, "hidden_act": c.hidden_act,
+ "rope_theta": rp.get("rope_theta"), "mrope_section": rp.get("mrope_section"),
+ "mrope_interleaved": rp.get("mrope_interleaved"),
+ "note": "text-only input: the three M-RoPE position components are equal (0..T-1), so the rotation is "
+ "plain RoPE over the full head_dim with rope_theta, rotate_half pairing (i, i + head_dim/2); "
+ "q_norm/k_norm are per-head RMSNorm before RoPE; attention is causal; the output read is the "
+ "last layer before the final norm",
+ }
+
+
+class DenoiseRecorder:
+ """Forward hooks on the transformer for one pipeline call.
+
+ Per call (= per step): the timestep as fed and the model output for the target tokens. Step 0 also
+ records txt_in, temb, the shared modulation, the RoPE table and block 0 in/out over the whole joint
+ sequence. With redundancy on, every block's target-token output (and its residual delta, output minus
+ input) is compared with the previous step's."""
+
+ def __init__(self, transformer, n_target: int, dump: bool, redundancy: bool):
+ self.tr = transformer
+ self.n = n_target
+ self.dump = dump
+ self.redundancy = redundancy
+ self.step = -1
+ self.timestep_in = []
+ self.noise_pred = []
+ self.latents = []
+ self.step_s = []
+ self.step0 = {}
+ self.stats = {"blocks": {}, "noise_pred": [], "latents": []}
+ self.prev_out = {}
+ self.prev_delta = {}
+ self.cur_in = {}
+ self.prev_noise = None
+ self.prev_latents = None
+ self._t0 = None
+ self.handles = []
+ tr = transformer
+ self.handles.append(tr.register_forward_pre_hook(self._model_pre, with_kwargs=True))
+ self.handles.append(tr.register_forward_hook(self._model_post))
+ if dump:
+ self.handles.append(tr.txt_in.register_forward_hook(self._step0("txt_in")))
+ self.handles.append(tr.time_text_embed.register_forward_hook(self._step0("temb")))
+ self.handles.append(tr.modulation.register_forward_hook(self._step0("modulation")))
+ self.handles.append(tr.pos_embed.register_forward_hook(self._rope))
+ for k, block in enumerate(tr.transformer_blocks):
+ if redundancy or (dump and k == 0):
+ self.handles.append(block.register_forward_pre_hook(self._block_pre(k), with_kwargs=True))
+ self.handles.append(block.register_forward_hook(self._block_post(k)))
+
+ def remove(self):
+ for h in self.handles:
+ h.remove()
+ self.handles = []
+
+ def _model_pre(self, module, args, kwargs):
+ self.step += 1
+ self.timestep_in.append(float(kwargs["timestep"].float()[0]))
+ self._t0 = time.time()
+
+ def _model_post(self, module, args, output):
+ self.step_s.append(time.time() - self._t0)
+ noise = f32(output[0][0, -self.n:])
+ if self.dump:
+ self.noise_pred.append(noise)
+ if self.redundancy:
+ if self.prev_noise is not None:
+ self.stats["noise_pred"].append({"step": self.step, **change_stats(noise, self.prev_noise)})
+ self.prev_noise = noise
+ log(f"step {self.step} transformer {self.step_s[-1]:.2f} s")
+
+ def _step0(self, name):
+ def hook(module, args, output):
+ if self.step == 0:
+ self.step0[name] = f32(output[0] if name == "txt_in" else output)
+ return hook
+
+ def _rope(self, module, args, output):
+ if self.step == 0:
+ self.step0["rope_cos"] = f32(output.real)
+ self.step0["rope_sin"] = f32(output.imag)
+
+ def _block_pre(self, k):
+ def hook(module, args, kwargs):
+ h = kwargs["hidden_states"][0]
+ if self.dump and k == 0 and self.step == 0:
+ self.step0["block0_in"] = f32(h)
+ if self.redundancy:
+ self.cur_in[k] = f32(h[-self.n:])
+ return hook
+
+ def _block_post(self, k):
+ def hook(module, args, output):
+ h = output[0]
+ if self.dump and k == 0 and self.step == 0:
+ self.step0["block0_out"] = f32(h)
+ if self.redundancy:
+ out = f32(h[-self.n:])
+ delta = out - self.cur_in.pop(k)
+ if k in self.prev_out:
+ entry = {"step": self.step, "out": change_stats(out, self.prev_out[k]),
+ "delta": change_stats(delta, self.prev_delta[k])}
+ self.stats["blocks"].setdefault(k, []).append(entry)
+ self.prev_out[k] = out
+ self.prev_delta[k] = delta
+ return hook
+
+ def on_step_end(self, pipe, i, t, kwargs):
+ lat = f32(kwargs["latents"][0])
+ if self.dump:
+ self.latents.append(lat)
+ if self.redundancy:
+ if self.prev_latents is not None:
+ self.stats["latents"].append({"step": i, **change_stats(lat, self.prev_latents)})
+ self.prev_latents = lat
+ return {}
+
+
+def change_stats(cur: torch.Tensor, prev: torch.Tensor) -> dict:
+ """Step-to-step change of a [tokens][dim] tensor: global cosine, and the per-token relative L2 change."""
+ cur = cur.double()
+ prev = prev.double()
+ cos = float((cur * prev).sum() / (cur.norm() * prev.norm()).clamp_min(1e-30))
+ rel = (cur - prev).norm(dim=-1) / prev.norm(dim=-1).clamp_min(1e-30)
+ q = torch.quantile(rel, torch.tensor([0.5, 0.9], dtype=rel.dtype))
+ out = {"cos": cos, "rel_mean": float(rel.mean()), "rel_median": float(q[0]), "rel_p90": float(q[1]),
+ "rel_global": float((cur - prev).norm() / prev.norm().clamp_min(1e-30))}
+ for edge in RATIO_EDGES:
+ out[f"frac_below_{int(round(edge * 100))}pct"] = float((rel < edge).double().mean())
+ return out
+
+
+def pack_initial_latents(seed: int, channels: int, width: int, height: int) -> torch.Tensor:
+ """The pipeline's prepare_latents with an explicit CPU generator: randn (1, 1, C, h, w), then packed."""
+ from diffusers import QwenImage21Pipeline
+
+ h = 2 * (height // (VAE_SCALE * 2))
+ w = 2 * (width // (VAE_SCALE * 2))
+ gen = torch.Generator(device="cpu").manual_seed(seed)
+ lat = torch.randn((1, 1, channels, h, w), generator=gen, dtype=torch.float32)
+ return QwenImage21Pipeline._pack_latents(lat, 1, channels, h, w)[0].contiguous()
+
+
+def denoise(model_dir: Path, runs: list[dict], dtype: torch.dtype, dump: bool, redundancy: bool,
+ guard: MemoryGuard | None = None):
+ """runs: dicts with prompt_embeds [T][D] f32, latents_init [N][C] f32, width, height, steps.
+ One transformer load serves every run. Returns per-run outputs and load info."""
+ from diffusers.pipelines.qwenimage21.pipeline_qwenimage21 import calculate_shift
+
+ processor = load_processor(model_dir)
+ t0 = time.time()
+ tr = load_transformer(model_dir, dtype, guard)
+ load_s = time.time() - t0
+ log(f"transformer loaded ({dtype}) in {load_s:.1f} s")
+ pipe = make_pipeline(model_dir, processor, transformer=tr)
+ outs = []
+ for run in runs:
+ width, height, steps = run["width"], run["height"], run["steps"]
+ n = run["latents_init"].shape[0]
+ rec = DenoiseRecorder(tr, n, dump and run.get("dump", True), redundancy and run.get("redundancy", True))
+ t1 = time.time()
+ try:
+ with torch.no_grad():
+ final = pipe(prompt_embeds=run["prompt_embeds"][None].to(dtype),
+ latents=run["latents_init"][None].to(dtype),
+ width=width, height=height, num_inference_steps=steps, true_cfg_scale=1.0,
+ output_type="latent", callback_on_step_end=rec.on_step_end,
+ return_dict=False)[0]
+ finally:
+ rec.remove()
+ total_s = time.time() - t1
+ sc = pipe.scheduler.config
+ mu = calculate_shift(n, sc.get("base_image_seq_len", 256), sc.get("max_image_seq_len", 4096),
+ sc.get("base_shift", 0.5), sc.get("max_shift", 1.15))
+ outs.append({
+ "final": f32(final[0]), "sigmas": f32(pipe.scheduler.sigmas), "timesteps": f32(pipe.scheduler.timesteps),
+ "mu": float(mu), "timestep_in": rec.timestep_in, "noise_pred": rec.noise_pred, "latents": rec.latents,
+ "step0": rec.step0, "step_s": rec.step_s, "denoise_s": total_s,
+ "redundancy": rec.stats if rec.redundancy else None,
+ })
+ rec = None # it holds the transformer; the model is freed below only if nothing else does
+ log(f"denoised {width}x{height}, {steps} steps in {total_s:.1f} s")
+ info = {"load_s": load_s, "dtype": str(dtype).replace("torch.", ""), "config": dict(tr.config),
+ "peak_rss_gb": peak_rss_gb()}
+ del pipe, tr, rec
+ release_memory()
+ log("transformer freed")
+ return outs, info
+
+
+def decode(model_dir: Path, items: list[dict], dump: bool):
+ """items: dicts with final [N][C] packed latents, width, height. The tail of QwenImage21Pipeline.__call__,
+ verbatim, plus a hook on the decoder for the pre-clamp output."""
+ processor = load_processor(model_dir)
+ t0 = time.time()
+ vae = load_vae(model_dir)
+ load_s = time.time() - t0
+ log(f"VAE loaded in {load_s:.1f} s")
+ pipe = make_pipeline(model_dir, processor, vae=vae)
+ captured = {}
+ handle = vae.decoder.register_forward_hook(lambda m, a, out: captured.__setitem__("raw", f32(out[0, :, 0])))
+ outs = []
+ try:
+ for item in items:
+ width, height = item["width"], item["height"]
+ t1 = time.time()
+ with torch.no_grad():
+ latents = pipe._unpack_latents(item["final"][None], height, width, pipe.vae_scale_factor)
+ latents = latents.to(pipe.vae.dtype)
+ latents_mean = (torch.tensor(pipe.vae.config.latents_mean)
+ .view(1, pipe.vae.config.z_dim, 1, 1, 1).to(latents.device, latents.dtype))
+ latents_std = (torch.tensor(pipe.vae.config.latents_std)
+ .view(1, pipe.vae.config.z_dim, 1, 1, 1).to(latents.device, latents.dtype))
+ latents = latents * latents_std + latents_mean
+ image = pipe.vae.decode(latents, return_dict=False)[0][:, :, 0]
+ pil = pipe.image_processor.postprocess(image, output_type="pil")[0]
+ dec_s = time.time() - t1
+ out = {"pil": pil, "rgba": torch.from_numpy(np.array(pil)).contiguous(), "decode_s": dec_s}
+ if dump:
+ out["vae_in"] = f32(latents[0, :, 0])
+ out["vae_out"] = f32(image[0])
+ out["vae_out_raw"] = captured["raw"]
+ outs.append(out)
+ log(f"decoded {width}x{height} in {dec_s:.1f} s")
+ finally:
+ handle.remove()
+ vcfg = dict(vae.config)
+ info = {"load_s": load_s, "dtype": "float32", "config": vcfg, "peak_rss_gb": peak_rss_gb()}
+ del pipe, vae, handle
+ release_memory()
+ return outs, info
+
+
+# ---------------------------------------------------------------- writing
+
+def versions() -> dict:
+ import diffusers
+ import safetensors
+ import tokenizers
+ import transformers
+
+ commit = None
+ try:
+ from importlib.metadata import distribution
+ du = distribution("diffusers").read_text("direct_url.json")
+ if du:
+ commit = json.loads(du).get("vcs_info", {}).get("commit_id")
+ except Exception:
+ pass
+ return {"diffusers": diffusers.__version__, "diffusers_commit": commit, "transformers": transformers.__version__,
+ "torch": torch.__version__, "tokenizers": tokenizers.__version__, "safetensors": safetensors.__version__,
+ "numpy": np.__version__, "python": platform.python_version()}
+
+
+def machine() -> dict:
+ cpu = platform.processor()
+ try:
+ with open("/proc/cpuinfo") as f:
+ for line in f:
+ if line.startswith("model name"):
+ cpu = line.split(":", 1)[1].strip()
+ break
+ except OSError:
+ pass
+ return {"host": platform.node(), "cpu": cpu, "torch_threads": torch.get_num_threads()}
+
+
+def model_constants(te_cfg: dict, tr_cfg: dict, vae_cfg: dict, sched_cfg: dict) -> dict:
+ """Everything beyond the config files a C implementation needs, stated once."""
+ inner = tr_cfg["num_attention_heads"] * tr_cfg["attention_head_dim"]
+ return {
+ "text_encoder": te_cfg,
+ "transformer": {
+ **{k: tr_cfg[k] for k in ("in_channels", "out_channels", "num_layers", "attention_head_dim",
+ "num_attention_heads", "context_in_dim", "mlp_ratio", "axes_dims_rope",
+ "eps", "causal_condition", "patch_size")},
+ "inner_dim": inner, "mlp_hidden": inner * tr_cfg["mlp_ratio"],
+ "rope_theta": 10000, "rope_pos_table": "index i >= 0 for 0..8191, negative index -k at row 8192+1024-k",
+ "timestep_embedding": {"dim": 256, "max_period": 10000, "time_factor": 1000.0,
+ "layout": "[cos(args) | sin(args)], args = 1000 * t * exp(-ln(10000) * i / 128)",
+ "input": "t = timestep / 1000 as fed (timestep_in); linear_1, SiLU, linear_2, "
+ "no biases"},
+ "modulation": "SiLU(temb) @ modulation.1.weight^T -> [scale1 | gate1 | scale2 | gate2], shared by "
+ "every block; row 0 = sampled t (target tokens), row 1 = t = 0 (text tokens)",
+ "block": "x += tanh(gate1) * attn(LN(x) * (1 + scale1)); x += tanh(gate2) * mlp(LN(x) * (1 + scale2)); "
+ "LN without affine, eps; mlp = out(silu(gate_layer(x)) * proj(x))",
+ "attention": "to_q/to_k/to_v, per-head RMSNorm (norm_q/norm_k, weight, eps), 3-axis RoPE as complex "
+ "pairs (x[2i], x[2i+1]), SDPA scale 1/sqrt(head_dim), to_out.0; no biases anywhere",
+ "mask": "text tokens causal among themselves and never see the image; image tokens see all text and "
+ "all image tokens. So the text K/V are step-independent (the pipeline's KV cache)",
+ "txt_in": "zero-centered RMSNorm (x * rsqrt(mean(x^2) + eps) * (1 + w)), in_layer, GELU tanh, out_layer",
+ "norm_out": "LN(x) * (1 + linear(SiLU(temb))), then proj_out",
+ "rope_positions": "text token j: (j, j, j); target image token (r, c) of an h x w grid: frame = T_text, "
+ "height = r - (h - h//2), width = c - (w - w//2)",
+ "sequence": "[prompt_embeds (after drop_idx) | target latents]",
+ },
+ "vae": {
+ **{k: vae_cfg[k] for k in ("z_dim", "base_dim", "decoder_base_dim", "dim_mult", "num_res_blocks",
+ "in_channels", "out_channels", "is_residual", "temperal_downsample",
+ "scale_factor_spatial", "latents_mean", "latents_std", "attn_scales")},
+ "denormalize": "z = latents * latents_std + latents_mean (per channel)",
+ "rms_norm": "F.normalize over channels (eps 1e-12) * sqrt(C) * gamma",
+ "clamp": [-1.0, 1.0],
+ "postprocess": "u8 = round(clamp(x / 2 + 0.5, 0, 1) * 255), RGBA, top row first",
+ },
+ "scheduler": dict(sched_cfg),
+ "pipeline": {"vae_scale_factor": VAE_SCALE, "multiple_of": 32, "true_cfg_scale": 1.0, "use_kv_cache": True,
+ "calculate_shift": "mu = N * m + b, m = (max_shift - base_shift) / (max_image_seq_len - "
+ "base_image_seq_len), b = base_shift - m * base_image_seq_len",
+ "sigmas": "s = linspace(1, 1/steps, steps) as f32; s = e^mu / (e^mu + (1/s - 1)); "
+ "stretch so the last is shift_terminal; append 0",
+ "euler": "latents += (sigma[i+1] - sigma[i]) * noise_pred",
+ "timestep_fed": "t = sigma * 1000, fed as t / 1000 (see timestep_in)"},
+ }
+
+
+def tensor_table(tensors: dict) -> dict:
+ return {k: {"shape": list(v.shape), "dtype": str(v.dtype).replace("torch.", "")} for k, v in tensors.items()}
+
+
+def write_run(out_dir: Path, enc: dict, den: dict, dec: dict, meta: dict, dump: bool):
+ from safetensors.torch import save_file
+
+ out_dir.mkdir(parents=True, exist_ok=True)
+ dec["pil"].save(out_dir / "image.png")
+ tensors = {}
+ if dump:
+ tensors["input_ids"] = torch.tensor(enc["input_ids"], dtype=torch.int32)
+ for i, h in enumerate(enc.get("te_hidden", [])):
+ tensors[f"te_hidden_{i:02d}"] = h
+ tensors["prompt_embeds"] = enc["prompt_embeds"]
+ tensors["latents_init"] = meta["latents_init"]
+ tensors["sigmas"] = den["sigmas"]
+ tensors["timesteps"] = den["timesteps"]
+ tensors["timestep_in"] = torch.tensor(den["timestep_in"], dtype=torch.float32)
+ for i, (npred, lat) in enumerate(zip(den["noise_pred"], den["latents"])):
+ tensors[f"noise_pred_{i:03d}"] = npred
+ tensors[f"latents_{i:03d}"] = lat
+ tensors["latents_final"] = den["final"]
+ for k, v in den["step0"].items():
+ tensors[f"step0_{k}"] = v
+ for k in ("vae_in", "vae_out_raw", "vae_out"):
+ tensors[k] = dec[k]
+ tensors["rgba"] = dec["rgba"]
+ save_file({k: v.contiguous() for k, v in tensors.items()}, str(out_dir / "ref.safetensors"))
+ n_text = len(enc["input_ids"]) - enc["drop_idx"]
+ n = meta["latents_init"].shape[0]
+ ref = {
+ "model": meta["model"],
+ "prompt": enc["prompt"],
+ "template_text": enc["template_text"],
+ "system_prompt": enc["system_prompt"],
+ "input_ids": enc["input_ids"],
+ "drop_idx": enc["drop_idx"],
+ "prompt_embeds_is_prenorm_last_layer": enc["prompt_embeds_is_prenorm_last_layer"],
+ "width": meta["width"], "height": meta["height"],
+ "latent_h": 2 * (meta["height"] // 32), "latent_w": 2 * (meta["width"] // 32),
+ "n_text_tokens": n_text, "n_image_tokens": n, "joint_seq_len": n_text + n,
+ "steps": meta["steps"], "seed": meta["seed"], "latents_seed_note": meta.get("latents_note"),
+ "mu": den["mu"],
+ "sigmas": [float(v) for v in den["sigmas"]],
+ "timesteps": [float(v) for v in den["timesteps"]],
+ "timestep_in": den["timestep_in"],
+ "compute_dtype": meta["compute_dtype"],
+ "processor": dict(PROCESSOR_NOTE),
+ "constants": meta["constants"],
+ "step0_notes": {
+ "step0_txt_in": "txt_in output [T_text][inner]",
+ "step0_temb": "time_text_embed output [2][inner]: row 0 sampled t, row 1 t = 0",
+ "step0_modulation": "modulation output [2][4 * inner]",
+ "step0_rope_cos/sin": "RoPE table for the joint sequence [T_text + N][head_dim / 2], axes concatenated",
+ "step0_block0_in/out": "block 0 input/output over the whole joint sequence [T_text + N][inner]; "
+ "the target tokens are the last N rows",
+ },
+ "timings_s": meta["timings"],
+ "peak_rss_gb": meta["peak_rss"],
+ "machine": machine(),
+ "versions": versions(),
+ "tensors": tensor_table(tensors) if dump else None,
+ }
+ (out_dir / "ref.json").write_text(json.dumps(ref, indent=1))
+
+
+def summarize_redundancy(stats: dict, n_layers: int) -> dict:
+ blocks = []
+ for k in range(n_layers):
+ entries = stats["blocks"].get(k, [])
+ blocks.append({"block": k, "steps": entries})
+ return {"blocks": blocks, "noise_pred": stats["noise_pred"], "latents": stats["latents"],
+ "definitions": {
+ "out": "block output, target image tokens only",
+ "delta": "block output minus block input (what a block-skip cache would reuse)",
+ "cos": "cosine similarity of step t vs step t-1 over the whole [N][D] matrix",
+ "rel": "per-token ||h_t - h_{t-1}|| / ||h_{t-1}||",
+ "frac_below_Xpct": "fraction of target tokens whose rel is under X percent",
+ }}
+
+
+# ---------------------------------------------------------------- main
+
+def load_tensor(path: Path, key: str) -> torch.Tensor:
+ from safetensors.torch import load_file
+
+ return load_file(str(path))[key].float()
+
+
+def parse_size(s: str) -> tuple[int, int]:
+ w, h = s.lower().split("x")
+ return int(w), int(h)
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ ap.add_argument("--model", type=Path, required=True)
+ ap.add_argument("--out", type=Path, required=True)
+ ap.add_argument("--prompt", default="a red fox sitting in the snow, photograph")
+ ap.add_argument("--size", default="512x512", help="WIDTHxHEIGHT")
+ ap.add_argument("--steps", default="8", help="one count, or a comma list (one run each, subdirs steps_N)")
+ ap.add_argument("--seed", type=int, default=42)
+ ap.add_argument("--dtype", choices=("f32", "bf16"), default="f32", help="transformer compute dtype")
+ ap.add_argument("--te-dtype", choices=("f32", "bf16"), default="f32", help="text encoder compute dtype")
+ ap.add_argument("--embeds", type=Path, help="reuse prompt_embeds (and ids) from a previous ref.safetensors")
+ ap.add_argument("--latents", type=Path, help="reuse latents_init from a previous ref.safetensors")
+ ap.add_argument("--dump", action="store_true", help="write ref.safetensors with every intermediate")
+ ap.add_argument("--redundancy", action="store_true", help="measure step-to-step change of every block")
+ ap.add_argument("--threads", type=int, default=8)
+ ap.add_argument("--max-rss-gb", type=float, default=35.0)
+ ap.add_argument("--wait-mem-minutes", type=float, default=30.0)
+ args = ap.parse_args()
+
+ torch.set_num_threads(args.threads)
+ torch.manual_seed(0)
+ guard = MemoryGuard(args.max_rss_gb, args.wait_mem_minutes)
+ width, height = parse_size(args.size)
+ if width % 32 or height % 32:
+ raise SystemExit("width and height must be multiples of 32")
+ steps_list = [int(s) for s in args.steps.split(",")]
+ dt = {"f32": torch.float32, "bf16": torch.bfloat16}
+ model_dir = args.model
+ timings = {}
+ peak = {}
+
+ # 1. text encoder
+ if args.embeds:
+ from safetensors.torch import load_file
+
+ prev = load_file(str(args.embeds))
+ prev_json = json.loads((args.embeds.parent / "ref.json").read_text())
+ enc = {"prompt": prev_json["prompt"], "template_text": prev_json["template_text"],
+ "system_prompt": prev_json["system_prompt"], "input_ids": prev_json["input_ids"],
+ "drop_idx": prev_json["drop_idx"], "prompt_embeds": prev["prompt_embeds"].float(),
+ "prompt_embeds_is_prenorm_last_layer": prev_json["prompt_embeds_is_prenorm_last_layer"]}
+ te_cfg = prev_json["constants"]["text_encoder"]
+ if args.prompt != enc["prompt"]:
+ log(f"--embeds overrides --prompt: using {enc['prompt']!r}")
+ timings["encode"] = {"reused_from": str(args.embeds)}
+ else:
+ te_need = dir_size_gb(model_dir / "text_encoder") * (2.0 if args.te_dtype == "f32" else 1.0)
+ te_dtype = dt[args.te_dtype]
+ if not guard.wait_for(te_need, "text encoder"):
+ if te_dtype == torch.float32:
+ log("falling back to bf16 compute for the text encoder")
+ te_dtype = torch.bfloat16
+ if not guard.wait_for(te_need / 2, "text encoder bf16"):
+ raise SystemExit("not enough memory for the text encoder")
+ else:
+ raise SystemExit("not enough memory for the text encoder")
+ encs, te_info = encode_prompts(model_dir, [args.prompt], te_dtype, args.dump, guard)
+ enc = encs[0]
+ te_cfg = te_info["config"]
+ timings["encode"] = {"load_s": te_info["load_s"], "compute_s": enc["encode_s"], "dtype": te_info["dtype"]}
+ peak["after_encode"] = te_info["peak_rss_gb"]
+
+ # 2. transformer
+ from diffusers import QwenImage21Transformer2DModel
+
+ tr_cfg = dict(QwenImage21Transformer2DModel.load_config(str(model_dir / "transformer")))
+ channels = tr_cfg["in_channels"]
+ if args.latents:
+ latents_init = load_tensor(args.latents, "latents_init")
+ latents_note = f"reused from {args.latents}"
+ else:
+ latents_init = pack_initial_latents(args.seed, channels, width, height)
+ latents_note = (f"torch.randn((1, 1, {channels}, h, w), generator=torch.Generator('cpu').manual_seed("
+ f"{args.seed}), float32), then QwenImage21Pipeline._pack_latents")
+ n = (height // VAE_SCALE) * (width // VAE_SCALE)
+ if latents_init.shape != (n, channels):
+ raise SystemExit(f"latents_init has shape {tuple(latents_init.shape)}, expected {(n, channels)}")
+ tr_need = dir_size_gb(model_dir / "transformer") * (2.0 if args.dtype == "f32" else 1.0) + 1.5
+ if not guard.wait_for(tr_need, "transformer"):
+ raise SystemExit("not enough memory for the transformer")
+ runs = [{"prompt_embeds": enc["prompt_embeds"], "latents_init": latents_init, "width": width, "height": height,
+ "steps": s} for s in steps_list]
+ dens, tr_info = denoise(model_dir, runs, dt[args.dtype], args.dump, args.redundancy, guard)
+ peak["after_denoise"] = tr_info["peak_rss_gb"]
+
+ # 3. VAE
+ if not guard.wait_for(dir_size_gb(model_dir / "vae") + 3.0, "VAE"):
+ raise SystemExit("not enough memory for the VAE")
+ decs, vae_info = decode(model_dir, [{"final": d["final"], "width": width, "height": height} for d in dens],
+ args.dump)
+ peak["after_decode"] = vae_info["peak_rss_gb"]
+
+ sched_cfg = dict(load_scheduler(model_dir).config)
+ constants = model_constants(te_cfg, tr_info["config"], vae_info["config"], sched_cfg)
+ for steps, den, dec in zip(steps_list, dens, decs):
+ out_dir = args.out if len(steps_list) == 1 else args.out / f"steps_{steps}"
+ t = {"encode": timings["encode"],
+ "denoise": {"load_s": tr_info["load_s"], "total_s": den["denoise_s"], "per_step_s": den["step_s"],
+ "dtype": tr_info["dtype"]},
+ "decode": {"load_s": vae_info["load_s"], "compute_s": dec["decode_s"], "dtype": vae_info["dtype"]},
+ "threads": args.threads}
+ meta = {"model": str(model_dir), "width": width, "height": height, "steps": steps, "seed": args.seed,
+ "latents_init": latents_init, "latents_note": latents_note,
+ "compute_dtype": {"text_encoder": timings["encode"].get("dtype", "reused"),
+ "transformer": tr_info["dtype"], "vae": "float32"},
+ "constants": constants, "timings": t, "peak_rss": peak}
+ write_run(out_dir, enc, den, dec, meta, args.dump)
+ if den["redundancy"] is not None:
+ red = summarize_redundancy(den["redundancy"], tr_info["config"]["num_layers"])
+ red.update({"width": width, "height": height, "steps": steps, "sigmas": [float(v) for v in den["sigmas"]],
+ "compute_dtype": tr_info["dtype"]})
+ (out_dir / "redundancy.json").write_text(json.dumps(red, indent=1))
+ log(f"wrote {out_dir}")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())