Skip to content

armgit / kokon

Public Imported from GitHub
0

Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.

Mac engine: fp64 DiT reference; unsloth's gap was torch fp32's drift

Gabriele Lozupone349deac61a15
diff --git a/engine-mac/README.md b/engine-mac/README.md
index 7d1efa6..b496ca6 100644
--- a/engine-mac/README.md
+++ b/engine-mac/README.md
@@ -49,8 +49,10 @@ python3 -m venv ~/kokon-bench/venv-oracle && ~/kokon-bench/venv-oracle/bin/pip i
256² text-to-image, σ 0.7, 2026-10-25: every stage within 2e-5 relative of diffusers fp32 (velocity 6e-6).
unsloth's Qwen-Image 2.1 Turbo Q4_K_M (the file Kokon will ship with this engine; ComfyUI layout, each block's MLP
-`gate_up` fused, read as two views): velocity within 7.2e-5, layers 0–1 within 2.4e-6, the gap growing in the last
-layers (not understood yet).
+`gate_up` fused, read as two views): **check it against an fp64 reference** (`--dtype float64 --all-layers`, ~1 min).
+On this file torch's own fp32 run drifts from layer 6 on, to 7.3e-5 of fp64 at the velocity, while the engine stays
+within 3.9e-6 (every layer within 6.3e-6); on the old uniform Q4_K file both are within 4.1e-6 of fp64. The 7.2e-5
+once measured against the fp32 reference was the reference's error, not the engine's.
**The text encoder against transformers.** The reference tokenizes with the processor of the official
Qwen/Qwen-Image-2.1-Turbo repository (`processor/` and `text_encoder/config.json`, a few MB) and runs transformers'
diff --git a/engine-mac/Sources/KokonEngine/QwenDiT.swift b/engine-mac/Sources/KokonEngine/QwenDiT.swift
index 8a770ab..82e309c 100644
--- a/engine-mac/Sources/KokonEngine/QwenDiT.swift
+++ b/engine-mac/Sources/KokonEngine/QwenDiT.swift
@@ -147,7 +147,7 @@ public final class QwenDiT {
// MARK: - one evaluation
/// Called with a stage's name and its rows (CPU pointer, floats) when `probe` is set; the pass then waits on
- /// the GPU at each stage. Stages: joint_in, layer0_out, layer1_out, layer31_out, norm_out, model_out.
+ /// the GPU at each stage. Stages: joint_in, l0_* (inside layer 0), layer0_out … layer31_out, norm_out, model_out.
public typealias Probe = (String, UnsafePointer<Float>, Int) -> Void
/// One evaluation: `encoder` the encoder's last layer `[T, 4096]` (all positions, image slots included),
@@ -198,7 +198,7 @@ public final class QwenDiT {
}
}
try encodeLayer(layer, commands: &commands, stage: stage)
- if let probe, layer == 0 || layer == 1 || layer == QwenDiT.layers - 1 {
+ if let probe {
try finish(commands); probe("layer\(layer)_out", arena.pointer(x), n * d); commands = try begin()
} else {
commands.commit()
diff --git a/engine-mac/tools/reference_dit.py b/engine-mac/tools/reference_dit.py
index b11093e..9a317e1 100644
--- a/engine-mac/tools/reference_dit.py
+++ b/engine-mac/tools/reference_dit.py
@@ -7,6 +7,10 @@ The whole model in fp32 is 28 GB, more than a 16 GB Mac holds: the script follow
cache) with one block module whose weights are swapped layer by layer.
python -I engine-mac/tools/reference_dit.py <dit.gguf> <out dir> [--grid 16] [--text 24] [--sigma 0.7]
+ [--all-layers] [--dtype float64]
+
+`--dtype float64` runs every module in fp64 (diffusers' own `.float()` casts made no-ops): the truth an fp32
+engine and the fp32 reference are both measured against, to tell a bug from the weights' own sensitivity.
Writes `<out>/meta.json` and one raw little-endian fp32 file per tensor (`<name>.f32`).
"""
@@ -31,8 +35,15 @@ from diffusers.models.transformers.transformer_qwenimage21 import (
PREFIX = "model.diffusion_model."
+def enable_float64():
+ """diffusers casts to fp32 in RoPE, its RMSNorms and the timestep path: `Tensor.float()` keeps fp64 tensors fp64."""
+ to_float = torch.Tensor.float
+ torch.Tensor.float = lambda self, *a, **k: self if self.dtype == torch.float64 else to_float(self, *a, **k)
+
+
class Weights:
- def __init__(self, path):
+ def __init__(self, path, dtype=torch.float32):
+ self.dtype = dtype
self.reader = gguf.GGUFReader(path)
self.tensors = {t.name.removeprefix(PREFIX): t for t in self.reader.tensors}
@@ -46,7 +57,7 @@ class Weights:
t = self.tensors[name]
values = gguf.quants.dequantize(t.data, t.tensor_type).astype(np.float32)
shape = [int(d) for d in reversed(t.shape)] # ggml ne order -> torch order
- return torch.from_numpy(values.reshape(shape).copy())
+ return torch.from_numpy(values.reshape(shape).copy()).to(self.dtype)
def load(self, module, prefix):
state = {name: self.get(prefix + name) for name, _ in module.named_parameters()}
@@ -81,8 +92,8 @@ def layer0_stages(block, x, modulation, rotary_emb, target_token_mask, save):
class ReferenceDiT:
"""QwenImage21Transformer2DModel.forward, batch 1, no cache, text-to-image, one block module swapped per layer."""
- def __init__(self, path, layers=32):
- self.w = w = Weights(path)
+ def __init__(self, path, layers=32, dtype=torch.float32):
+ self.w = w = Weights(path, dtype)
d, self.layers = 4096, layers
self.img_in = torch.nn.Linear(64, d, bias=False)
w.load(self.img_in, "img_in.")
@@ -98,6 +109,11 @@ class ReferenceDiT:
w.load(self.proj_out, "proj_out.")
self.rope = QwenImage21Rope(theta=10000, axes_dim=[16, 56, 56])
self.block = QwenImage21TransformerBlock(d, 32, 128, mlp_ratio=3, eps=1e-6)
+ for m in (self.img_in, self.txt_in, self.time_text_embed, self.modulation_linear, self.norm_out,
+ self.proj_out, self.block):
+ m.to(dtype)
+
+ all_layers = False
def forward(self, encoder, latent, h, w_, timestep, save=None, progress=True):
"""`encoder` [1, T, d], `latent` [1, h * w, 64], `timestep` a [1] tensor (σ); returns the target's velocity
@@ -133,7 +149,7 @@ class ReferenceDiT:
layer0_stages(self.block, joint, modulation, rotary_emb, target_token_mask, save)
joint = self.block(hidden_states=joint, modulation=modulation, rotary_emb=rotary_emb, attention_mask=None,
target_token_mask=target_token_mask, segments=segments)
- if save and (layer in (0, 1) or layer == self.layers - 1):
+ if save and (self.all_layers or layer in (0, 1) or layer == self.layers - 1):
save(f"layer{layer}_out", joint[0])
if progress:
print(f"layer {layer} done", flush=True)
@@ -155,7 +171,12 @@ def main():
parser.add_argument("--sigma", type=float, default=0.7)
parser.add_argument("--layers", type=int, default=32)
parser.add_argument("--seed", type=int, default=42)
+ parser.add_argument("--all-layers", action="store_true", help="save every layer's output")
+ parser.add_argument("--dtype", default="float32", choices=["float32", "float64"])
args = parser.parse_args()
+ dtype = getattr(torch, args.dtype)
+ if dtype == torch.float64:
+ enable_float64()
torch.manual_seed(args.seed)
torch.set_grad_enabled(False)
os.makedirs(args.out, exist_ok=True)
@@ -173,11 +194,12 @@ def main():
latent = torch.randn(1, h * w_, 64)
save("in_encoder", encoder[0])
save("in_latent", latent[0])
- dit = ReferenceDiT(args.gguf, layers=args.layers)
- dit.forward(encoder, latent, h, w_, torch.tensor([args.sigma], dtype=torch.float32), save=save)
+ dit = ReferenceDiT(args.gguf, layers=args.layers, dtype=dtype)
+ dit.all_layers = args.all_layers
+ dit.forward(encoder.to(dtype), latent.to(dtype), h, w_, torch.tensor([args.sigma], dtype=dtype), save=save)
meta = {"grid": [h, w_], "text": args.text, "sigma": args.sigma, "layers": args.layers, "prefix": args.text,
- "seed": args.seed, "tensors": saved, "gguf": os.path.basename(args.gguf),
+ "seed": args.seed, "dtype": args.dtype, "tensors": saved, "gguf": os.path.basename(args.gguf),
"diffusers": __import__("diffusers").__version__, "torch": torch.__version__}
with open(os.path.join(args.out, "meta.json"), "w") as f:
json.dump(meta, f, indent=1)
diff --git a/engine-mac/tools/reference_generate.py b/engine-mac/tools/reference_generate.py
index df4fd04..2237c83 100644
--- a/engine-mac/tools/reference_generate.py
+++ b/engine-mac/tools/reference_generate.py
@@ -10,7 +10,10 @@ Saves the noise, every step's velocity and latent, the decoded image and the fin
`kokon-engine-mac check-generate`. 256² takes ~5 min (8 DiT evaluations, each layer's weights unpacked by gguf-py).
python -I engine-mac/tools/reference_generate.py <dit.gguf> <vae.safetensors> <text reference dir> <repo dir> <out>
- [--grid 16] [--seed 1]
+ [--grid 16] [--seed 1] [--dtype float64]
+
+`--dtype float64`: the DiT and the latents in fp64 (the scheduler still rounds the latent to fp32 each step), the
+VAE in fp32. Needed on unsloth's DiT, where torch's fp32 drifts to 7e-5 of fp64 in one evaluation.
`<repo dir>`: the official Qwen/Qwen-Image-2.1-Turbo files (model_index.json: sample_sigmas; scheduler/).
"""
@@ -26,7 +29,7 @@ from diffusers.image_processor import VaeImageProcessor
from diffusers.pipelines.qwenimage21.pipeline_qwenimage21 import calculate_shift
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # the sibling references
-from reference_dit import ReferenceDiT # noqa: E402
+from reference_dit import ReferenceDiT, enable_float64 # noqa: E402
from reference_vae import load_vae # noqa: E402
@@ -39,7 +42,11 @@ def main():
parser.add_argument("out")
parser.add_argument("--grid", type=int, default=16, help="latent grid side (16 = a 256² image)")
parser.add_argument("--seed", type=int, default=1)
+ parser.add_argument("--dtype", default="float32", choices=["float32", "float64"])
args = parser.parse_args()
+ dtype = getattr(torch, args.dtype)
+ if dtype == torch.float64:
+ enable_float64()
torch.set_grad_enabled(False)
os.makedirs(args.out, exist_ok=True)
@@ -48,13 +55,13 @@ def main():
text_meta = json.load(open(os.path.join(args.text, "meta.json")))
conditioning = torch.from_numpy(np.fromfile(os.path.join(args.text, "conditioning.f32"), dtype=np.float32))
- conditioning = conditioning.view(1, -1, 4096)
+ conditioning = conditioning.view(1, -1, 4096).to(dtype)
h = w = args.grid
# prepare_latents + _pack_latents
noise = torch.randn((1, 1, 64, h, w), generator=torch.Generator().manual_seed(args.seed), dtype=torch.float32)
save("noise", noise[0, 0])
- latents = noise.view(1, 64, h * w).transpose(1, 2)
+ latents = noise.view(1, 64, h * w).transpose(1, 2).to(dtype)
# retrieve_timesteps with the checkpoint's sigmas
index = json.load(open(os.path.join(args.repo, "model_index.json")))
@@ -66,9 +73,9 @@ def main():
scheduler.set_timesteps(sigmas=index["sample_sigmas"], mu=mu)
scheduler.set_begin_index(0)
- dit = ReferenceDiT(args.dit)
+ dit = ReferenceDiT(args.dit, dtype=dtype)
for i, t in enumerate(scheduler.timesteps):
- velocity = dit.forward(conditioning, latents, h, w, t.expand(1).to(torch.float32) / 1000, progress=False)
+ velocity = dit.forward(conditioning, latents, h, w, t.expand(1).to(dtype) / 1000, progress=False)
save(f"velocity{i}", velocity[0])
latents = scheduler.step(velocity, t, latents, return_dict=False)[0]
save(f"latent{i}", latents[0])
@@ -76,7 +83,7 @@ def main():
# _unpack_latents, denormalize, decode, postprocess
vae = load_vae(args.vae)
- unpacked = latents.transpose(1, 2).reshape(1, 64, 1, h, w)
+ unpacked = latents.to(torch.float32).transpose(1, 2).reshape(1, 64, 1, h, w)
mean = torch.tensor(vae.config.latents_mean).view(1, 64, 1, 1, 1)
std = torch.tensor(vae.config.latents_std).view(1, 64, 1, 1, 1)
image = vae.decode(unpacked * std + mean, return_dict=False)[0][:, :, 0]
@@ -87,7 +94,7 @@ def main():
meta = {"prompt": text_meta["prompt"], "grid": [h, w], "seed": args.seed, "mode": pil.mode,
"sigmas": scheduler.sigmas.tolist(), "timesteps": (scheduler.timesteps / 1000).tolist(),
- "dit": os.path.basename(args.dit), "vae": os.path.basename(args.vae)}
+ "dit": os.path.basename(args.dit), "dtype": args.dtype, "vae": os.path.basename(args.vae)}
with open(os.path.join(args.out, "meta.json"), "w") as f:
json.dump(meta, f, indent=1)
print(f"image {pil.size} {pil.mode}")