Skip to content

armgit / kokon

Public Imported from GitHub
0

Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.

Mac engine: the Qwen-Image 2.1 text encoder

Gabriele Lozuponed50bf6053cba
diff --git a/engine-mac/README.md b/engine-mac/README.md
index 04b26a0..554536a 100644
--- a/engine-mac/README.md
+++ b/engine-mac/README.md
@@ -28,6 +28,8 @@ swift test # no model needed
| `Coexecution.swift`, `Kernels/ane.metal` | the AMX and the Neural Engine beside the GPU on the DiT's products |
| `Kernels/attention.metal` | fused block-causal attention |
| `QwenVAE.swift`, `Safetensors.swift` | the VAE decoder as one MPSGraph, from Kokon's VAE file (Wan 2.2 names) |
+| `QwenTokenizer.swift` | Qwen's byte-level BPE tokenizer, read from the encoder GGUF's metadata |
+| `QwenTextEncoder.swift`, `Kernels/text.metal` | the text encoder: Qwen3-VL-8B's text decoder on the GPU, diffusers' prompt template |
Tests check the CPU and GPU unpacking against gguf-py's values (`tools/make_quant_fixtures.py` writes the
fixture), and the sequence layout against diffusers' rules worked out by hand.
@@ -44,6 +46,22 @@ python3 -m venv ~/kokon-bench/venv-oracle && ~/kokon-bench/venv-oracle/bin/pip i
256² text-to-image, σ 0.7, 2026-10-25: every stage within 2e-5 relative of diffusers fp32 (velocity 6e-6).
+**The text encoder against transformers.** The reference tokenizes with the processor of the official
+Qwen/Qwen-Image-2.1-Turbo repository (`processor/` and `text_encoder/config.json`, a few MB) and runs transformers'
+Qwen3-VL decoder layers in fp32 on the GGUF's weights, one layer at a time; `--sanity` checks the language model
+head answers " Paris" to a factual prompt (it does), so the weights are read right and not only read alike.
+
+```bash
+~/kokon-bench/venv-oracle/bin/pip install torchvision huggingface_hub # the processor needs torchvision
+~/kokon-bench/venv-oracle/bin/python -I tools/reference_tokenizer.py <processor dir> ~/kokon-bench/ref/text/tokenizer.json
+.build/release/kokon-engine-mac check-tokenizer <encoder.gguf> ~/kokon-bench/ref/text/tokenizer.json
+~/kokon-bench/venv-oracle/bin/python -I tools/reference_text_encoder.py <encoder.gguf> <processor dir> <text_encoder dir> ~/kokon-bench/ref/text/fox --sanity # ~50 s
+.build/release/kokon-engine-mac check-text <encoder.gguf> ~/kokon-bench/ref/text/fox
+```
+
+2026-10-25: tokenizer identical on 42 cases (accents, CJK, emoji, decomposed Unicode, special tokens in the text);
+conditioning within 5.1e-6 relative (37 tokens) and 5.6e-6 (144 tokens). One encode: 0.45 s (37 tokens), 0.8 s (144).
+
## First measurements (M2 Pro 19-core GPU, 16 GB, macOS 27, 2026-10-25, `probe`)
m = 4 096 rows (a 1024² image), TFLOP/s:
diff --git a/engine-mac/Sources/KokonEngine/Kernels/text.metal b/engine-mac/Sources/KokonEngine/Kernels/text.metal
new file mode 100644
index 0000000..6a6ea99
--- /dev/null
+++ b/engine-mac/Sources/KokonEngine/Kernels/text.metal
@@ -0,0 +1,49 @@
+#include <metal_stdlib>
+using namespace metal;
+
+// The Qwen3-VL text decoder's kernels that the DiT's (dit.metal) do not cover, fp32, following transformers'
+// modeling_qwen3_vl.py (QwenTextEncoder.swift says which module each one replaces). HeadParams is dit.metal's.
+
+// self_attn.q_norm / k_norm (RMSNorm over head_dim 128, weight) then apply_rotary_pos_emb: rotate_half pairs
+// dimension i with i + 64, `rope` is [rows, 64, (cos, sin)]. One SIMD group per (row, head), each lane the
+// dimensions lane, lane + 32, lane + 64, lane + 96.
+kernel void head_rmsnorm_rope_neox(device float *x [[buffer(0)]], device const float *weight [[buffer(1)]],
+ device const float *rope [[buffer(2)]], constant HeadParams &p [[buffer(3)]],
+ uint group [[threadgroup_position_in_grid]], uint lane [[thread_index_in_simdgroup]],
+ uint simd [[simdgroup_index_in_threadgroup]], uint groups [[simdgroups_per_threadgroup]]) {
+ uint unit = group * groups + simd; // (row, head)
+ if (unit >= p.rows * p.heads) return;
+ uint row = unit / p.heads;
+ device float *h = x + (ulong)unit * 128;
+ float4 v = float4(h[lane], h[32 + lane], h[64 + lane], h[96 + lane]);
+ float inv = rsqrt(simd_sum(dot(v, v)) / 128.0f + p.eps);
+ v = (v * inv) * float4(weight[lane], weight[32 + lane], weight[64 + lane], weight[96 + lane]);
+ device const float *t = rope + (ulong)row * 128;
+ float c0 = t[lane * 2], s0 = t[lane * 2 + 1], c1 = t[(32 + lane) * 2], s1 = t[(32 + lane) * 2 + 1];
+ h[lane] = v.x * c0 - v.z * s0;
+ h[64 + lane] = v.z * c0 + v.x * s0;
+ h[32 + lane] = v.y * c1 - v.w * s1;
+ h[96 + lane] = v.w * c1 + v.y * s1;
+}
+
+struct RepeatParams {
+ uint count; // output floats: rows × heads × 128
+ uint heads; // query heads
+ uint groups; // query heads per key/value head
+};
+
+// repeat_kv: rows of `heads / groups` key or value heads widened to `heads`, each head repeated `groups` times.
+kernel void repeat_kv(device const float *kv [[buffer(0)]], device float *out [[buffer(1)]],
+ constant RepeatParams &p [[buffer(2)]], uint i [[thread_position_in_grid]]) {
+ if (i >= p.count) return;
+ uint width = p.heads * 128;
+ uint row = i / width, c = i % width;
+ out[i] = kv[(ulong)row * (width / p.groups) + (c / 128 / p.groups) * 128 + c % 128];
+}
+
+// x += y: the decoder layer's two residuals.
+kernel void add_inplace(device float *x [[buffer(0)]], device const float *y [[buffer(1)]],
+ constant uint &count [[buffer(2)]], uint i [[thread_position_in_grid]]) {
+ if (i >= count) return;
+ x[i] = x[i] + y[i];
+}
diff --git a/engine-mac/Sources/KokonEngine/QwenTextEncoder.swift b/engine-mac/Sources/KokonEngine/QwenTextEncoder.swift
new file mode 100644
index 0000000..6594b22
--- /dev/null
+++ b/engine-mac/Sources/KokonEngine/QwenTextEncoder.swift
@@ -0,0 +1,279 @@
+import Foundation
+import Metal
+import MetalPerformanceShaders
+
+/// **Qwen-Image 2.1's text encoder**: the text part of Qwen3-VL-8B-Instruct (`Qwen3VLTextModel`, transformers 5) on
+/// the GPU, fp32 activations, the GGUF weights unpacked on the GPU just before each product, as the DiT does.
+///
+/// prompt ─ template ─ tokenizer ─ token_embd rows (CPU)
+/// 36 × { x += o_proj(causal SDPA(RoPE(RMS(q)), RoPE(RMS(k)), v)) on RMS(x)·w (32 query heads, 8 key/value)
+/// x += down(SiLU(gate) ⊙ up) on RMS(x)·w }
+/// conditioning = x without the system message's rows (no final norm)
+///
+/// What diffusers' QwenImage21Pipeline does (0.42, `_get_qwen_prompt_embeds`): the raw template below, the hidden
+/// state of the last decoder layer *before* the model's final RMSNorm (the pipeline hooks the norm away), and the
+/// leading tokens of the system message dropped. Text only: the three M-RoPE axes carry the same position, so the
+/// interleaved M-RoPE is plain RoPE (θ = 5·10⁶) and the vision tower's deepstack inputs never apply.
+public final class QwenTextEncoder {
+ public static let dim = 4096, heads = 32, kvHeads = 8, headDim = 128, hidden = 12288, layers = 36
+ static let eps: Float = 1e-6, theta: Float = 5_000_000
+ public static let systemPrompt = "Comprehend and analyze the provided prompt."
+
+ /// `prompt_template_t2i`: the prompt is inserted as is, the processor tokenizes the whole string.
+ public static func textToImage(_ prompt: String) -> String {
+ // Qwen has no bos token, so an empty prompt leaves the encoder nothing to read: diffusers sends " ".
+ "<|im_start|>system\n\(systemPrompt)<|im_end|>\n<|im_start|>user\n\(prompt.isEmpty ? " " : prompt)<|im_end|>\n<|im_start|>assistant\n"
+ }
+
+ public let file: GGUFFile
+ public let tokenizer: QwenTokenizer
+ let context: MetalContext
+ let dequant: GPUDequant
+ /// Tokens of the system message as the chat template writes it alone (`_drop_idx`, 14).
+ public let dropCount: Int
+ private var products: [String: MPSMatrixMultiplication] = [:]
+ /// Seconds per phase of the last forward.
+ public private(set) var timings: [String: Double] = [:]
+
+ public init(file: GGUFFile, context: MetalContext) throws {
+ guard file.metadata["general.architecture"]?.string == "qwen3vl",
+ file.metadata["qwen3vl.block_count"]?.int == QwenTextEncoder.layers,
+ file.metadata["qwen3vl.embedding_length"]?.int == QwenTextEncoder.dim else {
+ throw GGUFFile.Failure.unsupported("\(file.path): not the Qwen3-VL-8B text encoder")
+ }
+ self.file = file
+ self.context = context
+ dequant = GPUDequant(context: context)
+ tokenizer = try QwenTokenizer(file: file)
+ dropCount = tokenizer.encode("<|im_start|>system\n\(QwenTextEncoder.systemPrompt)<|im_end|>\n").count
+ }
+
+ /// Token ids of the text-to-image prompt.
+ public func tokens(prompt: String) -> [Int] { tokenizer.encode(QwenTextEncoder.textToImage(prompt)) }
+
+ /// The DiT's encoder rows for `prompt`: `[T, 4096]`, T = tokens − the system message's.
+ public func conditioning(prompt: String) throws -> (rows: [Float], count: Int) {
+ let ids = tokens(prompt: prompt)
+ let all = try forward(ids: ids)
+ let d = QwenTextEncoder.dim
+ return (Array(all[(dropCount * d)...]), ids.count - dropCount)
+ }
+
+ /// Drops the GPU windows on the file (the caller drops the file to unmap it).
+ public func release() { context.release(file) }
+
+ // MARK: - forward
+
+ /// Called with a stage's name and its rows (CPU pointer, floats); the pass then waits on the GPU at each stage.
+ /// Stages: embeddings, layer0_out, layer1_out, layer17_out, layer35_out.
+ public typealias Probe = (String, UnsafePointer<Float>, Int) -> Void
+
+ struct Buffers {
+ let arena: Arena
+ let x, normed, q, k, v, kx, vx, attended, o, h1, h2, rope, norms: Arena.Slice
+ let reserves: [Arena.Slice]
+ }
+ private var nextReserve = 0
+
+ /// The last decoder layer's output for every token, `[ids.count, 4096]`.
+ public func forward(ids: [Int], probe: Probe? = nil) throws -> [Float] {
+ let d = QwenTextEncoder.dim, t = ids.count, kvWidth = QwenTextEncoder.kvHeads * QwenTextEncoder.headDim
+ let started = DispatchTime.now().uptimeNanoseconds
+ timings = [:]
+ // Rows padded as the DiT's (64, see QwenDiT.padded), plus 64 for the attention kernel's overrun.
+ let n = QwenDiT.padded(t) + 64
+ let sizes = [n * d, n * d, n * d, n * kvWidth, n * kvWidth, n * d, n * d, n * d, n * d,
+ n * QwenTextEncoder.hidden, n * QwenTextEncoder.hidden, n * 128,
+ QwenTextEncoder.layers * 4 * d, d * QwenTextEncoder.hidden, d * QwenTextEncoder.hidden]
+ let arena = try Arena(device: context.device, bytes: Arena.bytes(sizes.map { $0 * 4 }))
+ var s = try sizes.map { try arena.slice(floats: $0) }.makeIterator()
+ let b = Buffers(arena: arena, x: s.next()!, normed: s.next()!, q: s.next()!, k: s.next()!, v: s.next()!,
+ kx: s.next()!, vx: s.next()!, attended: s.next()!, o: s.next()!, h1: s.next()!, h2: s.next()!,
+ rope: s.next()!, norms: s.next()!, reserves: [s.next()!, s.next()!])
+
+ // Norm weights of every layer (F32, small), laid out per layer: input, post-attention, q (128), k (128).
+ for layer in 0..<QwenTextEncoder.layers {
+ let base = layer * 4 * d
+ for (i, name) in ["attn_norm", "ffn_norm"].enumerated() {
+ let w = try Dequant.array(file, "blk.\(layer).\(name).weight")
+ arena.pointer(b.norms, floatOffset: base + i * d).update(from: w, count: d)
+ }
+ for (i, name) in ["attn_q_norm", "attn_k_norm"].enumerated() {
+ let w = try Dequant.array(file, "blk.\(layer).\(name).weight")
+ arena.pointer(b.norms, floatOffset: base + 2 * d + i * QwenTextEncoder.headDim)
+ .update(from: w, count: QwenTextEncoder.headDim)
+ }
+ }
+ let table = QwenTextEncoder.ropeTable(rows: t)
+ arena.pointer(b.rope).update(from: table, count: table.count)
+ // Embedding rows, unpacked on the CPU.
+ let embedding = try file.tensor("token_embd.weight")
+ guard let type = embedding.type, embedding.columns == d else {
+ throw GGUFFile.Failure.unsupported("token_embd.weight: \(embedding.dims)")
+ }
+ let rowBytes = type.bytes(count: d)
+ for (row, id) in ids.enumerated() {
+ guard id >= 0, id < embedding.rows else { throw GGUFFile.Failure.unsupported("token id \(id)") }
+ Dequant.row(file.data(embedding) + id * rowBytes, type: type, count: d, into: arena.pointer(b.x, floatOffset: row * d))
+ }
+ probe?("embeddings", arena.pointer(b.x), t * d)
+
+ var commands = try begin()
+ for layer in 0..<QwenTextEncoder.layers {
+ try encodeLayer(layer, rows: t, b, commands: commands)
+ if let probe, [0, 1, 17, QwenTextEncoder.layers - 1].contains(layer) {
+ try MetalContext.finish(commands, "text encoder")
+ probe("layer\(layer)_out", arena.pointer(b.x), t * d)
+ } else {
+ commands.commit()
+ }
+ commands = try begin()
+ }
+ try MetalContext.finish(commands, "text encoder")
+ timings["total"] = Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9
+ return Array(UnsafeBufferPointer(start: arena.pointer(b.x), count: t * d))
+ }
+
+ /// `[rows, 64, (cos, sin)]` of `position · ω_i`, `ω_i = 1 / θ^(2i / 128)`, fp32 as Qwen3VLTextRotaryEmbedding.
+ static func ropeTable(rows: Int) -> [Float] {
+ let half = QwenTextEncoder.headDim / 2
+ let omegas = (0..<half).map { i in 1 / powf(theta, Float(2 * i) / Float(QwenTextEncoder.headDim)) }
+ var out = [Float](repeating: 0, count: rows * half * 2)
+ for row in 0..<rows {
+ for (i, w) in omegas.enumerated() {
+ let angle = Float(row) * w
+ out[(row * half + i) * 2] = cosf(angle)
+ out[(row * half + i) * 2 + 1] = sinf(angle)
+ }
+ }
+ return out
+ }
+
+ func encodeLayer(_ layer: Int, rows t: Int, _ b: Buffers, commands: MTLCommandBuffer) throws {
+ let d = QwenTextEncoder.dim, block = "blk.\(layer)."
+ let norms = layer * 4 * d
+ try rmsNorm(b, b.x, into: b.normed, rows: t, weight: norms, commands: commands)
+ try linear(block + "attn_q.weight", b, a: b.normed, c: b.q, m: t, commands: commands)
+ try linear(block + "attn_k.weight", b, a: b.normed, c: b.k, m: t, commands: commands)
+ try linear(block + "attn_v.weight", b, a: b.normed, c: b.v, m: t, commands: commands)
+ try headNormRope(b, b.q, heads: QwenTextEncoder.heads, weight: norms + 2 * d, rows: t, commands: commands)
+ try headNormRope(b, b.k, heads: QwenTextEncoder.kvHeads, weight: norms + 2 * d + QwenTextEncoder.headDim,
+ rows: t, commands: commands)
+ try repeatKV(b, b.k, into: b.kx, rows: t, commands: commands)
+ try repeatKV(b, b.v, into: b.vx, rows: t, commands: commands)
+ try attend(b, rows: t, commands: commands)
+ try linear(block + "attn_output.weight", b, a: b.attended, c: b.o, m: t, commands: commands)
+ try add(b, b.o, count: t * d, commands: commands)
+ try rmsNorm(b, b.x, into: b.normed, rows: t, weight: norms + d, commands: commands)
+ try linear(block + "ffn_gate.weight", b, a: b.normed, c: b.h1, m: t, commands: commands)
+ try linear(block + "ffn_up.weight", b, a: b.normed, c: b.h2, m: t, commands: commands)
+ try compute("swiglu", commands: commands) { e in
+ e.setBuffer(b.arena.buffer, offset: b.h1.offset, index: 0)
+ e.setBuffer(b.arena.buffer, offset: b.h2.offset, index: 1)
+ var c = UInt32(t * QwenTextEncoder.hidden)
+ e.setBytes(&c, length: 4, index: 2)
+ e.dispatchThreads(MTLSize(width: Int(c), height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ }
+ try linear(block + "ffn_down.weight", b, a: b.h1, c: b.o, m: t, commands: commands)
+ try add(b, b.o, count: t * d, commands: commands)
+ }
+
+ // MARK: - encoders
+
+ func begin() throws -> MTLCommandBuffer { MPSCommandBuffer(from: context.queue) }
+
+ func compute(_ name: String, commands: MTLCommandBuffer, _ body: (MTLComputeCommandEncoder) throws -> Void) throws {
+ let encoder = commands.makeComputeCommandEncoder()!
+ encoder.label = name
+ encoder.setComputePipelineState(try context.pipeline(name))
+ try body(encoder)
+ encoder.endEncoding()
+ }
+
+ /// `C[m, n] = A[m, k] · W`, W the file's Linear `[n, k]` unpacked to `[k, n]` fp32 in a reserve first; m padded
+ /// to 64 (QwenDiT.padded: small MPS products slow every later one).
+ func linear(_ name: String, _ b: Buffers, a: Arena.Slice, c: Arena.Slice, m: Int, commands: MTLCommandBuffer) throws {
+ let tensor = try file.tensor(name)
+ let (kk, nn) = (tensor.columns, tensor.rows)
+ let m = QwenDiT.padded(m)
+ guard m * kk * 4 <= a.bytes, m * nn * 4 <= c.bytes else {
+ throw MetalContext.Failure.buffer("\(name): no room for \(m) padded rows")
+ }
+ let reserve = b.reserves[nextReserve]
+ nextReserve = (nextReserve + 1) % b.reserves.count
+ try dequant.encode(tensor, of: file, into: b.arena.buffer, outOffset: reserve.offset, transposed: true, commands: commands)
+ let key = "\(m)×\(kk)×\(nn)"
+ let product = products[key] ?? MPSMatrixMultiplication(device: context.device, transposeLeft: false, transposeRight: false,
+ resultRows: m, resultColumns: nn, interiorColumns: kk, alpha: 1, beta: 0)
+ products[key] = product
+ func matrix(_ offset: Int, _ rows: Int, _ columns: Int) -> MPSMatrix {
+ MPSMatrix(buffer: b.arena.buffer, offset: offset,
+ descriptor: MPSMatrixDescriptor(rows: rows, columns: columns, rowBytes: columns * 4, dataType: .float32))
+ }
+ product.encode(commandBuffer: commands, leftMatrix: matrix(a.offset, m, kk), rightMatrix: matrix(reserve.offset, kk, nn),
+ resultMatrix: matrix(c.offset, m, nn))
+ }
+
+ func rmsNorm(_ b: Buffers, _ input: Arena.Slice, into out: Arena.Slice, rows: Int, weight: Int, commands: MTLCommandBuffer) throws {
+ try compute("rmsnorm_rows", commands: commands) { e in
+ e.setBuffer(b.arena.buffer, offset: input.offset, index: 0)
+ e.setBuffer(b.arena.buffer, offset: out.offset, index: 1)
+ e.setBuffer(b.arena.buffer, offset: b.norms.offset + weight * 4, index: 2)
+ var params = QwenDiT.RowParams(columns: UInt32(QwenTextEncoder.dim), targetStart: 0, eps: QwenTextEncoder.eps)
+ e.setBytes(&params, length: MemoryLayout<QwenDiT.RowParams>.stride, index: 3)
+ e.dispatchThreadgroups(MTLSize(width: rows, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ }
+ }
+
+ func headNormRope(_ b: Buffers, _ tensor: Arena.Slice, heads: Int, weight: Int, rows: Int, commands: MTLCommandBuffer) throws {
+ try compute("head_rmsnorm_rope_neox", commands: commands) { e in
+ e.setBuffer(b.arena.buffer, offset: tensor.offset, index: 0)
+ e.setBuffer(b.arena.buffer, offset: b.norms.offset + weight * 4, index: 1)
+ e.setBuffer(b.arena.buffer, offset: b.rope.offset, index: 2)
+ var params = QwenDiT.HeadParams(heads: UInt32(heads), rows: UInt32(rows), eps: QwenTextEncoder.eps)
+ e.setBytes(&params, length: MemoryLayout<QwenDiT.HeadParams>.stride, index: 3)
+ let units = rows * heads // one SIMD group each, 8 per threadgroup
+ e.dispatchThreadgroups(MTLSize(width: (units + 7) / 8, height: 1, depth: 1),
+ threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ }
+ }
+
+ struct RepeatParams { var count, heads, groups: UInt32 }
+
+ func repeatKV(_ b: Buffers, _ input: Arena.Slice, into out: Arena.Slice, rows: Int, commands: MTLCommandBuffer) throws {
+ try compute("repeat_kv", commands: commands) { e in
+ e.setBuffer(b.arena.buffer, offset: input.offset, index: 0)
+ e.setBuffer(b.arena.buffer, offset: out.offset, index: 1)
+ var params = RepeatParams(count: UInt32(rows * QwenTextEncoder.dim), heads: UInt32(QwenTextEncoder.heads),
+ groups: UInt32(QwenTextEncoder.heads / QwenTextEncoder.kvHeads))
+ e.setBytes(&params, length: MemoryLayout<RepeatParams>.stride, index: 2)
+ e.dispatchThreads(MTLSize(width: Int(params.count), height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ }
+ }
+
+ func add(_ b: Buffers, _ y: Arena.Slice, count: Int, commands: MTLCommandBuffer) throws {
+ try compute("add_inplace", commands: commands) { e in
+ e.setBuffer(b.arena.buffer, offset: b.x.offset, index: 0)
+ e.setBuffer(b.arena.buffer, offset: y.offset, index: 1)
+ var c = UInt32(count)
+ e.setBytes(&c, length: 4, index: 2)
+ e.dispatchThreads(MTLSize(width: count, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ }
+ }
+
+ /// Causal attention of every row on the DiT's flash kernel (`Kernels/attention.metal`), keys and values widened
+ /// to the 32 query heads first.
+ func attend(_ b: Buffers, rows: Int, commands: MTLCommandBuffer) throws {
+ try compute("flash_attention", commands: commands) { e in
+ e.setBuffer(b.arena.buffer, offset: b.q.offset, index: 0)
+ e.setBuffer(b.arena.buffer, offset: b.kx.offset, index: 1)
+ e.setBuffer(b.arena.buffer, offset: b.vx.offset, index: 2)
+ e.setBuffer(b.arena.buffer, offset: b.attended.offset, index: 3)
+ var params = QwenDiT.FlashParams(queries: UInt32(rows), keys: UInt32(rows), causal: 1, stride: UInt32(QwenTextEncoder.dim),
+ scale: 1 / Float(QwenTextEncoder.headDim).squareRoot() * Float(M_LOG2E), headBase: 0)
+ e.setBytes(&params, length: MemoryLayout<QwenDiT.FlashParams>.stride, index: 4)
+ e.dispatchThreadgroups(MTLSize(width: (rows + 63) / 64, height: QwenTextEncoder.heads, depth: 1),
+ threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ }
+ }
+}
diff --git a/engine-mac/Sources/KokonEngine/QwenTokenizer.swift b/engine-mac/Sources/KokonEngine/QwenTokenizer.swift
new file mode 100644
index 0000000..85dfe93
--- /dev/null
+++ b/engine-mac/Sources/KokonEngine/QwenTokenizer.swift
@@ -0,0 +1,135 @@
+import Foundation
+
+/// **Qwen's byte-level BPE tokenizer**, read from the GGUF metadata (`tokenizer.ggml.*`, written by llama.cpp from
+/// the model's `tokenizer.json`), doing what the Hugging Face `tokenizers` pipeline of Qwen3-VL does:
+///
+/// added tokens (<|im_start|>, <|image_pad|>, …) split out first, matched longest-first, never normalized
+/// each piece between them: NFC ─ the Qwen2 split regex ─ bytes as GPT-2's printable characters ─ BPE merges
+///
+/// Checked token for token against the processor of the official Qwen-Image 2.1 repository
+/// (`tools/reference_tokenizer.py`, `check-tokenizer`).
+public final class QwenTokenizer: @unchecked Sendable {
+ /// `tokenizer.json`'s pre-tokenizer pattern (Split, behavior Isolated).
+ static let pattern = #"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"#
+
+ public let tokens: [String]
+ let ids: [String: Int]
+ /// "left right" → rank (the merge's index).
+ let ranks: [String: Int]
+ /// Added tokens (GGUF token types 3 control and 4 user-defined) by their UTF-8 bytes, longest first.
+ let added: [(bytes: [UInt8], id: Int)]
+ let regex: NSRegularExpression
+ /// GPT-2's byte → printable character table.
+ static let byteCharacters: [Character] = {
+ let printable = Set(Array(33...126) + Array(161...172) + Array(174...255))
+ var table = [Character](repeating: " ", count: 256)
+ var next = 256
+ for b in 0..<256 {
+ if printable.contains(b) {
+ table[b] = Character(Unicode.Scalar(UInt32(b))!)
+ } else {
+ table[b] = Character(Unicode.Scalar(UInt32(next))!)
+ next += 1
+ }
+ }
+ return table
+ }()
+ private var cache: [String: [Int]] = [:]
+ private let lock = NSLock()
+
+ public init(file: GGUFFile) throws {
+ guard file.metadata["tokenizer.ggml.model"]?.string == "gpt2",
+ file.metadata["tokenizer.ggml.pre"]?.string == "qwen2",
+ case .array(let tokenValues)? = file.metadata["tokenizer.ggml.tokens"],
+ case .array(let mergeValues)? = file.metadata["tokenizer.ggml.merges"],
+ case .array(let typeValues)? = file.metadata["tokenizer.ggml.token_type"] else {
+ throw GGUFFile.Failure.unsupported("\(file.path): no Qwen2 BPE tokenizer in the metadata")
+ }
+ tokens = tokenValues.map { $0.string ?? "" }
+ var ids: [String: Int] = [:]
+ ids.reserveCapacity(tokens.count)
+ for (i, t) in tokens.enumerated() { ids[t] = i }
+ self.ids = ids
+ var ranks: [String: Int] = [:]
+ ranks.reserveCapacity(mergeValues.count)
+ for (i, m) in mergeValues.enumerated() { if let s = m.string { ranks[s] = i } }
+ self.ranks = ranks
+ let names = tokens
+ added = typeValues.enumerated().compactMap { i, type in
+ type.int == 3 || type.int == 4 ? (Array(names[i].utf8), i) : nil
+ }.sorted { $0.bytes.count > $1.bytes.count }
+ regex = try NSRegularExpression(pattern: QwenTokenizer.pattern)
+ }
+
+ /// The token id of an added token such as `<|im_start|>`.
+ public func id(of token: String) -> Int? { ids[token] }
+
+ public func encode(_ text: String) -> [Int] {
+ var out: [Int] = []
+ let bytes = Array(text.utf8)
+ var start = 0, i = 0
+ while i < bytes.count {
+ if let match = added.first(where: { a in
+ a.bytes.count <= bytes.count - i && a.bytes[0] == bytes[i] && bytes[i..<(i + a.bytes.count)].elementsEqual(a.bytes)
+ }) {
+ if start < i { encodeOrdinary(String(decoding: bytes[start..<i], as: UTF8.self), into: &out) }
+ out.append(match.id)
+ i += match.bytes.count
+ start = i
+ } else {
+ i += 1
+ }
+ }
+ if start < bytes.count { encodeOrdinary(String(decoding: bytes[start...], as: UTF8.self), into: &out) }
+ return out
+ }
+
+ /// Text without added tokens: NFC, the split regex (unmatched gaps kept as pieces too), then BPE per piece.
+ private func encodeOrdinary(_ text: String, into out: inout [Int]) {
+ let normalized = text.precomposedStringWithCanonicalMapping as NSString
+ var cursor = 0
+ func piece(_ range: NSRange) {
+ guard range.length > 0 else { return }
+ out += bpe(normalized.substring(with: range))
+ }
+ for match in regex.matches(in: normalized as String, range: NSRange(location: 0, length: normalized.length)) {
+ piece(NSRange(location: cursor, length: match.range.location - cursor))
+ piece(match.range)
+ cursor = match.range.location + match.range.length
+ }
+ piece(NSRange(location: cursor, length: normalized.length - cursor))
+ }
+
+ /// One piece: its bytes as characters, merged by rank (lowest first, leftmost first) until no merge applies.
+ func bpe(_ piece: String) -> [Int] {
+ lock.lock()
+ if let hit = cache[piece] { lock.unlock(); return hit }
+ lock.unlock()
+ var parts = piece.utf8.map { String(QwenTokenizer.byteCharacters[Int($0)]) }
+ while parts.count > 1 {
+ var best = Int.max, at = -1
+ for j in 0..<(parts.count - 1) {
+ if let r = ranks[parts[j] + " " + parts[j + 1]], r < best { best = r; at = j }
+ }
+ guard at >= 0 else { break }
+ parts[at] += parts[at + 1]
+ parts.remove(at: at + 1)
+ }
+ // Every byte is a token of its own, so a part is always in the vocabulary.
+ let result = parts.map { ids[$0] ?? 0 }
+ lock.lock(); cache[piece] = result; lock.unlock()
+ return result
+ }
+
+ /// Token ids back to text (byte-level characters to bytes); for messages and tests.
+ public func decode(_ ids: [Int]) -> String {
+ var reverse: [Character: UInt8] = [:]
+ for (b, c) in QwenTokenizer.byteCharacters.enumerated() { reverse[c] = UInt8(b) }
+ var bytes: [UInt8] = []
+ for id in ids where id >= 0 && id < tokens.count {
+ if added.contains(where: { $0.id == id }) { bytes += Array(tokens[id].utf8); continue }
+ for c in tokens[id] { if let b = reverse[c] { bytes.append(b) } }
+ }
+ return String(decoding: bytes, as: UTF8.self)
+ }
+}
diff --git a/engine-mac/Sources/KokonEngine/TextChecks.swift b/engine-mac/Sources/KokonEngine/TextChecks.swift
new file mode 100644
index 0000000..0c292d1
--- /dev/null
+++ b/engine-mac/Sources/KokonEngine/TextChecks.swift
@@ -0,0 +1,58 @@
+import Foundation
+
+extension Checks {
+ /// The tokenizer against `tools/reference_tokenizer.py` (the official processor): every case token for token.
+ public static func tokenizer(path: String, fixture: String) throws {
+ let file = try GGUFFile(path: path)
+ var started = DispatchTime.now().uptimeNanoseconds
+ let tokenizer = try QwenTokenizer(file: file)
+ print(String(format: "tokenizer: %d tokens, read in %.2f s", tokenizer.tokens.count,
+ Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9))
+ let cases = try JSONSerialization.jsonObject(with: Data(contentsOf: URL(fileURLWithPath: fixture))) as! [[String: Any]]
+ var failed = 0
+ started = DispatchTime.now().uptimeNanoseconds
+ for c in cases {
+ let text = c["text"] as! String, want = c["ids"] as! [Int]
+ let got = tokenizer.encode(text)
+ if got != want {
+ failed += 1
+ let at = (0..<min(got.count, want.count)).first { got[$0] != want[$0] } ?? min(got.count, want.count)
+ print("MISMATCH \(text.debugDescription.prefix(80))\n at \(at): got \(Array(got[at...].prefix(8))), want \(Array(want[at...].prefix(8)))")
+ }
+ }
+ print(String(format: "%d / %d cases identical (%.1f ms)", cases.count - failed, cases.count,
+ Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-6))
+ if failed > 0 { throw GGUFFile.Failure.unsupported("tokenizer differs from the reference") }
+ }
+
+ /// The text encoder against `tools/reference_text_encoder.py`: the token ids, each saved stage, the conditioning.
+ public static func textEncoder(path: String, reference directory: String) throws {
+ let meta = try JSONSerialization.jsonObject(with: Data(contentsOf: URL(fileURLWithPath: directory)
+ .appendingPathComponent("meta.json"))) as! [String: Any]
+ let prompt = meta["prompt"] as! String, wantIds = meta["ids"] as! [Int], wantDrop = meta["drop"] as! Int
+ let file = try GGUFFile(path: path)
+ let context = try MetalContext()
+ let encoder = try QwenTextEncoder(file: file, context: context)
+ let ids = encoder.tokens(prompt: prompt)
+ print("tokens: \(ids.count), \(ids == wantIds ? "identical" : "DIFFERENT") ids; drop \(encoder.dropCount)"
+ + (encoder.dropCount == wantDrop ? "" : " (reference \(wantDrop))"))
+ guard ids == wantIds, encoder.dropCount == wantDrop else { throw GGUFFile.Failure.unsupported("token ids differ") }
+
+ func report(_ name: String, _ got: UnsafePointer<Float>, _ want: [Float]) {
+ let r = compare(got, want)
+ print(String(format: " %-12@ rel %.2e max |Δ| %.3g%@", name as NSString, r.relative, r.maxAbs, r.finite ? "" : " NON-FINITE"))
+ }
+ _ = try encoder.forward(ids: ids) { name, pointer, count in
+ guard let want = try? reference(directory, name) else { return }
+ precondition(want.count == count, "\(name): \(count) floats, reference \(want.count)")
+ report(name, pointer, want)
+ }
+ for run in 1...2 {
+ let started = DispatchTime.now().uptimeNanoseconds
+ let (rows, count) = try encoder.conditioning(prompt: prompt)
+ let seconds = Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9
+ if run == 1 { rows.withUnsafeBufferPointer { report("conditioning", $0.baseAddress!, try! reference(directory, "conditioning")) } }
+ print(String(format: "encode %d (%d rows): %.3f s", run, count, seconds))
+ }
+ }
+}
diff --git a/engine-mac/Sources/kokon-engine-mac/main.swift b/engine-mac/Sources/kokon-engine-mac/main.swift
index 026e739..7167e32 100644
--- a/engine-mac/Sources/kokon-engine-mac/main.swift
+++ b/engine-mac/Sources/kokon-engine-mac/main.swift
@@ -9,6 +9,8 @@ kokon-engine-mac: Kokon's Apple Silicon engine (in development)
check-dit <dit.gguf> <reference dir> one DiT evaluation against tools/reference_dit.py
time-dit <dit.gguf> [grid] [text] [runs] DiT evaluations on random inputs (grid 64 = 1024²)
check-vae <vae.safetensors> <reference dir> [grid] the VAE decoder against tools/reference_vae.py
+ check-tokenizer <encoder.gguf> <fixture.json> the tokenizer against tools/reference_tokenizer.py
+ check-text <encoder.gguf> <reference dir> the text encoder against tools/reference_text_encoder.py
probe this Mac's matrix throughput: GPU, AMX, Neural Engine, together
"""
@@ -52,6 +54,12 @@ do {
case "check-vae":
guard arguments.count >= 3 else { fail(usage) }
try Checks.vae(path: arguments[1], reference: arguments[2], timeGrid: arguments.count > 3 ? Int(arguments[3]) : nil)
+ case "check-tokenizer":
+ guard arguments.count == 3 else { fail(usage) }
+ try Checks.tokenizer(path: arguments[1], fixture: arguments[2])
+ case "check-text":
+ guard arguments.count == 3 else { fail(usage) }
+ try Checks.textEncoder(path: arguments[1], reference: arguments[2])
case "probe":
try Probe.run()
default:
diff --git a/engine-mac/Tests/KokonEngineTests/QwenTokenizerTests.swift b/engine-mac/Tests/KokonEngineTests/QwenTokenizerTests.swift
new file mode 100644
index 0000000..6317fb5
--- /dev/null
+++ b/engine-mac/Tests/KokonEngineTests/QwenTokenizerTests.swift
@@ -0,0 +1,71 @@
+import Foundation
+import XCTest
+@testable import KokonEngine
+
+/// The tokenizer on a tiny vocabulary: the 256 byte characters, a few merges and one added token. The real
+/// vocabulary is checked against the official processor by `check-tokenizer`.
+final class QwenTokenizerTests: XCTestCase {
+ private var temporary: URL!
+ private var tokenizer: QwenTokenizer!
+ private var file: GGUFFile!
+
+ override func setUpWithError() throws {
+ temporary = FileManager.default.temporaryDirectory.appendingPathComponent("kokon-tokenizer-tests-\(UUID())")
+ try FileManager.default.createDirectory(at: temporary, withIntermediateDirectories: true)
+ let bytes = QwenTokenizer.byteCharacters.map { String($0) }
+ let merges = ["h e", "t he", "Ġ c", "Ġc a", "Ġca t", "' s"]
+ let tokens = bytes + ["he", "the", "Ġc", "Ġca", "Ġcat", "'s", "<|im_start|>"]
+ let types = [Int32](repeating: 1, count: tokens.count - 1) + [3]
+
+ func string(_ s: String) -> [UInt8] {
+ var out: [UInt8] = []
+ withUnsafeBytes(of: UInt64(s.utf8.count).littleEndian) { out += $0 }
+ return out + Array(s.utf8)
+ }
+ func array(_ itemType: UInt32, _ items: [[UInt8]]) -> [UInt8] {
+ var out: [UInt8] = []
+ withUnsafeBytes(of: itemType.littleEndian) { out += $0 }
+ withUnsafeBytes(of: UInt64(items.count).littleEndian) { out += $0 }
+ return out + items.flatMap { $0 }
+ }
+ let path = try TestGGUF.write(in: temporary, metadata: [
+ ("tokenizer.ggml.model", 8, string("gpt2")),
+ ("tokenizer.ggml.pre", 8, string("qwen2")),
+ ("tokenizer.ggml.tokens", 9, array(8, tokens.map(string))),
+ ("tokenizer.ggml.merges", 9, array(8, merges.map(string))),
+ ("tokenizer.ggml.token_type", 9, array(5, types.map { t in withUnsafeBytes(of: t.littleEndian) { Array($0) } })),
+ ], tensors: [])
+ file = try GGUFFile(path: path)
+ tokenizer = try QwenTokenizer(file: file)
+ }
+
+ override func tearDownWithError() throws { try? FileManager.default.removeItem(at: temporary) }
+
+ private func id(_ token: String) -> Int { tokenizer.tokens.firstIndex(of: token)! }
+ private func byte(_ b: UInt8) -> Int { id(String(QwenTokenizer.byteCharacters[Int(b)])) }
+
+ func testByteTableIsGPT2s() {
+ XCTAssertEqual(QwenTokenizer.byteCharacters[Int(UInt8(ascii: "a"))], "a")
+ XCTAssertEqual(QwenTokenizer.byteCharacters[32], "Ġ") // space
+ XCTAssertEqual(QwenTokenizer.byteCharacters[10], "Ċ") // newline
+ XCTAssertEqual(Set(QwenTokenizer.byteCharacters).count, 256)
+ }
+
+ func testMergesByRankAcrossSplitPieces() {
+ // "the" merges h+e first (rank 0), then t+he; " cat" is its own piece; "'s" is the contraction alternative.
+ XCTAssertEqual(tokenizer.encode("the cat's"), [id("the"), id("Ġcat"), id("'s")])
+ // No merge for "ht": bytes stay single.
+ XCTAssertEqual(tokenizer.encode("ht"), [byte(UInt8(ascii: "h")), byte(UInt8(ascii: "t"))])
+ }
+
+ func testAddedTokensAreSplitOutFirst() {
+ XCTAssertEqual(tokenizer.encode("the<|im_start|>the"), [id("the"), id("<|im_start|>"), id("the")])
+ XCTAssertEqual(tokenizer.encode("<|im_start|>"), [id("<|im_start|>")])
+ }
+
+ func testTextIsNFCNormalized() {
+ // e + combining acute becomes é (C3 A9) before the bytes are taken.
+ XCTAssertEqual(tokenizer.encode("e\u{301}"), [byte(0xC3), byte(0xA9)])
+ XCTAssertEqual(tokenizer.decode(tokenizer.encode("the cat é")), "the cat é")
+ }
+}
diff --git a/engine-mac/tools/reference_text_encoder.py b/engine-mac/tools/reference_text_encoder.py
new file mode 100644
index 0000000..d02b1a1
--- /dev/null
+++ b/engine-mac/tools/reference_text_encoder.py
@@ -0,0 +1,136 @@
+"""The Qwen-Image 2.1 text encoder reference for the Mac engine: transformers' own Qwen3-VL text decoder layers, in
+fp32 on the CPU, on the weights of Kokon's GGUF unpacked by gguf-py, run on the prompt exactly as diffusers'
+QwenImage21Pipeline builds it (0.42, `_get_qwen_prompt_embeds`, text-to-image):
+
+ template "<|im_start|>system\\n{sys}<|im_end|>\\n<|im_start|>user\\n{prompt}<|im_end|>\\n<|im_start|>assistant\\n"
+ tokenized by the official processor; conditioning = the last decoder layer's output (before the final RMSNorm),
+ without the system message's tokens
+
+The 8B model in fp32 is 33 GB, more than a 16 GB Mac holds: one decoder layer module, its weights swapped layer
+by layer (as `reference_dit.py` does for the DiT). `--sanity` also runs the language model head on a factual
+prompt and prints its next token: a check that the GGUF weights are read right, which a comparison of two readers
+of the same file cannot be.
+
+ python -I engine-mac/tools/reference_text_encoder.py <encoder.gguf> <processor dir> <text_encoder dir> <out dir>
+ [--prompt "..."] [--sanity]
+
+Writes `<out>/meta.json` (prompt, ids, drop) and one raw little-endian fp32 file per tensor (`<name>.f32`).
+"""
+import argparse
+import json
+import os
+
+import gguf
+import numpy as np
+import torch
+from transformers import AutoConfig, AutoProcessor
+from transformers.models.qwen3_vl.modeling_qwen3_vl import (
+ Qwen3VLTextDecoderLayer,
+ Qwen3VLTextRMSNorm,
+ Qwen3VLTextRotaryEmbedding,
+)
+
+SYSTEM = "Comprehend and analyze the provided prompt."
+TEMPLATE = f"<|im_start|>system\n{SYSTEM}<|im_end|>\n<|im_start|>user\n{{}}<|im_end|>\n<|im_start|>assistant\n"
+NAMES = {
+ "attn_q": "self_attn.q_proj", "attn_k": "self_attn.k_proj", "attn_v": "self_attn.v_proj",
+ "attn_output": "self_attn.o_proj", "attn_q_norm": "self_attn.q_norm", "attn_k_norm": "self_attn.k_norm",
+ "attn_norm": "input_layernorm", "ffn_norm": "post_attention_layernorm",
+ "ffn_gate": "mlp.gate_proj", "ffn_up": "mlp.up_proj", "ffn_down": "mlp.down_proj",
+}
+
+
+class Weights:
+ def __init__(self, path):
+ self.reader = gguf.GGUFReader(path)
+ self.tensors = {t.name: t for t in self.reader.tensors}
+
+ def get(self, name, rows=None):
+ t = self.tensors[name]
+ data = t.data if rows is None else t.data[rows]
+ values = gguf.quants.dequantize(data, t.tensor_type).astype(np.float32)
+ shape = [int(d) for d in reversed(t.shape)] # ggml ne order -> torch order
+ if rows is not None:
+ shape[0] = len(rows)
+ return torch.from_numpy(values.reshape(shape).copy())
+
+ def load_layer(self, layer, index):
+ state = {}
+ for gname, hname in NAMES.items():
+ state[hname + ".weight"] = self.get(f"blk.{index}.{gname}.weight")
+ layer.load_state_dict(state, strict=True)
+
+
+def run(weights, config, ids, save=None):
+ """The text model's forward on one unpadded prompt; returns the last layer's output (no final norm)."""
+ x = weights.get("token_embd.weight", rows=ids)[None]
+ if save:
+ save("embeddings", x[0])
+ t = x.shape[1]
+ position_ids = torch.arange(t).view(1, 1, -1).expand(3, 1, -1) # text only: the three axes agree
+ rope = Qwen3VLTextRotaryEmbedding(config)
+ position_embeddings = rope(x, position_ids)
+ mask = torch.full((t, t), torch.finfo(torch.float32).min).triu(1)[None, None]
+ layer = Qwen3VLTextDecoderLayer(config, 0)
+ for index in range(config.num_hidden_layers):
+ weights.load_layer(layer, index)
+ x = layer(x, position_embeddings=position_embeddings, attention_mask=mask)
+ if save and index in (0, 1, 17, config.num_hidden_layers - 1):
+ save(f"layer{index}_out", x[0])
+ return x
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("gguf")
+ parser.add_argument("processor")
+ parser.add_argument("text_encoder", help="text_encoder/ of the official repository (config.json)")
+ parser.add_argument("out")
+ parser.add_argument("--prompt", default="A red fox sitting in fresh snow at dawn, soft golden light, photo")
+ parser.add_argument("--sanity", action="store_true")
+ args = parser.parse_args()
+ torch.set_grad_enabled(False)
+ os.makedirs(args.out, exist_ok=True)
+ config = AutoConfig.from_pretrained(args.text_encoder).text_config
+ config._attn_implementation = "eager"
+ processor = AutoProcessor.from_pretrained(args.processor)
+ weights = Weights(args.gguf)
+
+ # QwenImage21Pipeline.__init__ / _get_qwen_prompt_embeds, one prompt, no images.
+ system = [{"role": "system", "content": [{"type": "text", "text": SYSTEM}]}]
+ drop = len(processor.apply_chat_template(system, tokenize=True, return_dict=False)[0])
+ prompt = args.prompt or " "
+ ids = processor(text=[TEMPLATE.format(prompt)], return_tensors="pt").input_ids[0].tolist()
+
+ saved = {}
+
+ def save(name, tensor):
+ array = tensor.detach().to(torch.float32).contiguous().numpy()
+ array.tofile(os.path.join(args.out, name + ".f32"))
+ saved[name] = list(array.shape)
+
+ hidden = run(weights, config, ids, save)
+ save("conditioning", hidden[0, drop:])
+ meta = {"prompt": prompt, "ids": ids, "drop": drop, "tensors": saved, "gguf": os.path.basename(args.gguf),
+ "transformers": __import__("transformers").__version__, "torch": torch.__version__}
+
+ if args.sanity:
+ question = ("<|im_start|>user\nWhat is the capital of France? Answer in one sentence.<|im_end|>\n"
+ "<|im_start|>assistant\nThe capital of France is")
+ qids = processor(text=[question], return_tensors="pt").input_ids[0].tolist()
+ h = run(weights, config, qids)
+ norm = Qwen3VLTextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+ norm.weight.copy_(weights.get("output_norm.weight"))
+ logits = weights.get("output.weight") @ norm(h[0, -1])
+ top = torch.topk(logits, 5)
+ tokens = [processor.tokenizer.decode([i]) for i in top.indices.tolist()]
+ print("sanity: next tokens after 'The capital of France is':", tokens)
+ meta["sanity"] = tokens
+
+ with open(os.path.join(args.out, "meta.json"), "w") as f:
+ json.dump(meta, f, indent=1)
+ print(f"{len(ids)} tokens, drop {drop}, conditioning {saved['conditioning']}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/engine-mac/tools/reference_tokenizer.py b/engine-mac/tools/reference_tokenizer.py
new file mode 100644
index 0000000..5bef82d
--- /dev/null
+++ b/engine-mac/tools/reference_tokenizer.py
@@ -0,0 +1,57 @@
+"""Token ids from the processor of the official Qwen-Image 2.1 repository (what diffusers' QwenImage21Pipeline
+tokenizes with), for `kokon-engine-mac check-tokenizer` to compare against.
+
+ python -I engine-mac/tools/reference_tokenizer.py <processor dir> <out.json>
+
+The processor dir is `processor/` of Qwen/Qwen-Image-2.1-Turbo (tokenizer.json, tokenizer_config.json, ...).
+Writes a list of {"text", "ids"}: the pipeline's full text-to-image prompts and harder strings (accents, CJK,
+emoji, digits, whitespace runs, contractions, special tokens inside the text).
+"""
+import json
+import sys
+
+from transformers import AutoProcessor
+
+SYSTEM = "Comprehend and analyze the provided prompt."
+TEMPLATE = f"<|im_start|>system\n{SYSTEM}<|im_end|>\n<|im_start|>user\n{{}}<|im_end|>\n<|im_start|>assistant\n"
+
+CASES = [
+ "A red fox in the snow",
+ " ",
+ "a cozy wooden cabin by a lake at sunset, warm light, 35mm photo, f/1.8",
+ "Replace the sky with a dramatic stormy sky; keep the house unchanged.",
+ "Un gatto che dorme sulla sedia, perché è stanco. Città, più, caffè",
+ "Ein schönes Haus — größer, weiß, Straße",
+ "一只在雪地里的红狐狸,电影感光线",
+ "日本語のテキスト「こんにちは」を看板に書いて",
+ "Emoji 🦊🔥👍🏽 and symbols ©®™ ½ ∑ ≤ →",
+ "Numbers 1234567890, 3.14159, 1,000,000 and 2026-10-25",
+ "Multiple spaces\tand\ttabs\n\nnew lines\r\nand trailing ",
+ " leading spaces",
+ "I'm sure it's they're we've you'll he'd IT'S I'M",
+ "Text on the sign: \"KOKON\" in bold, 'single quotes', `backticks`",
+ "café vs café (decomposed), Å vs Å",
+ "special <|im_start|> inside and <|image_pad|><think> tags",
+ "url https://kokon.app/docs?x=1&y=2#top and email [email protected]",
+ "naïve résumé coöperate façade jalapeño",
+ "Кириллица и ελληνικά και العربية עברית",
+ "x" * 300,
+ "The quick brown fox jumps over the lazy dog. " * 8,
+]
+
+
+def main():
+ processor = AutoProcessor.from_pretrained(sys.argv[1])
+ out = []
+ for text in CASES:
+ out.append({"text": text, "ids": processor.tokenizer.encode(text, add_special_tokens=False)})
+ prompt = TEMPLATE.format(text)
+ ids = processor(text=[prompt], return_tensors="pt").input_ids[0].tolist()
+ out.append({"text": prompt, "ids": ids})
+ with open(sys.argv[2], "w") as f:
+ json.dump(out, f, ensure_ascii=False, indent=0)
+ print(f"{len(out)} cases")
+
+
+if __name__ == "__main__":
+ main()