Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.
TypeScript ai-photo-editorcomfyui-alternativegenerative-fillimage-editingimage-generationinpaintinglinuxlocal-aimacosmcpmcp-servernano-banana-alternativeofflinephoto-editorprivacyqwen-imageqwen-image-2-1stable-diffusion-cppvulkanwindows
Mac engine: the Qwen-Image 2.1 text encoder
Gabriele Lozuponed50bf6053cba
diff --git a/engine-mac/README.md b/engine-mac/README.mdindex 04b26a0..554536a 100644--- a/engine-mac/README.md+++ b/engine-mac/README.md@@ -28,6 +28,8 @@ swift test # no model needed| `Coexecution.swift`, `Kernels/ane.metal` | the AMX and the Neural Engine beside the GPU on the DiT's products || `Kernels/attention.metal` | fused block-causal attention || `QwenVAE.swift`, `Safetensors.swift` | the VAE decoder as one MPSGraph, from Kokon's VAE file (Wan 2.2 names) |+| `QwenTokenizer.swift` | Qwen's byte-level BPE tokenizer, read from the encoder GGUF's metadata |+| `QwenTextEncoder.swift`, `Kernels/text.metal` | the text encoder: Qwen3-VL-8B's text decoder on the GPU, diffusers' prompt template |Tests check the CPU and GPU unpacking against gguf-py's values (`tools/make_quant_fixtures.py` writes thefixture), and the sequence layout against diffusers' rules worked out by hand.@@ -44,6 +46,22 @@ python3 -m venv ~/kokon-bench/venv-oracle && ~/kokon-bench/venv-oracle/bin/pip i256² text-to-image, σ 0.7, 2026-10-25: every stage within 2e-5 relative of diffusers fp32 (velocity 6e-6).+**The text encoder against transformers.** The reference tokenizes with the processor of the official+Qwen/Qwen-Image-2.1-Turbo repository (`processor/` and `text_encoder/config.json`, a few MB) and runs transformers'+Qwen3-VL decoder layers in fp32 on the GGUF's weights, one layer at a time; `--sanity` checks the language model+head answers " Paris" to a factual prompt (it does), so the weights are read right and not only read alike.++```bash+~/kokon-bench/venv-oracle/bin/pip install torchvision huggingface_hub # the processor needs torchvision+~/kokon-bench/venv-oracle/bin/python -I tools/reference_tokenizer.py <processor dir> ~/kokon-bench/ref/text/tokenizer.json+.build/release/kokon-engine-mac check-tokenizer <encoder.gguf> ~/kokon-bench/ref/text/tokenizer.json+~/kokon-bench/venv-oracle/bin/python -I tools/reference_text_encoder.py <encoder.gguf> <processor dir> <text_encoder dir> ~/kokon-bench/ref/text/fox --sanity # ~50 s+.build/release/kokon-engine-mac check-text <encoder.gguf> ~/kokon-bench/ref/text/fox+```++2026-10-25: tokenizer identical on 42 cases (accents, CJK, emoji, decomposed Unicode, special tokens in the text);+conditioning within 5.1e-6 relative (37 tokens) and 5.6e-6 (144 tokens). One encode: 0.45 s (37 tokens), 0.8 s (144).+## First measurements (M2 Pro 19-core GPU, 16 GB, macOS 27, 2026-10-25, `probe`)m = 4 096 rows (a 1024² image), TFLOP/s:diff --git a/engine-mac/Sources/KokonEngine/Kernels/text.metal b/engine-mac/Sources/KokonEngine/Kernels/text.metalnew file mode 100644index 0000000..6a6ea99--- /dev/null+++ b/engine-mac/Sources/KokonEngine/Kernels/text.metal@@ -0,0 +1,49 @@+#include <metal_stdlib>+using namespace metal;++// The Qwen3-VL text decoder's kernels that the DiT's (dit.metal) do not cover, fp32, following transformers'+// modeling_qwen3_vl.py (QwenTextEncoder.swift says which module each one replaces). HeadParams is dit.metal's.++// self_attn.q_norm / k_norm (RMSNorm over head_dim 128, weight) then apply_rotary_pos_emb: rotate_half pairs+// dimension i with i + 64, `rope` is [rows, 64, (cos, sin)]. One SIMD group per (row, head), each lane the+// dimensions lane, lane + 32, lane + 64, lane + 96.+kernel void head_rmsnorm_rope_neox(device float *x [[buffer(0)]], device const float *weight [[buffer(1)]],+ device const float *rope [[buffer(2)]], constant HeadParams &p [[buffer(3)]],+ uint group [[threadgroup_position_in_grid]], uint lane [[thread_index_in_simdgroup]],+ uint simd [[simdgroup_index_in_threadgroup]], uint groups [[simdgroups_per_threadgroup]]) {+ uint unit = group * groups + simd; // (row, head)+ if (unit >= p.rows * p.heads) return;+ uint row = unit / p.heads;+ device float *h = x + (ulong)unit * 128;+ float4 v = float4(h[lane], h[32 + lane], h[64 + lane], h[96 + lane]);+ float inv = rsqrt(simd_sum(dot(v, v)) / 128.0f + p.eps);+ v = (v * inv) * float4(weight[lane], weight[32 + lane], weight[64 + lane], weight[96 + lane]);+ device const float *t = rope + (ulong)row * 128;+ float c0 = t[lane * 2], s0 = t[lane * 2 + 1], c1 = t[(32 + lane) * 2], s1 = t[(32 + lane) * 2 + 1];+ h[lane] = v.x * c0 - v.z * s0;+ h[64 + lane] = v.z * c0 + v.x * s0;+ h[32 + lane] = v.y * c1 - v.w * s1;+ h[96 + lane] = v.w * c1 + v.y * s1;+}++struct RepeatParams {+ uint count; // output floats: rows × heads × 128+ uint heads; // query heads+ uint groups; // query heads per key/value head+};++// repeat_kv: rows of `heads / groups` key or value heads widened to `heads`, each head repeated `groups` times.+kernel void repeat_kv(device const float *kv [[buffer(0)]], device float *out [[buffer(1)]],+ constant RepeatParams &p [[buffer(2)]], uint i [[thread_position_in_grid]]) {+ if (i >= p.count) return;+ uint width = p.heads * 128;+ uint row = i / width, c = i % width;+ out[i] = kv[(ulong)row * (width / p.groups) + (c / 128 / p.groups) * 128 + c % 128];+}++// x += y: the decoder layer's two residuals.+kernel void add_inplace(device float *x [[buffer(0)]], device const float *y [[buffer(1)]],+ constant uint &count [[buffer(2)]], uint i [[thread_position_in_grid]]) {+ if (i >= count) return;+ x[i] = x[i] + y[i];+}diff --git a/engine-mac/Sources/KokonEngine/QwenTextEncoder.swift b/engine-mac/Sources/KokonEngine/QwenTextEncoder.swiftnew file mode 100644index 0000000..6594b22--- /dev/null+++ b/engine-mac/Sources/KokonEngine/QwenTextEncoder.swift@@ -0,0 +1,279 @@+import Foundation+import Metal+import MetalPerformanceShaders++/// **Qwen-Image 2.1's text encoder**: the text part of Qwen3-VL-8B-Instruct (`Qwen3VLTextModel`, transformers 5) on+/// the GPU, fp32 activations, the GGUF weights unpacked on the GPU just before each product, as the DiT does.+///+/// prompt ─ template ─ tokenizer ─ token_embd rows (CPU)+/// 36 × { x += o_proj(causal SDPA(RoPE(RMS(q)), RoPE(RMS(k)), v)) on RMS(x)·w (32 query heads, 8 key/value)+/// x += down(SiLU(gate) ⊙ up) on RMS(x)·w }+/// conditioning = x without the system message's rows (no final norm)+///+/// What diffusers' QwenImage21Pipeline does (0.42, `_get_qwen_prompt_embeds`): the raw template below, the hidden+/// state of the last decoder layer *before* the model's final RMSNorm (the pipeline hooks the norm away), and the+/// leading tokens of the system message dropped. Text only: the three M-RoPE axes carry the same position, so the+/// interleaved M-RoPE is plain RoPE (θ = 5·10⁶) and the vision tower's deepstack inputs never apply.+public final class QwenTextEncoder {+ public static let dim = 4096, heads = 32, kvHeads = 8, headDim = 128, hidden = 12288, layers = 36+ static let eps: Float = 1e-6, theta: Float = 5_000_000+ public static let systemPrompt = "Comprehend and analyze the provided prompt."++ /// `prompt_template_t2i`: the prompt is inserted as is, the processor tokenizes the whole string.+ public static func textToImage(_ prompt: String) -> String {+ // Qwen has no bos token, so an empty prompt leaves the encoder nothing to read: diffusers sends " ".+ "<|im_start|>system\n\(systemPrompt)<|im_end|>\n<|im_start|>user\n\(prompt.isEmpty ? " " : prompt)<|im_end|>\n<|im_start|>assistant\n"+ }++ public let file: GGUFFile+ public let tokenizer: QwenTokenizer+ let context: MetalContext+ let dequant: GPUDequant+ /// Tokens of the system message as the chat template writes it alone (`_drop_idx`, 14).+ public let dropCount: Int+ private var products: [String: MPSMatrixMultiplication] = [:]+ /// Seconds per phase of the last forward.+ public private(set) var timings: [String: Double] = [:]++ public init(file: GGUFFile, context: MetalContext) throws {+ guard file.metadata["general.architecture"]?.string == "qwen3vl",+ file.metadata["qwen3vl.block_count"]?.int == QwenTextEncoder.layers,+ file.metadata["qwen3vl.embedding_length"]?.int == QwenTextEncoder.dim else {+ throw GGUFFile.Failure.unsupported("\(file.path): not the Qwen3-VL-8B text encoder")+ }+ self.file = file+ self.context = context+ dequant = GPUDequant(context: context)+ tokenizer = try QwenTokenizer(file: file)+ dropCount = tokenizer.encode("<|im_start|>system\n\(QwenTextEncoder.systemPrompt)<|im_end|>\n").count+ }++ /// Token ids of the text-to-image prompt.+ public func tokens(prompt: String) -> [Int] { tokenizer.encode(QwenTextEncoder.textToImage(prompt)) }++ /// The DiT's encoder rows for `prompt`: `[T, 4096]`, T = tokens − the system message's.+ public func conditioning(prompt: String) throws -> (rows: [Float], count: Int) {+ let ids = tokens(prompt: prompt)+ let all = try forward(ids: ids)+ let d = QwenTextEncoder.dim+ return (Array(all[(dropCount * d)...]), ids.count - dropCount)+ }++ /// Drops the GPU windows on the file (the caller drops the file to unmap it).+ public func release() { context.release(file) }++ // MARK: - forward++ /// Called with a stage's name and its rows (CPU pointer, floats); the pass then waits on the GPU at each stage.+ /// Stages: embeddings, layer0_out, layer1_out, layer17_out, layer35_out.+ public typealias Probe = (String, UnsafePointer<Float>, Int) -> Void++ struct Buffers {+ let arena: Arena+ let x, normed, q, k, v, kx, vx, attended, o, h1, h2, rope, norms: Arena.Slice+ let reserves: [Arena.Slice]+ }+ private var nextReserve = 0++ /// The last decoder layer's output for every token, `[ids.count, 4096]`.+ public func forward(ids: [Int], probe: Probe? = nil) throws -> [Float] {+ let d = QwenTextEncoder.dim, t = ids.count, kvWidth = QwenTextEncoder.kvHeads * QwenTextEncoder.headDim+ let started = DispatchTime.now().uptimeNanoseconds+ timings = [:]+ // Rows padded as the DiT's (64, see QwenDiT.padded), plus 64 for the attention kernel's overrun.+ let n = QwenDiT.padded(t) + 64+ let sizes = [n * d, n * d, n * d, n * kvWidth, n * kvWidth, n * d, n * d, n * d, n * d,+ n * QwenTextEncoder.hidden, n * QwenTextEncoder.hidden, n * 128,+ QwenTextEncoder.layers * 4 * d, d * QwenTextEncoder.hidden, d * QwenTextEncoder.hidden]+ let arena = try Arena(device: context.device, bytes: Arena.bytes(sizes.map { $0 * 4 }))+ var s = try sizes.map { try arena.slice(floats: $0) }.makeIterator()+ let b = Buffers(arena: arena, x: s.next()!, normed: s.next()!, q: s.next()!, k: s.next()!, v: s.next()!,+ kx: s.next()!, vx: s.next()!, attended: s.next()!, o: s.next()!, h1: s.next()!, h2: s.next()!,+ rope: s.next()!, norms: s.next()!, reserves: [s.next()!, s.next()!])++ // Norm weights of every layer (F32, small), laid out per layer: input, post-attention, q (128), k (128).+ for layer in 0..<QwenTextEncoder.layers {+ let base = layer * 4 * d+ for (i, name) in ["attn_norm", "ffn_norm"].enumerated() {+ let w = try Dequant.array(file, "blk.\(layer).\(name).weight")+ arena.pointer(b.norms, floatOffset: base + i * d).update(from: w, count: d)+ }+ for (i, name) in ["attn_q_norm", "attn_k_norm"].enumerated() {+ let w = try Dequant.array(file, "blk.\(layer).\(name).weight")+ arena.pointer(b.norms, floatOffset: base + 2 * d + i * QwenTextEncoder.headDim)+ .update(from: w, count: QwenTextEncoder.headDim)+ }+ }+ let table = QwenTextEncoder.ropeTable(rows: t)+ arena.pointer(b.rope).update(from: table, count: table.count)+ // Embedding rows, unpacked on the CPU.+ let embedding = try file.tensor("token_embd.weight")+ guard let type = embedding.type, embedding.columns == d else {+ throw GGUFFile.Failure.unsupported("token_embd.weight: \(embedding.dims)")+ }+ let rowBytes = type.bytes(count: d)+ for (row, id) in ids.enumerated() {+ guard id >= 0, id < embedding.rows else { throw GGUFFile.Failure.unsupported("token id \(id)") }+ Dequant.row(file.data(embedding) + id * rowBytes, type: type, count: d, into: arena.pointer(b.x, floatOffset: row * d))+ }+ probe?("embeddings", arena.pointer(b.x), t * d)++ var commands = try begin()+ for layer in 0..<QwenTextEncoder.layers {+ try encodeLayer(layer, rows: t, b, commands: commands)+ if let probe, [0, 1, 17, QwenTextEncoder.layers - 1].contains(layer) {+ try MetalContext.finish(commands, "text encoder")+ probe("layer\(layer)_out", arena.pointer(b.x), t * d)+ } else {+ commands.commit()+ }+ commands = try begin()+ }+ try MetalContext.finish(commands, "text encoder")+ timings["total"] = Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9+ return Array(UnsafeBufferPointer(start: arena.pointer(b.x), count: t * d))+ }++ /// `[rows, 64, (cos, sin)]` of `position · ω_i`, `ω_i = 1 / θ^(2i / 128)`, fp32 as Qwen3VLTextRotaryEmbedding.+ static func ropeTable(rows: Int) -> [Float] {+ let half = QwenTextEncoder.headDim / 2+ let omegas = (0..<half).map { i in 1 / powf(theta, Float(2 * i) / Float(QwenTextEncoder.headDim)) }+ var out = [Float](repeating: 0, count: rows * half * 2)+ for row in 0..<rows {+ for (i, w) in omegas.enumerated() {+ let angle = Float(row) * w+ out[(row * half + i) * 2] = cosf(angle)+ out[(row * half + i) * 2 + 1] = sinf(angle)+ }+ }+ return out+ }++ func encodeLayer(_ layer: Int, rows t: Int, _ b: Buffers, commands: MTLCommandBuffer) throws {+ let d = QwenTextEncoder.dim, block = "blk.\(layer)."+ let norms = layer * 4 * d+ try rmsNorm(b, b.x, into: b.normed, rows: t, weight: norms, commands: commands)+ try linear(block + "attn_q.weight", b, a: b.normed, c: b.q, m: t, commands: commands)+ try linear(block + "attn_k.weight", b, a: b.normed, c: b.k, m: t, commands: commands)+ try linear(block + "attn_v.weight", b, a: b.normed, c: b.v, m: t, commands: commands)+ try headNormRope(b, b.q, heads: QwenTextEncoder.heads, weight: norms + 2 * d, rows: t, commands: commands)+ try headNormRope(b, b.k, heads: QwenTextEncoder.kvHeads, weight: norms + 2 * d + QwenTextEncoder.headDim,+ rows: t, commands: commands)+ try repeatKV(b, b.k, into: b.kx, rows: t, commands: commands)+ try repeatKV(b, b.v, into: b.vx, rows: t, commands: commands)+ try attend(b, rows: t, commands: commands)+ try linear(block + "attn_output.weight", b, a: b.attended, c: b.o, m: t, commands: commands)+ try add(b, b.o, count: t * d, commands: commands)+ try rmsNorm(b, b.x, into: b.normed, rows: t, weight: norms + d, commands: commands)+ try linear(block + "ffn_gate.weight", b, a: b.normed, c: b.h1, m: t, commands: commands)+ try linear(block + "ffn_up.weight", b, a: b.normed, c: b.h2, m: t, commands: commands)+ try compute("swiglu", commands: commands) { e in+ e.setBuffer(b.arena.buffer, offset: b.h1.offset, index: 0)+ e.setBuffer(b.arena.buffer, offset: b.h2.offset, index: 1)+ var c = UInt32(t * QwenTextEncoder.hidden)+ e.setBytes(&c, length: 4, index: 2)+ e.dispatchThreads(MTLSize(width: Int(c), height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))+ }+ try linear(block + "ffn_down.weight", b, a: b.h1, c: b.o, m: t, commands: commands)+ try add(b, b.o, count: t * d, commands: commands)+ }++ // MARK: - encoders++ func begin() throws -> MTLCommandBuffer { MPSCommandBuffer(from: context.queue) }++ func compute(_ name: String, commands: MTLCommandBuffer, _ body: (MTLComputeCommandEncoder) throws -> Void) throws {+ let encoder = commands.makeComputeCommandEncoder()!+ encoder.label = name+ encoder.setComputePipelineState(try context.pipeline(name))+ try body(encoder)+ encoder.endEncoding()+ }++ /// `C[m, n] = A[m, k] · W`, W the file's Linear `[n, k]` unpacked to `[k, n]` fp32 in a reserve first; m padded+ /// to 64 (QwenDiT.padded: small MPS products slow every later one).+ func linear(_ name: String, _ b: Buffers, a: Arena.Slice, c: Arena.Slice, m: Int, commands: MTLCommandBuffer) throws {+ let tensor = try file.tensor(name)+ let (kk, nn) = (tensor.columns, tensor.rows)+ let m = QwenDiT.padded(m)+ guard m * kk * 4 <= a.bytes, m * nn * 4 <= c.bytes else {+ throw MetalContext.Failure.buffer("\(name): no room for \(m) padded rows")+ }+ let reserve = b.reserves[nextReserve]+ nextReserve = (nextReserve + 1) % b.reserves.count+ try dequant.encode(tensor, of: file, into: b.arena.buffer, outOffset: reserve.offset, transposed: true, commands: commands)+ let key = "\(m)×\(kk)×\(nn)"+ let product = products[key] ?? MPSMatrixMultiplication(device: context.device, transposeLeft: false, transposeRight: false,+ resultRows: m, resultColumns: nn, interiorColumns: kk, alpha: 1, beta: 0)+ products[key] = product+ func matrix(_ offset: Int, _ rows: Int, _ columns: Int) -> MPSMatrix {+ MPSMatrix(buffer: b.arena.buffer, offset: offset,+ descriptor: MPSMatrixDescriptor(rows: rows, columns: columns, rowBytes: columns * 4, dataType: .float32))+ }+ product.encode(commandBuffer: commands, leftMatrix: matrix(a.offset, m, kk), rightMatrix: matrix(reserve.offset, kk, nn),+ resultMatrix: matrix(c.offset, m, nn))+ }++ func rmsNorm(_ b: Buffers, _ input: Arena.Slice, into out: Arena.Slice, rows: Int, weight: Int, commands: MTLCommandBuffer) throws {+ try compute("rmsnorm_rows", commands: commands) { e in+ e.setBuffer(b.arena.buffer, offset: input.offset, index: 0)+ e.setBuffer(b.arena.buffer, offset: out.offset, index: 1)+ e.setBuffer(b.arena.buffer, offset: b.norms.offset + weight * 4, index: 2)+ var params = QwenDiT.RowParams(columns: UInt32(QwenTextEncoder.dim), targetStart: 0, eps: QwenTextEncoder.eps)+ e.setBytes(¶ms, length: MemoryLayout<QwenDiT.RowParams>.stride, index: 3)+ e.dispatchThreadgroups(MTLSize(width: rows, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))+ }+ }++ func headNormRope(_ b: Buffers, _ tensor: Arena.Slice, heads: Int, weight: Int, rows: Int, commands: MTLCommandBuffer) throws {+ try compute("head_rmsnorm_rope_neox", commands: commands) { e in+ e.setBuffer(b.arena.buffer, offset: tensor.offset, index: 0)+ e.setBuffer(b.arena.buffer, offset: b.norms.offset + weight * 4, index: 1)+ e.setBuffer(b.arena.buffer, offset: b.rope.offset, index: 2)+ var params = QwenDiT.HeadParams(heads: UInt32(heads), rows: UInt32(rows), eps: QwenTextEncoder.eps)+ e.setBytes(¶ms, length: MemoryLayout<QwenDiT.HeadParams>.stride, index: 3)+ let units = rows * heads // one SIMD group each, 8 per threadgroup+ e.dispatchThreadgroups(MTLSize(width: (units + 7) / 8, height: 1, depth: 1),+ threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))+ }+ }++ struct RepeatParams { var count, heads, groups: UInt32 }++ func repeatKV(_ b: Buffers, _ input: Arena.Slice, into out: Arena.Slice, rows: Int, commands: MTLCommandBuffer) throws {+ try compute("repeat_kv", commands: commands) { e in+ e.setBuffer(b.arena.buffer, offset: input.offset, index: 0)+ e.setBuffer(b.arena.buffer, offset: out.offset, index: 1)+ var params = RepeatParams(count: UInt32(rows * QwenTextEncoder.dim), heads: UInt32(QwenTextEncoder.heads),+ groups: UInt32(QwenTextEncoder.heads / QwenTextEncoder.kvHeads))+ e.setBytes(¶ms, length: MemoryLayout<RepeatParams>.stride, index: 2)+ e.dispatchThreads(MTLSize(width: Int(params.count), height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))+ }+ }++ func add(_ b: Buffers, _ y: Arena.Slice, count: Int, commands: MTLCommandBuffer) throws {+ try compute("add_inplace", commands: commands) { e in+ e.setBuffer(b.arena.buffer, offset: b.x.offset, index: 0)+ e.setBuffer(b.arena.buffer, offset: y.offset, index: 1)+ var c = UInt32(count)+ e.setBytes(&c, length: 4, index: 2)+ e.dispatchThreads(MTLSize(width: count, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))+ }+ }++ /// Causal attention of every row on the DiT's flash kernel (`Kernels/attention.metal`), keys and values widened+ /// to the 32 query heads first.+ func attend(_ b: Buffers, rows: Int, commands: MTLCommandBuffer) throws {+ try compute("flash_attention", commands: commands) { e in+ e.setBuffer(b.arena.buffer, offset: b.q.offset, index: 0)+ e.setBuffer(b.arena.buffer, offset: b.kx.offset, index: 1)+ e.setBuffer(b.arena.buffer, offset: b.vx.offset, index: 2)+ e.setBuffer(b.arena.buffer, offset: b.attended.offset, index: 3)+ var params = QwenDiT.FlashParams(queries: UInt32(rows), keys: UInt32(rows), causal: 1, stride: UInt32(QwenTextEncoder.dim),+ scale: 1 / Float(QwenTextEncoder.headDim).squareRoot() * Float(M_LOG2E), headBase: 0)+ e.setBytes(¶ms, length: MemoryLayout<QwenDiT.FlashParams>.stride, index: 4)+ e.dispatchThreadgroups(MTLSize(width: (rows + 63) / 64, height: QwenTextEncoder.heads, depth: 1),+ threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))+ }+ }+}diff --git a/engine-mac/Sources/KokonEngine/QwenTokenizer.swift b/engine-mac/Sources/KokonEngine/QwenTokenizer.swiftnew file mode 100644index 0000000..85dfe93--- /dev/null+++ b/engine-mac/Sources/KokonEngine/QwenTokenizer.swift@@ -0,0 +1,135 @@+import Foundation++/// **Qwen's byte-level BPE tokenizer**, read from the GGUF metadata (`tokenizer.ggml.*`, written by llama.cpp from+/// the model's `tokenizer.json`), doing what the Hugging Face `tokenizers` pipeline of Qwen3-VL does:+///+/// added tokens (<|im_start|>, <|image_pad|>, …) split out first, matched longest-first, never normalized+/// each piece between them: NFC ─ the Qwen2 split regex ─ bytes as GPT-2's printable characters ─ BPE merges+///+/// Checked token for token against the processor of the official Qwen-Image 2.1 repository+/// (`tools/reference_tokenizer.py`, `check-tokenizer`).+public final class QwenTokenizer: @unchecked Sendable {+ /// `tokenizer.json`'s pre-tokenizer pattern (Split, behavior Isolated).+ static let pattern = #"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"#++ public let tokens: [String]+ let ids: [String: Int]+ /// "left right" → rank (the merge's index).+ let ranks: [String: Int]+ /// Added tokens (GGUF token types 3 control and 4 user-defined) by their UTF-8 bytes, longest first.+ let added: [(bytes: [UInt8], id: Int)]+ let regex: NSRegularExpression+ /// GPT-2's byte → printable character table.+ static let byteCharacters: [Character] = {+ let printable = Set(Array(33...126) + Array(161...172) + Array(174...255))+ var table = [Character](repeating: " ", count: 256)+ var next = 256+ for b in 0..<256 {+ if printable.contains(b) {+ table[b] = Character(Unicode.Scalar(UInt32(b))!)+ } else {+ table[b] = Character(Unicode.Scalar(UInt32(next))!)+ next += 1+ }+ }+ return table+ }()+ private var cache: [String: [Int]] = [:]+ private let lock = NSLock()++ public init(file: GGUFFile) throws {+ guard file.metadata["tokenizer.ggml.model"]?.string == "gpt2",+ file.metadata["tokenizer.ggml.pre"]?.string == "qwen2",+ case .array(let tokenValues)? = file.metadata["tokenizer.ggml.tokens"],+ case .array(let mergeValues)? = file.metadata["tokenizer.ggml.merges"],+ case .array(let typeValues)? = file.metadata["tokenizer.ggml.token_type"] else {+ throw GGUFFile.Failure.unsupported("\(file.path): no Qwen2 BPE tokenizer in the metadata")+ }+ tokens = tokenValues.map { $0.string ?? "" }+ var ids: [String: Int] = [:]+ ids.reserveCapacity(tokens.count)+ for (i, t) in tokens.enumerated() { ids[t] = i }+ self.ids = ids+ var ranks: [String: Int] = [:]+ ranks.reserveCapacity(mergeValues.count)+ for (i, m) in mergeValues.enumerated() { if let s = m.string { ranks[s] = i } }+ self.ranks = ranks+ let names = tokens+ added = typeValues.enumerated().compactMap { i, type in+ type.int == 3 || type.int == 4 ? (Array(names[i].utf8), i) : nil+ }.sorted { $0.bytes.count > $1.bytes.count }+ regex = try NSRegularExpression(pattern: QwenTokenizer.pattern)+ }++ /// The token id of an added token such as `<|im_start|>`.+ public func id(of token: String) -> Int? { ids[token] }++ public func encode(_ text: String) -> [Int] {+ var out: [Int] = []+ let bytes = Array(text.utf8)+ var start = 0, i = 0+ while i < bytes.count {+ if let match = added.first(where: { a in+ a.bytes.count <= bytes.count - i && a.bytes[0] == bytes[i] && bytes[i..<(i + a.bytes.count)].elementsEqual(a.bytes)+ }) {+ if start < i { encodeOrdinary(String(decoding: bytes[start..<i], as: UTF8.self), into: &out) }+ out.append(match.id)+ i += match.bytes.count+ start = i+ } else {+ i += 1+ }+ }+ if start < bytes.count { encodeOrdinary(String(decoding: bytes[start...], as: UTF8.self), into: &out) }+ return out+ }++ /// Text without added tokens: NFC, the split regex (unmatched gaps kept as pieces too), then BPE per piece.+ private func encodeOrdinary(_ text: String, into out: inout [Int]) {+ let normalized = text.precomposedStringWithCanonicalMapping as NSString+ var cursor = 0+ func piece(_ range: NSRange) {+ guard range.length > 0 else { return }+ out += bpe(normalized.substring(with: range))+ }+ for match in regex.matches(in: normalized as String, range: NSRange(location: 0, length: normalized.length)) {+ piece(NSRange(location: cursor, length: match.range.location - cursor))+ piece(match.range)+ cursor = match.range.location + match.range.length+ }+ piece(NSRange(location: cursor, length: normalized.length - cursor))+ }++ /// One piece: its bytes as characters, merged by rank (lowest first, leftmost first) until no merge applies.+ func bpe(_ piece: String) -> [Int] {+ lock.lock()+ if let hit = cache[piece] { lock.unlock(); return hit }+ lock.unlock()+ var parts = piece.utf8.map { String(QwenTokenizer.byteCharacters[Int($0)]) }+ while parts.count > 1 {+ var best = Int.max, at = -1+ for j in 0..<(parts.count - 1) {+ if let r = ranks[parts[j] + " " + parts[j + 1]], r < best { best = r; at = j }+ }+ guard at >= 0 else { break }+ parts[at] += parts[at + 1]+ parts.remove(at: at + 1)+ }+ // Every byte is a token of its own, so a part is always in the vocabulary.+ let result = parts.map { ids[$0] ?? 0 }+ lock.lock(); cache[piece] = result; lock.unlock()+ return result+ }++ /// Token ids back to text (byte-level characters to bytes); for messages and tests.+ public func decode(_ ids: [Int]) -> String {+ var reverse: [Character: UInt8] = [:]+ for (b, c) in QwenTokenizer.byteCharacters.enumerated() { reverse[c] = UInt8(b) }+ var bytes: [UInt8] = []+ for id in ids where id >= 0 && id < tokens.count {+ if added.contains(where: { $0.id == id }) { bytes += Array(tokens[id].utf8); continue }+ for c in tokens[id] { if let b = reverse[c] { bytes.append(b) } }+ }+ return String(decoding: bytes, as: UTF8.self)+ }+}diff --git a/engine-mac/Sources/KokonEngine/TextChecks.swift b/engine-mac/Sources/KokonEngine/TextChecks.swiftnew file mode 100644index 0000000..0c292d1--- /dev/null+++ b/engine-mac/Sources/KokonEngine/TextChecks.swift@@ -0,0 +1,58 @@+import Foundation++extension Checks {+ /// The tokenizer against `tools/reference_tokenizer.py` (the official processor): every case token for token.+ public static func tokenizer(path: String, fixture: String) throws {+ let file = try GGUFFile(path: path)+ var started = DispatchTime.now().uptimeNanoseconds+ let tokenizer = try QwenTokenizer(file: file)+ print(String(format: "tokenizer: %d tokens, read in %.2f s", tokenizer.tokens.count,+ Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9))+ let cases = try JSONSerialization.jsonObject(with: Data(contentsOf: URL(fileURLWithPath: fixture))) as! [[String: Any]]+ var failed = 0+ started = DispatchTime.now().uptimeNanoseconds+ for c in cases {+ let text = c["text"] as! String, want = c["ids"] as! [Int]+ let got = tokenizer.encode(text)+ if got != want {+ failed += 1+ let at = (0..<min(got.count, want.count)).first { got[$0] != want[$0] } ?? min(got.count, want.count)+ print("MISMATCH \(text.debugDescription.prefix(80))\n at \(at): got \(Array(got[at...].prefix(8))), want \(Array(want[at...].prefix(8)))")+ }+ }+ print(String(format: "%d / %d cases identical (%.1f ms)", cases.count - failed, cases.count,+ Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-6))+ if failed > 0 { throw GGUFFile.Failure.unsupported("tokenizer differs from the reference") }+ }++ /// The text encoder against `tools/reference_text_encoder.py`: the token ids, each saved stage, the conditioning.+ public static func textEncoder(path: String, reference directory: String) throws {+ let meta = try JSONSerialization.jsonObject(with: Data(contentsOf: URL(fileURLWithPath: directory)+ .appendingPathComponent("meta.json"))) as! [String: Any]+ let prompt = meta["prompt"] as! String, wantIds = meta["ids"] as! [Int], wantDrop = meta["drop"] as! Int+ let file = try GGUFFile(path: path)+ let context = try MetalContext()+ let encoder = try QwenTextEncoder(file: file, context: context)+ let ids = encoder.tokens(prompt: prompt)+ print("tokens: \(ids.count), \(ids == wantIds ? "identical" : "DIFFERENT") ids; drop \(encoder.dropCount)"+ + (encoder.dropCount == wantDrop ? "" : " (reference \(wantDrop))"))+ guard ids == wantIds, encoder.dropCount == wantDrop else { throw GGUFFile.Failure.unsupported("token ids differ") }++ func report(_ name: String, _ got: UnsafePointer<Float>, _ want: [Float]) {+ let r = compare(got, want)+ print(String(format: " %-12@ rel %.2e max |Δ| %.3g%@", name as NSString, r.relative, r.maxAbs, r.finite ? "" : " NON-FINITE"))+ }+ _ = try encoder.forward(ids: ids) { name, pointer, count in+ guard let want = try? reference(directory, name) else { return }+ precondition(want.count == count, "\(name): \(count) floats, reference \(want.count)")+ report(name, pointer, want)+ }+ for run in 1...2 {+ let started = DispatchTime.now().uptimeNanoseconds+ let (rows, count) = try encoder.conditioning(prompt: prompt)+ let seconds = Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9+ if run == 1 { rows.withUnsafeBufferPointer { report("conditioning", $0.baseAddress!, try! reference(directory, "conditioning")) } }+ print(String(format: "encode %d (%d rows): %.3f s", run, count, seconds))+ }+ }+}diff --git a/engine-mac/Sources/kokon-engine-mac/main.swift b/engine-mac/Sources/kokon-engine-mac/main.swiftindex 026e739..7167e32 100644--- a/engine-mac/Sources/kokon-engine-mac/main.swift+++ b/engine-mac/Sources/kokon-engine-mac/main.swift@@ -9,6 +9,8 @@ kokon-engine-mac: Kokon's Apple Silicon engine (in development)check-dit <dit.gguf> <reference dir> one DiT evaluation against tools/reference_dit.pytime-dit <dit.gguf> [grid] [text] [runs] DiT evaluations on random inputs (grid 64 = 1024²)check-vae <vae.safetensors> <reference dir> [grid] the VAE decoder against tools/reference_vae.py+ check-tokenizer <encoder.gguf> <fixture.json> the tokenizer against tools/reference_tokenizer.py+ check-text <encoder.gguf> <reference dir> the text encoder against tools/reference_text_encoder.pyprobe this Mac's matrix throughput: GPU, AMX, Neural Engine, together"""@@ -52,6 +54,12 @@ do {case "check-vae":guard arguments.count >= 3 else { fail(usage) }try Checks.vae(path: arguments[1], reference: arguments[2], timeGrid: arguments.count > 3 ? Int(arguments[3]) : nil)+ case "check-tokenizer":+ guard arguments.count == 3 else { fail(usage) }+ try Checks.tokenizer(path: arguments[1], fixture: arguments[2])+ case "check-text":+ guard arguments.count == 3 else { fail(usage) }+ try Checks.textEncoder(path: arguments[1], reference: arguments[2])case "probe":try Probe.run()default:diff --git a/engine-mac/Tests/KokonEngineTests/QwenTokenizerTests.swift b/engine-mac/Tests/KokonEngineTests/QwenTokenizerTests.swiftnew file mode 100644index 0000000..6317fb5--- /dev/null+++ b/engine-mac/Tests/KokonEngineTests/QwenTokenizerTests.swift@@ -0,0 +1,71 @@+import Foundation+import XCTest+@testable import KokonEngine++/// The tokenizer on a tiny vocabulary: the 256 byte characters, a few merges and one added token. The real+/// vocabulary is checked against the official processor by `check-tokenizer`.+final class QwenTokenizerTests: XCTestCase {+ private var temporary: URL!+ private var tokenizer: QwenTokenizer!+ private var file: GGUFFile!++ override func setUpWithError() throws {+ temporary = FileManager.default.temporaryDirectory.appendingPathComponent("kokon-tokenizer-tests-\(UUID())")+ try FileManager.default.createDirectory(at: temporary, withIntermediateDirectories: true)+ let bytes = QwenTokenizer.byteCharacters.map { String($0) }+ let merges = ["h e", "t he", "Ġ c", "Ġc a", "Ġca t", "' s"]+ let tokens = bytes + ["he", "the", "Ġc", "Ġca", "Ġcat", "'s", "<|im_start|>"]+ let types = [Int32](repeating: 1, count: tokens.count - 1) + [3]++ func string(_ s: String) -> [UInt8] {+ var out: [UInt8] = []+ withUnsafeBytes(of: UInt64(s.utf8.count).littleEndian) { out += $0 }+ return out + Array(s.utf8)+ }+ func array(_ itemType: UInt32, _ items: [[UInt8]]) -> [UInt8] {+ var out: [UInt8] = []+ withUnsafeBytes(of: itemType.littleEndian) { out += $0 }+ withUnsafeBytes(of: UInt64(items.count).littleEndian) { out += $0 }+ return out + items.flatMap { $0 }+ }+ let path = try TestGGUF.write(in: temporary, metadata: [+ ("tokenizer.ggml.model", 8, string("gpt2")),+ ("tokenizer.ggml.pre", 8, string("qwen2")),+ ("tokenizer.ggml.tokens", 9, array(8, tokens.map(string))),+ ("tokenizer.ggml.merges", 9, array(8, merges.map(string))),+ ("tokenizer.ggml.token_type", 9, array(5, types.map { t in withUnsafeBytes(of: t.littleEndian) { Array($0) } })),+ ], tensors: [])+ file = try GGUFFile(path: path)+ tokenizer = try QwenTokenizer(file: file)+ }++ override func tearDownWithError() throws { try? FileManager.default.removeItem(at: temporary) }++ private func id(_ token: String) -> Int { tokenizer.tokens.firstIndex(of: token)! }+ private func byte(_ b: UInt8) -> Int { id(String(QwenTokenizer.byteCharacters[Int(b)])) }++ func testByteTableIsGPT2s() {+ XCTAssertEqual(QwenTokenizer.byteCharacters[Int(UInt8(ascii: "a"))], "a")+ XCTAssertEqual(QwenTokenizer.byteCharacters[32], "Ġ") // space+ XCTAssertEqual(QwenTokenizer.byteCharacters[10], "Ċ") // newline+ XCTAssertEqual(Set(QwenTokenizer.byteCharacters).count, 256)+ }++ func testMergesByRankAcrossSplitPieces() {+ // "the" merges h+e first (rank 0), then t+he; " cat" is its own piece; "'s" is the contraction alternative.+ XCTAssertEqual(tokenizer.encode("the cat's"), [id("the"), id("Ġcat"), id("'s")])+ // No merge for "ht": bytes stay single.+ XCTAssertEqual(tokenizer.encode("ht"), [byte(UInt8(ascii: "h")), byte(UInt8(ascii: "t"))])+ }++ func testAddedTokensAreSplitOutFirst() {+ XCTAssertEqual(tokenizer.encode("the<|im_start|>the"), [id("the"), id("<|im_start|>"), id("the")])+ XCTAssertEqual(tokenizer.encode("<|im_start|>"), [id("<|im_start|>")])+ }++ func testTextIsNFCNormalized() {+ // e + combining acute becomes é (C3 A9) before the bytes are taken.+ XCTAssertEqual(tokenizer.encode("e\u{301}"), [byte(0xC3), byte(0xA9)])+ XCTAssertEqual(tokenizer.decode(tokenizer.encode("the cat é")), "the cat é")+ }+}diff --git a/engine-mac/tools/reference_text_encoder.py b/engine-mac/tools/reference_text_encoder.pynew file mode 100644index 0000000..d02b1a1--- /dev/null+++ b/engine-mac/tools/reference_text_encoder.py@@ -0,0 +1,136 @@+"""The Qwen-Image 2.1 text encoder reference for the Mac engine: transformers' own Qwen3-VL text decoder layers, in+fp32 on the CPU, on the weights of Kokon's GGUF unpacked by gguf-py, run on the prompt exactly as diffusers'+QwenImage21Pipeline builds it (0.42, `_get_qwen_prompt_embeds`, text-to-image):++ template "<|im_start|>system\\n{sys}<|im_end|>\\n<|im_start|>user\\n{prompt}<|im_end|>\\n<|im_start|>assistant\\n"+ tokenized by the official processor; conditioning = the last decoder layer's output (before the final RMSNorm),+ without the system message's tokens++The 8B model in fp32 is 33 GB, more than a 16 GB Mac holds: one decoder layer module, its weights swapped layer+by layer (as `reference_dit.py` does for the DiT). `--sanity` also runs the language model head on a factual+prompt and prints its next token: a check that the GGUF weights are read right, which a comparison of two readers+of the same file cannot be.++ python -I engine-mac/tools/reference_text_encoder.py <encoder.gguf> <processor dir> <text_encoder dir> <out dir>+ [--prompt "..."] [--sanity]++Writes `<out>/meta.json` (prompt, ids, drop) and one raw little-endian fp32 file per tensor (`<name>.f32`).+"""+import argparse+import json+import os++import gguf+import numpy as np+import torch+from transformers import AutoConfig, AutoProcessor+from transformers.models.qwen3_vl.modeling_qwen3_vl import (+ Qwen3VLTextDecoderLayer,+ Qwen3VLTextRMSNorm,+ Qwen3VLTextRotaryEmbedding,+)++SYSTEM = "Comprehend and analyze the provided prompt."+TEMPLATE = f"<|im_start|>system\n{SYSTEM}<|im_end|>\n<|im_start|>user\n{{}}<|im_end|>\n<|im_start|>assistant\n"+NAMES = {+ "attn_q": "self_attn.q_proj", "attn_k": "self_attn.k_proj", "attn_v": "self_attn.v_proj",+ "attn_output": "self_attn.o_proj", "attn_q_norm": "self_attn.q_norm", "attn_k_norm": "self_attn.k_norm",+ "attn_norm": "input_layernorm", "ffn_norm": "post_attention_layernorm",+ "ffn_gate": "mlp.gate_proj", "ffn_up": "mlp.up_proj", "ffn_down": "mlp.down_proj",+}+++class Weights:+ def __init__(self, path):+ self.reader = gguf.GGUFReader(path)+ self.tensors = {t.name: t for t in self.reader.tensors}++ def get(self, name, rows=None):+ t = self.tensors[name]+ data = t.data if rows is None else t.data[rows]+ values = gguf.quants.dequantize(data, t.tensor_type).astype(np.float32)+ shape = [int(d) for d in reversed(t.shape)] # ggml ne order -> torch order+ if rows is not None:+ shape[0] = len(rows)+ return torch.from_numpy(values.reshape(shape).copy())++ def load_layer(self, layer, index):+ state = {}+ for gname, hname in NAMES.items():+ state[hname + ".weight"] = self.get(f"blk.{index}.{gname}.weight")+ layer.load_state_dict(state, strict=True)+++def run(weights, config, ids, save=None):+ """The text model's forward on one unpadded prompt; returns the last layer's output (no final norm)."""+ x = weights.get("token_embd.weight", rows=ids)[None]+ if save:+ save("embeddings", x[0])+ t = x.shape[1]+ position_ids = torch.arange(t).view(1, 1, -1).expand(3, 1, -1) # text only: the three axes agree+ rope = Qwen3VLTextRotaryEmbedding(config)+ position_embeddings = rope(x, position_ids)+ mask = torch.full((t, t), torch.finfo(torch.float32).min).triu(1)[None, None]+ layer = Qwen3VLTextDecoderLayer(config, 0)+ for index in range(config.num_hidden_layers):+ weights.load_layer(layer, index)+ x = layer(x, position_embeddings=position_embeddings, attention_mask=mask)+ if save and index in (0, 1, 17, config.num_hidden_layers - 1):+ save(f"layer{index}_out", x[0])+ return x+++def main():+ parser = argparse.ArgumentParser()+ parser.add_argument("gguf")+ parser.add_argument("processor")+ parser.add_argument("text_encoder", help="text_encoder/ of the official repository (config.json)")+ parser.add_argument("out")+ parser.add_argument("--prompt", default="A red fox sitting in fresh snow at dawn, soft golden light, photo")+ parser.add_argument("--sanity", action="store_true")+ args = parser.parse_args()+ torch.set_grad_enabled(False)+ os.makedirs(args.out, exist_ok=True)+ config = AutoConfig.from_pretrained(args.text_encoder).text_config+ config._attn_implementation = "eager"+ processor = AutoProcessor.from_pretrained(args.processor)+ weights = Weights(args.gguf)++ # QwenImage21Pipeline.__init__ / _get_qwen_prompt_embeds, one prompt, no images.+ system = [{"role": "system", "content": [{"type": "text", "text": SYSTEM}]}]+ drop = len(processor.apply_chat_template(system, tokenize=True, return_dict=False)[0])+ prompt = args.prompt or " "+ ids = processor(text=[TEMPLATE.format(prompt)], return_tensors="pt").input_ids[0].tolist()++ saved = {}++ def save(name, tensor):+ array = tensor.detach().to(torch.float32).contiguous().numpy()+ array.tofile(os.path.join(args.out, name + ".f32"))+ saved[name] = list(array.shape)++ hidden = run(weights, config, ids, save)+ save("conditioning", hidden[0, drop:])+ meta = {"prompt": prompt, "ids": ids, "drop": drop, "tensors": saved, "gguf": os.path.basename(args.gguf),+ "transformers": __import__("transformers").__version__, "torch": torch.__version__}++ if args.sanity:+ question = ("<|im_start|>user\nWhat is the capital of France? Answer in one sentence.<|im_end|>\n"+ "<|im_start|>assistant\nThe capital of France is")+ qids = processor(text=[question], return_tensors="pt").input_ids[0].tolist()+ h = run(weights, config, qids)+ norm = Qwen3VLTextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)+ norm.weight.copy_(weights.get("output_norm.weight"))+ logits = weights.get("output.weight") @ norm(h[0, -1])+ top = torch.topk(logits, 5)+ tokens = [processor.tokenizer.decode([i]) for i in top.indices.tolist()]+ print("sanity: next tokens after 'The capital of France is':", tokens)+ meta["sanity"] = tokens++ with open(os.path.join(args.out, "meta.json"), "w") as f:+ json.dump(meta, f, indent=1)+ print(f"{len(ids)} tokens, drop {drop}, conditioning {saved['conditioning']}")+++if __name__ == "__main__":+ main()diff --git a/engine-mac/tools/reference_tokenizer.py b/engine-mac/tools/reference_tokenizer.pynew file mode 100644index 0000000..5bef82d--- /dev/null+++ b/engine-mac/tools/reference_tokenizer.py@@ -0,0 +1,57 @@+"""Token ids from the processor of the official Qwen-Image 2.1 repository (what diffusers' QwenImage21Pipeline+tokenizes with), for `kokon-engine-mac check-tokenizer` to compare against.++ python -I engine-mac/tools/reference_tokenizer.py <processor dir> <out.json>++The processor dir is `processor/` of Qwen/Qwen-Image-2.1-Turbo (tokenizer.json, tokenizer_config.json, ...).+Writes a list of {"text", "ids"}: the pipeline's full text-to-image prompts and harder strings (accents, CJK,+emoji, digits, whitespace runs, contractions, special tokens inside the text).+"""+import json+import sys++from transformers import AutoProcessor++SYSTEM = "Comprehend and analyze the provided prompt."+TEMPLATE = f"<|im_start|>system\n{SYSTEM}<|im_end|>\n<|im_start|>user\n{{}}<|im_end|>\n<|im_start|>assistant\n"++CASES = [+ "A red fox in the snow",+ " ",+ "a cozy wooden cabin by a lake at sunset, warm light, 35mm photo, f/1.8",+ "Replace the sky with a dramatic stormy sky; keep the house unchanged.",+ "Un gatto che dorme sulla sedia, perché è stanco. Città, più, caffè",+ "Ein schönes Haus — größer, weiß, Straße",+ "一只在雪地里的红狐狸,电影感光线",+ "日本語のテキスト「こんにちは」を看板に書いて",+ "Emoji 🦊🔥👍🏽 and symbols ©®™ ½ ∑ ≤ →",+ "Numbers 1234567890, 3.14159, 1,000,000 and 2026-10-25",+ "Multiple spaces\tand\ttabs\n\nnew lines\r\nand trailing ",+ " leading spaces",+ "I'm sure it's they're we've you'll he'd IT'S I'M",+ "Text on the sign: \"KOKON\" in bold, 'single quotes', `backticks`",+ "café vs café (decomposed), Å vs Å",+ "special <|im_start|> inside and <|image_pad|><think> tags",+ "url https://kokon.app/docs?x=1&y=2#top and email [email protected]",+ "naïve résumé coöperate façade jalapeño",+ "Кириллица и ελληνικά και العربية עברית",+ "x" * 300,+ "The quick brown fox jumps over the lazy dog. " * 8,+]+++def main():+ processor = AutoProcessor.from_pretrained(sys.argv[1])+ out = []+ for text in CASES:+ out.append({"text": text, "ids": processor.tokenizer.encode(text, add_special_tokens=False)})+ prompt = TEMPLATE.format(text)+ ids = processor(text=[prompt], return_tensors="pt").input_ids[0].tolist()+ out.append({"text": prompt, "ids": ids})+ with open(sys.argv[2], "w") as f:+ json.dump(out, f, ensure_ascii=False, indent=0)+ print(f"{len(out)} cases")+++if __name__ == "__main__":+ main()