Skip to content

armgit / kokon

Public Imported from GitHub
0

Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.

Mac engine: the text encoder with images, checked against transformers

Gabriele Lozupone415ee8ff2c9d
diff --git a/engine-mac/README.md b/engine-mac/README.md
index 4e58c89..10d3ef1 100644
--- a/engine-mac/README.md
+++ b/engine-mac/README.md
@@ -91,6 +91,16 @@ head answers " Paris" to a factual prompt (it does), so the weights are read rig
2026-10-25: tokenizer identical on 42 cases (accents, CJK, emoji, decomposed Unicode, special tokens in the text);
conditioning within 5.1e-6 relative (37 tokens) and 5.6e-6 (144 tokens). One encode: 0.45 s (37 tokens), 0.8 s (144).
+**With images** (an edit's prompt, `prompt_template_ti2i`): `<image1><|vision_start|><|image_pad|><|vision_end|>` before
+the prompt, each `<|image_pad|>` expanded to the image's slots (one per 32 × 32 pixels), those rows the vision tower's
+merged rows, Qwen3-VL's 3D M-RoPE positions (`get_rope_index`; interleaved sections 24/20/20), the three deepstack outputs
+added to the image rows after layers 0–2, and the DiT told which conditioning rows are image slots (`image_pad_mask`).
+`reference_text_encoder.py --image <png> --mmproj <mmproj.gguf>` runs the same in transformers; `--sanity` then also asks
+the model the color of a red disk (it answers "Red": the mmproj is read right). `check-text <encoder> <dir> <mmproj>`,
+the street cat at 384 × 256 (96 slots, 139 tokens; M2 Pro, 2026-10-25): ids, positions and image slots identical; the
+conditioning within 7.5e-6 with the reference's own vision rows, 3.3e-5 with the engine's, whose tower is 7.9e-6 from fp64
+on this photo where torch's fp32 is 3.0e-5 from it. One encode with the image: 0.82 s, its vision tower 0.56 s.
+
**Text to image against diffusers.** `tools/reference_generate.py` runs QwenImage21Pipeline's loop with the
references above (diffusers' scheduler on the checkpoint's sigmas, the DiT layer by layer, the VAE, the image
processor) and saves its noise; `check-generate` starts the engine from that noise:
diff --git a/engine-mac/Sources/KokonEngine/QwenPipeline.swift b/engine-mac/Sources/KokonEngine/QwenPipeline.swift
index b5c0ab1..b26c56a 100644
--- a/engine-mac/Sources/KokonEngine/QwenPipeline.swift
+++ b/engine-mac/Sources/KokonEngine/QwenPipeline.swift
@@ -103,12 +103,14 @@ public final class QwenPipeline {
(encoderFile, encoder) = (file, try QwenTextEncoder(file: file, context: context))
}
defer { encoder!.release() }
- return try encoder!.conditioning(prompt: prompt)
+ let c = try encoder!.conditioning(prompt: prompt)
+ return (c.rows, c.count)
}
let encoder = try handedOver ?? QwenTextEncoder(file: try GGUFFile(path: encoderPath), context: context)
handedOver = nil
defer { encoder.release() }
- return try encoder.conditioning(prompt: prompt)
+ let c = try encoder.conditioning(prompt: prompt)
+ return (c.rows, c.count)
}()
timings["encode"] = since(t)
log(String(format: "text encoder: %d tokens in %.2f s", rows, timings["encode"]!))
diff --git a/engine-mac/Sources/KokonEngine/QwenTextEncoder.swift b/engine-mac/Sources/KokonEngine/QwenTextEncoder.swift
index fae6d4a..e8b0326 100644
--- a/engine-mac/Sources/KokonEngine/QwenTextEncoder.swift
+++ b/engine-mac/Sources/KokonEngine/QwenTextEncoder.swift
@@ -12,8 +12,15 @@ import MetalPerformanceShaders
///
/// What diffusers' QwenImage21Pipeline does (0.42, `_get_qwen_prompt_embeds`): the raw template below, the hidden
/// state of the last decoder layer *before* the model's final RMSNorm (the pipeline hooks the norm away), and the
-/// leading tokens of the system message dropped. Text only: the three M-RoPE axes carry the same position, so the
-/// interleaved M-RoPE is plain RoPE (θ = 5·10⁶) and the vision tower's deepstack inputs never apply.
+/// leading tokens of the system message dropped. Text only, the three M-RoPE axes carry the same position, so the
+/// interleaved M-RoPE is plain RoPE (θ = 5·10⁶).
+///
+/// **With images** (an edit, `prompt_template_ti2i`): "<image1><|vision_start|><|image_pad|><|vision_end|>" before the
+/// prompt (" <imageN>…" for more), each `<|image_pad|>` expanded to one token per 32 × 32 pixels of its image; those rows
+/// start as the vision tower's merged rows instead of token embeddings, positions are Qwen3-VL's 3D M-RoPE
+/// (`get_rope_index`: an image's rows at (start, start + row, start + column), the text after it from start + max(rows,
+/// columns)), and the tower's three deepstack outputs are added to the image rows after layers 0, 1 and 2
+/// (`_deepstack_process`). The DiT gets which conditioning rows are image slots (`image_pad_mask`).
public final class QwenTextEncoder {
public static let dim = 4096, heads = 32, kvHeads = 8, headDim = 128, hidden = 12288, layers = 36
static let eps: Float = 1e-6, theta: Float = 5_000_000
@@ -48,15 +55,66 @@ public final class QwenTextEncoder {
dropCount = tokenizer.encode("<|im_start|>system\n\(QwenTextEncoder.systemPrompt)<|im_end|>\n").count
}
- /// Token ids of the text-to-image prompt.
- public func tokens(prompt: String) -> [Int] { tokenizer.encode(QwenTextEncoder.textToImage(prompt)) }
+ /// `prompt_template_ti2i` for `images` condition images.
+ public static func imageToImage(_ prompt: String, images: Int) -> String {
+ let markers = (1...max(images, 1)).map { "<image\($0)><|vision_start|><|image_pad|><|vision_end|>" }.joined(separator: " ")
+ return textToImage(markers + (prompt.isEmpty ? " " : prompt))
+ }
+
+ /// `<|image_pad|>`'s id (151655).
+ public var imagePad: Int { tokenizer.encode("<|image_pad|>")[0] }
+
+ /// Token ids of the prompt: text to image, or with images (each `<|image_pad|>` expanded to the image's slots, as
+ /// the processor does before tokenizing).
+ public func tokens(prompt: String, images: [QwenVisionTower.Output] = []) -> [Int] {
+ guard !images.isEmpty else { return tokenizer.encode(QwenTextEncoder.textToImage(prompt)) }
+ let pad = imagePad
+ var out: [Int] = [], next = 0
+ for id in tokenizer.encode(QwenTextEncoder.imageToImage(prompt, images: images.count)) {
+ if id == pad, next < images.count {
+ out += [Int](repeating: pad, count: images[next].slots)
+ next += 1
+ } else {
+ out.append(id)
+ }
+ }
+ return out
+ }
- /// The DiT's encoder rows for `prompt`: `[T, 4096]`, T = tokens − the system message's.
- public func conditioning(prompt: String) throws -> (rows: [Float], count: Int) {
- let ids = tokens(prompt: prompt)
- let all = try forward(ids: ids)
- let d = QwenTextEncoder.dim
- return (Array(all[(dropCount * d)...]), ids.count - dropCount)
+ /// The DiT's encoder rows for `prompt`: `[T, 4096]`, T = tokens − the system message's; `imageSlots`: which of
+ /// those rows are image slots (diffusers' `image_pad_mask`), in the order of `images`.
+ public func conditioning(prompt: String, images: [QwenVisionTower.Output] = []) throws
+ -> (rows: [Float], count: Int, imageSlots: [Bool]) {
+ let ids = tokens(prompt: prompt, images: images)
+ let all = try forward(ids: ids, images: images)
+ let d = QwenTextEncoder.dim, pad = imagePad
+ return (Array(all[(dropCount * d)...]), ids.count - dropCount, ids[dropCount...].map { $0 == pad && !images.isEmpty })
+ }
+
+ /// Qwen3-VL's `get_rope_index` for one prompt: (t, h, w) per token. Text advances all three by one; the k-th run of
+ /// `<|image_pad|>` is image k's grid of (rows, columns) slots at (start, start + row, start + column), and the text
+ /// after it starts at start + max(rows, columns).
+ public static func positions(ids: [Int], imagePad: Int, grids: [(rows: Int, columns: Int)]) throws -> [SIMD3<Int32>] {
+ var out: [SIMD3<Int32>] = [], position = 0, i = 0, image = 0
+ while i < ids.count {
+ if ids[i] != imagePad {
+ out.append(SIMD3(repeating: Int32(position)))
+ position += 1
+ i += 1
+ continue
+ }
+ guard image < grids.count else { throw QwenSequence.Failure.layout("more image slots than images") }
+ let (rows, columns) = grids[image]
+ guard i + rows * columns <= ids.count, ids[i..<(i + rows * columns)].allSatisfy({ $0 == imagePad }) else {
+ throw QwenSequence.Failure.layout("image \(image + 1): its slots do not match its \(rows) × \(columns) grid")
+ }
+ for r in 0..<rows { for c in 0..<columns { out.append(SIMD3(Int32(position), Int32(position + r), Int32(position + c))) } }
+ position += max(rows, columns)
+ i += rows * columns
+ image += 1
+ }
+ guard image == grids.count else { throw QwenSequence.Failure.layout("\(grids.count) images, \(image) runs of slots") }
+ return out
}
/// Drops the GPU windows on the file (the caller drops the file to unmap it).
@@ -70,27 +128,35 @@ public final class QwenTextEncoder {
struct Buffers {
let arena: Arena
- let x, normed, q, k, v, kx, vx, attended, o, h1, h2, rope, norms: Arena.Slice
+ let x, normed, q, k, v, kx, vx, attended, o, h1, h2, rope, norms, deepstack: Arena.Slice
let reserves: [Arena.Slice]
}
private var nextReserve = 0
/// The last decoder layer's output for every token, `[ids.count, 4096]`. `cache`: each layer's K (after its norm
- /// and RoPE) and V are kept there too, for text generation (`QwenTextGenerator`).
- public func forward(ids: [Int], probe: Probe? = nil, cache: KVCache? = nil) throws -> [Float] {
+ /// and RoPE) and V are kept there too, for text generation (`QwenTextGenerator`). `images`: the vision tower's
+ /// outputs for the runs of `<|image_pad|>` in `ids`, in order.
+ public func forward(ids: [Int], probe: Probe? = nil, cache: KVCache? = nil, images: [QwenVisionTower.Output] = []) throws -> [Float] {
let d = QwenTextEncoder.dim, t = ids.count, kvWidth = QwenTextEncoder.kvHeads * QwenTextEncoder.headDim
let started = DispatchTime.now().uptimeNanoseconds
timings = [:]
+ // Image k's slots: the k-th run of <|image_pad|> (`positions` checks each run against its grid).
+ let pad = imagePad
+ let positions = try QwenTextEncoder.positions(ids: ids, imagePad: images.isEmpty ? -1 : pad,
+ grids: images.map { ($0.height / 32, $0.width / 32) })
+ var starts: [Int] = []
+ for i in ids.indices where !images.isEmpty && ids[i] == pad && (i == 0 || ids[i - 1] != pad) { starts.append(i) }
+ let slots = images.reduce(0) { $0 + $1.slots }
// Rows padded as the DiT's (64, see QwenDiT.padded), plus 64 for the attention kernel's overrun.
let n = QwenDiT.padded(t) + 64
let sizes = [n * d, n * d, n * d, n * kvWidth, n * kvWidth, n * d, n * d, n * d, n * d,
n * QwenTextEncoder.hidden, n * QwenTextEncoder.hidden, n * 128,
- QwenTextEncoder.layers * 4 * d, d * QwenTextEncoder.hidden, d * QwenTextEncoder.hidden]
+ QwenTextEncoder.layers * 4 * d, max(1, 3 * slots * d), d * QwenTextEncoder.hidden, d * QwenTextEncoder.hidden]
let arena = try Arena(device: context.device, bytes: Arena.bytes(sizes.map { $0 * 4 }))
var s = try sizes.map { try arena.slice(floats: $0) }.makeIterator()
let b = Buffers(arena: arena, x: s.next()!, normed: s.next()!, q: s.next()!, k: s.next()!, v: s.next()!,
kx: s.next()!, vx: s.next()!, attended: s.next()!, o: s.next()!, h1: s.next()!, h2: s.next()!,
- rope: s.next()!, norms: s.next()!, reserves: [s.next()!, s.next()!])
+ rope: s.next()!, norms: s.next()!, deepstack: s.next()!, reserves: [s.next()!, s.next()!])
// Norm weights of every layer (F32, small), laid out per layer: input, post-attention, q (128), k (128).
for layer in 0..<QwenTextEncoder.layers {
@@ -105,7 +171,7 @@ public final class QwenTextEncoder {
.update(from: w, count: QwenTextEncoder.headDim)
}
}
- let table = QwenTextEncoder.ropeTable(rows: t)
+ let table = QwenTextEncoder.ropeTable(positions: positions)
arena.pointer(b.rope).update(from: table, count: table.count)
// Embedding rows, unpacked on the CPU.
let embedding = try file.tensor("token_embd.weight")
@@ -117,11 +183,27 @@ public final class QwenTextEncoder {
guard id >= 0, id < embedding.rows else { throw GGUFFile.Failure.unsupported("token id \(id)") }
Dequant.row(file.data(embedding) + id * rowBytes, type: type, count: d, into: arena.pointer(b.x, floatOffset: row * d))
}
+ // Image slots: the tower's merged rows replace the embeddings; its deepstack rows wait in their slice.
+ var at = 0
+ for (image, start) in zip(images, starts) {
+ arena.pointer(b.x, floatOffset: start * d).update(from: image.merged, count: image.slots * d)
+ for (j, rows) in image.deepstack.enumerated() {
+ arena.pointer(b.deepstack, floatOffset: (j * slots + at) * d).update(from: rows, count: image.slots * d)
+ }
+ at += image.slots
+ }
probe?("embeddings", arena.pointer(b.x), t * d)
var commands = try begin()
for layer in 0..<QwenTextEncoder.layers {
try encodeLayer(layer, rows: t, b, commands: commands, cache: cache)
+ if layer < 3, !images.isEmpty {
+ var at = 0
+ for (image, start) in zip(images, starts) {
+ try addRows(b, at: start, from: (layer * slots + at) * d, count: image.slots * d, commands: commands)
+ at += image.slots
+ }
+ }
if let probe, [0, 1, 17, QwenTextEncoder.layers - 1].contains(layer) {
try MetalContext.finish(commands, "text encoder")
probe("layer\(layer)_out", arena.pointer(b.x), t * d)
@@ -137,12 +219,19 @@ public final class QwenTextEncoder {
/// `[rows, 64, (cos, sin)]` of `position · ω_i`, `ω_i = 1 / θ^(2i / 128)`, fp32 as Qwen3VLTextRotaryEmbedding.
static func ropeTable(rows: Int) -> [Float] {
+ ropeTable(positions: (0..<rows).map { SIMD3(repeating: Int32($0)) })
+ }
+
+ /// The same for 3D positions, interleaved as `mrope_section` [24, 20, 20] with `mrope_interleaved`: frequency i
+ /// turns with the height when i % 3 == 1, with the width when i % 3 == 2 (both below 60), else with t.
+ static func ropeTable(positions: [SIMD3<Int32>]) -> [Float] {
let half = QwenTextEncoder.headDim / 2
let omegas = (0..<half).map { i in 1 / powf(theta, Float(2 * i) / Float(QwenTextEncoder.headDim)) }
- var out = [Float](repeating: 0, count: rows * half * 2)
- for row in 0..<rows {
+ let axes = (0..<half).map { i in i < 60 && i % 3 == 1 ? 1 : i < 60 && i % 3 == 2 ? 2 : 0 }
+ var out = [Float](repeating: 0, count: positions.count * half * 2)
+ for (row, p) in positions.enumerated() {
for (i, w) in omegas.enumerated() {
- let angle = Float(row) * w
+ let angle = Float(p[axes[i]]) * w
out[(row * half + i) * 2] = cosf(angle)
out[(row * half + i) * 2 + 1] = sinf(angle)
}
@@ -256,6 +345,17 @@ public final class QwenTextEncoder {
}
}
+ /// x[row `at`…] += deepstack floats from `offset` (`_deepstack_process`).
+ func addRows(_ b: Buffers, at row: Int, from offset: Int, count: Int, commands: MTLCommandBuffer) throws {
+ try compute("add_inplace", commands: commands) { e in
+ e.setBuffer(b.arena.buffer, offset: b.x.offset + row * QwenTextEncoder.dim * 4, index: 0)
+ e.setBuffer(b.arena.buffer, offset: b.deepstack.offset + offset * 4, index: 1)
+ var c = UInt32(count)
+ e.setBytes(&c, length: 4, index: 2)
+ e.dispatchThreads(MTLSize(width: count, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ }
+ }
+
func add(_ b: Buffers, _ y: Arena.Slice, count: Int, commands: MTLCommandBuffer) throws {
try compute("add_inplace", commands: commands) { e in
e.setBuffer(b.arena.buffer, offset: b.x.offset, index: 0)
diff --git a/engine-mac/Sources/KokonEngine/TextChecks.swift b/engine-mac/Sources/KokonEngine/TextChecks.swift
index 333b844..5140f7e 100644
--- a/engine-mac/Sources/KokonEngine/TextChecks.swift
+++ b/engine-mac/Sources/KokonEngine/TextChecks.swift
@@ -26,31 +26,74 @@ extension Checks {
}
/// The text encoder against `tools/reference_text_encoder.py`: the token ids, each saved stage, the conditioning.
- public static func textEncoder(path: String, reference directory: String) throws {
+ /// A reference with images (`--image`) needs the vision tower (`mmproj`): then also the positions, the image slots
+ /// and the tower's outputs on the processor's input.
+ public static func textEncoder(path: String, reference directory: String, mmproj: String? = nil) throws {
let meta = try JSONSerialization.jsonObject(with: Data(contentsOf: URL(fileURLWithPath: directory)
.appendingPathComponent("meta.json"))) as! [String: Any]
let prompt = meta["prompt"] as! String, wantIds = meta["ids"] as! [Int], wantDrop = meta["drop"] as! Int
- let file = try GGUFFile(path: path)
let context = try MetalContext()
+ var images: [QwenVisionTower.Output] = []
+ if let sizes = meta["images"] as? [[Int]], !sizes.isEmpty {
+ guard let mmproj else { throw GGUFFile.Failure.unsupported("this reference has images: pass the mmproj GGUF") }
+ let tower = try QwenVisionTower(file: try GGUFFile(path: mmproj), context: context)
+ for i in sizes.indices {
+ let png = try QwenPipeline.decodeImage(Data(contentsOf: URL(fileURLWithPath: directory).appendingPathComponent("image\(i + 1).png")))
+ let input = QwenVisionTower.pixels(rgba: png.rgba, width: png.width, height: png.height)
+ images.append(try tower.encode(rgb: input.rgb, width: input.width, height: input.height))
+ print(String(format: "image %d: %d×%d, %d slots, vision tower %.2f s", i + 1, input.width, input.height,
+ images[i].slots, tower.seconds))
+ }
+ let merged = images.flatMap(\.merged)
+ merged.withUnsafeBufferPointer {
+ let r = compare($0.baseAddress!, try! reference(directory, "vision_merged"))
+ print(String(format: " vision merged rel %.2e", r.relative))
+ }
+ tower.release()
+ }
+ let file = try GGUFFile(path: path)
let encoder = try QwenTextEncoder(file: file, context: context)
- let ids = encoder.tokens(prompt: prompt)
+ let ids = encoder.tokens(prompt: prompt, images: images)
print("tokens: \(ids.count), \(ids == wantIds ? "identical" : "DIFFERENT") ids; drop \(encoder.dropCount)"
+ (encoder.dropCount == wantDrop ? "" : " (reference \(wantDrop))"))
guard ids == wantIds, encoder.dropCount == wantDrop else { throw GGUFFile.Failure.unsupported("token ids differ") }
+ if !images.isEmpty {
+ let positions = try QwenTextEncoder.positions(ids: ids, imagePad: encoder.imagePad,
+ grids: images.map { ($0.height / 32, $0.width / 32) })
+ let want = try reference(directory, "positions") // [3, T]
+ let same = positions.indices.allSatisfy { i in (0..<3).allSatisfy { Float(positions[i][$0]) == want[$0 * ids.count + i] } }
+ print("positions: \(same ? "identical" : "DIFFERENT")")
+ }
func report(_ name: String, _ got: UnsafePointer<Float>, _ want: [Float]) {
let r = compare(got, want)
print(String(format: " %-12@ rel %.2e max |Δ| %.3g%@", name as NSString, r.relative, r.maxAbs, r.finite ? "" : " NON-FINITE"))
}
- _ = try encoder.forward(ids: ids) { name, pointer, count in
+ _ = try encoder.forward(ids: ids, probe: { name, pointer, count in
guard let want = try? reference(directory, name) else { return }
precondition(want.count == count, "\(name): \(count) floats, reference \(want.count)")
report(name, pointer, want)
+ }, images: images)
+ if !images.isEmpty {
+ // The text encoder alone: the reference's own vision rows in the slots.
+ var at = 0
+ let merged = try reference(directory, "vision_merged"), deep = try (0..<3).map { try reference(directory, "vision_deepstack\($0)") }
+ let given = images.map { image -> QwenVisionTower.Output in
+ defer { at += image.slots * QwenTextEncoder.dim }
+ let range = at..<(at + image.slots * QwenTextEncoder.dim)
+ return QwenVisionTower.Output(merged: Array(merged[range]), deepstack: deep.map { Array($0[range]) },
+ height: image.height, width: image.width)
+ }
+ let (rows, _, _) = try encoder.conditioning(prompt: prompt, images: given)
+ rows.withUnsafeBufferPointer { report("conditioning, the reference's vision rows", $0.baseAddress!, try! reference(directory, "conditioning")) }
}
for run in 1...2 {
let started = DispatchTime.now().uptimeNanoseconds
- let (rows, count) = try encoder.conditioning(prompt: prompt)
+ let (rows, count, slots) = try encoder.conditioning(prompt: prompt, images: images)
let seconds = Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9
+ if run == 1, let want = meta["image_pad_mask"] as? [Int] {
+ print("image slots: \(slots.map { $0 ? 1 : 0 } == want ? "identical" : "DIFFERENT") (\(slots.filter { $0 }.count))")
+ }
if run == 1 { rows.withUnsafeBufferPointer { report("conditioning", $0.baseAddress!, try! reference(directory, "conditioning")) } }
print(String(format: "encode %d (%d rows): %.3f s", run, count, seconds))
}
diff --git a/engine-mac/Sources/kokon-engine-mac/main.swift b/engine-mac/Sources/kokon-engine-mac/main.swift
index e542430..4a71f4e 100644
--- a/engine-mac/Sources/kokon-engine-mac/main.swift
+++ b/engine-mac/Sources/kokon-engine-mac/main.swift
@@ -12,7 +12,7 @@ kokon-engine-mac: Kokon's Apple Silicon engine (in development)
check-vae-encoder <vae.safetensors> <reference dir> [WxH] the VAE encoder against tools/reference_vae_encoder.py, then its time
time-vae <vae.safetensors> [grid] the VAE decoder's time and memory (64 = 1024²; KOKON_VAE_TILE)
check-tokenizer <encoder.gguf> <fixture.json> the tokenizer against tools/reference_tokenizer.py
- check-text <encoder.gguf> <reference dir> the text encoder against tools/reference_text_encoder.py
+ check-text <encoder.gguf> <reference dir> [mmproj.gguf] the text encoder against tools/reference_text_encoder.py
check-vision <mmproj.gguf> <reference dir> the vision tower against tools/reference_vision.py, then its time
check-generate <encoder.gguf> <dit.gguf> <vae.safetensors> <reference dir> [text reference dir]
text to image against tools/reference_generate.py, from its noise
@@ -93,8 +93,8 @@ do {
guard arguments.count == 3 else { fail(usage) }
try Checks.tokenizer(path: arguments[1], fixture: arguments[2])
case "check-text":
- guard arguments.count == 3 else { fail(usage) }
- try Checks.textEncoder(path: arguments[1], reference: arguments[2])
+ guard arguments.count == 3 || arguments.count == 4 else { fail(usage) }
+ try Checks.textEncoder(path: arguments[1], reference: arguments[2], mmproj: arguments.count == 4 ? arguments[3] : nil)
case "check-vision":
guard arguments.count == 3 else { fail(usage) }
try Checks.vision(path: arguments[1], reference: arguments[2])
diff --git a/engine-mac/tools/reference_text_encoder.py b/engine-mac/tools/reference_text_encoder.py
index d02b1a1..daa760c 100644
--- a/engine-mac/tools/reference_text_encoder.py
+++ b/engine-mac/tools/reference_text_encoder.py
@@ -1,37 +1,52 @@
"""The Qwen-Image 2.1 text encoder reference for the Mac engine: transformers' own Qwen3-VL text decoder layers, in
fp32 on the CPU, on the weights of Kokon's GGUF unpacked by gguf-py, run on the prompt exactly as diffusers'
-QwenImage21Pipeline builds it (0.42, `_get_qwen_prompt_embeds`, text-to-image):
+QwenImage21Pipeline builds it (0.42, `_get_qwen_prompt_embeds`):
template "<|im_start|>system\\n{sys}<|im_end|>\\n<|im_start|>user\\n{prompt}<|im_end|>\\n<|im_start|>assistant\\n"
tokenized by the official processor; conditioning = the last decoder layer's output (before the final RMSNorm),
without the system message's tokens
+With `--image` (an edit, `prompt_template_ti2i`), "<image1><|vision_start|><|image_pad|><|vision_end|>" before the
+prompt (" <imageN>…" for more images); the processor expands each `<|image_pad|>` to one token per 32 × 32 pixels, the
+vision tower (`reference_vision.py`, on the mmproj GGUF) fills those rows, positions are Qwen3-VL's 3D M-RoPE
+(`get_rope_index`), and its three deepstack outputs are added to the image rows after decoder layers 0, 1, 2. Also
+saved: `image_pad_mask` (which conditioning rows are image slots) and the vision tower's outputs.
+
The 8B model in fp32 is 33 GB, more than a 16 GB Mac holds: one decoder layer module, its weights swapped layer
by layer (as `reference_dit.py` does for the DiT). `--sanity` also runs the language model head on a factual
prompt and prints its next token: a check that the GGUF weights are read right, which a comparison of two readers
of the same file cannot be.
python -I engine-mac/tools/reference_text_encoder.py <encoder.gguf> <processor dir> <text_encoder dir> <out dir>
- [--prompt "..."] [--sanity]
+ [--prompt "..."] [--sanity] [--image in.png --mmproj mmproj.gguf]...
Writes `<out>/meta.json` (prompt, ids, drop) and one raw little-endian fp32 file per tensor (`<name>.f32`).
"""
import argparse
import json
import os
+import sys
+from types import SimpleNamespace
import gguf
import numpy as np
import torch
from transformers import AutoConfig, AutoProcessor
from transformers.models.qwen3_vl.modeling_qwen3_vl import (
+ Qwen3VLModel,
Qwen3VLTextDecoderLayer,
Qwen3VLTextRMSNorm,
Qwen3VLTextRotaryEmbedding,
+ Qwen3VLVisionModel,
)
+from PIL import Image
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from reference_vision import state_dict as vision_state_dict # noqa: E402
SYSTEM = "Comprehend and analyze the provided prompt."
TEMPLATE = f"<|im_start|>system\n{SYSTEM}<|im_end|>\n<|im_start|>user\n{{}}<|im_end|>\n<|im_start|>assistant\n"
+IMAGE = "<image{}><|vision_start|><|image_pad|><|vision_end|>"
NAMES = {
"attn_q": "self_attn.q_proj", "attn_k": "self_attn.k_proj", "attn_v": "self_attn.v_proj",
"attn_output": "self_attn.o_proj", "attn_q_norm": "self_attn.q_norm", "attn_k_norm": "self_attn.k_norm",
@@ -61,13 +76,18 @@ class Weights:
layer.load_state_dict(state, strict=True)
-def run(weights, config, ids, save=None):
- """The text model's forward on one unpadded prompt; returns the last layer's output (no final norm)."""
+def run(weights, config, ids, save=None, vision=None):
+ """The text model's forward on one unpadded prompt; returns the last layer's output (no final norm). `vision`:
+ (image rows mask, merged rows, deepstack list, position ids [3, 1, T]) for a prompt with images."""
x = weights.get("token_embd.weight", rows=ids)[None]
+ t = x.shape[1]
+ if vision is None:
+ position_ids = torch.arange(t).view(1, 1, -1).expand(3, 1, -1) # text only: the three axes agree
+ else:
+ visual, merged, deepstack, position_ids = vision
+ x[0, visual] = merged
if save:
save("embeddings", x[0])
- t = x.shape[1]
- position_ids = torch.arange(t).view(1, 1, -1).expand(3, 1, -1) # text only: the three axes agree
rope = Qwen3VLTextRotaryEmbedding(config)
position_embeddings = rope(x, position_ids)
mask = torch.full((t, t), torch.finfo(torch.float32).min).triu(1)[None, None]
@@ -75,6 +95,9 @@ def run(weights, config, ids, save=None):
for index in range(config.num_hidden_layers):
weights.load_layer(layer, index)
x = layer(x, position_embeddings=position_embeddings, attention_mask=mask)
+ if vision is not None and index < len(deepstack): # Qwen3VLTextModel._deepstack_process
+ x = x.clone()
+ x[0, visual] = x[0, visual] + deepstack[index]
if save and index in (0, 1, 17, config.num_hidden_layers - 1):
save(f"layer{index}_out", x[0])
return x
@@ -88,6 +111,8 @@ def main():
parser.add_argument("out")
parser.add_argument("--prompt", default="A red fox sitting in fresh snow at dawn, soft golden light, photo")
parser.add_argument("--sanity", action="store_true")
+ parser.add_argument("--image", action="append", default=[], help="a condition image (sides multiples of 32)")
+ parser.add_argument("--mmproj", help="the vision tower's GGUF, with --image")
args = parser.parse_args()
torch.set_grad_enabled(False)
os.makedirs(args.out, exist_ok=True)
@@ -100,8 +125,6 @@ def main():
system = [{"role": "system", "content": [{"type": "text", "text": SYSTEM}]}]
drop = len(processor.apply_chat_template(system, tokenize=True, return_dict=False)[0])
prompt = args.prompt or " "
- ids = processor(text=[TEMPLATE.format(prompt)], return_tensors="pt").input_ids[0].tolist()
-
saved = {}
def save(name, tensor):
@@ -109,23 +132,84 @@ def main():
array.tofile(os.path.join(args.out, name + ".f32"))
saved[name] = list(array.shape)
- hidden = run(weights, config, ids, save)
+ full_config = AutoConfig.from_pretrained(args.text_encoder)
+ tower = None
+
+ def with_images(text, images, save=None):
+ """Token ids and run()'s `vision` for `text` (a template with its image markers) and its images."""
+ nonlocal tower
+ flat = []
+ for image in images: # the pipeline: RGBA composited over white for the vision encoder
+ if image.mode == "RGBA":
+ white = Image.new("RGB", image.size, (255, 255, 255))
+ white.paste(image, mask=image.getchannel("A"))
+ image = white
+ flat.append(image.convert("RGB"))
+ inputs = processor(text=[text], images=flat, padding=True, padding_side="left", return_tensors="pt")
+ if tower is None:
+ vconfig = full_config.vision_config
+ vconfig._attn_implementation = "eager"
+ tower = Qwen3VLVisionModel(vconfig).eval()
+ tower.load_state_dict(vision_state_dict(args.mmproj, vconfig), strict=True)
+ out = tower(inputs.pixel_values.to(torch.float32), grid_thw=inputs.image_grid_thw, return_dict=True)
+ shim = SimpleNamespace(config=full_config)
+ shim.get_vision_position_ids = lambda *a, **k: Qwen3VLModel.get_vision_position_ids(shim, *a, **k)
+ position_ids, _ = Qwen3VLModel.get_rope_index(shim, inputs.input_ids, inputs.mm_token_type_ids,
+ image_grid_thw=inputs.image_grid_thw,
+ attention_mask=inputs.attention_mask)
+ if save:
+ save("vision_merged", out.pooler_output)
+ for j, feature in enumerate(out.deepstack_features):
+ save(f"vision_deepstack{j}", feature)
+ save("positions", position_ids[:, 0].to(torch.float32))
+ ids = inputs.input_ids[0].tolist()
+ visual = torch.tensor(ids) == full_config.image_token_id
+ return ids, (visual, out.pooler_output, out.deepstack_features, position_ids)
+
+ vision, images = None, []
+ if args.image:
+ images = [Image.open(p) for p in args.image]
+ for i, image in enumerate(images):
+ image.save(os.path.join(args.out, f"image{i + 1}.png"))
+ template = TEMPLATE.replace("{}", " ".join(IMAGE.format(i + 1) for i in range(len(images))) + "{}")
+ ids, vision = with_images(template.format(prompt), images, save)
+ else:
+ ids = processor(text=[TEMPLATE.format(prompt)], return_tensors="pt").input_ids[0].tolist()
+
+ hidden = run(weights, config, ids, save, vision)
save("conditioning", hidden[0, drop:])
meta = {"prompt": prompt, "ids": ids, "drop": drop, "tensors": saved, "gguf": os.path.basename(args.gguf),
+ "images": [[i.width, i.height] for i in images],
"transformers": __import__("transformers").__version__, "torch": torch.__version__}
+ if args.image:
+ meta["image_pad_mask"] = [int(v) for v in vision[0][drop:].tolist()]
if args.sanity:
- question = ("<|im_start|>user\nWhat is the capital of France? Answer in one sentence.<|im_end|>\n"
- "<|im_start|>assistant\nThe capital of France is")
- qids = processor(text=[question], return_tensors="pt").input_ids[0].tolist()
- h = run(weights, config, qids)
norm = Qwen3VLTextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
norm.weight.copy_(weights.get("output_norm.weight"))
- logits = weights.get("output.weight") @ norm(h[0, -1])
- top = torch.topk(logits, 5)
- tokens = [processor.tokenizer.decode([i]) for i in top.indices.tolist()]
+
+ def next_tokens(text, images=None):
+ if images:
+ qids, qvision = with_images(text, images)
+ else:
+ qids, qvision = processor(text=[text], return_tensors="pt").input_ids[0].tolist(), None
+ h = run(weights, config, qids, vision=qvision)
+ logits = weights.get("output.weight") @ norm(h[0, -1])
+ return [processor.tokenizer.decode([i]) for i in torch.topk(logits, 5).indices.tolist()]
+
+ tokens = next_tokens("<|im_start|>user\nWhat is the capital of France? Answer in one sentence.<|im_end|>\n"
+ "<|im_start|>assistant\nThe capital of France is")
print("sanity: next tokens after 'The capital of France is':", tokens)
meta["sanity"] = tokens
+ if args.mmproj:
+ # A red disk on white: the vision tower's weights are read right if the model sees it.
+ yy, xx = np.mgrid[0:320, 0:320]
+ disk = np.full((320, 320, 3), 255, np.uint8)
+ disk[(yy - 160) ** 2 + (xx - 160) ** 2 < 100 ** 2] = (220, 30, 30)
+ tokens = next_tokens("<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>What color is the shape? "
+ "Answer in one word.<|im_end|>\n<|im_start|>assistant\n", [Image.fromarray(disk)])
+ print("sanity: the shape's color:", tokens)
+ meta["vision_sanity"] = tokens
with open(os.path.join(args.out, "meta.json"), "w") as f:
json.dump(meta, f, indent=1)