Skip to content

armgit / kokon

Public Imported from GitHub
0

Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.

Mac engine: edits with reference images, and Prompt+ looking at them

Gabriele Lozuponef60f53e2a18e
diff --git a/crates/qls-core/src/process.rs b/crates/qls-core/src/process.rs
index ec3289f..7a0fb83 100644
--- a/crates/qls-core/src/process.rs
+++ b/crates/qls-core/src/process.rs
@@ -126,7 +126,7 @@ pub fn build_server_args(cfg: &AppConfig, profile: &ModelProfile) -> Vec<String>
/// none of sd-server's placement or backend flags apply; the vision projector is for edits, not supported yet.
pub fn build_mac_engine_args(cfg: &AppConfig, profile: &ModelProfile) -> Vec<String> {
let p = |p: &Path| p.to_string_lossy().into_owned();
- vec![
+ let mut a: Vec<String> = vec![
"serve".into(),
"--listen-ip".into(),
"127.0.0.1".into(),
@@ -138,7 +138,13 @@ pub fn build_mac_engine_args(cfg: &AppConfig, profile: &ModelProfile) -> Vec<Str
p(&profile.vae),
"--llm".into(),
p(&profile.text_encoder),
- ]
+ ];
+ // The vision tower: edits need it, as sd-server's.
+ if let Some(v) = &profile.vision_projector {
+ a.push("--llm_vision".into());
+ a.push(p(v));
+ }
+ a
}
/// A process this module may reuse or stop: sd-server, or Kokon's Mac engine.
@@ -990,8 +996,13 @@ mod tests {
let a = build_server_args(&cfg, cfg.profile());
let p = |p: &Path| p.to_string_lossy().into_owned();
let profile = cfg.profile();
- assert_eq!(a, ["serve", "--listen-ip", "127.0.0.1", "--listen-port", &cfg.server_port.to_string(), "--diffusion-model",
- &p(&profile.diffusion_model), "--vae", &p(&profile.vae), "--llm", &p(&profile.text_encoder)]);
+ let mut want: Vec<String> = ["serve", "--listen-ip", "127.0.0.1", "--listen-port", &cfg.server_port.to_string(), "--diffusion-model",
+ &p(&profile.diffusion_model), "--vae", &p(&profile.vae), "--llm", &p(&profile.text_encoder)].map(String::from).to_vec();
+ if let Some(v) = &profile.vision_projector {
+ want.extend(["--llm_vision".to_string(), p(v)]); // edits
+ }
+ assert_eq!(a, want);
+ assert!(profile.vision_projector.is_some(), "the default profile has the mmproj, so the engine can edit");
assert_ne!(a, sdcpp, "a different launch: switching restarts the server");
// Only on the Metal backend: elsewhere sd.cpp stays the engine.
cfg.gpu_backend = crate::config::GpuBackend::Cuda;
diff --git a/crates/qls-core/src/sdcpp.rs b/crates/qls-core/src/sdcpp.rs
index d068efb..f53f081 100644
--- a/crates/qls-core/src/sdcpp.rs
+++ b/crates/qls-core/src/sdcpp.rs
@@ -219,12 +219,15 @@ impl Capabilities {
pub fn is_mac_engine(&self) -> bool {
self.engine == crate::config::MAC_ENGINE_EXE
}
- /// What in `req` Kokon's Mac engine cannot do yet (it runs text to image only), in the user's words; None when it
- /// can, and always for sd-server.
+ /// What in `req` Kokon's Mac engine cannot do yet, in the user's words; None when it can, and always for
+ /// sd-server. It edits with reference images when it has the vision projector (`ref_images` in its features), but
+ /// has no init image or mask yet (brushed edits, extending an image).
pub fn mac_engine_unsupported(&self, req: &ImgGenRequest) -> Option<&'static str> {
if !self.is_mac_engine() {
None
- } else if !req.ref_images.is_empty() || req.init_image.is_some() || req.mask_image.is_some() {
+ } else if req.init_image.is_some() || req.mask_image.is_some() {
+ Some("edits of a brushed area or extending images")
+ } else if !req.ref_images.is_empty() && !self.img_feature("ref_images") {
Some("edits")
} else if req.hires.as_ref().is_some_and(|h| h.enabled) {
Some("second passes")
@@ -910,9 +913,14 @@ mod tests {
let t2i = ImgGenRequest { prompt: "a fox".into(), width: 1024, height: 1024, ..Default::default() };
assert_eq!(caps.mac_engine_unsupported(&t2i), None);
let edit = ImgGenRequest { ref_images: vec!["png".into()], ..t2i.clone() };
- assert_eq!(caps.mac_engine_unsupported(&edit), Some("edits"));
+ assert_eq!(caps.mac_engine_unsupported(&edit), Some("edits"), "no vision projector");
let inpaint = ImgGenRequest { mask_image: Some("png".into()), ..t2i.clone() };
- assert_eq!(caps.mac_engine_unsupported(&inpaint), Some("edits"));
+ assert_eq!(caps.mac_engine_unsupported(&inpaint), Some("edits of a brushed area or extending images"));
+ // Started with --llm_vision: edits with references run, a mask still does not.
+ let vision: Capabilities = serde_json::from_str(r#"{"engine":"kokon-engine-mac","supported_modes":["img_gen"],
+ "features_by_mode":{"img_gen":{"ref_images":true}}}"#).unwrap();
+ assert_eq!(vision.mac_engine_unsupported(&edit), None);
+ assert!(vision.mac_engine_unsupported(&inpaint).is_some());
let hires = ImgGenRequest { hires: Some(HiresParams { enabled: true, ..Default::default() }), ..t2i.clone() };
assert_eq!(caps.mac_engine_unsupported(&hires), Some("second passes"));
// sd-server: no engine field, nothing refused here.
diff --git a/docs/STATUS.md b/docs/STATUS.md
index 0fbea5a..816958a 100644
--- a/docs/STATUS.md
+++ b/docs/STATUS.md
@@ -24,10 +24,11 @@ Windows, macOS (Apple Silicon, 15+) and Linux (`.deb`, `.rpm`). Installed copies
- **LoRA is beta** and tested on Windows only; reports from macOS and Linux are welcome
([#33](https://github.com/GabrieleLozupone/kokon/issues/33) has a product-shot test).
- **Macs are slow** compared with NVIDIA cards (about 4 minutes for 1 MP on an M2 Pro 16 GB). Kokon's own Mac
- engine is in development on the `macos-optimization` branch (`engine-mac/`, not released): text to image and
- Prompt+ (a prompt written in about 7–9 s), about 64–67 s for 1 MP on the same Mac through the app's API (peak memory 5 GB),
- tested from the app, as an experimental switch in Settings → Performance (off by default). Edits, LoRAs and the
- upscaler still need the standard engine.
+ engine is in development on the `macos-optimization` branch (`engine-mac/`, not released): text to image,
+ Prompt+ (a prompt written in about 7–9 s) and edits with reference images (Prompt+ looking at them too), about 64–67 s
+ for 1 MP text to image and ~150 s for a 1 MP edit on the same Mac (peak memory 5 GB), as an experimental switch in
+ Settings → Performance (off by default). Brushed edits, extending an image, LoRAs and the upscaler still need the
+ standard engine.
- The Windows installer is not code-signed: SmartScreen or Smart App Control may block it (README FAQ: Unblock).
- HEIC photos can't be opened on Linux yet.
diff --git a/engine-mac/README.md b/engine-mac/README.md
index 10d3ef1..3ffc88d 100644
--- a/engine-mac/README.md
+++ b/engine-mac/README.md
@@ -14,9 +14,9 @@ swift test # no model needed
.build/release/kokon-engine-mac inspect <file.gguf>
.build/release/kokon-engine-mac check-dequant <file.gguf> [tensor...]
.build/release/kokon-engine-mac generate <encoder.gguf> <dit.gguf> <vae.safetensors> "a red fox in the snow" \
- [--size 1024x1024] [--seed 42] [--out out.png]
+ [--size 1024x1024] [--seed 42] [--out out.png] [--ref in.png --mmproj <mmproj.gguf>]
.build/release/kokon-engine-mac serve --diffusion-model <dit.gguf> --vae <vae.safetensors> --llm <encoder.gguf> \
- [--listen-port 1234] # sd-server's /sdcpp/v1 API, text to image and Prompt+
+ [--llm_vision <mmproj.gguf>] [--listen-port 1234] # sd-server's /sdcpp/v1 API: text to image, edits, Prompt+
.build/release/kokon-engine-mac bench # a few minutes: speed and accuracy of this Mac's paths, a report
.build/release/kokon-engine-mac check-na # the Metal 4 matrix product against MPS
.build/release/kokon-engine-mac check-ane8 # the int8 Neural Engine programs against the fp16 one
@@ -260,9 +260,10 @@ Engine's programs, are not pipelines an archive holds. Only an M2 Pro has been m
`kokon-engine-mac serve` answers sd-server's `/sdcpp/v1` API (capabilities, img_gen, jobs/{id}, cancel), so
qls-core, the app, remote access and MCP use it as they use sd-server; it prints the log lines qls-core's
-`ProgressParser` reads. Text to image only for now: reference images, masks, LoRAs and second passes are refused
-with 400 (qls-core says so before submitting), upscaling falls back to Lanczos. Prompt+ works on text-to-image prompts
-(`text_gen`, below); with images (edits, Compose) it needs the vision tower. A running image
+`ProgressParser` reads. Text to image, and edits with reference images when started with `--llm_vision` (qls-core passes
+the mmproj; below); an init image or a mask (brushed edits, extending an image), LoRAs and second passes are refused with
+400 (qls-core says so before submitting), upscaling falls back to Lanczos. Prompt+ works on every prompt (`text_gen`,
+below), looking at the images of an edit with the vision tower. A running image
cancels within a DiT layer (sd-server answers 409 and qls-core restarts it). A new prompt length rebuilds the DiT's
working memory in ~0.5 s.
@@ -305,6 +306,59 @@ the API at 1024², alternated (`tools/api_check.py --enhance`): the image's text
2.1–2.65 s for a plain image, peak footprint 5.1 GB either way; Prompt+ adds 8.7–9.0 s to an image (75.4–77.2 s against
65.6–71.3).
+## Edits (reference images)
+
+As `QwenImage21Pipeline` with `image`: each reference goes through the vision tower into the text encoder's slots (the
+ti2i template, deepstack, 3D positions; above) and through the VAE encoder (`_encode_vae_image`) into the DiT's prefix:
+its latent tokens sit in the encoder's slots of that image (2 × 2 tokens a slot, `QwenSequence`), are modulated from
+t = 0 like the text, and see only what precedes them plus themselves (block-causal), the target last. sd-server's
+`ref_images` as Kokon sends them: Standard detail (`resize_before_vae=false`) encodes each reference at its own size;
+otherwise sd.cpp's rule, resized to the output's area at its aspect, sides rounded to 32, nearest neighbour; the vision
+tower sees what the VAE encodes (sd.cpp's `vlm_use_vae_input`). Sides must be multiples of 32.
+
+Against diffusers (`reference_generate.py` on a `reference_text_encoder.py --image` directory: the street cat at
+384 × 256, "Turn the cat into a small red fox, keep the street and the light", output 384 × 256, fp32, M2 Pro,
+2026-10-25), `check-generate <encoder> <dit> <vae> <dir> <text dir> [mmproj]`:
+
+| | final latent | image | 8-bit values off |
+|---|---|---|---|
+| the reference's conditioning and latents, fp32 path | 1.7e-6 | 4.0e-6 | 68 of 393 216, by 1 |
+| the engine's vision tower, text encoder, VAE encoder, fp32 path | 2.3e-6 | 4.8e-6 | 89, by 1 |
+| the same, default path (fp16 products and attention) | 1.8e-4 | 3.6e-4 | 5 083, by at most 2 |
+
+**The prefix K/V cache holds the references' rows too**, in fp16 when the attention reads fp16 (the default): the same
+values to the bit for the GPU's heads. A 1 MP reference is ~4 100 prefix rows, 2.1 GB of cache. M2 Pro 16 GB, edits with
+the reference at the output's size, alternated:
+
+| reference and output | cache | steps 2–8 | image | page-ins an image |
+|---|---|---|---|---|
+| 768 × 448 | off | 4.9–5.0 s | 46.2 s | 12.1 GB |
+| 768 × 448 | 0.72 GB | 3.5 s | 38.1 s | 13.3 GB |
+| 1344 × 768 | off | 16.6–17.1 s | 147.7–150.1 s | 20.9 GB |
+| 1344 × 768 | 2.13 GB | 14.8–20.4 s | 144.2–158.7 s | 40.4 GB |
+
+On 16 GB the 2.1 GB cache evicts the DiT's file pages, read again every step, and gains nothing; the cache is on up to
+1 GB below 32 GB of memory and to an eighth of it from 32 GB (not measured there; `KOKON_PREFIX_CACHE_MB` to choose). A 1 MP
+edit: vision tower 2.5 s, text encoder 3.7 s (1 037 tokens), VAE encoder 0.8 s, 8 steps of ~17 s, VAE 4.6 s; peak footprint
+5.0 GB. Through the server (`tools/api_check.py --ref … --enhance`, 768 × 448): Prompt+ looking at the reference 5.5–6.8 s
+(vision 0.6 s, 742 prompt tokens), the edit 36.4–37.3 s, peak footprint 3.6 GB.
+
+**Prompt+ with images**: sd.cpp's template (`<|vision_start|><|image_pad|><|vision_end|>` before the prompt, `<imageN>`
+labels when several, each side rounded to 32), the prefill with the images' rows, positions and deepstack, then the
+decode at positions offset from the cache slots (an image's slots advance the position by max(rows, columns), not by
+their count). `check-textgen <encoder> <dir> 48 <image> <mmproj>` against `reference_text_generation.py --mmproj` (the
+cat, Kokon's edit system prompt, "make the cat a small red fox"): the first token's logits within 2.5e-6, every step
+within 6.8e-4 (the fp16 cache), the greedy choice the same at 23 of 23.
+
+```bash
+~/kokon-bench/venv-oracle/bin/python -I tools/reference_text_encoder.py <encoder.gguf> <processor> <text_encoder> ~/kokon-bench/ref/text/cat-edit --prompt "..." --image cat-384x256.png --mmproj <mmproj.gguf> --sanity
+~/kokon-bench/venv-oracle/bin/python -I tools/reference_generate.py <dit.gguf> <vae> ~/kokon-bench/ref/text/cat-edit <repo dir> ~/kokon-bench/ref/edit-cat-384 --size 384x256 # ~6 min
+.build/release/kokon-engine-mac check-generate <encoder.gguf> <dit.gguf> <vae> ~/kokon-bench/ref/edit-cat-384 ~/kokon-bench/ref/text/cat-edit [<mmproj.gguf>]
+.build/release/kokon-engine-mac generate <encoder.gguf> <dit.gguf> <vae> "make the cat a fox" --ref in.png --mmproj <mmproj.gguf>
+```
+
+Not yet: an init image with a mask (brushed edits, extending an image: sd.cpp blends the masked latent each step).
+
## Per-chip paths
Selected at run time from the GPU family and the chip (`Chip.swift`), each with the M2 path as its fallback and a
diff --git a/engine-mac/Sources/KokonEngine/EngineServer.swift b/engine-mac/Sources/KokonEngine/EngineServer.swift
index 7394acf..7a67c66 100644
--- a/engine-mac/Sources/KokonEngine/EngineServer.swift
+++ b/engine-mac/Sources/KokonEngine/EngineServer.swift
@@ -9,8 +9,9 @@ import Foundation
/// POST /sdcpp/v1/jobs/{id}/cancel 200 {status: cancelled}: a running image stops within a DiT layer
/// POST /sdcpp/v1/text_gen Prompt+: the text encoder's LLM writes, streamed as Server-Sent Events
///
-/// Text to image only for now: requests with reference images, a mask, an init image, LoRAs or a second pass are
-/// refused with 400 and say so, and so is text generation with images. Jobs and text generation run one at a time;
+/// Text to image and edits (`ref_images`, with the vision projector given as `--llm_vision`; Prompt+ then also looks at
+/// the images); requests with a mask, an
+/// init image, LoRAs or a second pass are refused with 400 and say so. Jobs and text generation run one at a time;
/// what stays loaded between them depends on the Mac (`QwenPipeline.keepModels`).
///
/// Progress goes to stdout as sd-server's log lines, the ones qls-core's `ProgressParser` reads (process.rs):
@@ -19,10 +20,13 @@ import Foundation
public final class EngineServer: @unchecked Sendable {
public struct Paths: Sendable {
public let diffusionModel, vae, textEncoder: String
- public init(diffusionModel: String, vae: String, textEncoder: String) {
+ /// The vision tower (mmproj): edits need it.
+ public let vision: String?
+ public init(diffusionModel: String, vae: String, textEncoder: String, vision: String? = nil) {
self.diffusionModel = diffusionModel
self.vae = vae
self.textEncoder = textEncoder
+ self.vision = vision
}
}
@@ -42,21 +46,27 @@ public final class EngineServer: @unchecked Sendable {
}
/// The fields of `img_gen` the engine reads (sd-server's names), checked.
- struct ImageRequest: Sendable {
+ struct ImageRequest: @unchecked Sendable {
let prompt: String
let width, height: Int
let seed: Int64
let batch: Int
let sigmas: [Float]
+ /// An edit's condition images, decoded, at the size they are encoded at.
+ let references: [QwenPipeline.Reference]
- /// nil and the reason when the engine cannot do it (yet).
- static func parse(_ body: Data) -> (ImageRequest?, String?) {
+ /// nil and the reason when the engine cannot do it (yet). `vision`: the engine has the vision tower (edits).
+ static func parse(_ body: Data, vision: Bool = true) -> (ImageRequest?, String?) {
guard let json = (try? JSONSerialization.jsonObject(with: body)) as? [String: Any] else {
return (nil, "the body is not a JSON object")
}
func array(_ key: String) -> [Any] { json[key] as? [Any] ?? [] }
- if !array("ref_images").isEmpty || json["init_image"] is String || json["mask_image"] is String {
- return (nil, "Kokon's Mac engine does text to image only for now (no reference images, init image or mask)")
+ if json["init_image"] is String || json["mask_image"] is String {
+ return (nil, "Kokon's Mac engine has no init image or mask yet (brushed edits, extending images)")
+ }
+ let refs = array("ref_images")
+ if !refs.isEmpty && !vision {
+ return (nil, "Qwen Image 2.1 editing requires Qwen3-VL vision weights; start the engine with --llm_vision")
}
if !array("lora").isEmpty { return (nil, "Kokon's Mac engine does not run LoRAs yet") }
if let hires = json["hires"] as? [String: Any], hires["enabled"] as? Bool == true {
@@ -83,11 +93,43 @@ public final class EngineServer: @unchecked Sendable {
if seed < 0 { seed = Int64.random(in: 0..<(1 << 31)) }
let batch = json["batch_count"] as? Int ?? 1
guard (1...EngineServer.maxBatch).contains(batch) else { return (nil, "batch_count \(batch): 1 to \(EngineServer.maxBatch)") }
+ // ref_image_args: sd.cpp's key=value list; only resize_before_vae changes what the engine does.
+ var resize = true
+ for pair in (json["ref_image_args"] as? String ?? "").split(separator: ",") {
+ let kv = pair.split(separator: "=", maxSplits: 1).map { $0.trimmingCharacters(in: .whitespaces) }
+ if kv.count == 2, kv[0] == "resize_before_vae" { resize = !["false", "0", "no", "off"].contains(kv[1].lowercased()) }
+ }
+ var references: [QwenPipeline.Reference] = []
+ for (i, item) in refs.enumerated() {
+ guard let text = item as? String, let data = Data(base64Encoded: String(text.split(separator: ",").last ?? ""),
+ options: .ignoreUnknownCharacters),
+ let image = try? QwenPipeline.decodeImage(data) else {
+ return (nil, "ref_images[\(i)] is not a base64 image")
+ }
+ var r = QwenPipeline.Reference(rgba: image.rgba, width: image.width, height: image.height)
+ if resize { r = EngineServer.resizeToTarget(r, width: width, height: height) }
+ guard r.width % 32 == 0, r.height % 32 == 0 else {
+ return (nil, "Qwen Image 2.1 reference dimensions must be multiples of 32 (\(r.width)×\(r.height))")
+ }
+ references.append(r)
+ }
return (ImageRequest(prompt: json["prompt"] as? String ?? "", width: width, height: height, seed: seed,
- batch: batch, sigmas: sigmas), nil)
+ batch: batch, sigmas: sigmas, references: references), nil)
}
}
+ /// sd.cpp's `resize_before_vae` for Qwen-Image 2.1 (pipeline/image.cpp): the reference resized to the output's area
+ /// at its own aspect, each side rounded to 32, nearest neighbour (`sd::ops::interpolate`'s default); the vision
+ /// tower then sees the same image (`vlm_use_vae_input`).
+ static func resizeToTarget(_ r: QwenPipeline.Reference, width: Int, height: Int) -> QwenPipeline.Reference {
+ let area = Double(width * height)
+ let w = (area * Double(r.width) / Double(r.height)).squareRoot()
+ let h = w * Double(r.height) / Double(r.width)
+ let (ow, oh) = (Int((w / 32).rounded() * 32), Int((h / 32).rounded() * 32))
+ guard ow > 0, oh > 0, (ow, oh) != (r.width, r.height) else { return r }
+ return resize(r, width: ow, height: oh)
+ }
+
static let maxBatch = 4, maxQueue = 8
public let paths: Paths
let pipeline: QwenPipeline
@@ -108,7 +150,9 @@ public final class EngineServer: @unchecked Sendable {
for path in [paths.diffusionModel, paths.vae, paths.textEncoder] where !FileManager.default.fileExists(atPath: path) {
throw GGUFFile.Failure.open(path)
}
+ if let vision = paths.vision, !FileManager.default.fileExists(atPath: vision) { throw GGUFFile.Failure.open(vision) }
pipeline = try QwenPipeline(encoder: paths.textEncoder, dit: paths.diffusionModel, vae: paths.vae)
+ pipeline.visionPath = paths.vision
pipeline.keepModels = true
pipeline.log = { _ in }
let worker = Thread { [self] in work() }
@@ -137,7 +181,7 @@ public final class EngineServer: @unchecked Sendable {
func capabilities() -> [String: Any] {
let url = URL(fileURLWithPath: paths.diffusionModel)
- let feature: [String: Any] = ["ref_images": false, "init_image": false, "mask_image": false, "lora": false,
+ let feature: [String: Any] = ["ref_images": paths.vision != nil, "init_image": false, "mask_image": false, "lora": false,
"hires": false, "custom_sigmas": true, "cancel_generating": true]
return [
"engine": "kokon-engine-mac",
@@ -148,7 +192,7 @@ public final class EngineServer: @unchecked Sendable {
"output_formats": ["png"],
"limits": ["min_width": 64, "max_width": 4096, "min_height": 64, "max_height": 4096,
"max_batch_count": EngineServer.maxBatch, "max_queue_size": EngineServer.maxQueue,
- "max_text_tokens": EngineServer.maxTextTokens, "max_text_images": 0],
+ "max_text_tokens": EngineServer.maxTextTokens, "max_text_images": paths.vision != nil ? EngineServer.maxTextImages : 0],
"features_by_mode": ["img_gen": feature],
"defaults_by_mode": ["img_gen": ["sample_steps": QwenPipeline.turboSigmas.count - 1, "txt_cfg": 1]],
"upscalers": [Any](),
@@ -158,7 +202,7 @@ public final class EngineServer: @unchecked Sendable {
}
func submit(_ body: Data) -> HTTPServer.Response {
- let (parsed, reason) = ImageRequest.parse(body)
+ let (parsed, reason) = ImageRequest.parse(body, vision: paths.vision != nil)
guard let parsed else { return .error(400, reason ?? "bad request") }
return lock.withLock {
guard queue.count < EngineServer.maxQueue else { return .error(429, "the queue is full") }
@@ -201,14 +245,16 @@ public final class EngineServer: @unchecked Sendable {
// MARK: - text generation (Prompt+)
/// The fields of `text_gen` (sd-server's names and checks, `examples/server/routes_sdcpp.cpp`).
- struct TextRequest: Sendable {
+ struct TextRequest: @unchecked Sendable {
let system, prompt: String
+ /// Shown before the prompt, sides rounded to 32 as sd.cpp's `vision_size`.
+ let images: [QwenPipeline.Reference]
let maxTokens: Int
let sampling: TextSampler.Parameters
let seed: Int64
let stream: Bool
- static func parse(_ body: Data) -> (TextRequest?, String?) {
+ static func parse(_ body: Data, vision: Bool = true) -> (TextRequest?, String?) {
guard let json = (try? JSONSerialization.jsonObject(with: body)) as? [String: Any] else { return (nil, "body must be an object") }
guard let prompt = json["prompt"] as? String, !prompt.isEmpty else { return (nil, "prompt is required") }
guard let system = json["system_prompt"] as? String else {
@@ -216,8 +262,20 @@ public final class EngineServer: @unchecked Sendable {
}
let maxTokens = json["max_tokens"] as? Int ?? 512
guard (1...EngineServer.maxTextTokens).contains(maxTokens) else { return (nil, "max_tokens must be between 1 and \(EngineServer.maxTextTokens)") }
- if let images = json["images"] as? [Any], !images.isEmpty {
- return (nil, "Kokon's Mac engine cannot look at images yet (no vision tower): Prompt+ works on text-to-image prompts only")
+ var images: [QwenPipeline.Reference] = []
+ if let list = json["images"] {
+ guard let list = list as? [Any], list.count <= EngineServer.maxTextImages else {
+ return (nil, "images must be an array of at most \(EngineServer.maxTextImages) base64 images")
+ }
+ if !list.isEmpty && !vision { return (nil, "images need the Qwen3-VL vision weights (--llm_vision)") }
+ for (i, item) in list.enumerated() {
+ guard let text = item as? String, let data = Data(base64Encoded: String(text.split(separator: ",").last ?? ""),
+ options: .ignoreUnknownCharacters),
+ let image = try? QwenPipeline.decodeImage(data) else {
+ return (nil, "images[\(i)] is not a base64 image")
+ }
+ images.append(EngineServer.roundedTo32(.init(rgba: image.rgba, width: image.width, height: image.height)))
+ }
}
var sampling = TextSampler.Parameters(temperature: 0, topP: 1, topK: 0, presencePenalty: 0)
var seed: Int64 = -1
@@ -234,18 +292,38 @@ public final class EngineServer: @unchecked Sendable {
return (nil, "sampling needs temperature >= 0, 0 < top_p <= 1, top_k >= 0 and a finite presence_penalty")
}
}
- return (TextRequest(system: system, prompt: prompt, maxTokens: maxTokens, sampling: sampling, seed: seed,
+ return (TextRequest(system: system, prompt: prompt, images: images, maxTokens: maxTokens, sampling: sampling, seed: seed,
stream: json["stream"] as? Bool ?? true), nil)
}
}
- static let maxTextTokens = 4096
+ static let maxTextTokens = 4096, maxTextImages = 4
+
+ /// sd.cpp's `vision_size` for text generation (no `image_max_pixels`): each side rounded to a multiple of 32, the
+ /// image resized when that changes it (nearest neighbour, as the edit references).
+ static func roundedTo32(_ r: QwenPipeline.Reference) -> QwenPipeline.Reference {
+ let (w, h) = (max(32, Int((Double(r.width) / 32).rounded()) * 32), max(32, Int((Double(r.height) / 32).rounded()) * 32))
+ return (w, h) == (r.width, r.height) ? r : resize(r, width: w, height: h)
+ }
+
+ /// Nearest-neighbour resize of 8-bit RGBA (`sd::ops::interpolate`'s default mode).
+ static func resize(_ r: QwenPipeline.Reference, width ow: Int, height oh: Int) -> QwenPipeline.Reference {
+ var out = [UInt8](repeating: 0, count: ow * oh * 4)
+ for y in 0..<oh {
+ let sy = min(r.height - 1, y * r.height / oh)
+ for x in 0..<ow {
+ let sx = min(r.width - 1, x * r.width / ow)
+ for c in 0..<4 { out[(y * ow + x) * 4 + c] = r.rgba[(sy * r.width + sx) * 4 + c] }
+ }
+ }
+ return QwenPipeline.Reference(rgba: out, width: ow, height: oh)
+ }
/// Synchronous like sd-server's: it waits for a running image (one GPU user at a time) and holds the models while it
/// writes. Streamed (the default) as Server-Sent Events: `delta` {text} per piece, then `done` {text, finish_reason,
/// seed, usage, timings} or `error` {error}; closing the connection stops it.
func textGen(_ body: Data) -> HTTPServer.Response {
- let (parsed, reason) = TextRequest.parse(body)
+ let (parsed, reason) = TextRequest.parse(body, vision: paths.vision != nil)
guard let request = parsed else { return .error(400, reason ?? "bad request") }
guard request.stream else {
let outcome = Result { try gpu.withLock { try generateText(request) { _ in true } } }
@@ -274,10 +352,11 @@ public final class EngineServer: @unchecked Sendable {
}
private func generateText(_ r: TextRequest, onPiece: (String) -> Bool) throws -> QwenTextGenerator.Result {
- let result = try pipeline.generateText(system: r.system, prompt: r.prompt, maxTokens: r.maxTokens, sampling: r.sampling,
- seed: r.seed, onPiece: onPiece)
- log(String(format: "[INFO ] engine-mac - text generation: %d prompt tokens, %d generated in %.2fs (prefill %.2fs, %.1f tok/s), seed %lld",
- result.promptTokens, result.generatedTokens, result.prepare + result.prefill + result.decode, result.prefill,
+ let result = try pipeline.generateText(system: r.system, prompt: r.prompt, images: r.images, maxTokens: r.maxTokens,
+ sampling: r.sampling, seed: r.seed, onPiece: onPiece)
+ log(String(format: "[INFO ] engine-mac - text generation: %d image(s) in %.2fs, %d prompt tokens, %d generated in %.2fs (prefill %.2fs, %.1f tok/s), seed %lld",
+ r.images.count, result.image, result.promptTokens, result.generatedTokens,
+ result.image + result.prepare + result.prefill + result.decode, result.prefill,
result.decode > 0 ? Double(result.tokens.count - 1) / result.decode : 0, result.seed))
return result
}
@@ -286,7 +365,7 @@ public final class EngineServer: @unchecked Sendable {
static func done(_ r: QwenTextGenerator.Result) -> [String: Any] {
["text": r.text, "finish_reason": r.finish.rawValue, "seed": r.seed,
"usage": ["prompt_tokens": r.promptTokens, "completion_tokens": r.generatedTokens],
- "timings": ["prepare_s": r.prepare, "image_s": 0, "prefill_s": r.prefill, "decode_s": r.decode]]
+ "timings": ["prepare_s": r.prepare, "image_s": r.image, "prefill_s": r.prefill, "decode_s": r.decode]]
}
// MARK: - the jobs
@@ -330,6 +409,9 @@ public final class EngineServer: @unchecked Sendable {
let started = DispatchTime.now().uptimeNanoseconds
func seconds(since t: UInt64) -> Double { Double(DispatchTime.now().uptimeNanoseconds - t) * 1e-9 }
log("[INFO ] engine-mac - generate_image \(r.width)x\(r.height)")
+ if !r.references.isEmpty {
+ log("[INFO ] engine-mac - EDIT mode: \(r.references.map { "\($0.width)x\($0.height)" }.joined(separator: ", "))")
+ }
log("[INFO ] engine-mac - sampling using Euler method")
pipeline.cancelled = { [weak self, job] in self?.lock.withLock { job.status == "cancelled" } ?? true }
var images: [String] = []
@@ -353,13 +435,13 @@ public final class EngineServer: @unchecked Sendable {
log("[INFO ] engine-mac - decoding 1 latents")
}
}
- let image = try pipeline.textToImage(prompt: r.prompt, width: r.width, height: r.height, seed: UInt64(bitPattern: seed),
- sigmas: r.sigmas)
+ let image = try pipeline.generate(prompt: r.prompt, references: r.references, width: r.width, height: r.height,
+ seed: UInt64(bitPattern: seed), sigmas: r.sigmas)
let png = try QwenPipeline.pngData(QwenPipeline.pixels(image, width: r.width, height: r.height), width: r.width, height: r.height)
images.append(png.base64EncodedString())
let t = pipeline.timings
- log(String(format: "[INFO ] engine-mac - image %d/%d: text encoder %.2fs, DiT ready %.2fs, VAE %.2fs, total %.2fs", i + 1, r.batch,
- t["encode"] ?? 0, t["dit load"] ?? 0, t["vae"] ?? 0, t["total"] ?? 0))
+ log(String(format: "[INFO ] engine-mac - image %d/%d: vision %.2fs, text encoder %.2fs, VAE encoder %.2fs, DiT ready %.2fs, VAE %.2fs, total %.2fs",
+ i + 1, r.batch, t["vision"] ?? 0, t["encode"] ?? 0, t["vae encode"] ?? 0, t["dit load"] ?? 0, t["vae"] ?? 0, t["total"] ?? 0))
}
pipeline.events = nil
let memory = MemoryUsage.now()
diff --git a/engine-mac/Sources/KokonEngine/Kernels/dit.metal b/engine-mac/Sources/KokonEngine/Kernels/dit.metal
index 3930547..a96fe4c 100644
--- a/engine-mac/Sources/KokonEngine/Kernels/dit.metal
+++ b/engine-mac/Sources/KokonEngine/Kernels/dit.metal
@@ -159,3 +159,9 @@ kernel void gelu_tanh(device float *x [[buffer(0)]], constant uint &count [[buff
float v = x[i];
x[i] = 0.5f * v * (1.0f + precise::tanh(0.7978845608028654f * (v + 0.044715f * v * v * v)));
}
+
+// fp16 -> fp32: the prefix K/V cache read back (QwenDiT.copyPrefix; `pack_half` writes it).
+kernel void widen_half(device const half *src [[buffer(0)]], device float *dst [[buffer(1)]],
+ constant uint &count [[buffer(2)]], uint i [[thread_position_in_grid]]) {
+ if (i < count) dst[i] = float(src[i]);
+}
diff --git a/engine-mac/Sources/KokonEngine/QwenDiT.swift b/engine-mac/Sources/KokonEngine/QwenDiT.swift
index 1827015..a67b222 100644
--- a/engine-mac/Sources/KokonEngine/QwenDiT.swift
+++ b/engine-mac/Sources/KokonEngine/QwenDiT.swift
@@ -18,8 +18,8 @@ import MetalPerformanceShaders
/// Attention is block-causal: each prefix segment sees the keys before its end (a text segment with a causal
/// triangle), the target sees every key.
///
-/// This first version runs the full pass every evaluation (no K/V cache of the prefix yet) on the GPU alone;
-/// the AMX and the Neural Engine join in the products later.
+/// An edit's condition images are prefix too: their latent tokens go through img_in like the target's and sit in
+/// their slots of the encoder's sequence (`QwenSequence`), modulated from t = 0.
public final class QwenDiT {
public static let dim = 4096, heads = 32, headDim = 128, hidden = 12288, layers = 32, channels = 64
static let eps: Float = 1e-6
@@ -41,7 +41,7 @@ public final class QwenDiT {
// Arena slices, rows of the joint sequence (N = prefix + target).
let x, normed, q, k, v, attended, o, h1, h2: Arena.Slice
let reserves: [Arena.Slice]
- let rope, modulation, qkNorms, textNorm, textIn, textOut, latent, output: Arena.Slice
+ let rope, modulation, qkNorms, textNorm, textIn, textOut, latent, output, referenceLatent: Arena.Slice
/// K and V packed for the flash kernel (`attention_pack_kv`): fp16, heads first, keys padded to 64.
let kPacked, vPacked: Arena.Slice
let paddedKeys: Int
@@ -49,10 +49,20 @@ public final class QwenDiT {
/// the modulation of t = 0, so its rows do not change from one evaluation of an image to the next. They are
/// saved during the first evaluation of a conditioning and every later one runs only the target's rows
/// (`KOKON_PREFIX_CACHE=0`: the whole sequence every time). nil when off or when it would exceed `prefixCacheLimit`.
+ /// **fp16** (`prefixHalf`) when the attention reads K and V as fp16 anyway (the packed kernel, the default): the
+ /// same values to the bit for the GPU's heads (the CPU's heads read them rounded); fp32 with any `KOKON_ATTENTION`
+ /// (the checks) or `KOKON_PREFIX_CACHE=fp32`. `KOKON_PREFIX_CACHE_MB` sets the limit.
let prefixCache: MTLBuffer?
- static let prefixCacheLimit = 512 << 20
- /// The conditioning whose prefix the cache holds.
- private var cachedEncoder: [Float]?
+ let prefixHalf: Bool
+ /// Up to 1 GB below 32 GB of memory, an eighth of it from there. M2 Pro 16 GB, edits, fp16 cache, 2026-10-25: a
+ /// 768 × 448 reference (0.72 GB) made the steps after the first 4.95 → 3.48 s (image 46.2 → 38.1 s); a 1344 × 768
+ /// one (2.13 GB) gained nothing, the DiT's file pages evicted and read again (40 GB of page-ins an image against
+ /// 21 without the cache). The value from 32 GB is not measured.
+ static var prefixCacheLimit: Int { Chip.current.memoryGB < 30 ? 1 << 30 : Int(Chip.current.memoryGB * Double(1 << 30)) / 8 }
+ /// The conditioning (encoder rows, then the references' tokens) whose prefix the cache holds.
+ private var cachedConditioning: [Float]?
+ /// Latent tokens of the condition images (every image but the last), in order.
+ public var referenceTokens: Int { sequence.images.dropLast().reduce(0) { $0 + $1.tokens } }
private var nextReserve = 0
private var products: [String: MPSMatrixMultiplication] = [:]
/// The AMX and the Neural Engine beside the GPU on the large products and the target's attention
@@ -85,7 +95,8 @@ public final class QwenDiT {
d * QwenDiT.hidden, d * QwenDiT.hidden,
n * 128, 2 * 5 * d, QwenDiT.layers * 2 * QwenDiT.headDim, d, t * d, t * d,
s * QwenDiT.channels, s * QwenDiT.channels,
- QwenDiT.heads * (n + 63) / 64 * 64 * QwenDiT.headDim / 2, QwenDiT.heads * (n + 63) / 64 * 64 * QwenDiT.headDim / 2]
+ QwenDiT.heads * (n + 63) / 64 * 64 * QwenDiT.headDim / 2, QwenDiT.heads * (n + 63) / 64 * 64 * QwenDiT.headDim / 2,
+ (QwenDiT.padded(sequence.images.dropLast().reduce(0) { $0 + $1.tokens }) + 64) * QwenDiT.channels]
let arena = try Arena(device: context.device, bytes: Arena.bytes(sizes.map { $0 * 4 }))
self.arena = arena
var slices = try sizes.map { try arena.slice(floats: $0) }.makeIterator()
@@ -94,12 +105,14 @@ public final class QwenDiT {
reserves = [slices.next()!, slices.next()!]
rope = slices.next()!; modulation = slices.next()!; qkNorms = slices.next()!; textNorm = slices.next()!
textIn = slices.next()!; textOut = slices.next()!; latent = slices.next()!; output = slices.next()!
- kPacked = slices.next()!; vPacked = slices.next()!
+ kPacked = slices.next()!; vPacked = slices.next()!; referenceLatent = slices.next()!
paddedKeys = (n + 63) / 64 * 64
let environment = ProcessInfo.processInfo.environment
- let cacheBytes = QwenDiT.layers * 2 * sequence.prefix * d * 4
- if environment["KOKON_PREFIX_CACHE"] != "0", sequence.prefix > 0, cacheBytes <= QwenDiT.prefixCacheLimit {
+ prefixHalf = environment["KOKON_ATTENTION"] == nil && environment["KOKON_PREFIX_CACHE"] != "fp32"
+ let cacheBytes = QwenDiT.layers * 2 * sequence.prefix * d * (prefixHalf ? 2 : 4)
+ let limit = Int(environment["KOKON_PREFIX_CACHE_MB"] ?? "").map { $0 << 20 } ?? QwenDiT.prefixCacheLimit
+ if environment["KOKON_PREFIX_CACHE"] != "0", sequence.prefix > 0, cacheBytes <= limit {
prefixCache = try context.scratch(cacheBytes, label: "prefix K/V")
} else {
prefixCache = nil
@@ -200,9 +213,13 @@ public final class QwenDiT {
public typealias Probe = (String, UnsafePointer<Float>, Int) -> Void
/// One evaluation: `encoder` the encoder's last layer `[T, 4096]` (all positions, image slots included),
- /// `target` the target latent `[S, 64]` as packed tokens, `sigma` in [0, 1]. Returns the velocity `[S, 64]`.
- public func forward(encoder: [Float], target: [Float], sigma: Float, probe: Probe? = nil) throws -> [Float] {
+ /// `references` the condition images' packed latent tokens `[R, 64]` (an edit; in order), `target` the target
+ /// latent `[S, 64]` as packed tokens, `sigma` in [0, 1]. Returns the velocity `[S, 64]`.
+ public func forward(encoder: [Float], references: [Float] = [], target: [Float], sigma: Float, probe: Probe? = nil) throws -> [Float] {
let d = QwenDiT.dim, n = sequence.count, p = sequence.prefix, s = sequence.target
+ guard references.count == referenceTokens * QwenDiT.channels else {
+ throw QwenSequence.Failure.layout("\(references.count / QwenDiT.channels) reference tokens for \(referenceTokens)")
+ }
let started = DispatchTime.now().uptimeNanoseconds
timings = [:]
castInput = nil
@@ -215,6 +232,7 @@ public final class QwenDiT {
arena.pointer(textIn, floatOffset: i * d).update(from: Array(encoder[row.encoder * d..<(row.encoder + 1) * d]), count: d)
}
arena.pointer(latent).update(from: target, count: s * QwenDiT.channels)
+ arena.pointer(referenceLatent).update(from: references, count: references.count)
// The cached path when there is a cache and no probe (the probes compare every row of every stage).
let cached = prefixCache != nil && probe == nil
@@ -223,8 +241,9 @@ public final class QwenDiT {
// A new conditioning: the whole sequence once, the prefix's K and V saved on the way. A pass of the prefix
// alone took 1.86 s at 1024² (products of 64 rows run far below the GPU's rate); riding along the first
// evaluation it costs its 64 rows.
- let saving = cached && cachedEncoder != encoder
- if !cached || saving { try embedText(commands: commands) }
+ let conditioning = encoder + references
+ let saving = cached && cachedConditioning != conditioning
+ if !cached || saving { try embedText(commands: commands); try embedReferences(commands: commands) }
try linear("img_in.weight", a: latent, c: x, m: s, cRow: p, commands: commands)
if let probe { try finish(commands); probe("joint_in", arena.pointer(x), n * d); commands = try begin() }
@@ -254,7 +273,7 @@ public final class QwenDiT {
try layerNorm(x, into: normed, rows: rows, prefixMod: 4 * d, targetMod: 5 * d + 4 * d, commands: commands)
if let probe { try finish(commands); probe("norm_out", arena.pointer(normed), n * d); commands = try begin() }
try linear("proj_out.weight", a: normed, aRow: p, c: output, m: s, commands: commands)
- if saving { cachedEncoder = encoder }
+ if saving { cachedConditioning = conditioning }
try finish(commands)
if let coexecutor, !profiling, balancing { try coexecutor.rebalance() }
let out = Array(UnsafeBufferPointer(start: arena.pointer(output), count: s * QwenDiT.channels))
@@ -292,9 +311,38 @@ public final class QwenDiT {
blit.endEncoding()
}
+ /// The condition images' tokens through img_in (into `o`, free before the first layer), copied to their joint rows.
+ func embedReferences(commands: MTLCommandBuffer) throws {
+ let d = QwenDiT.dim, tokens = referenceTokens
+ guard tokens > 0 else { return }
+ try linear("img_in.weight", a: referenceLatent, c: o, m: tokens, commands: commands)
+ let blit = commands.makeBlitCommandEncoder()!
+ var at = 0
+ for (i, grid) in sequence.images.dropLast().enumerated() {
+ blit.copy(from: arena.buffer, sourceOffset: o.offset + at * d * 4, to: arena.buffer,
+ destinationOffset: x.offset + sequence.blockStarts[i] * d * 4, size: grid.tokens * d * 4)
+ at += grid.tokens
+ }
+ blit.endEncoding()
+ }
+
/// The prefix's K and V of `layer` between k, v and the cache (`save`: into the cache).
- func copyPrefix(layer: Int, save: Bool, commands: MTLCommandBuffer) {
+ func copyPrefix(layer: Int, save: Bool, commands: MTLCommandBuffer) throws {
guard let prefixCache else { return }
+ if prefixHalf {
+ let count = sequence.prefix * QwenDiT.dim
+ for (i, slice) in [k, v].enumerated() {
+ let cached = (layer * 2 + i) * count * 2
+ try compute(save ? "pack_half" : "widen_half", commands: commands) { e in
+ e.setBuffer(save ? arena.buffer : prefixCache, offset: save ? slice.offset : cached, index: 0)
+ e.setBuffer(save ? prefixCache : arena.buffer, offset: save ? cached : slice.offset, index: 1)
+ var c = UInt32(count)
+ e.setBytes(&c, length: 4, index: 2)
+ e.dispatchThreads(MTLSize(width: count, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))
+ }
+ }
+ return
+ }
let bytes = sequence.prefix * QwenDiT.dim * 4
let blit = commands.makeBlitCommandEncoder()!
for (i, slice) in [k, v].enumerated() {
@@ -332,8 +380,8 @@ public final class QwenDiT {
causal: segment.isText, commands: commands)
}
}
- if mode == .saving { copyPrefix(layer: layer, save: true, commands: commands) }
- if mode == .target { copyPrefix(layer: layer, save: false, commands: commands) }
+ if mode == .saving { try copyPrefix(layer: layer, save: true, commands: commands) }
+ if mode == .target { try copyPrefix(layer: layer, save: false, commands: commands) }
try attend(queries: p, count: n - p, keys: n, causal: false, commands: commands)
try linear(block + "attn.to_out.0.weight", a: attended, aRow: first, c: o, m: count, cRow: first,
next: block + "img_mlp.gate_layer.weight", commands: commands)
diff --git a/engine-mac/Sources/KokonEngine/QwenPipeline.swift b/engine-mac/Sources/KokonEngine/QwenPipeline.swift
index b26c56a..aa6371a 100644
--- a/engine-mac/Sources/KokonEngine/QwenPipeline.swift
+++ b/engine-mac/Sources/KokonEngine/QwenPipeline.swift
@@ -77,6 +77,14 @@ public final class QwenPipeline {
(width / 32 * 32, height / 32 * 32)
}
+ /// An edit's condition image: interleaved 8-bit RGBA (straight alpha) at the size it is encoded at (sides multiples
+ /// of 32: the vision tower's grid).
+ public struct Reference {
+ public let rgba: [UInt8]
+ public let width, height: Int
+ public init(rgba: [UInt8], width: Int, height: Int) { (self.rgba, self.width, self.height) = (rgba, width, height) }
+ }
+
/// Text to image at `width` × `height` (multiples of 32). `noise`: a latent `[64, h, w]` to start from instead of
/// the seed's; `conditioning`: the encoder's output `[T, 4096]` to use instead of encoding `prompt` (the checks);
/// `step`: called after each step with its index and the packed latent `[h·w, 64]`.
@@ -84,48 +92,106 @@ public final class QwenPipeline {
public func textToImage(prompt: String, width: Int, height: Int, seed: UInt64, noise: [Float]? = nil,
conditioning given: [Float]? = nil, sigmas: [Float] = QwenPipeline.turboSigmas,
step: ((Int, [Float]) -> Void)? = nil) throws -> [Float] {
+ try generate(prompt: prompt, width: width, height: height, seed: seed, noise: noise,
+ conditioning: given.map { ($0, [Bool](repeating: false, count: $0.count / QwenDiT.dim)) },
+ sigmas: sigmas, step: step)
+ }
+
+ /// **The vision tower's file** (mmproj, `--llm_vision`): edits need it.
+ public var visionPath: String?
+
+ /// Text to image, or **an edit** with `references` (`QwenImage21Pipeline.__call__` with `image`): each reference goes
+ /// through the vision tower into the text encoder's slots and through the VAE encoder into the DiT's prefix, in
+ /// the slots the encoder marks (`img_shapes`: the references, then the target). sd.cpp's `vlm_use_vae_input`: the
+ /// tower sees the image the VAE encodes. `conditioning`: the encoder's rows and which are image slots, instead of
+ /// encoding (the checks); `referenceLatents`: the references' packed tokens instead of the VAE encoder's.
+ public func generate(prompt: String, references: [Reference] = [], width: Int, height: Int, seed: UInt64,
+ noise: [Float]? = nil, conditioning given: (rows: [Float], slots: [Bool])? = nil,
+ referenceLatents givenLatents: [Float]? = nil, sigmas: [Float] = QwenPipeline.turboSigmas,
+ step: ((Int, [Float]) -> Void)? = nil) throws -> [Float] {
guard width % 32 == 0, height % 32 == 0, width >= 64, height >= 64 else {
throw QwenSequence.Failure.layout("\(width)×\(height): each side must be a multiple of 32, at least 64")
}
+ for r in references where r.width % 32 != 0 || r.height % 32 != 0 || r.rgba.count != 4 * r.width * r.height {
+ throw QwenSequence.Failure.layout("a reference of \(r.width)×\(r.height): sides must be multiples of 32")
+ }
let started = DispatchTime.now().uptimeNanoseconds
timings = [:]
func since(_ t: UInt64) -> Double { Double(DispatchTime.now().uptimeNanoseconds - t) * 1e-9 }
let (h, w) = (height / 16, width / 16)
let channels = QwenDiT.channels
- // 1. The prompt. The encoder and its 5 GB mapping are gone when this scope ends.
+ // 1. The references through the vision tower (its 1.1 GB file unmapped after).
var t = DispatchTime.now().uptimeNanoseconds
- let (conditioning, rows): ([Float], Int) = try {
- if let given { return (given, given.count / QwenDiT.dim) }
+ var images: [QwenVisionTower.Output] = []
+ if !references.isEmpty && given == nil {
+ guard let visionPath else {
+ throw GGUFFile.Failure.unsupported("edits need the vision projector (mmproj): start the engine with --llm_vision")
+ }
+ let tower = try QwenVisionTower(file: try GGUFFile(path: visionPath), context: context)
+ defer { tower.release() }
+ for r in references {
+ let input = QwenVisionTower.pixels(rgba: r.rgba, width: r.width, height: r.height)
+ images.append(try tower.encode(rgb: input.rgb, width: input.width, height: input.height))
+ }
+ timings["vision"] = since(t)
+ log(String(format: "vision tower: %d image(s), %d slots in %.2f s", images.count, images.reduce(0) { $0 + $1.slots },
+ timings["vision"]!))
+ }
+
+ // 2. The prompt. The encoder and its 5 GB mapping are gone when this scope ends.
+ t = DispatchTime.now().uptimeNanoseconds
+ let (conditioning, rows, slots): ([Float], Int, [Bool]) = try {
+ if let given { return (given.rows, given.rows.count / QwenDiT.dim, given.slots) }
if keepModels && keepEncoder {
if encoder == nil {
let file = try GGUFFile(path: encoderPath)
(encoderFile, encoder) = (file, try QwenTextEncoder(file: file, context: context))
}
defer { encoder!.release() }
- let c = try encoder!.conditioning(prompt: prompt)
- return (c.rows, c.count)
+ let c = try encoder!.conditioning(prompt: prompt, images: images)
+ return (c.rows, c.count, c.imageSlots)
}
let encoder = try handedOver ?? QwenTextEncoder(file: try GGUFFile(path: encoderPath), context: context)
handedOver = nil
defer { encoder.release() }
- let c = try encoder.conditioning(prompt: prompt)
- return (c.rows, c.count)
+ let c = try encoder.conditioning(prompt: prompt, images: images)
+ return (c.rows, c.count, c.imageSlots)
}()
timings["encode"] = since(t)
log(String(format: "text encoder: %d tokens in %.2f s", rows, timings["encode"]!))
events?(.encoded(tokens: rows, seconds: timings["encode"]!))
- // 2. Noise, packed: token y·w + x holds the 64 channels of cell (y, x).
+ // 3. The references' latents (`_encode_vae_image`), packed and in order.
+ t = DispatchTime.now().uptimeNanoseconds
+ var referenceTokens: [Float] = givenLatents ?? []
+ if givenLatents == nil, !references.isEmpty {
+ if vae == nil { vae = try QwenVAE(path: vaePath, context: context) }
+ for r in references {
+ // VaeImageProcessor.preprocess: x / 255, then 2x − 1; planar RGBA.
+ let plane = r.width * r.height
+ var input = [Float](repeating: 0, count: 4 * plane)
+ for p in 0..<plane { for c in 0..<4 { input[c * plane + p] = Float(r.rgba[p * 4 + c]) / 255 * 2 - 1 } }
+ let latent = try vae!.encode(input, height: r.height, width: r.width)
+ referenceTokens += QwenPipeline.pack(latent, height: r.height / 16, width: r.width / 16)
+ }
+ if !keepVAE { vae = nil }
+ timings["vae encode"] = since(t)
+ log(String(format: "VAE encoder: %d reference(s) in %.2f s", references.count, timings["vae encode"]!))
+ }
+
+ // 4. Noise, packed: token y·w + x holds the 64 channels of cell (y, x).
let start = noise ?? Noise.philox(seed: seed, count: channels * h * w) // sd.cpp's, value for value
guard start.count == channels * h * w else {
throw QwenSequence.Failure.layout("noise of \(start.count) values for a \(channels)×\(h)×\(w) latent")
}
var x = QwenPipeline.pack(start, height: h, width: w)
- // 3. The steps.
+ // 5. The steps.
t = DispatchTime.now().uptimeNanoseconds
- let sequence = try QwenSequence.textToImage(text: rows, target: .init(height: h, width: w))
+ let target = QwenSequence.Grid(height: h, width: w)
+ let grids = references.map { QwenSequence.Grid(height: $0.height / 16, width: $0.width / 16) }
+ let sequence = try QwenSequence(slots: slots + [Bool](repeating: true, count: target.tokens / 4), images: grids + [target])
try {
let file = try ditFile ?? GGUFFile(path: ditPath)
defer { context.release(file) }
@@ -144,10 +210,12 @@ public final class QwenPipeline {
defer { dit.cancelled = { false } }
try dit.prepareModulation([0] + sigmas.dropLast())
timings["dit load"] = since(t)
+ log(String(format: "DiT: %d rows, prefix %d, prefix K/V cache %@", sequence.count, sequence.prefix,
+ dit.prefixCache.map { String(format: "%.2f GB (%@)", Double($0.length) / 1e9, dit.prefixHalf ? "fp16" : "fp32") } ?? "off"))
for i in 0..<(sigmas.count - 1) {
if cancelled() { throw QwenDiT.Cancelled() }
let stepStart = DispatchTime.now().uptimeNanoseconds
- let velocity = try dit.forward(encoder: conditioning, target: x, sigma: sigmas[i])
+ let velocity = try dit.forward(encoder: conditioning, references: referenceTokens, target: x, sigma: sigmas[i])
let dt = sigmas[i + 1] - sigmas[i]
for j in x.indices { x[j] += dt * velocity[j] }
timings["step \(i + 1)"] = since(stepStart)
@@ -158,7 +226,7 @@ public final class QwenPipeline {
shares = dit.coexecutor?.shares
}()
- // 4. The image.
+ // 6. The image.
if !keepDiT { dit = nil } // its working memory goes before the VAE's
t = DispatchTime.now().uptimeNanoseconds
events?(.decoding)
@@ -173,9 +241,24 @@ public final class QwenPipeline {
/// **Prompt+**: the text encoder's LLM writes the assistant's reply to `prompt` (`QwenTextGenerator`), on the same
/// file and with the same keeping rules as the image's encoder.
- public func generateText(system: String, prompt: String, maxTokens: Int, sampling: TextSampler.Parameters, seed: Int64,
+ /// `images`: shown before the prompt (an edit's references), through the vision tower first (`visionPath`).
+ public func generateText(system: String, prompt: String, images: [Reference] = [], maxTokens: Int,
+ sampling: TextSampler.Parameters, seed: Int64,
onPiece: (String) -> Bool) throws -> QwenTextGenerator.Result {
let started = DispatchTime.now().uptimeNanoseconds
+ var outputs: [QwenVisionTower.Output] = []
+ if !images.isEmpty {
+ guard let visionPath else {
+ throw GGUFFile.Failure.unsupported("images need the vision projector (mmproj): start the engine with --llm_vision")
+ }
+ let tower = try QwenVisionTower(file: try GGUFFile(path: visionPath), context: context)
+ defer { tower.release() }
+ for r in images {
+ let input = QwenVisionTower.pixels(rgba: r.rgba, width: r.width, height: r.height)
+ outputs.append(try tower.encode(rgb: input.rgb, width: input.width, height: input.height))
+ }
+ }
+ let imageSeconds = Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9
let encoder: QwenTextEncoder
if keepModels && keepEncoder {
if self.encoder == nil {
@@ -188,10 +271,12 @@ public final class QwenPipeline {
handedOver = encoder // for the image that follows
}
defer { encoder.release() }
- var result = try QwenTextGenerator(encoder: encoder).generate(system: system, prompt: prompt, maxTokens: maxTokens,
- sampling: sampling, seed: seed, onPiece: onPiece)
+ var result = try QwenTextGenerator(encoder: encoder).generate(system: system, prompt: prompt, images: outputs,
+ maxTokens: maxTokens, sampling: sampling, seed: seed,
+ onPiece: onPiece)
// With opening the file and reading its tokenizer.
- result.prepare = Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9 - result.prefill - result.decode
+ result.image = imageSeconds
+ result.prepare = Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9 - result.prefill - result.decode - imageSeconds
return result
}
diff --git a/engine-mac/Sources/KokonEngine/QwenTextEncoder.swift b/engine-mac/Sources/KokonEngine/QwenTextEncoder.swift
index e8b0326..39ee8a3 100644
--- a/engine-mac/Sources/KokonEngine/QwenTextEncoder.swift
+++ b/engine-mac/Sources/KokonEngine/QwenTextEncoder.swift
@@ -68,9 +68,14 @@ public final class QwenTextEncoder {
/// the processor does before tokenizing).
public func tokens(prompt: String, images: [QwenVisionTower.Output] = []) -> [Int] {
guard !images.isEmpty else { return tokenizer.encode(QwenTextEncoder.textToImage(prompt)) }
+ return expand(tokenizer.encode(QwenTextEncoder.imageToImage(prompt, images: images.count)), images: images)
+ }
+
+ /// Each `<|image_pad|>` of `ids` repeated to its image's slots, in order.
+ public func expand(_ ids: [Int], images: [QwenVisionTower.Output]) -> [Int] {
let pad = imagePad
var out: [Int] = [], next = 0
- for id in tokenizer.encode(QwenTextEncoder.imageToImage(prompt, images: images.count)) {
+ for id in ids {
if id == pad, next < images.count {
out += [Int](repeating: pad, count: images[next].slots)
next += 1
diff --git a/engine-mac/Sources/KokonEngine/QwenTextGeneration.swift b/engine-mac/Sources/KokonEngine/QwenTextGeneration.swift
index d5326c7..096b7e6 100644
--- a/engine-mac/Sources/KokonEngine/QwenTextGeneration.swift
+++ b/engine-mac/Sources/KokonEngine/QwenTextGeneration.swift
@@ -22,8 +22,8 @@ public final class QwenTextGenerator {
public var finish = Finish.stop
public var seed: Int64 = 0
public var promptTokens = 0, generatedTokens = 0
- /// Seconds: tokenizing and buffers, prefill (to the first token), decode (first token to last).
- public var prepare = 0.0, prefill = 0.0, decode = 0.0
+ /// Seconds: tokenizing and buffers, prefill (to the first token), decode (first token to last), the vision tower.
+ public var prepare = 0.0, prefill = 0.0, decode = 0.0, image = 0.0
/// Every token sampled, the stop token included.
public var tokens: [Int] = []
}
@@ -37,15 +37,22 @@ public final class QwenTextGenerator {
context = encoder.context
}
- /// sd.cpp's chat template for text generation without images: the system turn when `system` is not empty.
- public static func template(system: String, prompt: String) -> String {
- (system.isEmpty ? "" : "<|im_start|>system\n\(system)<|im_end|>\n") + "<|im_start|>user\n\(prompt)<|im_end|>\n<|im_start|>assistant\n"
+ /// sd.cpp's chat template for text generation (src/pipeline/text.cpp): the system turn when `system` is not empty;
+ /// each image before the prompt as `<|vision_start|><|image_pad|><|vision_end|>`, labelled `<image1>`, ` <image2>`…
+ /// when there are several.
+ public static func template(system: String, prompt: String, images: Int = 0) -> String {
+ let markers = (0..<images).map { i in
+ (images > 1 ? (i == 0 ? "" : " ") + "<image\(i + 1)>" : "") + "<|vision_start|><|image_pad|><|vision_end|>"
+ }.joined()
+ return (system.isEmpty ? "" : "<|im_start|>system\n\(system)<|im_end|>\n") + "<|im_start|>user\n\(markers)\(prompt)<|im_end|>\n<|im_start|>assistant\n"
}
/// Writes the assistant's reply to `prompt`. `onPiece` gets the text as it is written (whole UTF-8 characters);
- /// returning false stops it (`cancelled`). `logits`: each step's logits, for the checks.
- public func generate(system: String, prompt: String, maxTokens: Int, sampling: TextSampler.Parameters, seed: Int64,
- onPiece: (String) -> Bool = { _ in true },
+ /// returning false stops it (`cancelled`). `logits`: each step's logits, for the checks. `images`: the vision
+ /// tower's outputs of the images shown before the prompt; their slots take 3D positions (`QwenTextEncoder.positions`),
+ /// so the reply's positions run ahead of its cache slots by a fixed offset.
+ public func generate(system: String, prompt: String, images: [QwenVisionTower.Output] = [], maxTokens: Int,
+ sampling: TextSampler.Parameters, seed: Int64, onPiece: (String) -> Bool = { _ in true },
logits: ((Int, UnsafePointer<Float>) -> Void)? = nil) throws -> Result {
var result = Result()
let started = DispatchTime.now().uptimeNanoseconds
@@ -55,7 +62,11 @@ public final class QwenTextGenerator {
throw GGUFFile.Failure.unsupported("the tokenizer has no <|im_end|> or <|endoftext|>")
}
let stops: Set<Int> = [imEnd, endOfText]
- let ids = tokenizer.encode(QwenTextGenerator.template(system: system, prompt: prompt))
+ let ids = encoder.expand(tokenizer.encode(QwenTextGenerator.template(system: system, prompt: prompt, images: images.count)),
+ images: images)
+ let positions = try QwenTextEncoder.positions(ids: ids, imagePad: images.isEmpty ? -1 : encoder.imagePad,
+ grids: images.map { ($0.height / 32, $0.width / 32) })
+ let offset = Int(positions.last.map { $0[0] + 1 } ?? 0) - ids.count // 0 for text: position = slot
let capacity = ids.count + maxTokens
guard capacity <= QwenTextGenerator.maxContext else {
throw GGUFFile.Failure.unsupported("prompt (\(ids.count) tokens) + max_tokens (\(maxTokens)) exceeds the \(QwenTextGenerator.maxContext)-token limit")
@@ -64,12 +75,12 @@ public final class QwenTextGenerator {
result.seed = seed >= 0 ? seed : Int64.random(in: 0...0x7fff_ffff)
var sampler = TextSampler(sampling, seed: UInt64(bitPattern: result.seed))
let cache = try KVCache(context: context, layers: QwenTextEncoder.layers, kvHeads: QwenTextEncoder.kvHeads, capacity: capacity)
- let step = try Decoder(generator: self, cache: cache)
+ let step = try Decoder(generator: self, cache: cache, positionOffset: offset)
result.prepare = since(started)
// Prefill: the prompt's K and V into the cache, the last row's hidden state to the head.
var t = DispatchTime.now().uptimeNanoseconds
- let hidden = try encoder.forward(ids: ids, cache: cache)
+ let hidden = try encoder.forward(ids: ids, cache: cache, images: images)
var next = try step.head(hidden: Array(hidden[((ids.count - 1) * QwenTextEncoder.dim)...]))
var stream = TokenStreamDecoder(tokenizer: tokenizer)
while true {
@@ -112,7 +123,8 @@ public final class QwenTextGenerator {
/// The GPU's seconds for the last token.
var lastGPU = 0.0
- init(generator: QwenTextGenerator, cache: KVCache) throws {
+ /// `positionOffset`: a slot's RoPE position minus the slot (images before the prompt).
+ init(generator: QwenTextGenerator, cache: KVCache, positionOffset: Int = 0) throws {
self.generator = generator
self.cache = cache
context = generator.context
@@ -136,7 +148,7 @@ public final class QwenTextGenerator {
}
}
(n + QwenTextEncoder.layers * (2 * d + 256)).update(from: try Dequant.array(file, "output_norm.weight"), count: d)
- let table = QwenTextEncoder.ropeTable(rows: cache.capacity)
+ let table = QwenTextEncoder.ropeTable(positions: (0..<cache.capacity).map { SIMD3(repeating: Int32($0 + positionOffset)) })
rope = try context.scratch(table.count * 4, label: "decode rope")
rope.contents().copyMemory(from: table, byteCount: table.count * 4)
logits = try context.scratch(QwenTextGenerator.vocabulary * 4, label: "logits")
diff --git a/engine-mac/Sources/KokonEngine/TextChecks.swift b/engine-mac/Sources/KokonEngine/TextChecks.swift
index 5140f7e..ca7c6a0 100644
--- a/engine-mac/Sources/KokonEngine/TextChecks.swift
+++ b/engine-mac/Sources/KokonEngine/TextChecks.swift
@@ -105,19 +105,45 @@ extension Checks {
/// the 8-bit pixels.
/// `text`: a `reference_text_encoder.py` directory whose conditioning replaces the engine's encoder, to measure
/// the DiT and the sampler alone.
- public static func generate(encoder: String, dit: String, vae: String, reference directory: String, text: String? = nil) throws {
+ /// **An edit** (the reference has `references`): `text` is needed (its images). Without `mmproj` the text reference's
+ /// conditioning and the generate reference's latents of the images go in (the DiT, the sampler and the decode alone);
+ /// with it, the engine's whole edit: vision tower, text encoder, VAE encoder on the images.
+ public static func generate(encoder: String, dit: String, vae: String, reference directory: String, text: String? = nil,
+ mmproj: String? = nil) throws {
let meta = try JSONSerialization.jsonObject(with: Data(contentsOf: URL(fileURLWithPath: directory)
.appendingPathComponent("meta.json"))) as! [String: Any]
let prompt = meta["prompt"] as! String, grid = meta["grid"] as! [Int]
let (h, w) = (grid[0], grid[1])
let pipeline = try QwenPipeline(encoder: encoder, dit: dit, vae: vae)
+ pipeline.visionPath = mmproj
func report(_ name: String, _ got: UnsafePointer<Float>, _ want: [Float]) {
let r = compare(got, want)
print(String(format: " %-12@ rel %.2e max |Δ| %.3g%@", name as NSString, r.relative, r.maxAbs, r.finite ? "" : " NON-FINITE"))
}
- let image = try pipeline.textToImage(prompt: prompt, width: w * 16, height: h * 16, seed: 0,
- noise: try reference(directory, "noise"),
- conditioning: try text.map { try reference($0, "conditioning") }) { i, latent in
+ var references: [QwenPipeline.Reference] = [], given: (rows: [Float], slots: [Bool])?, latents: [Float]?
+ let sizes = meta["references"] as? [[Int]] ?? []
+ if !sizes.isEmpty {
+ guard let text else { throw GGUFFile.Failure.unsupported("an edit reference: pass its text reference directory") }
+ for i in sizes.indices {
+ let png = try QwenPipeline.decodeImage(Data(contentsOf: URL(fileURLWithPath: text).appendingPathComponent("image\(i + 1).png")))
+ references.append(.init(rgba: png.rgba, width: png.width, height: png.height))
+ }
+ if mmproj == nil {
+ let textMeta = try JSONSerialization.jsonObject(with: Data(contentsOf: URL(fileURLWithPath: text)
+ .appendingPathComponent("meta.json"))) as! [String: Any]
+ given = (try reference(text, "conditioning"), (textMeta["image_pad_mask"] as! [Int]).map { $0 != 0 })
+ latents = try sizes.indices.flatMap { try reference(directory, "reference\($0)") }
+ print("edit: the references' conditioning and latents")
+ } else {
+ print("edit: the engine's vision tower, text encoder and VAE encoder")
+ }
+ } else if let text {
+ let rows = try reference(text, "conditioning")
+ given = (rows, [Bool](repeating: false, count: rows.count / QwenDiT.dim))
+ }
+ let image = try pipeline.generate(prompt: prompt, references: references, width: w * 16, height: h * 16, seed: 0,
+ noise: try reference(directory, "noise"), conditioning: given,
+ referenceLatents: latents) { i, latent in
latent.withUnsafeBufferPointer { report("latent\(i)", $0.baseAddress!, try! reference(directory, "latent\(i)")) }
}
image.withUnsafeBufferPointer { report("image", $0.baseAddress!, try! reference(directory, "image")) }
@@ -183,24 +209,37 @@ extension Checks {
/// `<dir>/engine.json`, its logits beside); once the reference has run on those tokens, each step's logits are
/// compared with transformers' (relative error, the greedy choice, the margin between the two best) and the
/// engine's speed is reported. `system` / `prompt` from `<dir>/engine.json` when it exists.
- public static func textGeneration(path: String, directory: String, tokens: Int = 48) throws {
+ /// `image` + `mmproj`: Prompt+ looking at an image (sd.cpp's template with it before the prompt).
+ public static func textGeneration(path: String, directory: String, tokens: Int = 48, image: String? = nil, mmproj: String? = nil) throws {
let dir = URL(fileURLWithPath: directory)
try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
let previous = (try? JSONSerialization.jsonObject(with: Data(contentsOf: dir.appendingPathComponent("engine.json")))) as? [String: Any]
let system = previous?["system"] as? String
?? "You write prompts for an image generation model. Expand the user's request into one detailed English paragraph that describes the finished image: subject, setting, composition, lighting and style. Output only the prompt."
let prompt = previous?["prompt"] as? String ?? "un gatto rosso che dorme su un davanzale, sera"
- let file = try GGUFFile(path: path)
+ let imagePath = image ?? previous?["image"] as? String
let context = try MetalContext()
+ var images: [QwenVisionTower.Output] = []
+ if let imagePath {
+ guard let mmproj else { throw GGUFFile.Failure.unsupported("an image needs the mmproj GGUF") }
+ let png = try QwenPipeline.decodeImage(Data(contentsOf: URL(fileURLWithPath: imagePath)))
+ let input = QwenVisionTower.pixels(rgba: png.rgba, width: png.width, height: png.height)
+ let tower = try QwenVisionTower(file: try GGUFFile(path: mmproj), context: context)
+ images = [try tower.encode(rgb: input.rgb, width: input.width, height: input.height)]
+ tower.release()
+ print(String(format: "image %@: %d slots, vision tower %.2f s", imagePath, images[0].slots, tower.seconds))
+ }
+ let file = try GGUFFile(path: path)
let encoder = try QwenTextEncoder(file: file, context: context)
let generator = QwenTextGenerator(encoder: encoder)
let vocabulary = QwenTextGenerator.vocabulary
var logits = [Float](repeating: 0, count: tokens * vocabulary)
- let result = try generator.generate(system: system, prompt: prompt, maxTokens: tokens, sampling: .greedy, seed: 0, logits: { step, l in
+ let result = try generator.generate(system: system, prompt: prompt, images: images, maxTokens: tokens, sampling: .greedy, seed: 0, logits: { step, l in
logits.withUnsafeMutableBufferPointer { ($0.baseAddress! + step * vocabulary).update(from: l, count: vocabulary) }
})
let steps = result.tokens.count
- let promptIDs = encoder.tokenizer.encode(QwenTextGenerator.template(system: system, prompt: prompt))
+ let promptIDs = encoder.expand(encoder.tokenizer.encode(QwenTextGenerator.template(system: system, prompt: prompt, images: images.count)),
+ images: images)
print(String(format: "%d prompt tokens, %d generated (%@): prefill %.2f s, decode %.2f s, %.1f tokens/s",
result.promptTokens, result.generatedTokens, result.finish.rawValue, result.prefill, result.decode,
Double(steps - 1) / max(result.decode, 1e-9)))
@@ -208,7 +247,8 @@ extension Checks {
if let previous, let generated = previous["generated"] as? [Int], generated != result.tokens {
print("NOTE: not the tokens of the engine's previous run (\(generated.count) then, \(steps) now): the reference must be run again")
}
- let meta: [String: Any] = ["system": system, "prompt": prompt, "prompt_ids": promptIDs, "generated": result.tokens, "text": result.text]
+ var meta: [String: Any] = ["system": system, "prompt": prompt, "prompt_ids": promptIDs, "generated": result.tokens, "text": result.text]
+ if let imagePath { meta["image"] = imagePath }
try JSONSerialization.data(withJSONObject: meta, options: [.prettyPrinted, .sortedKeys]).write(to: dir.appendingPathComponent("engine.json"))
try logits.prefix(steps * vocabulary).withUnsafeBufferPointer { Data(buffer: $0) }.write(to: dir.appendingPathComponent("engine_logits.f32"))
diff --git a/engine-mac/Sources/kokon-engine-mac/main.swift b/engine-mac/Sources/kokon-engine-mac/main.swift
index 4a71f4e..2bf452d 100644
--- a/engine-mac/Sources/kokon-engine-mac/main.swift
+++ b/engine-mac/Sources/kokon-engine-mac/main.swift
@@ -14,18 +14,24 @@ kokon-engine-mac: Kokon's Apple Silicon engine (in development)
check-tokenizer <encoder.gguf> <fixture.json> the tokenizer against tools/reference_tokenizer.py
check-text <encoder.gguf> <reference dir> [mmproj.gguf] the text encoder against tools/reference_text_encoder.py
check-vision <mmproj.gguf> <reference dir> the vision tower against tools/reference_vision.py, then its time
- check-generate <encoder.gguf> <dit.gguf> <vae.safetensors> <reference dir> [text reference dir]
- text to image against tools/reference_generate.py, from its noise
- (and from the text reference's conditioning when given)
- check-textgen <encoder.gguf> <dir> [tokens] text generation (Prompt+), greedy, against tools/reference_text_generation.py
+ check-generate <encoder.gguf> <dit.gguf> <vae.safetensors> <reference dir> [text reference dir] [mmproj.gguf]
+ text to image or an edit against tools/reference_generate.py, from its noise
+ (and from the text reference's conditioning when given; an edit needs the
+ text reference, and with the mmproj runs the engine's whole edit)
+ check-textgen <encoder.gguf> <dir> [tokens] [image.png mmproj.gguf]
+ text generation (Prompt+), greedy, against tools/reference_text_generation.py
time-matvec <encoder.gguf> the decode's products on the encoder's weights: ms and GB/s
check-steps <dit.gguf> <generate reference dir> <text reference dir>
each step alone against tools/reference_generate.py (its latents in)
check-noise <philox.bin> the seed's noise against sd.cpp's generator (tools/philox_reference.cpp)
generate <encoder.gguf> <dit.gguf> <vae.safetensors> "<prompt>" [--size WxH] [--seed N] [--out file.png]
- text to image, Turbo's 8 steps (default 1024x1024, seed 42, out.png)
- serve --diffusion-model <dit.gguf> --vae <vae.safetensors> --llm <encoder.gguf> [--listen-ip 127.0.0.1] [--listen-port 1234]
- the engine as a server speaking sd-server's /sdcpp/v1 API (text to image)
+ [--ref image.png ... --mmproj mmproj.gguf]
+ text to image, Turbo's 8 steps (default 1024x1024, seed 42, out.png); an edit
+ with --ref (sides multiples of 32; the size defaults to the last reference's)
+ serve --diffusion-model <dit.gguf> --vae <vae.safetensors> --llm <encoder.gguf> [--llm_vision <mmproj.gguf>]
+ [--listen-ip 127.0.0.1] [--listen-port 1234]
+ the engine as a server speaking sd-server's /sdcpp/v1 API (text to image;
+ edits with --llm_vision)
bench [--quick] [--out report.md] [--encoder P --dit P --vae P]
a few minutes: this Mac's paths, speed at 1024², accuracy against the fp32 path;
prints a report to paste in a GitHub issue (Kokon's model files by default)
@@ -72,7 +78,8 @@ do {
runs: numbers.dropFirst(2).first ?? 3)
case "check-textgen":
guard arguments.count >= 3 else { fail(usage) }
- try Checks.textGeneration(path: arguments[1], directory: arguments[2], tokens: arguments.count > 3 ? Int(arguments[3]) ?? 48 : 48)
+ try Checks.textGeneration(path: arguments[1], directory: arguments[2], tokens: arguments.count > 3 ? Int(arguments[3]) ?? 48 : 48,
+ image: arguments.count > 5 ? arguments[4] : nil, mmproj: arguments.count > 5 ? arguments[5] : nil)
case "time-matvec":
guard arguments.count == 2 else { fail(usage) }
try Checks.timeMatvec(path: arguments[1])
@@ -99,12 +106,13 @@ do {
guard arguments.count == 3 else { fail(usage) }
try Checks.vision(path: arguments[1], reference: arguments[2])
case "check-generate":
- guard arguments.count == 5 || arguments.count == 6 else { fail(usage) }
+ guard (5...7).contains(arguments.count) else { fail(usage) }
try Checks.generate(encoder: arguments[1], dit: arguments[2], vae: arguments[3], reference: arguments[4],
- text: arguments.count == 6 ? arguments[5] : nil)
+ text: arguments.count >= 6 ? arguments[5] : nil, mmproj: arguments.count == 7 ? arguments[6] : nil)
case "generate":
guard arguments.count >= 5 else { fail(usage) }
- var (width, height, seed, out) = (1024, 1024, UInt64(42), "out.png")
+ var (width, height, seed, out) = (0, 0, UInt64(42), "out.png")
+ var (references, mmproj) = ([QwenPipeline.Reference](), String?.none)
var options = arguments.dropFirst(5).makeIterator()
while let option = options.next() {
guard let value = options.next() else { fail(usage) }
@@ -115,13 +123,21 @@ do {
(width, height) = (parts[0], parts[1])
case "--seed": seed = UInt64(value) ?? seed
case "--out": out = value
+ case "--ref":
+ let image = try QwenPipeline.decodeImage(Data(contentsOf: URL(fileURLWithPath: value)))
+ references.append(.init(rgba: image.rgba, width: image.width, height: image.height))
+ case "--mmproj": mmproj = value
default: fail(usage)
}
}
+ if width == 0 { (width, height) = references.last.map { ($0.width, $0.height) } ?? (1024, 1024) }
let size = QwenPipeline.workingSize(width: width, height: height)
if size != (width, height) { print("size \(width)×\(height) → \(size.width)×\(size.height) (multiples of 32)") }
let pipeline = try QwenPipeline(encoder: arguments[1], dit: arguments[2], vae: arguments[3])
- let image = try pipeline.textToImage(prompt: arguments[4], width: size.width, height: size.height, seed: seed)
+ pipeline.visionPath = mmproj
+ let image = try pipeline.generate(prompt: arguments[4], references: references, width: size.width, height: size.height, seed: seed)
+ let m = MemoryUsage.now()
+ print(String(format: "peak footprint %.2f GB", m.footprintPeak))
try QwenPipeline.writePNG(QwenPipeline.pixels(image, width: size.width, height: size.height),
width: size.width, height: size.height, to: out)
print("wrote \(out) (\(size.width)×\(size.height), seed \(seed))")
@@ -132,7 +148,7 @@ do {
guard arguments.count == 2 else { fail(usage) }
try Checks.noise(dump: arguments[1])
case "serve":
- var (ip, port, dit, vae, llm) = ("127.0.0.1", UInt16(1234), "", "", "")
+ var (ip, port, dit, vae, llm, vision) = ("127.0.0.1", UInt16(1234), "", "", "", String?.none)
var options = arguments.dropFirst().makeIterator()
while let option = options.next() {
guard let value = options.next() else { fail("\(option) needs a value\n\n" + usage) }
@@ -142,14 +158,14 @@ do {
case "--diffusion-model": dit = value
case "--vae": vae = value
case "--llm": llm = value
- case "--llm_vision": break // edits, later
+ case "--llm_vision": vision = value
default: fail("unknown option \(option)\n\n" + usage)
}
}
guard !dit.isEmpty, !vae.isEmpty, !llm.isEmpty else { fail("serve needs --diffusion-model, --vae and --llm\n\n" + usage) }
setvbuf(stdout, nil, _IOLBF, 0) // qls-core follows the log file stdout goes to
print("kokon-engine-mac (experimental), Qwen-Image 2.1 Turbo")
- let engine = try EngineServer(paths: .init(diffusionModel: dit, vae: vae, textEncoder: llm))
+ let engine = try EngineServer(paths: .init(diffusionModel: dit, vae: vae, textEncoder: llm, vision: vision))
let server = try HTTPServer(host: ip, port: port) { engine.handle($0) }
print("[INFO ] engine-mac - listening on: http://\(ip):\(server.port)")
server.run()
diff --git a/engine-mac/Tests/KokonEngineTests/VisionTests.swift b/engine-mac/Tests/KokonEngineTests/VisionTests.swift
index acf329a..1bad88b 100644
--- a/engine-mac/Tests/KokonEngineTests/VisionTests.swift
+++ b/engine-mac/Tests/KokonEngineTests/VisionTests.swift
@@ -38,3 +38,82 @@ final class VisionTests: XCTestCase {
XCTAssert(zip(row, row.dropFirst()).allSatisfy { $0 <= $1 })
}
}
+
+/// The server's edit requests, without a model: references decoded, sd.cpp's resize rule, what is refused.
+final class EditRequestTests: XCTestCase {
+ func png(_ width: Int, _ height: Int) throws -> String {
+ let rgba = (0..<(width * height * 4)).map { UInt8(truncatingIfNeeded: $0 * 7) }
+ return try QwenPipeline.pngData(rgba, width: width, height: height).base64EncodedString()
+ }
+
+ func body(_ extra: [String: Any]) throws -> Data {
+ var json: [String: Any] = ["prompt": "make it blue", "width": 512, "height": 384, "seed": 3]
+ json.merge(extra) { $1 }
+ return try JSONSerialization.data(withJSONObject: json)
+ }
+
+ func testReferencesAreDecodedAndKeptAtTheirSizeWithoutResize() throws {
+ let image = try png(320, 224)
+ let (r, reason) = EngineServer.ImageRequest.parse(try body(["ref_images": ["data:image/png;base64," + image],
+ "ref_image_args": "resize_before_vae=false"]))
+ XCTAssertNil(reason)
+ XCTAssertEqual(r?.references.count, 1)
+ XCTAssertEqual(r.map { [$0.references[0].width, $0.references[0].height] }, [320, 224])
+ XCTAssertEqual(r?.references[0].rgba[4 * 5 + 1], UInt8(truncatingIfNeeded: (4 * 5 + 1) * 7)) // straight RGBA as sent
+ }
+
+ func testResizeBeforeVAEScalesToTheOutputsArea() throws {
+ // 512 × 384 output, a 320 × 224 reference: √(196 608 · 320 / 224) = 530 → 544; 530 · 224 / 320 = 371 → 384.
+ let (r, reason) = EngineServer.ImageRequest.parse(try body(["ref_images": [try png(320, 224)]]))
+ XCTAssertNil(reason)
+ XCTAssertEqual(r.map { [$0.references[0].width, $0.references[0].height] }, [544, 384])
+ }
+
+ func testRefusals() throws {
+ XCTAssertNotNil(EngineServer.ImageRequest.parse(try body(["ref_images": [try png(320, 224)]]), vision: false).1)
+ XCTAssertNotNil(EngineServer.ImageRequest.parse(try body(["ref_images": [try png(320, 224)], "mask_image": "x"])).1)
+ XCTAssertNotNil(EngineServer.ImageRequest.parse(try body(["ref_images": [try png(336, 224)],
+ "ref_image_args": "resize_before_vae=false"])).1) // 336 % 32
+ XCTAssertNotNil(EngineServer.ImageRequest.parse(try body(["ref_images": ["not an image"]])).1)
+ }
+}
+
+/// The text encoder's side of images, without a model: Qwen3-VL's 3D positions and the templates.
+final class ImagePromptTests: XCTestCase {
+ /// get_rope_index, worked out by hand: 2 text tokens, a 2 × 3 grid of slots, 2 text tokens.
+ func testPositionsOfAnImageAndTheTextAfterIt() throws {
+ let pad = 9
+ let ids = [1, 2] + [Int](repeating: pad, count: 6) + [3, 4]
+ let p = try QwenTextEncoder.positions(ids: ids, imagePad: pad, grids: [(rows: 2, columns: 3)])
+ let want: [[Int32]] = [[0, 0, 0], [1, 1, 1],
+ [2, 2, 2], [2, 2, 3], [2, 2, 4], [2, 3, 2], [2, 3, 3], [2, 3, 4],
+ [5, 5, 5], [6, 6, 6]] // after the image: 2 + max(2, 3)
+ XCTAssertEqual(p.map { [$0[0], $0[1], $0[2]] }, want)
+ XCTAssertThrowsError(try QwenTextEncoder.positions(ids: ids, imagePad: pad, grids: [(rows: 2, columns: 2)]))
+ XCTAssertEqual(try QwenTextEncoder.positions(ids: [1, 2, 3], imagePad: -1, grids: []).map { $0[1] }, [0, 1, 2])
+ }
+
+ /// The M-RoPE table: text rows equal the 1D table; an image row turns frequency 1 with its row, 2 with its column.
+ func testMRopeInterleavesTheAxes() {
+ let text = QwenTextEncoder.ropeTable(rows: 3)
+ XCTAssertEqual(QwenTextEncoder.ropeTable(positions: (0..<3).map { SIMD3(repeating: Int32($0)) }), text)
+ let image = QwenTextEncoder.ropeTable(positions: [SIMD3(4, 7, 9)])
+ let omega = (0..<64).map { 1 / powf(5_000_000, Float(2 * $0) / 128) }
+ for (i, axis) in [(0, 4), (1, 7), (2, 9), (3, 4), (60, 4), (61, 4), (62, 4), (63, 4)] {
+ XCTAssertEqual(image[2 * i], cosf(Float(axis) * omega[i]), accuracy: 1e-7, "frequency \(i)")
+ }
+ }
+
+ func testTemplates() {
+ XCTAssertEqual(QwenTextEncoder.imageToImage("make it blue", images: 2),
+ "<|im_start|>system\nComprehend and analyze the provided prompt.<|im_end|>\n<|im_start|>user\n"
+ + "<image1><|vision_start|><|image_pad|><|vision_end|> <image2><|vision_start|><|image_pad|><|vision_end|>"
+ + "make it blue<|im_end|>\n<|im_start|>assistant\n")
+ // sd.cpp's text generation: one image unlabelled, several labelled.
+ XCTAssertEqual(QwenTextGenerator.template(system: "", prompt: "p", images: 1),
+ "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>p<|im_end|>\n<|im_start|>assistant\n")
+ XCTAssertEqual(QwenTextGenerator.template(system: "s", prompt: "p", images: 2),
+ "<|im_start|>system\ns<|im_end|>\n<|im_start|>user\n<image1><|vision_start|><|image_pad|><|vision_end|>"
+ + " <image2><|vision_start|><|image_pad|><|vision_end|>p<|im_end|>\n<|im_start|>assistant\n")
+ }
+}
diff --git a/engine-mac/tools/api_check.py b/engine-mac/tools/api_check.py
index 8c1d2aa..dc6ad0d 100644
--- a/engine-mac/tools/api_check.py
+++ b/engine-mac/tools/api_check.py
@@ -2,9 +2,11 @@
"""Text to image through a running `kokon-engine-mac serve` (or sd-server) as qls-core sends it: capabilities,
img_gen, polling jobs/{id}, optionally a cancel; with `--enhance`, Prompt+ first (text_gen with Kokon's built-in
system prompt, read from crates/qls-core/src/enhancer.rs, and its sampling). Prints the time of each image and writes
-the PNGs.
+the PNGs. `--ref image.png` (repeatable): an edit, the references sent as Standard detail sends them
+(`resize_before_vae=false`; sides multiples of 32), and Prompt+ then looks at them with the edit system prompt.
python3 -I tools/api_check.py --port 1234 --images 2 --out /tmp/api [--size 1024x1024] [--cancel-after 10] [--enhance]
+ [--ref image.png]
"""
import argparse, base64, json, os, re, time, urllib.request, urllib.error
@@ -32,9 +34,12 @@ def main():
p.add_argument("--out", default=".")
p.add_argument("--cancel-after", type=float, default=0, help="cancel the last image after this many seconds")
p.add_argument("--enhance", action="store_true", help="Prompt+ writes each image's prompt first")
+ p.add_argument("--ref", action="append", default=[], help="a reference image: an edit")
a = p.parse_args()
enhancer = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../crates/qls-core/src/enhancer.rs")
- system = re.search(r'pub const T2I_SYSTEM_PROMPT: &str = r#"(.*?)"#;', open(enhancer).read(), re.S).group(1) if a.enhance else ""
+ which = "EDIT_SYSTEM_PROMPT" if a.ref else "T2I_SYSTEM_PROMPT"
+ system = re.search(rf'pub const {which}: &str = r#"(.*?)"#;', open(enhancer).read(), re.S).group(1) if a.enhance else ""
+ refs = [base64.b64encode(open(path, "rb").read()).decode() for path in a.ref]
width, height = (int(v) for v in a.size.split("x"))
status, caps = call(a.port, "GET", "/sdcpp/v1/capabilities", timeout=10)
print(f"capabilities {status}: engine {caps.get('engine', 'sd-server')}, model {caps.get('model', {}).get('name')}")
@@ -44,7 +49,7 @@ def main():
prompt = a.prompt
if a.enhance:
status, done = call(a.port, "POST", "/sdcpp/v1/text_gen", {
- "prompt": a.prompt, "system_prompt": system, "max_tokens": 1024, "stream": False,
+ "prompt": a.prompt, "system_prompt": system, "images": refs, "max_tokens": 1024, "stream": False,
"sampling": {"temperature": 0.7, "top_p": 0.8, "top_k": 20, "presence_penalty": 1.5, "seed": a.seed + i}}, timeout=300)
if status != 200:
print(f"text_gen {status}: {done}")
@@ -55,6 +60,8 @@ def main():
"batch_count": 1, "sample_params": {"sample_method": "euler", "sample_steps": 8,
"custom_sigmas": TURBO_SIGMAS, "guidance": {"txt_cfg": 1.0}},
"output_format": "png", "embed_image_metadata": False}
+ if refs:
+ body.update(ref_images=refs, ref_image_args="resize_before_vae=false", image_preprocess=["target=ref,mode=none"])
status, accepted = call(a.port, "POST", "/sdcpp/v1/img_gen", body)
if status != 202:
print(f"img_gen {status}: {accepted}")
diff --git a/engine-mac/tools/reference_dit.py b/engine-mac/tools/reference_dit.py
index 9a317e1..4bf9c27 100644
--- a/engine-mac/tools/reference_dit.py
+++ b/engine-mac/tools/reference_dit.py
@@ -64,8 +64,8 @@ class Weights:
module.load_state_dict(state, strict=True)
-def layer0_stages(block, x, modulation, rotary_emb, target_token_mask, save):
- """QwenImage21TransformerBlock.forward and its processor, unrolled, saving each stage (t2i: one text segment)."""
+def layer0_stages(block, x, modulation, rotary_emb, target_token_mask, segments, save):
+ """QwenImage21TransformerBlock.forward and its processor, unrolled, saving each stage."""
from diffusers.models.transformers.transformer_qwenimage21 import apply_rotary_emb_qwen
mod1, mod2 = modulation.chunk(2, dim=-1)
normed, gate1 = block._modulate(block.img_norm1(x), mod1, target_token_mask)
@@ -79,8 +79,7 @@ def layer0_stages(block, x, modulation, rotary_emb, target_token_mask, save):
save("l0_q", q[0].flatten(1))
save("l0_k", k[0].flatten(1))
save("l0_v", v[0].flatten(1))
- out = attn(hidden_states=normed, rotary_emb=rotary_emb,
- segments=[(0, int((~target_token_mask).sum()), True)])
+ out = attn(hidden_states=normed, rotary_emb=rotary_emb, segments=segments)
save("l0_attn_out", out[0])
x = x + gate1.tanh() * out
save("l0_res1", x[0])
@@ -90,7 +89,7 @@ def layer0_stages(block, x, modulation, rotary_emb, target_token_mask, save):
class ReferenceDiT:
- """QwenImage21Transformer2DModel.forward, batch 1, no cache, text-to-image, one block module swapped per layer."""
+ """QwenImage21Transformer2DModel.forward, batch 1, no cache, one block module swapped per layer."""
def __init__(self, path, layers=32, dtype=torch.float32):
self.w = w = Weights(path, dtype)
@@ -115,13 +114,16 @@ class ReferenceDiT:
all_layers = False
- def forward(self, encoder, latent, h, w_, timestep, save=None, progress=True):
+ def forward(self, encoder, latent, h, w_, timestep, save=None, progress=True, refs=(), slots=None):
"""`encoder` [1, T, d], `latent` [1, h * w, 64], `timestep` a [1] tensor (σ); returns the target's velocity
- [1, h * w, 64]. `save(name, tensor)` receives every stage when given."""
+ [1, h * w, 64]. `save(name, tensor)` receives every stage when given. An edit: `refs` the condition images'
+ packed latents with their grids [(latent [1, h_i * w_i, 64], h_i, w_i)], `slots` which encoder rows are their
+ slots (the text encoder's `image_pad_mask`)."""
d, text, tokens = 4096, encoder.shape[1], h * w_
- img_shapes = [(1, h, w_)]
- img_mask = torch.tensor([[False] * text + [True] * (tokens // 4)])
- hidden_states = self.img_in(latent)
+ img_shapes = [(1, rh, rw) for _, rh, rw in refs] + [(1, h, w_)]
+ slots = list(slots) if slots is not None else [False] * text
+ img_mask = torch.tensor([slots + [True] * (tokens // 4)])
+ hidden_states = self.img_in(torch.cat([r for r, _, _ in refs] + [latent], dim=1))
encoder_hidden_states = self.txt_in(encoder)
repeats = torch.where(img_mask, 4, 1)[0]
image_pad_mask = torch.repeat_interleave(img_mask[0], repeats)
@@ -146,7 +148,7 @@ class ReferenceDiT:
for layer in range(self.layers):
self.w.load(self.block, f"transformer_blocks.{layer}.")
if layer == 0 and save:
- layer0_stages(self.block, joint, modulation, rotary_emb, target_token_mask, save)
+ layer0_stages(self.block, joint, modulation, rotary_emb, target_token_mask, segments, save)
joint = self.block(hidden_states=joint, modulation=modulation, rotary_emb=rotary_emb, attention_mask=None,
target_token_mask=target_token_mask, segments=segments)
if save and (self.all_layers or layer in (0, 1) or layer == self.layers - 1):
diff --git a/engine-mac/tools/reference_generate.py b/engine-mac/tools/reference_generate.py
index 2237c83..205c9e6 100644
--- a/engine-mac/tools/reference_generate.py
+++ b/engine-mac/tools/reference_generate.py
@@ -9,8 +9,13 @@ the pieces the other references check, in fp32 on the CPU:
Saves the noise, every step's velocity and latent, the decoded image and the final pixels for
`kokon-engine-mac check-generate`. 256² takes ~5 min (8 DiT evaluations, each layer's weights unpacked by gguf-py).
+**An edit** when the text reference has images (`reference_text_encoder.py --image`): each image as the pipeline hands it
+to the VAE (RGBA, `VaeImageProcessor.preprocess` at its own size, which `resize_before_vae=false` keeps), encoded
+(`_encode_vae_image`: the mode, (z − mean) / std), packed, and prepended to the target's tokens at every step; the
+text reference's `image_pad_mask` places their slots. Saved as `reference<i>`.
+
python -I engine-mac/tools/reference_generate.py <dit.gguf> <vae.safetensors> <text reference dir> <repo dir> <out>
- [--grid 16] [--seed 1] [--dtype float64]
+ [--grid 16 | --size WxH] [--seed 1] [--dtype float64]
`--dtype float64`: the DiT and the latents in fp64 (the scheduler still rounds the latent to fp32 each step), the
VAE in fp32. Needed on unsloth's DiT, where torch's fp32 drifts to 7e-5 of fp64 in one evaluation.
@@ -24,6 +29,7 @@ import sys
import numpy as np
import torch
+from PIL import Image
from diffusers import FlowMatchEulerDiscreteScheduler
from diffusers.image_processor import VaeImageProcessor
from diffusers.pipelines.qwenimage21.pipeline_qwenimage21 import calculate_shift
@@ -31,6 +37,7 @@ from diffusers.pipelines.qwenimage21.pipeline_qwenimage21 import calculate_shift
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # the sibling references
from reference_dit import ReferenceDiT, enable_float64 # noqa: E402
from reference_vae import load_vae # noqa: E402
+from reference_vae_encoder import load_full_vae # noqa: E402
def main():
@@ -41,6 +48,7 @@ def main():
parser.add_argument("repo")
parser.add_argument("out")
parser.add_argument("--grid", type=int, default=16, help="latent grid side (16 = a 256² image)")
+ parser.add_argument("--size", help="WxH of the image instead (multiples of 32)")
parser.add_argument("--seed", type=int, default=1)
parser.add_argument("--dtype", default="float32", choices=["float32", "float64"])
args = parser.parse_args()
@@ -57,6 +65,25 @@ def main():
conditioning = torch.from_numpy(np.fromfile(os.path.join(args.text, "conditioning.f32"), dtype=np.float32))
conditioning = conditioning.view(1, -1, 4096).to(dtype)
h = w = args.grid
+ if args.size:
+ w, h = (int(v) // 16 for v in args.size.split("x"))
+
+ # The edit's condition images (prepare_latents with `images`).
+ refs, slots, sizes = [], None, text_meta.get("images") or []
+ if sizes:
+ slots = [bool(v) for v in text_meta["image_pad_mask"]]
+ encoder_vae = load_full_vae(args.vae)
+ processor = VaeImageProcessor(vae_scale_factor=16, vae_latent_channels=64)
+ mean = torch.tensor(encoder_vae.config.latents_mean).view(1, 64, 1, 1, 1)
+ std = torch.tensor(encoder_vae.config.latents_std).view(1, 64, 1, 1, 1)
+ for i, (rw, rh) in enumerate(sizes):
+ image = Image.open(os.path.join(args.text, f"image{i + 1}.png")).convert("RGBA")
+ pixels = processor.preprocess(image, width=rw, height=rh).unsqueeze(2)
+ latent = (encoder_vae.encode(pixels).latent_dist.mode() - mean) / std
+ packed = latent[0, :, 0].reshape(64, -1).transpose(0, 1)[None]
+ save(f"reference{i}", packed[0])
+ refs.append((packed.to(dtype), rh // 16, rw // 16))
+ del encoder_vae
# prepare_latents + _pack_latents
noise = torch.randn((1, 1, 64, h, w), generator=torch.Generator().manual_seed(args.seed), dtype=torch.float32)
@@ -75,7 +102,8 @@ def main():
dit = ReferenceDiT(args.dit, dtype=dtype)
for i, t in enumerate(scheduler.timesteps):
- velocity = dit.forward(conditioning, latents, h, w, t.expand(1).to(dtype) / 1000, progress=False)
+ velocity = dit.forward(conditioning, latents, h, w, t.expand(1).to(dtype) / 1000, progress=False,
+ refs=refs, slots=slots)
save(f"velocity{i}", velocity[0])
latents = scheduler.step(velocity, t, latents, return_dict=False)[0]
save(f"latent{i}", latents[0])
@@ -94,7 +122,8 @@ def main():
meta = {"prompt": text_meta["prompt"], "grid": [h, w], "seed": args.seed, "mode": pil.mode,
"sigmas": scheduler.sigmas.tolist(), "timesteps": (scheduler.timesteps / 1000).tolist(),
- "dit": os.path.basename(args.dit), "dtype": args.dtype, "vae": os.path.basename(args.vae)}
+ "dit": os.path.basename(args.dit), "dtype": args.dtype, "vae": os.path.basename(args.vae),
+ "references": sizes}
with open(os.path.join(args.out, "meta.json"), "w") as f:
json.dump(meta, f, indent=1)
print(f"image {pil.size} {pil.mode}")
diff --git a/engine-mac/tools/reference_text_encoder.py b/engine-mac/tools/reference_text_encoder.py
index daa760c..7d0b9c0 100644
--- a/engine-mac/tools/reference_text_encoder.py
+++ b/engine-mac/tools/reference_text_encoder.py
@@ -76,6 +76,44 @@ class Weights:
layer.load_state_dict(state, strict=True)
+class VisionPrompt:
+ """The processor and the vision tower on a prompt with images: token ids and run()'s `vision` (image rows, merged
+ rows, deepstack, 3D positions). `extra` token ids appended after the prompt (text) get positions too."""
+
+ def __init__(self, processor, full_config, mmproj):
+ self.processor, self.config, self.mmproj, self.tower = processor, full_config, mmproj, None
+
+ def __call__(self, text, images, save=None, extra=()):
+ flat = []
+ for image in images: # the pipeline: RGBA composited over white for the vision encoder
+ if image.mode == "RGBA":
+ white = Image.new("RGB", image.size, (255, 255, 255))
+ white.paste(image, mask=image.getchannel("A"))
+ image = white
+ flat.append(image.convert("RGB"))
+ inputs = self.processor(text=[text], images=flat, padding=True, padding_side="left", return_tensors="pt")
+ if self.tower is None:
+ vconfig = self.config.vision_config
+ vconfig._attn_implementation = "eager"
+ self.tower = Qwen3VLVisionModel(vconfig).eval()
+ self.tower.load_state_dict(vision_state_dict(self.mmproj, vconfig), strict=True)
+ out = self.tower(inputs.pixel_values.to(torch.float32), grid_thw=inputs.image_grid_thw, return_dict=True)
+ input_ids = torch.cat([inputs.input_ids, torch.tensor([list(extra)], dtype=inputs.input_ids.dtype)], dim=1)
+ types = torch.cat([inputs.mm_token_type_ids, torch.zeros(1, len(extra), dtype=inputs.mm_token_type_ids.dtype)], dim=1)
+ shim = SimpleNamespace(config=self.config)
+ shim.get_vision_position_ids = lambda *a, **k: Qwen3VLModel.get_vision_position_ids(shim, *a, **k)
+ position_ids, _ = Qwen3VLModel.get_rope_index(shim, input_ids, types, image_grid_thw=inputs.image_grid_thw,
+ attention_mask=torch.ones_like(input_ids))
+ if save:
+ save("vision_merged", out.pooler_output)
+ for j, feature in enumerate(out.deepstack_features):
+ save(f"vision_deepstack{j}", feature)
+ save("positions", position_ids[:, 0].to(torch.float32))
+ ids = input_ids[0].tolist()
+ visual = torch.tensor(ids) == self.config.image_token_id
+ return ids, (visual, out.pooler_output, out.deepstack_features, position_ids)
+
+
def run(weights, config, ids, save=None, vision=None):
"""The text model's forward on one unpadded prompt; returns the last layer's output (no final norm). `vision`:
(image rows mask, merged rows, deepstack list, position ids [3, 1, T]) for a prompt with images."""
@@ -133,38 +171,10 @@ def main():
saved[name] = list(array.shape)
full_config = AutoConfig.from_pretrained(args.text_encoder)
- tower = None
+ vision_prompt = VisionPrompt(processor, full_config, args.mmproj) if args.mmproj else None
def with_images(text, images, save=None):
- """Token ids and run()'s `vision` for `text` (a template with its image markers) and its images."""
- nonlocal tower
- flat = []
- for image in images: # the pipeline: RGBA composited over white for the vision encoder
- if image.mode == "RGBA":
- white = Image.new("RGB", image.size, (255, 255, 255))
- white.paste(image, mask=image.getchannel("A"))
- image = white
- flat.append(image.convert("RGB"))
- inputs = processor(text=[text], images=flat, padding=True, padding_side="left", return_tensors="pt")
- if tower is None:
- vconfig = full_config.vision_config
- vconfig._attn_implementation = "eager"
- tower = Qwen3VLVisionModel(vconfig).eval()
- tower.load_state_dict(vision_state_dict(args.mmproj, vconfig), strict=True)
- out = tower(inputs.pixel_values.to(torch.float32), grid_thw=inputs.image_grid_thw, return_dict=True)
- shim = SimpleNamespace(config=full_config)
- shim.get_vision_position_ids = lambda *a, **k: Qwen3VLModel.get_vision_position_ids(shim, *a, **k)
- position_ids, _ = Qwen3VLModel.get_rope_index(shim, inputs.input_ids, inputs.mm_token_type_ids,
- image_grid_thw=inputs.image_grid_thw,
- attention_mask=inputs.attention_mask)
- if save:
- save("vision_merged", out.pooler_output)
- for j, feature in enumerate(out.deepstack_features):
- save(f"vision_deepstack{j}", feature)
- save("positions", position_ids[:, 0].to(torch.float32))
- ids = inputs.input_ids[0].tolist()
- visual = torch.tensor(ids) == full_config.image_token_id
- return ids, (visual, out.pooler_output, out.deepstack_features, position_ids)
+ return vision_prompt(text, images, save)
vision, images = None, []
if args.image:
diff --git a/engine-mac/tools/reference_text_generation.py b/engine-mac/tools/reference_text_generation.py
index 20f567d..5a58d78 100644
--- a/engine-mac/tools/reference_text_generation.py
+++ b/engine-mac/tools/reference_text_generation.py
@@ -6,6 +6,9 @@ engine generated (teacher forcing), with the final RMSNorm and the language mode
python -I engine-mac/tools/reference_text_generation.py <encoder.gguf> <processor dir> <text_encoder dir> <dir>
kokon-engine-mac check-textgen <encoder.gguf> <dir> # 2. the same tokens again, compared with the reference
+With an image (`check-textgen <encoder> <dir> <tokens> <image.png> <mmproj>`, then `--mmproj` here): sd.cpp's template
+with the image before the prompt, the vision tower and 3D positions as `reference_text_encoder.py --image`.
+
The prompt is tokenized by the official processor as sd.cpp's template writes it (system turn, user turn, assistant),
and must give the engine's ids. Writes `<dir>/reference_logits.f32` ([generated, vocabulary] fp32: the logits that
predict each generated token) and `<dir>/reference.json` (its own greedy choice at each position).
@@ -17,10 +20,11 @@ import sys
import torch
from transformers import AutoConfig, AutoProcessor
+from PIL import Image
from transformers.models.qwen3_vl.modeling_qwen3_vl import Qwen3VLTextRMSNorm
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from reference_text_encoder import Weights, run # noqa: E402
+from reference_text_encoder import VisionPrompt, Weights, run # noqa: E402
def main():
@@ -29,6 +33,7 @@ def main():
parser.add_argument("processor")
parser.add_argument("text_encoder", help="text_encoder/ of the official repository (config.json)")
parser.add_argument("dir", help="where check-textgen wrote engine.json")
+ parser.add_argument("--mmproj", help="the vision tower's GGUF, when engine.json has an image")
args = parser.parse_args()
torch.set_grad_enabled(False)
with open(os.path.join(args.dir, "engine.json")) as f:
@@ -37,17 +42,25 @@ def main():
config._attn_implementation = "eager"
processor = AutoProcessor.from_pretrained(args.processor)
- # sd.cpp's template (src/pipeline/text.cpp), text only.
+ # sd.cpp's template (src/pipeline/text.cpp); one image goes before the prompt, unlabelled.
system, prompt = engine["system"], engine["prompt"]
- text = (f"<|im_start|>system\n{system}<|im_end|>\n" if system else "") + f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n"
- ids = processor(text=[text], return_tensors="pt").input_ids[0].tolist()
+ marker = "<|vision_start|><|image_pad|><|vision_end|>" if engine.get("image") else ""
+ text = (f"<|im_start|>system\n{system}<|im_end|>\n" if system else "") + f"<|im_start|>user\n{marker}{prompt}<|im_end|>\n<|im_start|>assistant\n"
+ generated = engine["generated"]
+ vision = None
+ if marker:
+ full_config = AutoConfig.from_pretrained(args.text_encoder)
+ sequence, vision = VisionPrompt(processor, full_config, args.mmproj)(text, [Image.open(engine["image"])],
+ extra=generated[:-1])
+ ids = sequence[:len(sequence) - len(generated) + 1]
+ else:
+ ids = processor(text=[text], return_tensors="pt").input_ids[0].tolist()
+ sequence = ids + generated[:-1] # each position predicts the next token
if ids != engine["prompt_ids"]:
sys.exit(f"prompt ids differ from the engine's: {len(ids)} against {len(engine['prompt_ids'])}")
- generated = engine["generated"]
- sequence = ids + generated[:-1] # each position predicts the next token
weights = Weights(args.gguf)
- hidden = run(weights, config, sequence)[0]
+ hidden = run(weights, config, sequence, vision=vision)[0]
norm = Qwen3VLTextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
norm.weight.copy_(weights.get("output_norm.weight"))
head = weights.get("output.weight")
diff --git a/engine-mac/tools/reference_vae_encoder.py b/engine-mac/tools/reference_vae_encoder.py
index 2e17a08..33c1159 100644
--- a/engine-mac/tools/reference_vae_encoder.py
+++ b/engine-mac/tools/reference_vae_encoder.py
@@ -47,6 +47,22 @@ def encoder_name(key):
raise KeyError(key)
+def load_full_vae(path):
+ """diffusers' AutoencoderKLQwenImage21, encoder and decoder, with the weights of Kokon's file, fp32, eval mode."""
+ from diffusers import AutoencoderKLQwenImage21
+ vae = AutoencoderKLQwenImage21() # the defaults are Qwen-Image 2.1's config
+ model = vae.state_dict()
+ converted = {}
+ for key, value in load_file(path).items():
+ name = encoder_name(key) or decoder_name(key)
+ if name is not None:
+ converted[name] = value.float().reshape(model[name].shape)
+ missing, unexpected = vae.load_state_dict(converted, strict=False)
+ missing = [k for k in missing if k.startswith("encoder.") or k.startswith("quant_conv")]
+ assert not missing and not unexpected, (missing[:10], unexpected[:10])
+ return vae.eval()
+
+
def main():
parser = argparse.ArgumentParser()
parser.add_argument("vae")
@@ -54,23 +70,12 @@ def main():
parser.add_argument("--size", default="256x256", help="WxH, multiples of 16")
parser.add_argument("--seed", type=int, default=11)
args = parser.parse_args()
- from diffusers import AutoencoderKLQwenImage21
torch.manual_seed(args.seed)
torch.set_grad_enabled(False)
os.makedirs(args.out, exist_ok=True)
width, height = (int(v) for v in args.size.split("x"))
- vae = AutoencoderKLQwenImage21() # the defaults are Qwen-Image 2.1's config
- model = vae.state_dict()
- converted = {}
- for key, value in load_file(args.vae).items():
- name = encoder_name(key) or decoder_name(key)
- if name is not None:
- converted[name] = value.float().reshape(model[name].shape)
- missing, unexpected = vae.load_state_dict(converted, strict=False)
- missing = [k for k in missing if k.startswith("encoder.") or k.startswith("quant_conv")]
- assert not missing and not unexpected, (missing[:10], unexpected[:10])
- vae.eval()
+ vae = load_full_vae(args.vae)
image = torch.rand(1, 4, 1, height // 16, width // 16) * 2 - 1
image = F.interpolate(image[:, :, 0], size=(height, width), mode="bilinear", align_corners=False)[:, :, None]