Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.
TypeScript ai-photo-editorcomfyui-alternativegenerative-fillimage-editingimage-generationinpaintinglinuxlocal-aimacosmcpmcp-servernano-banana-alternativeofflinephoto-editorprivacyqwen-imageqwen-image-2-1stable-diffusion-cppvulkanwindows
Mac engine: edits with reference images, and Prompt+ looking at them
Gabriele Lozuponef60f53e2a18e
diff --git a/crates/qls-core/src/process.rs b/crates/qls-core/src/process.rsindex ec3289f..7a0fb83 100644--- a/crates/qls-core/src/process.rs+++ b/crates/qls-core/src/process.rs@@ -126,7 +126,7 @@ pub fn build_server_args(cfg: &AppConfig, profile: &ModelProfile) -> Vec<String>/// none of sd-server's placement or backend flags apply; the vision projector is for edits, not supported yet.pub fn build_mac_engine_args(cfg: &AppConfig, profile: &ModelProfile) -> Vec<String> {let p = |p: &Path| p.to_string_lossy().into_owned();- vec![+ let mut a: Vec<String> = vec!["serve".into(),"--listen-ip".into(),"127.0.0.1".into(),@@ -138,7 +138,13 @@ pub fn build_mac_engine_args(cfg: &AppConfig, profile: &ModelProfile) -> Vec<Strp(&profile.vae),"--llm".into(),p(&profile.text_encoder),- ]+ ];+ // The vision tower: edits need it, as sd-server's.+ if let Some(v) = &profile.vision_projector {+ a.push("--llm_vision".into());+ a.push(p(v));+ }+ a}/// A process this module may reuse or stop: sd-server, or Kokon's Mac engine.@@ -990,8 +996,13 @@ mod tests {let a = build_server_args(&cfg, cfg.profile());let p = |p: &Path| p.to_string_lossy().into_owned();let profile = cfg.profile();- assert_eq!(a, ["serve", "--listen-ip", "127.0.0.1", "--listen-port", &cfg.server_port.to_string(), "--diffusion-model",- &p(&profile.diffusion_model), "--vae", &p(&profile.vae), "--llm", &p(&profile.text_encoder)]);+ let mut want: Vec<String> = ["serve", "--listen-ip", "127.0.0.1", "--listen-port", &cfg.server_port.to_string(), "--diffusion-model",+ &p(&profile.diffusion_model), "--vae", &p(&profile.vae), "--llm", &p(&profile.text_encoder)].map(String::from).to_vec();+ if let Some(v) = &profile.vision_projector {+ want.extend(["--llm_vision".to_string(), p(v)]); // edits+ }+ assert_eq!(a, want);+ assert!(profile.vision_projector.is_some(), "the default profile has the mmproj, so the engine can edit");assert_ne!(a, sdcpp, "a different launch: switching restarts the server");// Only on the Metal backend: elsewhere sd.cpp stays the engine.cfg.gpu_backend = crate::config::GpuBackend::Cuda;diff --git a/crates/qls-core/src/sdcpp.rs b/crates/qls-core/src/sdcpp.rsindex d068efb..f53f081 100644--- a/crates/qls-core/src/sdcpp.rs+++ b/crates/qls-core/src/sdcpp.rs@@ -219,12 +219,15 @@ impl Capabilities {pub fn is_mac_engine(&self) -> bool {self.engine == crate::config::MAC_ENGINE_EXE}- /// What in `req` Kokon's Mac engine cannot do yet (it runs text to image only), in the user's words; None when it- /// can, and always for sd-server.+ /// What in `req` Kokon's Mac engine cannot do yet, in the user's words; None when it can, and always for+ /// sd-server. It edits with reference images when it has the vision projector (`ref_images` in its features), but+ /// has no init image or mask yet (brushed edits, extending an image).pub fn mac_engine_unsupported(&self, req: &ImgGenRequest) -> Option<&'static str> {if !self.is_mac_engine() {None- } else if !req.ref_images.is_empty() || req.init_image.is_some() || req.mask_image.is_some() {+ } else if req.init_image.is_some() || req.mask_image.is_some() {+ Some("edits of a brushed area or extending images")+ } else if !req.ref_images.is_empty() && !self.img_feature("ref_images") {Some("edits")} else if req.hires.as_ref().is_some_and(|h| h.enabled) {Some("second passes")@@ -910,9 +913,14 @@ mod tests {let t2i = ImgGenRequest { prompt: "a fox".into(), width: 1024, height: 1024, ..Default::default() };assert_eq!(caps.mac_engine_unsupported(&t2i), None);let edit = ImgGenRequest { ref_images: vec!["png".into()], ..t2i.clone() };- assert_eq!(caps.mac_engine_unsupported(&edit), Some("edits"));+ assert_eq!(caps.mac_engine_unsupported(&edit), Some("edits"), "no vision projector");let inpaint = ImgGenRequest { mask_image: Some("png".into()), ..t2i.clone() };- assert_eq!(caps.mac_engine_unsupported(&inpaint), Some("edits"));+ assert_eq!(caps.mac_engine_unsupported(&inpaint), Some("edits of a brushed area or extending images"));+ // Started with --llm_vision: edits with references run, a mask still does not.+ let vision: Capabilities = serde_json::from_str(r#"{"engine":"kokon-engine-mac","supported_modes":["img_gen"],+ "features_by_mode":{"img_gen":{"ref_images":true}}}"#).unwrap();+ assert_eq!(vision.mac_engine_unsupported(&edit), None);+ assert!(vision.mac_engine_unsupported(&inpaint).is_some());let hires = ImgGenRequest { hires: Some(HiresParams { enabled: true, ..Default::default() }), ..t2i.clone() };assert_eq!(caps.mac_engine_unsupported(&hires), Some("second passes"));// sd-server: no engine field, nothing refused here.diff --git a/docs/STATUS.md b/docs/STATUS.mdindex 0fbea5a..816958a 100644--- a/docs/STATUS.md+++ b/docs/STATUS.md@@ -24,10 +24,11 @@ Windows, macOS (Apple Silicon, 15+) and Linux (`.deb`, `.rpm`). Installed copies- **LoRA is beta** and tested on Windows only; reports from macOS and Linux are welcome([#33](https://github.com/GabrieleLozupone/kokon/issues/33) has a product-shot test).- **Macs are slow** compared with NVIDIA cards (about 4 minutes for 1 MP on an M2 Pro 16 GB). Kokon's own Mac- engine is in development on the `macos-optimization` branch (`engine-mac/`, not released): text to image and- Prompt+ (a prompt written in about 7–9 s), about 64–67 s for 1 MP on the same Mac through the app's API (peak memory 5 GB),- tested from the app, as an experimental switch in Settings → Performance (off by default). Edits, LoRAs and the- upscaler still need the standard engine.+ engine is in development on the `macos-optimization` branch (`engine-mac/`, not released): text to image,+ Prompt+ (a prompt written in about 7–9 s) and edits with reference images (Prompt+ looking at them too), about 64–67 s+ for 1 MP text to image and ~150 s for a 1 MP edit on the same Mac (peak memory 5 GB), as an experimental switch in+ Settings → Performance (off by default). Brushed edits, extending an image, LoRAs and the upscaler still need the+ standard engine.- The Windows installer is not code-signed: SmartScreen or Smart App Control may block it (README FAQ: Unblock).- HEIC photos can't be opened on Linux yet.diff --git a/engine-mac/README.md b/engine-mac/README.mdindex 10d3ef1..3ffc88d 100644--- a/engine-mac/README.md+++ b/engine-mac/README.md@@ -14,9 +14,9 @@ swift test # no model needed.build/release/kokon-engine-mac inspect <file.gguf>.build/release/kokon-engine-mac check-dequant <file.gguf> [tensor...].build/release/kokon-engine-mac generate <encoder.gguf> <dit.gguf> <vae.safetensors> "a red fox in the snow" \- [--size 1024x1024] [--seed 42] [--out out.png]+ [--size 1024x1024] [--seed 42] [--out out.png] [--ref in.png --mmproj <mmproj.gguf>].build/release/kokon-engine-mac serve --diffusion-model <dit.gguf> --vae <vae.safetensors> --llm <encoder.gguf> \- [--listen-port 1234] # sd-server's /sdcpp/v1 API, text to image and Prompt++ [--llm_vision <mmproj.gguf>] [--listen-port 1234] # sd-server's /sdcpp/v1 API: text to image, edits, Prompt+.build/release/kokon-engine-mac bench # a few minutes: speed and accuracy of this Mac's paths, a report.build/release/kokon-engine-mac check-na # the Metal 4 matrix product against MPS.build/release/kokon-engine-mac check-ane8 # the int8 Neural Engine programs against the fp16 one@@ -260,9 +260,10 @@ Engine's programs, are not pipelines an archive holds. Only an M2 Pro has been m`kokon-engine-mac serve` answers sd-server's `/sdcpp/v1` API (capabilities, img_gen, jobs/{id}, cancel), soqls-core, the app, remote access and MCP use it as they use sd-server; it prints the log lines qls-core's-`ProgressParser` reads. Text to image only for now: reference images, masks, LoRAs and second passes are refused-with 400 (qls-core says so before submitting), upscaling falls back to Lanczos. Prompt+ works on text-to-image prompts-(`text_gen`, below); with images (edits, Compose) it needs the vision tower. A running image+`ProgressParser` reads. Text to image, and edits with reference images when started with `--llm_vision` (qls-core passes+the mmproj; below); an init image or a mask (brushed edits, extending an image), LoRAs and second passes are refused with+400 (qls-core says so before submitting), upscaling falls back to Lanczos. Prompt+ works on every prompt (`text_gen`,+below), looking at the images of an edit with the vision tower. A running imagecancels within a DiT layer (sd-server answers 409 and qls-core restarts it). A new prompt length rebuilds the DiT'sworking memory in ~0.5 s.@@ -305,6 +306,59 @@ the API at 1024², alternated (`tools/api_check.py --enhance`): the image's text2.1–2.65 s for a plain image, peak footprint 5.1 GB either way; Prompt+ adds 8.7–9.0 s to an image (75.4–77.2 s against65.6–71.3).+## Edits (reference images)++As `QwenImage21Pipeline` with `image`: each reference goes through the vision tower into the text encoder's slots (the+ti2i template, deepstack, 3D positions; above) and through the VAE encoder (`_encode_vae_image`) into the DiT's prefix:+its latent tokens sit in the encoder's slots of that image (2 × 2 tokens a slot, `QwenSequence`), are modulated from+t = 0 like the text, and see only what precedes them plus themselves (block-causal), the target last. sd-server's+`ref_images` as Kokon sends them: Standard detail (`resize_before_vae=false`) encodes each reference at its own size;+otherwise sd.cpp's rule, resized to the output's area at its aspect, sides rounded to 32, nearest neighbour; the vision+tower sees what the VAE encodes (sd.cpp's `vlm_use_vae_input`). Sides must be multiples of 32.++Against diffusers (`reference_generate.py` on a `reference_text_encoder.py --image` directory: the street cat at+384 × 256, "Turn the cat into a small red fox, keep the street and the light", output 384 × 256, fp32, M2 Pro,+2026-10-25), `check-generate <encoder> <dit> <vae> <dir> <text dir> [mmproj]`:++| | final latent | image | 8-bit values off |+|---|---|---|---|+| the reference's conditioning and latents, fp32 path | 1.7e-6 | 4.0e-6 | 68 of 393 216, by 1 |+| the engine's vision tower, text encoder, VAE encoder, fp32 path | 2.3e-6 | 4.8e-6 | 89, by 1 |+| the same, default path (fp16 products and attention) | 1.8e-4 | 3.6e-4 | 5 083, by at most 2 |++**The prefix K/V cache holds the references' rows too**, in fp16 when the attention reads fp16 (the default): the same+values to the bit for the GPU's heads. A 1 MP reference is ~4 100 prefix rows, 2.1 GB of cache. M2 Pro 16 GB, edits with+the reference at the output's size, alternated:++| reference and output | cache | steps 2–8 | image | page-ins an image |+|---|---|---|---|---|+| 768 × 448 | off | 4.9–5.0 s | 46.2 s | 12.1 GB |+| 768 × 448 | 0.72 GB | 3.5 s | 38.1 s | 13.3 GB |+| 1344 × 768 | off | 16.6–17.1 s | 147.7–150.1 s | 20.9 GB |+| 1344 × 768 | 2.13 GB | 14.8–20.4 s | 144.2–158.7 s | 40.4 GB |++On 16 GB the 2.1 GB cache evicts the DiT's file pages, read again every step, and gains nothing; the cache is on up to+1 GB below 32 GB of memory and to an eighth of it from 32 GB (not measured there; `KOKON_PREFIX_CACHE_MB` to choose). A 1 MP+edit: vision tower 2.5 s, text encoder 3.7 s (1 037 tokens), VAE encoder 0.8 s, 8 steps of ~17 s, VAE 4.6 s; peak footprint+5.0 GB. Through the server (`tools/api_check.py --ref … --enhance`, 768 × 448): Prompt+ looking at the reference 5.5–6.8 s+(vision 0.6 s, 742 prompt tokens), the edit 36.4–37.3 s, peak footprint 3.6 GB.++**Prompt+ with images**: sd.cpp's template (`<|vision_start|><|image_pad|><|vision_end|>` before the prompt, `<imageN>`+labels when several, each side rounded to 32), the prefill with the images' rows, positions and deepstack, then the+decode at positions offset from the cache slots (an image's slots advance the position by max(rows, columns), not by+their count). `check-textgen <encoder> <dir> 48 <image> <mmproj>` against `reference_text_generation.py --mmproj` (the+cat, Kokon's edit system prompt, "make the cat a small red fox"): the first token's logits within 2.5e-6, every step+within 6.8e-4 (the fp16 cache), the greedy choice the same at 23 of 23.++```bash+~/kokon-bench/venv-oracle/bin/python -I tools/reference_text_encoder.py <encoder.gguf> <processor> <text_encoder> ~/kokon-bench/ref/text/cat-edit --prompt "..." --image cat-384x256.png --mmproj <mmproj.gguf> --sanity+~/kokon-bench/venv-oracle/bin/python -I tools/reference_generate.py <dit.gguf> <vae> ~/kokon-bench/ref/text/cat-edit <repo dir> ~/kokon-bench/ref/edit-cat-384 --size 384x256 # ~6 min+.build/release/kokon-engine-mac check-generate <encoder.gguf> <dit.gguf> <vae> ~/kokon-bench/ref/edit-cat-384 ~/kokon-bench/ref/text/cat-edit [<mmproj.gguf>]+.build/release/kokon-engine-mac generate <encoder.gguf> <dit.gguf> <vae> "make the cat a fox" --ref in.png --mmproj <mmproj.gguf>+```++Not yet: an init image with a mask (brushed edits, extending an image: sd.cpp blends the masked latent each step).+## Per-chip pathsSelected at run time from the GPU family and the chip (`Chip.swift`), each with the M2 path as its fallback and adiff --git a/engine-mac/Sources/KokonEngine/EngineServer.swift b/engine-mac/Sources/KokonEngine/EngineServer.swiftindex 7394acf..7a67c66 100644--- a/engine-mac/Sources/KokonEngine/EngineServer.swift+++ b/engine-mac/Sources/KokonEngine/EngineServer.swift@@ -9,8 +9,9 @@ import Foundation/// POST /sdcpp/v1/jobs/{id}/cancel 200 {status: cancelled}: a running image stops within a DiT layer/// POST /sdcpp/v1/text_gen Prompt+: the text encoder's LLM writes, streamed as Server-Sent Events///-/// Text to image only for now: requests with reference images, a mask, an init image, LoRAs or a second pass are-/// refused with 400 and say so, and so is text generation with images. Jobs and text generation run one at a time;+/// Text to image and edits (`ref_images`, with the vision projector given as `--llm_vision`; Prompt+ then also looks at+/// the images); requests with a mask, an+/// init image, LoRAs or a second pass are refused with 400 and say so. Jobs and text generation run one at a time;/// what stays loaded between them depends on the Mac (`QwenPipeline.keepModels`).////// Progress goes to stdout as sd-server's log lines, the ones qls-core's `ProgressParser` reads (process.rs):@@ -19,10 +20,13 @@ import Foundationpublic final class EngineServer: @unchecked Sendable {public struct Paths: Sendable {public let diffusionModel, vae, textEncoder: String- public init(diffusionModel: String, vae: String, textEncoder: String) {+ /// The vision tower (mmproj): edits need it.+ public let vision: String?+ public init(diffusionModel: String, vae: String, textEncoder: String, vision: String? = nil) {self.diffusionModel = diffusionModelself.vae = vaeself.textEncoder = textEncoder+ self.vision = vision}}@@ -42,21 +46,27 @@ public final class EngineServer: @unchecked Sendable {}/// The fields of `img_gen` the engine reads (sd-server's names), checked.- struct ImageRequest: Sendable {+ struct ImageRequest: @unchecked Sendable {let prompt: Stringlet width, height: Intlet seed: Int64let batch: Intlet sigmas: [Float]+ /// An edit's condition images, decoded, at the size they are encoded at.+ let references: [QwenPipeline.Reference]- /// nil and the reason when the engine cannot do it (yet).- static func parse(_ body: Data) -> (ImageRequest?, String?) {+ /// nil and the reason when the engine cannot do it (yet). `vision`: the engine has the vision tower (edits).+ static func parse(_ body: Data, vision: Bool = true) -> (ImageRequest?, String?) {guard let json = (try? JSONSerialization.jsonObject(with: body)) as? [String: Any] else {return (nil, "the body is not a JSON object")}func array(_ key: String) -> [Any] { json[key] as? [Any] ?? [] }- if !array("ref_images").isEmpty || json["init_image"] is String || json["mask_image"] is String {- return (nil, "Kokon's Mac engine does text to image only for now (no reference images, init image or mask)")+ if json["init_image"] is String || json["mask_image"] is String {+ return (nil, "Kokon's Mac engine has no init image or mask yet (brushed edits, extending images)")+ }+ let refs = array("ref_images")+ if !refs.isEmpty && !vision {+ return (nil, "Qwen Image 2.1 editing requires Qwen3-VL vision weights; start the engine with --llm_vision")}if !array("lora").isEmpty { return (nil, "Kokon's Mac engine does not run LoRAs yet") }if let hires = json["hires"] as? [String: Any], hires["enabled"] as? Bool == true {@@ -83,11 +93,43 @@ public final class EngineServer: @unchecked Sendable {if seed < 0 { seed = Int64.random(in: 0..<(1 << 31)) }let batch = json["batch_count"] as? Int ?? 1guard (1...EngineServer.maxBatch).contains(batch) else { return (nil, "batch_count \(batch): 1 to \(EngineServer.maxBatch)") }+ // ref_image_args: sd.cpp's key=value list; only resize_before_vae changes what the engine does.+ var resize = true+ for pair in (json["ref_image_args"] as? String ?? "").split(separator: ",") {+ let kv = pair.split(separator: "=", maxSplits: 1).map { $0.trimmingCharacters(in: .whitespaces) }+ if kv.count == 2, kv[0] == "resize_before_vae" { resize = !["false", "0", "no", "off"].contains(kv[1].lowercased()) }+ }+ var references: [QwenPipeline.Reference] = []+ for (i, item) in refs.enumerated() {+ guard let text = item as? String, let data = Data(base64Encoded: String(text.split(separator: ",").last ?? ""),+ options: .ignoreUnknownCharacters),+ let image = try? QwenPipeline.decodeImage(data) else {+ return (nil, "ref_images[\(i)] is not a base64 image")+ }+ var r = QwenPipeline.Reference(rgba: image.rgba, width: image.width, height: image.height)+ if resize { r = EngineServer.resizeToTarget(r, width: width, height: height) }+ guard r.width % 32 == 0, r.height % 32 == 0 else {+ return (nil, "Qwen Image 2.1 reference dimensions must be multiples of 32 (\(r.width)×\(r.height))")+ }+ references.append(r)+ }return (ImageRequest(prompt: json["prompt"] as? String ?? "", width: width, height: height, seed: seed,- batch: batch, sigmas: sigmas), nil)+ batch: batch, sigmas: sigmas, references: references), nil)}}+ /// sd.cpp's `resize_before_vae` for Qwen-Image 2.1 (pipeline/image.cpp): the reference resized to the output's area+ /// at its own aspect, each side rounded to 32, nearest neighbour (`sd::ops::interpolate`'s default); the vision+ /// tower then sees the same image (`vlm_use_vae_input`).+ static func resizeToTarget(_ r: QwenPipeline.Reference, width: Int, height: Int) -> QwenPipeline.Reference {+ let area = Double(width * height)+ let w = (area * Double(r.width) / Double(r.height)).squareRoot()+ let h = w * Double(r.height) / Double(r.width)+ let (ow, oh) = (Int((w / 32).rounded() * 32), Int((h / 32).rounded() * 32))+ guard ow > 0, oh > 0, (ow, oh) != (r.width, r.height) else { return r }+ return resize(r, width: ow, height: oh)+ }+static let maxBatch = 4, maxQueue = 8public let paths: Pathslet pipeline: QwenPipeline@@ -108,7 +150,9 @@ public final class EngineServer: @unchecked Sendable {for path in [paths.diffusionModel, paths.vae, paths.textEncoder] where !FileManager.default.fileExists(atPath: path) {throw GGUFFile.Failure.open(path)}+ if let vision = paths.vision, !FileManager.default.fileExists(atPath: vision) { throw GGUFFile.Failure.open(vision) }pipeline = try QwenPipeline(encoder: paths.textEncoder, dit: paths.diffusionModel, vae: paths.vae)+ pipeline.visionPath = paths.visionpipeline.keepModels = truepipeline.log = { _ in }let worker = Thread { [self] in work() }@@ -137,7 +181,7 @@ public final class EngineServer: @unchecked Sendable {func capabilities() -> [String: Any] {let url = URL(fileURLWithPath: paths.diffusionModel)- let feature: [String: Any] = ["ref_images": false, "init_image": false, "mask_image": false, "lora": false,+ let feature: [String: Any] = ["ref_images": paths.vision != nil, "init_image": false, "mask_image": false, "lora": false,"hires": false, "custom_sigmas": true, "cancel_generating": true]return ["engine": "kokon-engine-mac",@@ -148,7 +192,7 @@ public final class EngineServer: @unchecked Sendable {"output_formats": ["png"],"limits": ["min_width": 64, "max_width": 4096, "min_height": 64, "max_height": 4096,"max_batch_count": EngineServer.maxBatch, "max_queue_size": EngineServer.maxQueue,- "max_text_tokens": EngineServer.maxTextTokens, "max_text_images": 0],+ "max_text_tokens": EngineServer.maxTextTokens, "max_text_images": paths.vision != nil ? EngineServer.maxTextImages : 0],"features_by_mode": ["img_gen": feature],"defaults_by_mode": ["img_gen": ["sample_steps": QwenPipeline.turboSigmas.count - 1, "txt_cfg": 1]],"upscalers": [Any](),@@ -158,7 +202,7 @@ public final class EngineServer: @unchecked Sendable {}func submit(_ body: Data) -> HTTPServer.Response {- let (parsed, reason) = ImageRequest.parse(body)+ let (parsed, reason) = ImageRequest.parse(body, vision: paths.vision != nil)guard let parsed else { return .error(400, reason ?? "bad request") }return lock.withLock {guard queue.count < EngineServer.maxQueue else { return .error(429, "the queue is full") }@@ -201,14 +245,16 @@ public final class EngineServer: @unchecked Sendable {// MARK: - text generation (Prompt+)/// The fields of `text_gen` (sd-server's names and checks, `examples/server/routes_sdcpp.cpp`).- struct TextRequest: Sendable {+ struct TextRequest: @unchecked Sendable {let system, prompt: String+ /// Shown before the prompt, sides rounded to 32 as sd.cpp's `vision_size`.+ let images: [QwenPipeline.Reference]let maxTokens: Intlet sampling: TextSampler.Parameterslet seed: Int64let stream: Bool- static func parse(_ body: Data) -> (TextRequest?, String?) {+ static func parse(_ body: Data, vision: Bool = true) -> (TextRequest?, String?) {guard let json = (try? JSONSerialization.jsonObject(with: body)) as? [String: Any] else { return (nil, "body must be an object") }guard let prompt = json["prompt"] as? String, !prompt.isEmpty else { return (nil, "prompt is required") }guard let system = json["system_prompt"] as? String else {@@ -216,8 +262,20 @@ public final class EngineServer: @unchecked Sendable {}let maxTokens = json["max_tokens"] as? Int ?? 512guard (1...EngineServer.maxTextTokens).contains(maxTokens) else { return (nil, "max_tokens must be between 1 and \(EngineServer.maxTextTokens)") }- if let images = json["images"] as? [Any], !images.isEmpty {- return (nil, "Kokon's Mac engine cannot look at images yet (no vision tower): Prompt+ works on text-to-image prompts only")+ var images: [QwenPipeline.Reference] = []+ if let list = json["images"] {+ guard let list = list as? [Any], list.count <= EngineServer.maxTextImages else {+ return (nil, "images must be an array of at most \(EngineServer.maxTextImages) base64 images")+ }+ if !list.isEmpty && !vision { return (nil, "images need the Qwen3-VL vision weights (--llm_vision)") }+ for (i, item) in list.enumerated() {+ guard let text = item as? String, let data = Data(base64Encoded: String(text.split(separator: ",").last ?? ""),+ options: .ignoreUnknownCharacters),+ let image = try? QwenPipeline.decodeImage(data) else {+ return (nil, "images[\(i)] is not a base64 image")+ }+ images.append(EngineServer.roundedTo32(.init(rgba: image.rgba, width: image.width, height: image.height)))+ }}var sampling = TextSampler.Parameters(temperature: 0, topP: 1, topK: 0, presencePenalty: 0)var seed: Int64 = -1@@ -234,18 +292,38 @@ public final class EngineServer: @unchecked Sendable {return (nil, "sampling needs temperature >= 0, 0 < top_p <= 1, top_k >= 0 and a finite presence_penalty")}}- return (TextRequest(system: system, prompt: prompt, maxTokens: maxTokens, sampling: sampling, seed: seed,+ return (TextRequest(system: system, prompt: prompt, images: images, maxTokens: maxTokens, sampling: sampling, seed: seed,stream: json["stream"] as? Bool ?? true), nil)}}- static let maxTextTokens = 4096+ static let maxTextTokens = 4096, maxTextImages = 4++ /// sd.cpp's `vision_size` for text generation (no `image_max_pixels`): each side rounded to a multiple of 32, the+ /// image resized when that changes it (nearest neighbour, as the edit references).+ static func roundedTo32(_ r: QwenPipeline.Reference) -> QwenPipeline.Reference {+ let (w, h) = (max(32, Int((Double(r.width) / 32).rounded()) * 32), max(32, Int((Double(r.height) / 32).rounded()) * 32))+ return (w, h) == (r.width, r.height) ? r : resize(r, width: w, height: h)+ }++ /// Nearest-neighbour resize of 8-bit RGBA (`sd::ops::interpolate`'s default mode).+ static func resize(_ r: QwenPipeline.Reference, width ow: Int, height oh: Int) -> QwenPipeline.Reference {+ var out = [UInt8](repeating: 0, count: ow * oh * 4)+ for y in 0..<oh {+ let sy = min(r.height - 1, y * r.height / oh)+ for x in 0..<ow {+ let sx = min(r.width - 1, x * r.width / ow)+ for c in 0..<4 { out[(y * ow + x) * 4 + c] = r.rgba[(sy * r.width + sx) * 4 + c] }+ }+ }+ return QwenPipeline.Reference(rgba: out, width: ow, height: oh)+ }/// Synchronous like sd-server's: it waits for a running image (one GPU user at a time) and holds the models while it/// writes. Streamed (the default) as Server-Sent Events: `delta` {text} per piece, then `done` {text, finish_reason,/// seed, usage, timings} or `error` {error}; closing the connection stops it.func textGen(_ body: Data) -> HTTPServer.Response {- let (parsed, reason) = TextRequest.parse(body)+ let (parsed, reason) = TextRequest.parse(body, vision: paths.vision != nil)guard let request = parsed else { return .error(400, reason ?? "bad request") }guard request.stream else {let outcome = Result { try gpu.withLock { try generateText(request) { _ in true } } }@@ -274,10 +352,11 @@ public final class EngineServer: @unchecked Sendable {}private func generateText(_ r: TextRequest, onPiece: (String) -> Bool) throws -> QwenTextGenerator.Result {- let result = try pipeline.generateText(system: r.system, prompt: r.prompt, maxTokens: r.maxTokens, sampling: r.sampling,- seed: r.seed, onPiece: onPiece)- log(String(format: "[INFO ] engine-mac - text generation: %d prompt tokens, %d generated in %.2fs (prefill %.2fs, %.1f tok/s), seed %lld",- result.promptTokens, result.generatedTokens, result.prepare + result.prefill + result.decode, result.prefill,+ let result = try pipeline.generateText(system: r.system, prompt: r.prompt, images: r.images, maxTokens: r.maxTokens,+ sampling: r.sampling, seed: r.seed, onPiece: onPiece)+ log(String(format: "[INFO ] engine-mac - text generation: %d image(s) in %.2fs, %d prompt tokens, %d generated in %.2fs (prefill %.2fs, %.1f tok/s), seed %lld",+ r.images.count, result.image, result.promptTokens, result.generatedTokens,+ result.image + result.prepare + result.prefill + result.decode, result.prefill,result.decode > 0 ? Double(result.tokens.count - 1) / result.decode : 0, result.seed))return result}@@ -286,7 +365,7 @@ public final class EngineServer: @unchecked Sendable {static func done(_ r: QwenTextGenerator.Result) -> [String: Any] {["text": r.text, "finish_reason": r.finish.rawValue, "seed": r.seed,"usage": ["prompt_tokens": r.promptTokens, "completion_tokens": r.generatedTokens],- "timings": ["prepare_s": r.prepare, "image_s": 0, "prefill_s": r.prefill, "decode_s": r.decode]]+ "timings": ["prepare_s": r.prepare, "image_s": r.image, "prefill_s": r.prefill, "decode_s": r.decode]]}// MARK: - the jobs@@ -330,6 +409,9 @@ public final class EngineServer: @unchecked Sendable {let started = DispatchTime.now().uptimeNanosecondsfunc seconds(since t: UInt64) -> Double { Double(DispatchTime.now().uptimeNanoseconds - t) * 1e-9 }log("[INFO ] engine-mac - generate_image \(r.width)x\(r.height)")+ if !r.references.isEmpty {+ log("[INFO ] engine-mac - EDIT mode: \(r.references.map { "\($0.width)x\($0.height)" }.joined(separator: ", "))")+ }log("[INFO ] engine-mac - sampling using Euler method")pipeline.cancelled = { [weak self, job] in self?.lock.withLock { job.status == "cancelled" } ?? true }var images: [String] = []@@ -353,13 +435,13 @@ public final class EngineServer: @unchecked Sendable {log("[INFO ] engine-mac - decoding 1 latents")}}- let image = try pipeline.textToImage(prompt: r.prompt, width: r.width, height: r.height, seed: UInt64(bitPattern: seed),- sigmas: r.sigmas)+ let image = try pipeline.generate(prompt: r.prompt, references: r.references, width: r.width, height: r.height,+ seed: UInt64(bitPattern: seed), sigmas: r.sigmas)let png = try QwenPipeline.pngData(QwenPipeline.pixels(image, width: r.width, height: r.height), width: r.width, height: r.height)images.append(png.base64EncodedString())let t = pipeline.timings- log(String(format: "[INFO ] engine-mac - image %d/%d: text encoder %.2fs, DiT ready %.2fs, VAE %.2fs, total %.2fs", i + 1, r.batch,- t["encode"] ?? 0, t["dit load"] ?? 0, t["vae"] ?? 0, t["total"] ?? 0))+ log(String(format: "[INFO ] engine-mac - image %d/%d: vision %.2fs, text encoder %.2fs, VAE encoder %.2fs, DiT ready %.2fs, VAE %.2fs, total %.2fs",+ i + 1, r.batch, t["vision"] ?? 0, t["encode"] ?? 0, t["vae encode"] ?? 0, t["dit load"] ?? 0, t["vae"] ?? 0, t["total"] ?? 0))}pipeline.events = nillet memory = MemoryUsage.now()diff --git a/engine-mac/Sources/KokonEngine/Kernels/dit.metal b/engine-mac/Sources/KokonEngine/Kernels/dit.metalindex 3930547..a96fe4c 100644--- a/engine-mac/Sources/KokonEngine/Kernels/dit.metal+++ b/engine-mac/Sources/KokonEngine/Kernels/dit.metal@@ -159,3 +159,9 @@ kernel void gelu_tanh(device float *x [[buffer(0)]], constant uint &count [[bufffloat v = x[i];x[i] = 0.5f * v * (1.0f + precise::tanh(0.7978845608028654f * (v + 0.044715f * v * v * v)));}++// fp16 -> fp32: the prefix K/V cache read back (QwenDiT.copyPrefix; `pack_half` writes it).+kernel void widen_half(device const half *src [[buffer(0)]], device float *dst [[buffer(1)]],+ constant uint &count [[buffer(2)]], uint i [[thread_position_in_grid]]) {+ if (i < count) dst[i] = float(src[i]);+}diff --git a/engine-mac/Sources/KokonEngine/QwenDiT.swift b/engine-mac/Sources/KokonEngine/QwenDiT.swiftindex 1827015..a67b222 100644--- a/engine-mac/Sources/KokonEngine/QwenDiT.swift+++ b/engine-mac/Sources/KokonEngine/QwenDiT.swift@@ -18,8 +18,8 @@ import MetalPerformanceShaders/// Attention is block-causal: each prefix segment sees the keys before its end (a text segment with a causal/// triangle), the target sees every key.///-/// This first version runs the full pass every evaluation (no K/V cache of the prefix yet) on the GPU alone;-/// the AMX and the Neural Engine join in the products later.+/// An edit's condition images are prefix too: their latent tokens go through img_in like the target's and sit in+/// their slots of the encoder's sequence (`QwenSequence`), modulated from t = 0.public final class QwenDiT {public static let dim = 4096, heads = 32, headDim = 128, hidden = 12288, layers = 32, channels = 64static let eps: Float = 1e-6@@ -41,7 +41,7 @@ public final class QwenDiT {// Arena slices, rows of the joint sequence (N = prefix + target).let x, normed, q, k, v, attended, o, h1, h2: Arena.Slicelet reserves: [Arena.Slice]- let rope, modulation, qkNorms, textNorm, textIn, textOut, latent, output: Arena.Slice+ let rope, modulation, qkNorms, textNorm, textIn, textOut, latent, output, referenceLatent: Arena.Slice/// K and V packed for the flash kernel (`attention_pack_kv`): fp16, heads first, keys padded to 64.let kPacked, vPacked: Arena.Slicelet paddedKeys: Int@@ -49,10 +49,20 @@ public final class QwenDiT {/// the modulation of t = 0, so its rows do not change from one evaluation of an image to the next. They are/// saved during the first evaluation of a conditioning and every later one runs only the target's rows/// (`KOKON_PREFIX_CACHE=0`: the whole sequence every time). nil when off or when it would exceed `prefixCacheLimit`.+ /// **fp16** (`prefixHalf`) when the attention reads K and V as fp16 anyway (the packed kernel, the default): the+ /// same values to the bit for the GPU's heads (the CPU's heads read them rounded); fp32 with any `KOKON_ATTENTION`+ /// (the checks) or `KOKON_PREFIX_CACHE=fp32`. `KOKON_PREFIX_CACHE_MB` sets the limit.let prefixCache: MTLBuffer?- static let prefixCacheLimit = 512 << 20- /// The conditioning whose prefix the cache holds.- private var cachedEncoder: [Float]?+ let prefixHalf: Bool+ /// Up to 1 GB below 32 GB of memory, an eighth of it from there. M2 Pro 16 GB, edits, fp16 cache, 2026-10-25: a+ /// 768 × 448 reference (0.72 GB) made the steps after the first 4.95 → 3.48 s (image 46.2 → 38.1 s); a 1344 × 768+ /// one (2.13 GB) gained nothing, the DiT's file pages evicted and read again (40 GB of page-ins an image against+ /// 21 without the cache). The value from 32 GB is not measured.+ static var prefixCacheLimit: Int { Chip.current.memoryGB < 30 ? 1 << 30 : Int(Chip.current.memoryGB * Double(1 << 30)) / 8 }+ /// The conditioning (encoder rows, then the references' tokens) whose prefix the cache holds.+ private var cachedConditioning: [Float]?+ /// Latent tokens of the condition images (every image but the last), in order.+ public var referenceTokens: Int { sequence.images.dropLast().reduce(0) { $0 + $1.tokens } }private var nextReserve = 0private var products: [String: MPSMatrixMultiplication] = [:]/// The AMX and the Neural Engine beside the GPU on the large products and the target's attention@@ -85,7 +95,8 @@ public final class QwenDiT {d * QwenDiT.hidden, d * QwenDiT.hidden,n * 128, 2 * 5 * d, QwenDiT.layers * 2 * QwenDiT.headDim, d, t * d, t * d,s * QwenDiT.channels, s * QwenDiT.channels,- QwenDiT.heads * (n + 63) / 64 * 64 * QwenDiT.headDim / 2, QwenDiT.heads * (n + 63) / 64 * 64 * QwenDiT.headDim / 2]+ QwenDiT.heads * (n + 63) / 64 * 64 * QwenDiT.headDim / 2, QwenDiT.heads * (n + 63) / 64 * 64 * QwenDiT.headDim / 2,+ (QwenDiT.padded(sequence.images.dropLast().reduce(0) { $0 + $1.tokens }) + 64) * QwenDiT.channels]let arena = try Arena(device: context.device, bytes: Arena.bytes(sizes.map { $0 * 4 }))self.arena = arenavar slices = try sizes.map { try arena.slice(floats: $0) }.makeIterator()@@ -94,12 +105,14 @@ public final class QwenDiT {reserves = [slices.next()!, slices.next()!]rope = slices.next()!; modulation = slices.next()!; qkNorms = slices.next()!; textNorm = slices.next()!textIn = slices.next()!; textOut = slices.next()!; latent = slices.next()!; output = slices.next()!- kPacked = slices.next()!; vPacked = slices.next()!+ kPacked = slices.next()!; vPacked = slices.next()!; referenceLatent = slices.next()!paddedKeys = (n + 63) / 64 * 64let environment = ProcessInfo.processInfo.environment- let cacheBytes = QwenDiT.layers * 2 * sequence.prefix * d * 4- if environment["KOKON_PREFIX_CACHE"] != "0", sequence.prefix > 0, cacheBytes <= QwenDiT.prefixCacheLimit {+ prefixHalf = environment["KOKON_ATTENTION"] == nil && environment["KOKON_PREFIX_CACHE"] != "fp32"+ let cacheBytes = QwenDiT.layers * 2 * sequence.prefix * d * (prefixHalf ? 2 : 4)+ let limit = Int(environment["KOKON_PREFIX_CACHE_MB"] ?? "").map { $0 << 20 } ?? QwenDiT.prefixCacheLimit+ if environment["KOKON_PREFIX_CACHE"] != "0", sequence.prefix > 0, cacheBytes <= limit {prefixCache = try context.scratch(cacheBytes, label: "prefix K/V")} else {prefixCache = nil@@ -200,9 +213,13 @@ public final class QwenDiT {public typealias Probe = (String, UnsafePointer<Float>, Int) -> Void/// One evaluation: `encoder` the encoder's last layer `[T, 4096]` (all positions, image slots included),- /// `target` the target latent `[S, 64]` as packed tokens, `sigma` in [0, 1]. Returns the velocity `[S, 64]`.- public func forward(encoder: [Float], target: [Float], sigma: Float, probe: Probe? = nil) throws -> [Float] {+ /// `references` the condition images' packed latent tokens `[R, 64]` (an edit; in order), `target` the target+ /// latent `[S, 64]` as packed tokens, `sigma` in [0, 1]. Returns the velocity `[S, 64]`.+ public func forward(encoder: [Float], references: [Float] = [], target: [Float], sigma: Float, probe: Probe? = nil) throws -> [Float] {let d = QwenDiT.dim, n = sequence.count, p = sequence.prefix, s = sequence.target+ guard references.count == referenceTokens * QwenDiT.channels else {+ throw QwenSequence.Failure.layout("\(references.count / QwenDiT.channels) reference tokens for \(referenceTokens)")+ }let started = DispatchTime.now().uptimeNanosecondstimings = [:]castInput = nil@@ -215,6 +232,7 @@ public final class QwenDiT {arena.pointer(textIn, floatOffset: i * d).update(from: Array(encoder[row.encoder * d..<(row.encoder + 1) * d]), count: d)}arena.pointer(latent).update(from: target, count: s * QwenDiT.channels)+ arena.pointer(referenceLatent).update(from: references, count: references.count)// The cached path when there is a cache and no probe (the probes compare every row of every stage).let cached = prefixCache != nil && probe == nil@@ -223,8 +241,9 @@ public final class QwenDiT {// A new conditioning: the whole sequence once, the prefix's K and V saved on the way. A pass of the prefix// alone took 1.86 s at 1024² (products of 64 rows run far below the GPU's rate); riding along the first// evaluation it costs its 64 rows.- let saving = cached && cachedEncoder != encoder- if !cached || saving { try embedText(commands: commands) }+ let conditioning = encoder + references+ let saving = cached && cachedConditioning != conditioning+ if !cached || saving { try embedText(commands: commands); try embedReferences(commands: commands) }try linear("img_in.weight", a: latent, c: x, m: s, cRow: p, commands: commands)if let probe { try finish(commands); probe("joint_in", arena.pointer(x), n * d); commands = try begin() }@@ -254,7 +273,7 @@ public final class QwenDiT {try layerNorm(x, into: normed, rows: rows, prefixMod: 4 * d, targetMod: 5 * d + 4 * d, commands: commands)if let probe { try finish(commands); probe("norm_out", arena.pointer(normed), n * d); commands = try begin() }try linear("proj_out.weight", a: normed, aRow: p, c: output, m: s, commands: commands)- if saving { cachedEncoder = encoder }+ if saving { cachedConditioning = conditioning }try finish(commands)if let coexecutor, !profiling, balancing { try coexecutor.rebalance() }let out = Array(UnsafeBufferPointer(start: arena.pointer(output), count: s * QwenDiT.channels))@@ -292,9 +311,38 @@ public final class QwenDiT {blit.endEncoding()}+ /// The condition images' tokens through img_in (into `o`, free before the first layer), copied to their joint rows.+ func embedReferences(commands: MTLCommandBuffer) throws {+ let d = QwenDiT.dim, tokens = referenceTokens+ guard tokens > 0 else { return }+ try linear("img_in.weight", a: referenceLatent, c: o, m: tokens, commands: commands)+ let blit = commands.makeBlitCommandEncoder()!+ var at = 0+ for (i, grid) in sequence.images.dropLast().enumerated() {+ blit.copy(from: arena.buffer, sourceOffset: o.offset + at * d * 4, to: arena.buffer,+ destinationOffset: x.offset + sequence.blockStarts[i] * d * 4, size: grid.tokens * d * 4)+ at += grid.tokens+ }+ blit.endEncoding()+ }+/// The prefix's K and V of `layer` between k, v and the cache (`save`: into the cache).- func copyPrefix(layer: Int, save: Bool, commands: MTLCommandBuffer) {+ func copyPrefix(layer: Int, save: Bool, commands: MTLCommandBuffer) throws {guard let prefixCache else { return }+ if prefixHalf {+ let count = sequence.prefix * QwenDiT.dim+ for (i, slice) in [k, v].enumerated() {+ let cached = (layer * 2 + i) * count * 2+ try compute(save ? "pack_half" : "widen_half", commands: commands) { e in+ e.setBuffer(save ? arena.buffer : prefixCache, offset: save ? slice.offset : cached, index: 0)+ e.setBuffer(save ? prefixCache : arena.buffer, offset: save ? cached : slice.offset, index: 1)+ var c = UInt32(count)+ e.setBytes(&c, length: 4, index: 2)+ e.dispatchThreads(MTLSize(width: count, height: 1, depth: 1), threadsPerThreadgroup: MTLSize(width: 256, height: 1, depth: 1))+ }+ }+ return+ }let bytes = sequence.prefix * QwenDiT.dim * 4let blit = commands.makeBlitCommandEncoder()!for (i, slice) in [k, v].enumerated() {@@ -332,8 +380,8 @@ public final class QwenDiT {causal: segment.isText, commands: commands)}}- if mode == .saving { copyPrefix(layer: layer, save: true, commands: commands) }- if mode == .target { copyPrefix(layer: layer, save: false, commands: commands) }+ if mode == .saving { try copyPrefix(layer: layer, save: true, commands: commands) }+ if mode == .target { try copyPrefix(layer: layer, save: false, commands: commands) }try attend(queries: p, count: n - p, keys: n, causal: false, commands: commands)try linear(block + "attn.to_out.0.weight", a: attended, aRow: first, c: o, m: count, cRow: first,next: block + "img_mlp.gate_layer.weight", commands: commands)diff --git a/engine-mac/Sources/KokonEngine/QwenPipeline.swift b/engine-mac/Sources/KokonEngine/QwenPipeline.swiftindex b26c56a..aa6371a 100644--- a/engine-mac/Sources/KokonEngine/QwenPipeline.swift+++ b/engine-mac/Sources/KokonEngine/QwenPipeline.swift@@ -77,6 +77,14 @@ public final class QwenPipeline {(width / 32 * 32, height / 32 * 32)}+ /// An edit's condition image: interleaved 8-bit RGBA (straight alpha) at the size it is encoded at (sides multiples+ /// of 32: the vision tower's grid).+ public struct Reference {+ public let rgba: [UInt8]+ public let width, height: Int+ public init(rgba: [UInt8], width: Int, height: Int) { (self.rgba, self.width, self.height) = (rgba, width, height) }+ }+/// Text to image at `width` × `height` (multiples of 32). `noise`: a latent `[64, h, w]` to start from instead of/// the seed's; `conditioning`: the encoder's output `[T, 4096]` to use instead of encoding `prompt` (the checks);/// `step`: called after each step with its index and the packed latent `[h·w, 64]`.@@ -84,48 +92,106 @@ public final class QwenPipeline {public func textToImage(prompt: String, width: Int, height: Int, seed: UInt64, noise: [Float]? = nil,conditioning given: [Float]? = nil, sigmas: [Float] = QwenPipeline.turboSigmas,step: ((Int, [Float]) -> Void)? = nil) throws -> [Float] {+ try generate(prompt: prompt, width: width, height: height, seed: seed, noise: noise,+ conditioning: given.map { ($0, [Bool](repeating: false, count: $0.count / QwenDiT.dim)) },+ sigmas: sigmas, step: step)+ }++ /// **The vision tower's file** (mmproj, `--llm_vision`): edits need it.+ public var visionPath: String?++ /// Text to image, or **an edit** with `references` (`QwenImage21Pipeline.__call__` with `image`): each reference goes+ /// through the vision tower into the text encoder's slots and through the VAE encoder into the DiT's prefix, in+ /// the slots the encoder marks (`img_shapes`: the references, then the target). sd.cpp's `vlm_use_vae_input`: the+ /// tower sees the image the VAE encodes. `conditioning`: the encoder's rows and which are image slots, instead of+ /// encoding (the checks); `referenceLatents`: the references' packed tokens instead of the VAE encoder's.+ public func generate(prompt: String, references: [Reference] = [], width: Int, height: Int, seed: UInt64,+ noise: [Float]? = nil, conditioning given: (rows: [Float], slots: [Bool])? = nil,+ referenceLatents givenLatents: [Float]? = nil, sigmas: [Float] = QwenPipeline.turboSigmas,+ step: ((Int, [Float]) -> Void)? = nil) throws -> [Float] {guard width % 32 == 0, height % 32 == 0, width >= 64, height >= 64 else {throw QwenSequence.Failure.layout("\(width)×\(height): each side must be a multiple of 32, at least 64")}+ for r in references where r.width % 32 != 0 || r.height % 32 != 0 || r.rgba.count != 4 * r.width * r.height {+ throw QwenSequence.Failure.layout("a reference of \(r.width)×\(r.height): sides must be multiples of 32")+ }let started = DispatchTime.now().uptimeNanosecondstimings = [:]func since(_ t: UInt64) -> Double { Double(DispatchTime.now().uptimeNanoseconds - t) * 1e-9 }let (h, w) = (height / 16, width / 16)let channels = QwenDiT.channels- // 1. The prompt. The encoder and its 5 GB mapping are gone when this scope ends.+ // 1. The references through the vision tower (its 1.1 GB file unmapped after).var t = DispatchTime.now().uptimeNanoseconds- let (conditioning, rows): ([Float], Int) = try {- if let given { return (given, given.count / QwenDiT.dim) }+ var images: [QwenVisionTower.Output] = []+ if !references.isEmpty && given == nil {+ guard let visionPath else {+ throw GGUFFile.Failure.unsupported("edits need the vision projector (mmproj): start the engine with --llm_vision")+ }+ let tower = try QwenVisionTower(file: try GGUFFile(path: visionPath), context: context)+ defer { tower.release() }+ for r in references {+ let input = QwenVisionTower.pixels(rgba: r.rgba, width: r.width, height: r.height)+ images.append(try tower.encode(rgb: input.rgb, width: input.width, height: input.height))+ }+ timings["vision"] = since(t)+ log(String(format: "vision tower: %d image(s), %d slots in %.2f s", images.count, images.reduce(0) { $0 + $1.slots },+ timings["vision"]!))+ }++ // 2. The prompt. The encoder and its 5 GB mapping are gone when this scope ends.+ t = DispatchTime.now().uptimeNanoseconds+ let (conditioning, rows, slots): ([Float], Int, [Bool]) = try {+ if let given { return (given.rows, given.rows.count / QwenDiT.dim, given.slots) }if keepModels && keepEncoder {if encoder == nil {let file = try GGUFFile(path: encoderPath)(encoderFile, encoder) = (file, try QwenTextEncoder(file: file, context: context))}defer { encoder!.release() }- let c = try encoder!.conditioning(prompt: prompt)- return (c.rows, c.count)+ let c = try encoder!.conditioning(prompt: prompt, images: images)+ return (c.rows, c.count, c.imageSlots)}let encoder = try handedOver ?? QwenTextEncoder(file: try GGUFFile(path: encoderPath), context: context)handedOver = nildefer { encoder.release() }- let c = try encoder.conditioning(prompt: prompt)- return (c.rows, c.count)+ let c = try encoder.conditioning(prompt: prompt, images: images)+ return (c.rows, c.count, c.imageSlots)}()timings["encode"] = since(t)log(String(format: "text encoder: %d tokens in %.2f s", rows, timings["encode"]!))events?(.encoded(tokens: rows, seconds: timings["encode"]!))- // 2. Noise, packed: token y·w + x holds the 64 channels of cell (y, x).+ // 3. The references' latents (`_encode_vae_image`), packed and in order.+ t = DispatchTime.now().uptimeNanoseconds+ var referenceTokens: [Float] = givenLatents ?? []+ if givenLatents == nil, !references.isEmpty {+ if vae == nil { vae = try QwenVAE(path: vaePath, context: context) }+ for r in references {+ // VaeImageProcessor.preprocess: x / 255, then 2x − 1; planar RGBA.+ let plane = r.width * r.height+ var input = [Float](repeating: 0, count: 4 * plane)+ for p in 0..<plane { for c in 0..<4 { input[c * plane + p] = Float(r.rgba[p * 4 + c]) / 255 * 2 - 1 } }+ let latent = try vae!.encode(input, height: r.height, width: r.width)+ referenceTokens += QwenPipeline.pack(latent, height: r.height / 16, width: r.width / 16)+ }+ if !keepVAE { vae = nil }+ timings["vae encode"] = since(t)+ log(String(format: "VAE encoder: %d reference(s) in %.2f s", references.count, timings["vae encode"]!))+ }++ // 4. Noise, packed: token y·w + x holds the 64 channels of cell (y, x).let start = noise ?? Noise.philox(seed: seed, count: channels * h * w) // sd.cpp's, value for valueguard start.count == channels * h * w else {throw QwenSequence.Failure.layout("noise of \(start.count) values for a \(channels)×\(h)×\(w) latent")}var x = QwenPipeline.pack(start, height: h, width: w)- // 3. The steps.+ // 5. The steps.t = DispatchTime.now().uptimeNanoseconds- let sequence = try QwenSequence.textToImage(text: rows, target: .init(height: h, width: w))+ let target = QwenSequence.Grid(height: h, width: w)+ let grids = references.map { QwenSequence.Grid(height: $0.height / 16, width: $0.width / 16) }+ let sequence = try QwenSequence(slots: slots + [Bool](repeating: true, count: target.tokens / 4), images: grids + [target])try {let file = try ditFile ?? GGUFFile(path: ditPath)defer { context.release(file) }@@ -144,10 +210,12 @@ public final class QwenPipeline {defer { dit.cancelled = { false } }try dit.prepareModulation([0] + sigmas.dropLast())timings["dit load"] = since(t)+ log(String(format: "DiT: %d rows, prefix %d, prefix K/V cache %@", sequence.count, sequence.prefix,+ dit.prefixCache.map { String(format: "%.2f GB (%@)", Double($0.length) / 1e9, dit.prefixHalf ? "fp16" : "fp32") } ?? "off"))for i in 0..<(sigmas.count - 1) {if cancelled() { throw QwenDiT.Cancelled() }let stepStart = DispatchTime.now().uptimeNanoseconds- let velocity = try dit.forward(encoder: conditioning, target: x, sigma: sigmas[i])+ let velocity = try dit.forward(encoder: conditioning, references: referenceTokens, target: x, sigma: sigmas[i])let dt = sigmas[i + 1] - sigmas[i]for j in x.indices { x[j] += dt * velocity[j] }timings["step \(i + 1)"] = since(stepStart)@@ -158,7 +226,7 @@ public final class QwenPipeline {shares = dit.coexecutor?.shares}()- // 4. The image.+ // 6. The image.if !keepDiT { dit = nil } // its working memory goes before the VAE'st = DispatchTime.now().uptimeNanosecondsevents?(.decoding)@@ -173,9 +241,24 @@ public final class QwenPipeline {/// **Prompt+**: the text encoder's LLM writes the assistant's reply to `prompt` (`QwenTextGenerator`), on the same/// file and with the same keeping rules as the image's encoder.- public func generateText(system: String, prompt: String, maxTokens: Int, sampling: TextSampler.Parameters, seed: Int64,+ /// `images`: shown before the prompt (an edit's references), through the vision tower first (`visionPath`).+ public func generateText(system: String, prompt: String, images: [Reference] = [], maxTokens: Int,+ sampling: TextSampler.Parameters, seed: Int64,onPiece: (String) -> Bool) throws -> QwenTextGenerator.Result {let started = DispatchTime.now().uptimeNanoseconds+ var outputs: [QwenVisionTower.Output] = []+ if !images.isEmpty {+ guard let visionPath else {+ throw GGUFFile.Failure.unsupported("images need the vision projector (mmproj): start the engine with --llm_vision")+ }+ let tower = try QwenVisionTower(file: try GGUFFile(path: visionPath), context: context)+ defer { tower.release() }+ for r in images {+ let input = QwenVisionTower.pixels(rgba: r.rgba, width: r.width, height: r.height)+ outputs.append(try tower.encode(rgb: input.rgb, width: input.width, height: input.height))+ }+ }+ let imageSeconds = Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9let encoder: QwenTextEncoderif keepModels && keepEncoder {if self.encoder == nil {@@ -188,10 +271,12 @@ public final class QwenPipeline {handedOver = encoder // for the image that follows}defer { encoder.release() }- var result = try QwenTextGenerator(encoder: encoder).generate(system: system, prompt: prompt, maxTokens: maxTokens,- sampling: sampling, seed: seed, onPiece: onPiece)+ var result = try QwenTextGenerator(encoder: encoder).generate(system: system, prompt: prompt, images: outputs,+ maxTokens: maxTokens, sampling: sampling, seed: seed,+ onPiece: onPiece)// With opening the file and reading its tokenizer.- result.prepare = Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9 - result.prefill - result.decode+ result.image = imageSeconds+ result.prepare = Double(DispatchTime.now().uptimeNanoseconds - started) * 1e-9 - result.prefill - result.decode - imageSecondsreturn result}diff --git a/engine-mac/Sources/KokonEngine/QwenTextEncoder.swift b/engine-mac/Sources/KokonEngine/QwenTextEncoder.swiftindex e8b0326..39ee8a3 100644--- a/engine-mac/Sources/KokonEngine/QwenTextEncoder.swift+++ b/engine-mac/Sources/KokonEngine/QwenTextEncoder.swift@@ -68,9 +68,14 @@ public final class QwenTextEncoder {/// the processor does before tokenizing).public func tokens(prompt: String, images: [QwenVisionTower.Output] = []) -> [Int] {guard !images.isEmpty else { return tokenizer.encode(QwenTextEncoder.textToImage(prompt)) }+ return expand(tokenizer.encode(QwenTextEncoder.imageToImage(prompt, images: images.count)), images: images)+ }++ /// Each `<|image_pad|>` of `ids` repeated to its image's slots, in order.+ public func expand(_ ids: [Int], images: [QwenVisionTower.Output]) -> [Int] {let pad = imagePadvar out: [Int] = [], next = 0- for id in tokenizer.encode(QwenTextEncoder.imageToImage(prompt, images: images.count)) {+ for id in ids {if id == pad, next < images.count {out += [Int](repeating: pad, count: images[next].slots)next += 1diff --git a/engine-mac/Sources/KokonEngine/QwenTextGeneration.swift b/engine-mac/Sources/KokonEngine/QwenTextGeneration.swiftindex d5326c7..096b7e6 100644--- a/engine-mac/Sources/KokonEngine/QwenTextGeneration.swift+++ b/engine-mac/Sources/KokonEngine/QwenTextGeneration.swift@@ -22,8 +22,8 @@ public final class QwenTextGenerator {public var finish = Finish.stoppublic var seed: Int64 = 0public var promptTokens = 0, generatedTokens = 0- /// Seconds: tokenizing and buffers, prefill (to the first token), decode (first token to last).- public var prepare = 0.0, prefill = 0.0, decode = 0.0+ /// Seconds: tokenizing and buffers, prefill (to the first token), decode (first token to last), the vision tower.+ public var prepare = 0.0, prefill = 0.0, decode = 0.0, image = 0.0/// Every token sampled, the stop token included.public var tokens: [Int] = []}@@ -37,15 +37,22 @@ public final class QwenTextGenerator {context = encoder.context}- /// sd.cpp's chat template for text generation without images: the system turn when `system` is not empty.- public static func template(system: String, prompt: String) -> String {- (system.isEmpty ? "" : "<|im_start|>system\n\(system)<|im_end|>\n") + "<|im_start|>user\n\(prompt)<|im_end|>\n<|im_start|>assistant\n"+ /// sd.cpp's chat template for text generation (src/pipeline/text.cpp): the system turn when `system` is not empty;+ /// each image before the prompt as `<|vision_start|><|image_pad|><|vision_end|>`, labelled `<image1>`, ` <image2>`…+ /// when there are several.+ public static func template(system: String, prompt: String, images: Int = 0) -> String {+ let markers = (0..<images).map { i in+ (images > 1 ? (i == 0 ? "" : " ") + "<image\(i + 1)>" : "") + "<|vision_start|><|image_pad|><|vision_end|>"+ }.joined()+ return (system.isEmpty ? "" : "<|im_start|>system\n\(system)<|im_end|>\n") + "<|im_start|>user\n\(markers)\(prompt)<|im_end|>\n<|im_start|>assistant\n"}/// Writes the assistant's reply to `prompt`. `onPiece` gets the text as it is written (whole UTF-8 characters);- /// returning false stops it (`cancelled`). `logits`: each step's logits, for the checks.- public func generate(system: String, prompt: String, maxTokens: Int, sampling: TextSampler.Parameters, seed: Int64,- onPiece: (String) -> Bool = { _ in true },+ /// returning false stops it (`cancelled`). `logits`: each step's logits, for the checks. `images`: the vision+ /// tower's outputs of the images shown before the prompt; their slots take 3D positions (`QwenTextEncoder.positions`),+ /// so the reply's positions run ahead of its cache slots by a fixed offset.+ public func generate(system: String, prompt: String, images: [QwenVisionTower.Output] = [], maxTokens: Int,+ sampling: TextSampler.Parameters, seed: Int64, onPiece: (String) -> Bool = { _ in true },logits: ((Int, UnsafePointer<Float>) -> Void)? = nil) throws -> Result {var result = Result()let started = DispatchTime.now().uptimeNanoseconds@@ -55,7 +62,11 @@ public final class QwenTextGenerator {throw GGUFFile.Failure.unsupported("the tokenizer has no <|im_end|> or <|endoftext|>")}let stops: Set<Int> = [imEnd, endOfText]- let ids = tokenizer.encode(QwenTextGenerator.template(system: system, prompt: prompt))+ let ids = encoder.expand(tokenizer.encode(QwenTextGenerator.template(system: system, prompt: prompt, images: images.count)),+ images: images)+ let positions = try QwenTextEncoder.positions(ids: ids, imagePad: images.isEmpty ? -1 : encoder.imagePad,+ grids: images.map { ($0.height / 32, $0.width / 32) })+ let offset = Int(positions.last.map { $0[0] + 1 } ?? 0) - ids.count // 0 for text: position = slotlet capacity = ids.count + maxTokensguard capacity <= QwenTextGenerator.maxContext else {throw GGUFFile.Failure.unsupported("prompt (\(ids.count) tokens) + max_tokens (\(maxTokens)) exceeds the \(QwenTextGenerator.maxContext)-token limit")@@ -64,12 +75,12 @@ public final class QwenTextGenerator {result.seed = seed >= 0 ? seed : Int64.random(in: 0...0x7fff_ffff)var sampler = TextSampler(sampling, seed: UInt64(bitPattern: result.seed))let cache = try KVCache(context: context, layers: QwenTextEncoder.layers, kvHeads: QwenTextEncoder.kvHeads, capacity: capacity)- let step = try Decoder(generator: self, cache: cache)+ let step = try Decoder(generator: self, cache: cache, positionOffset: offset)result.prepare = since(started)// Prefill: the prompt's K and V into the cache, the last row's hidden state to the head.var t = DispatchTime.now().uptimeNanoseconds- let hidden = try encoder.forward(ids: ids, cache: cache)+ let hidden = try encoder.forward(ids: ids, cache: cache, images: images)var next = try step.head(hidden: Array(hidden[((ids.count - 1) * QwenTextEncoder.dim)...]))var stream = TokenStreamDecoder(tokenizer: tokenizer)while true {@@ -112,7 +123,8 @@ public final class QwenTextGenerator {/// The GPU's seconds for the last token.var lastGPU = 0.0- init(generator: QwenTextGenerator, cache: KVCache) throws {+ /// `positionOffset`: a slot's RoPE position minus the slot (images before the prompt).+ init(generator: QwenTextGenerator, cache: KVCache, positionOffset: Int = 0) throws {self.generator = generatorself.cache = cachecontext = generator.context@@ -136,7 +148,7 @@ public final class QwenTextGenerator {}}(n + QwenTextEncoder.layers * (2 * d + 256)).update(from: try Dequant.array(file, "output_norm.weight"), count: d)- let table = QwenTextEncoder.ropeTable(rows: cache.capacity)+ let table = QwenTextEncoder.ropeTable(positions: (0..<cache.capacity).map { SIMD3(repeating: Int32($0 + positionOffset)) })rope = try context.scratch(table.count * 4, label: "decode rope")rope.contents().copyMemory(from: table, byteCount: table.count * 4)logits = try context.scratch(QwenTextGenerator.vocabulary * 4, label: "logits")diff --git a/engine-mac/Sources/KokonEngine/TextChecks.swift b/engine-mac/Sources/KokonEngine/TextChecks.swiftindex 5140f7e..ca7c6a0 100644--- a/engine-mac/Sources/KokonEngine/TextChecks.swift+++ b/engine-mac/Sources/KokonEngine/TextChecks.swift@@ -105,19 +105,45 @@ extension Checks {/// the 8-bit pixels./// `text`: a `reference_text_encoder.py` directory whose conditioning replaces the engine's encoder, to measure/// the DiT and the sampler alone.- public static func generate(encoder: String, dit: String, vae: String, reference directory: String, text: String? = nil) throws {+ /// **An edit** (the reference has `references`): `text` is needed (its images). Without `mmproj` the text reference's+ /// conditioning and the generate reference's latents of the images go in (the DiT, the sampler and the decode alone);+ /// with it, the engine's whole edit: vision tower, text encoder, VAE encoder on the images.+ public static func generate(encoder: String, dit: String, vae: String, reference directory: String, text: String? = nil,+ mmproj: String? = nil) throws {let meta = try JSONSerialization.jsonObject(with: Data(contentsOf: URL(fileURLWithPath: directory).appendingPathComponent("meta.json"))) as! [String: Any]let prompt = meta["prompt"] as! String, grid = meta["grid"] as! [Int]let (h, w) = (grid[0], grid[1])let pipeline = try QwenPipeline(encoder: encoder, dit: dit, vae: vae)+ pipeline.visionPath = mmprojfunc report(_ name: String, _ got: UnsafePointer<Float>, _ want: [Float]) {let r = compare(got, want)print(String(format: " %-12@ rel %.2e max |Δ| %.3g%@", name as NSString, r.relative, r.maxAbs, r.finite ? "" : " NON-FINITE"))}- let image = try pipeline.textToImage(prompt: prompt, width: w * 16, height: h * 16, seed: 0,- noise: try reference(directory, "noise"),- conditioning: try text.map { try reference($0, "conditioning") }) { i, latent in+ var references: [QwenPipeline.Reference] = [], given: (rows: [Float], slots: [Bool])?, latents: [Float]?+ let sizes = meta["references"] as? [[Int]] ?? []+ if !sizes.isEmpty {+ guard let text else { throw GGUFFile.Failure.unsupported("an edit reference: pass its text reference directory") }+ for i in sizes.indices {+ let png = try QwenPipeline.decodeImage(Data(contentsOf: URL(fileURLWithPath: text).appendingPathComponent("image\(i + 1).png")))+ references.append(.init(rgba: png.rgba, width: png.width, height: png.height))+ }+ if mmproj == nil {+ let textMeta = try JSONSerialization.jsonObject(with: Data(contentsOf: URL(fileURLWithPath: text)+ .appendingPathComponent("meta.json"))) as! [String: Any]+ given = (try reference(text, "conditioning"), (textMeta["image_pad_mask"] as! [Int]).map { $0 != 0 })+ latents = try sizes.indices.flatMap { try reference(directory, "reference\($0)") }+ print("edit: the references' conditioning and latents")+ } else {+ print("edit: the engine's vision tower, text encoder and VAE encoder")+ }+ } else if let text {+ let rows = try reference(text, "conditioning")+ given = (rows, [Bool](repeating: false, count: rows.count / QwenDiT.dim))+ }+ let image = try pipeline.generate(prompt: prompt, references: references, width: w * 16, height: h * 16, seed: 0,+ noise: try reference(directory, "noise"), conditioning: given,+ referenceLatents: latents) { i, latent inlatent.withUnsafeBufferPointer { report("latent\(i)", $0.baseAddress!, try! reference(directory, "latent\(i)")) }}image.withUnsafeBufferPointer { report("image", $0.baseAddress!, try! reference(directory, "image")) }@@ -183,24 +209,37 @@ extension Checks {/// `<dir>/engine.json`, its logits beside); once the reference has run on those tokens, each step's logits are/// compared with transformers' (relative error, the greedy choice, the margin between the two best) and the/// engine's speed is reported. `system` / `prompt` from `<dir>/engine.json` when it exists.- public static func textGeneration(path: String, directory: String, tokens: Int = 48) throws {+ /// `image` + `mmproj`: Prompt+ looking at an image (sd.cpp's template with it before the prompt).+ public static func textGeneration(path: String, directory: String, tokens: Int = 48, image: String? = nil, mmproj: String? = nil) throws {let dir = URL(fileURLWithPath: directory)try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)let previous = (try? JSONSerialization.jsonObject(with: Data(contentsOf: dir.appendingPathComponent("engine.json")))) as? [String: Any]let system = previous?["system"] as? String?? "You write prompts for an image generation model. Expand the user's request into one detailed English paragraph that describes the finished image: subject, setting, composition, lighting and style. Output only the prompt."let prompt = previous?["prompt"] as? String ?? "un gatto rosso che dorme su un davanzale, sera"- let file = try GGUFFile(path: path)+ let imagePath = image ?? previous?["image"] as? Stringlet context = try MetalContext()+ var images: [QwenVisionTower.Output] = []+ if let imagePath {+ guard let mmproj else { throw GGUFFile.Failure.unsupported("an image needs the mmproj GGUF") }+ let png = try QwenPipeline.decodeImage(Data(contentsOf: URL(fileURLWithPath: imagePath)))+ let input = QwenVisionTower.pixels(rgba: png.rgba, width: png.width, height: png.height)+ let tower = try QwenVisionTower(file: try GGUFFile(path: mmproj), context: context)+ images = [try tower.encode(rgb: input.rgb, width: input.width, height: input.height)]+ tower.release()+ print(String(format: "image %@: %d slots, vision tower %.2f s", imagePath, images[0].slots, tower.seconds))+ }+ let file = try GGUFFile(path: path)let encoder = try QwenTextEncoder(file: file, context: context)let generator = QwenTextGenerator(encoder: encoder)let vocabulary = QwenTextGenerator.vocabularyvar logits = [Float](repeating: 0, count: tokens * vocabulary)- let result = try generator.generate(system: system, prompt: prompt, maxTokens: tokens, sampling: .greedy, seed: 0, logits: { step, l in+ let result = try generator.generate(system: system, prompt: prompt, images: images, maxTokens: tokens, sampling: .greedy, seed: 0, logits: { step, l inlogits.withUnsafeMutableBufferPointer { ($0.baseAddress! + step * vocabulary).update(from: l, count: vocabulary) }})let steps = result.tokens.count- let promptIDs = encoder.tokenizer.encode(QwenTextGenerator.template(system: system, prompt: prompt))+ let promptIDs = encoder.expand(encoder.tokenizer.encode(QwenTextGenerator.template(system: system, prompt: prompt, images: images.count)),+ images: images)print(String(format: "%d prompt tokens, %d generated (%@): prefill %.2f s, decode %.2f s, %.1f tokens/s",result.promptTokens, result.generatedTokens, result.finish.rawValue, result.prefill, result.decode,Double(steps - 1) / max(result.decode, 1e-9)))@@ -208,7 +247,8 @@ extension Checks {if let previous, let generated = previous["generated"] as? [Int], generated != result.tokens {print("NOTE: not the tokens of the engine's previous run (\(generated.count) then, \(steps) now): the reference must be run again")}- let meta: [String: Any] = ["system": system, "prompt": prompt, "prompt_ids": promptIDs, "generated": result.tokens, "text": result.text]+ var meta: [String: Any] = ["system": system, "prompt": prompt, "prompt_ids": promptIDs, "generated": result.tokens, "text": result.text]+ if let imagePath { meta["image"] = imagePath }try JSONSerialization.data(withJSONObject: meta, options: [.prettyPrinted, .sortedKeys]).write(to: dir.appendingPathComponent("engine.json"))try logits.prefix(steps * vocabulary).withUnsafeBufferPointer { Data(buffer: $0) }.write(to: dir.appendingPathComponent("engine_logits.f32"))diff --git a/engine-mac/Sources/kokon-engine-mac/main.swift b/engine-mac/Sources/kokon-engine-mac/main.swiftindex 4a71f4e..2bf452d 100644--- a/engine-mac/Sources/kokon-engine-mac/main.swift+++ b/engine-mac/Sources/kokon-engine-mac/main.swift@@ -14,18 +14,24 @@ kokon-engine-mac: Kokon's Apple Silicon engine (in development)check-tokenizer <encoder.gguf> <fixture.json> the tokenizer against tools/reference_tokenizer.pycheck-text <encoder.gguf> <reference dir> [mmproj.gguf] the text encoder against tools/reference_text_encoder.pycheck-vision <mmproj.gguf> <reference dir> the vision tower against tools/reference_vision.py, then its time- check-generate <encoder.gguf> <dit.gguf> <vae.safetensors> <reference dir> [text reference dir]- text to image against tools/reference_generate.py, from its noise- (and from the text reference's conditioning when given)- check-textgen <encoder.gguf> <dir> [tokens] text generation (Prompt+), greedy, against tools/reference_text_generation.py+ check-generate <encoder.gguf> <dit.gguf> <vae.safetensors> <reference dir> [text reference dir] [mmproj.gguf]+ text to image or an edit against tools/reference_generate.py, from its noise+ (and from the text reference's conditioning when given; an edit needs the+ text reference, and with the mmproj runs the engine's whole edit)+ check-textgen <encoder.gguf> <dir> [tokens] [image.png mmproj.gguf]+ text generation (Prompt+), greedy, against tools/reference_text_generation.pytime-matvec <encoder.gguf> the decode's products on the encoder's weights: ms and GB/scheck-steps <dit.gguf> <generate reference dir> <text reference dir>each step alone against tools/reference_generate.py (its latents in)check-noise <philox.bin> the seed's noise against sd.cpp's generator (tools/philox_reference.cpp)generate <encoder.gguf> <dit.gguf> <vae.safetensors> "<prompt>" [--size WxH] [--seed N] [--out file.png]- text to image, Turbo's 8 steps (default 1024x1024, seed 42, out.png)- serve --diffusion-model <dit.gguf> --vae <vae.safetensors> --llm <encoder.gguf> [--listen-ip 127.0.0.1] [--listen-port 1234]- the engine as a server speaking sd-server's /sdcpp/v1 API (text to image)+ [--ref image.png ... --mmproj mmproj.gguf]+ text to image, Turbo's 8 steps (default 1024x1024, seed 42, out.png); an edit+ with --ref (sides multiples of 32; the size defaults to the last reference's)+ serve --diffusion-model <dit.gguf> --vae <vae.safetensors> --llm <encoder.gguf> [--llm_vision <mmproj.gguf>]+ [--listen-ip 127.0.0.1] [--listen-port 1234]+ the engine as a server speaking sd-server's /sdcpp/v1 API (text to image;+ edits with --llm_vision)bench [--quick] [--out report.md] [--encoder P --dit P --vae P]a few minutes: this Mac's paths, speed at 1024², accuracy against the fp32 path;prints a report to paste in a GitHub issue (Kokon's model files by default)@@ -72,7 +78,8 @@ do {runs: numbers.dropFirst(2).first ?? 3)case "check-textgen":guard arguments.count >= 3 else { fail(usage) }- try Checks.textGeneration(path: arguments[1], directory: arguments[2], tokens: arguments.count > 3 ? Int(arguments[3]) ?? 48 : 48)+ try Checks.textGeneration(path: arguments[1], directory: arguments[2], tokens: arguments.count > 3 ? Int(arguments[3]) ?? 48 : 48,+ image: arguments.count > 5 ? arguments[4] : nil, mmproj: arguments.count > 5 ? arguments[5] : nil)case "time-matvec":guard arguments.count == 2 else { fail(usage) }try Checks.timeMatvec(path: arguments[1])@@ -99,12 +106,13 @@ do {guard arguments.count == 3 else { fail(usage) }try Checks.vision(path: arguments[1], reference: arguments[2])case "check-generate":- guard arguments.count == 5 || arguments.count == 6 else { fail(usage) }+ guard (5...7).contains(arguments.count) else { fail(usage) }try Checks.generate(encoder: arguments[1], dit: arguments[2], vae: arguments[3], reference: arguments[4],- text: arguments.count == 6 ? arguments[5] : nil)+ text: arguments.count >= 6 ? arguments[5] : nil, mmproj: arguments.count == 7 ? arguments[6] : nil)case "generate":guard arguments.count >= 5 else { fail(usage) }- var (width, height, seed, out) = (1024, 1024, UInt64(42), "out.png")+ var (width, height, seed, out) = (0, 0, UInt64(42), "out.png")+ var (references, mmproj) = ([QwenPipeline.Reference](), String?.none)var options = arguments.dropFirst(5).makeIterator()while let option = options.next() {guard let value = options.next() else { fail(usage) }@@ -115,13 +123,21 @@ do {(width, height) = (parts[0], parts[1])case "--seed": seed = UInt64(value) ?? seedcase "--out": out = value+ case "--ref":+ let image = try QwenPipeline.decodeImage(Data(contentsOf: URL(fileURLWithPath: value)))+ references.append(.init(rgba: image.rgba, width: image.width, height: image.height))+ case "--mmproj": mmproj = valuedefault: fail(usage)}}+ if width == 0 { (width, height) = references.last.map { ($0.width, $0.height) } ?? (1024, 1024) }let size = QwenPipeline.workingSize(width: width, height: height)if size != (width, height) { print("size \(width)×\(height) → \(size.width)×\(size.height) (multiples of 32)") }let pipeline = try QwenPipeline(encoder: arguments[1], dit: arguments[2], vae: arguments[3])- let image = try pipeline.textToImage(prompt: arguments[4], width: size.width, height: size.height, seed: seed)+ pipeline.visionPath = mmproj+ let image = try pipeline.generate(prompt: arguments[4], references: references, width: size.width, height: size.height, seed: seed)+ let m = MemoryUsage.now()+ print(String(format: "peak footprint %.2f GB", m.footprintPeak))try QwenPipeline.writePNG(QwenPipeline.pixels(image, width: size.width, height: size.height),width: size.width, height: size.height, to: out)print("wrote \(out) (\(size.width)×\(size.height), seed \(seed))")@@ -132,7 +148,7 @@ do {guard arguments.count == 2 else { fail(usage) }try Checks.noise(dump: arguments[1])case "serve":- var (ip, port, dit, vae, llm) = ("127.0.0.1", UInt16(1234), "", "", "")+ var (ip, port, dit, vae, llm, vision) = ("127.0.0.1", UInt16(1234), "", "", "", String?.none)var options = arguments.dropFirst().makeIterator()while let option = options.next() {guard let value = options.next() else { fail("\(option) needs a value\n\n" + usage) }@@ -142,14 +158,14 @@ do {case "--diffusion-model": dit = valuecase "--vae": vae = valuecase "--llm": llm = value- case "--llm_vision": break // edits, later+ case "--llm_vision": vision = valuedefault: fail("unknown option \(option)\n\n" + usage)}}guard !dit.isEmpty, !vae.isEmpty, !llm.isEmpty else { fail("serve needs --diffusion-model, --vae and --llm\n\n" + usage) }setvbuf(stdout, nil, _IOLBF, 0) // qls-core follows the log file stdout goes toprint("kokon-engine-mac (experimental), Qwen-Image 2.1 Turbo")- let engine = try EngineServer(paths: .init(diffusionModel: dit, vae: vae, textEncoder: llm))+ let engine = try EngineServer(paths: .init(diffusionModel: dit, vae: vae, textEncoder: llm, vision: vision))let server = try HTTPServer(host: ip, port: port) { engine.handle($0) }print("[INFO ] engine-mac - listening on: http://\(ip):\(server.port)")server.run()diff --git a/engine-mac/Tests/KokonEngineTests/VisionTests.swift b/engine-mac/Tests/KokonEngineTests/VisionTests.swiftindex acf329a..1bad88b 100644--- a/engine-mac/Tests/KokonEngineTests/VisionTests.swift+++ b/engine-mac/Tests/KokonEngineTests/VisionTests.swift@@ -38,3 +38,82 @@ final class VisionTests: XCTestCase {XCTAssert(zip(row, row.dropFirst()).allSatisfy { $0 <= $1 })}}++/// The server's edit requests, without a model: references decoded, sd.cpp's resize rule, what is refused.+final class EditRequestTests: XCTestCase {+ func png(_ width: Int, _ height: Int) throws -> String {+ let rgba = (0..<(width * height * 4)).map { UInt8(truncatingIfNeeded: $0 * 7) }+ return try QwenPipeline.pngData(rgba, width: width, height: height).base64EncodedString()+ }++ func body(_ extra: [String: Any]) throws -> Data {+ var json: [String: Any] = ["prompt": "make it blue", "width": 512, "height": 384, "seed": 3]+ json.merge(extra) { $1 }+ return try JSONSerialization.data(withJSONObject: json)+ }++ func testReferencesAreDecodedAndKeptAtTheirSizeWithoutResize() throws {+ let image = try png(320, 224)+ let (r, reason) = EngineServer.ImageRequest.parse(try body(["ref_images": ["data:image/png;base64," + image],+ "ref_image_args": "resize_before_vae=false"]))+ XCTAssertNil(reason)+ XCTAssertEqual(r?.references.count, 1)+ XCTAssertEqual(r.map { [$0.references[0].width, $0.references[0].height] }, [320, 224])+ XCTAssertEqual(r?.references[0].rgba[4 * 5 + 1], UInt8(truncatingIfNeeded: (4 * 5 + 1) * 7)) // straight RGBA as sent+ }++ func testResizeBeforeVAEScalesToTheOutputsArea() throws {+ // 512 × 384 output, a 320 × 224 reference: √(196 608 · 320 / 224) = 530 → 544; 530 · 224 / 320 = 371 → 384.+ let (r, reason) = EngineServer.ImageRequest.parse(try body(["ref_images": [try png(320, 224)]]))+ XCTAssertNil(reason)+ XCTAssertEqual(r.map { [$0.references[0].width, $0.references[0].height] }, [544, 384])+ }++ func testRefusals() throws {+ XCTAssertNotNil(EngineServer.ImageRequest.parse(try body(["ref_images": [try png(320, 224)]]), vision: false).1)+ XCTAssertNotNil(EngineServer.ImageRequest.parse(try body(["ref_images": [try png(320, 224)], "mask_image": "x"])).1)+ XCTAssertNotNil(EngineServer.ImageRequest.parse(try body(["ref_images": [try png(336, 224)],+ "ref_image_args": "resize_before_vae=false"])).1) // 336 % 32+ XCTAssertNotNil(EngineServer.ImageRequest.parse(try body(["ref_images": ["not an image"]])).1)+ }+}++/// The text encoder's side of images, without a model: Qwen3-VL's 3D positions and the templates.+final class ImagePromptTests: XCTestCase {+ /// get_rope_index, worked out by hand: 2 text tokens, a 2 × 3 grid of slots, 2 text tokens.+ func testPositionsOfAnImageAndTheTextAfterIt() throws {+ let pad = 9+ let ids = [1, 2] + [Int](repeating: pad, count: 6) + [3, 4]+ let p = try QwenTextEncoder.positions(ids: ids, imagePad: pad, grids: [(rows: 2, columns: 3)])+ let want: [[Int32]] = [[0, 0, 0], [1, 1, 1],+ [2, 2, 2], [2, 2, 3], [2, 2, 4], [2, 3, 2], [2, 3, 3], [2, 3, 4],+ [5, 5, 5], [6, 6, 6]] // after the image: 2 + max(2, 3)+ XCTAssertEqual(p.map { [$0[0], $0[1], $0[2]] }, want)+ XCTAssertThrowsError(try QwenTextEncoder.positions(ids: ids, imagePad: pad, grids: [(rows: 2, columns: 2)]))+ XCTAssertEqual(try QwenTextEncoder.positions(ids: [1, 2, 3], imagePad: -1, grids: []).map { $0[1] }, [0, 1, 2])+ }++ /// The M-RoPE table: text rows equal the 1D table; an image row turns frequency 1 with its row, 2 with its column.+ func testMRopeInterleavesTheAxes() {+ let text = QwenTextEncoder.ropeTable(rows: 3)+ XCTAssertEqual(QwenTextEncoder.ropeTable(positions: (0..<3).map { SIMD3(repeating: Int32($0)) }), text)+ let image = QwenTextEncoder.ropeTable(positions: [SIMD3(4, 7, 9)])+ let omega = (0..<64).map { 1 / powf(5_000_000, Float(2 * $0) / 128) }+ for (i, axis) in [(0, 4), (1, 7), (2, 9), (3, 4), (60, 4), (61, 4), (62, 4), (63, 4)] {+ XCTAssertEqual(image[2 * i], cosf(Float(axis) * omega[i]), accuracy: 1e-7, "frequency \(i)")+ }+ }++ func testTemplates() {+ XCTAssertEqual(QwenTextEncoder.imageToImage("make it blue", images: 2),+ "<|im_start|>system\nComprehend and analyze the provided prompt.<|im_end|>\n<|im_start|>user\n"+ + "<image1><|vision_start|><|image_pad|><|vision_end|> <image2><|vision_start|><|image_pad|><|vision_end|>"+ + "make it blue<|im_end|>\n<|im_start|>assistant\n")+ // sd.cpp's text generation: one image unlabelled, several labelled.+ XCTAssertEqual(QwenTextGenerator.template(system: "", prompt: "p", images: 1),+ "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>p<|im_end|>\n<|im_start|>assistant\n")+ XCTAssertEqual(QwenTextGenerator.template(system: "s", prompt: "p", images: 2),+ "<|im_start|>system\ns<|im_end|>\n<|im_start|>user\n<image1><|vision_start|><|image_pad|><|vision_end|>"+ + " <image2><|vision_start|><|image_pad|><|vision_end|>p<|im_end|>\n<|im_start|>assistant\n")+ }+}diff --git a/engine-mac/tools/api_check.py b/engine-mac/tools/api_check.pyindex 8c1d2aa..dc6ad0d 100644--- a/engine-mac/tools/api_check.py+++ b/engine-mac/tools/api_check.py@@ -2,9 +2,11 @@"""Text to image through a running `kokon-engine-mac serve` (or sd-server) as qls-core sends it: capabilities,img_gen, polling jobs/{id}, optionally a cancel; with `--enhance`, Prompt+ first (text_gen with Kokon's built-insystem prompt, read from crates/qls-core/src/enhancer.rs, and its sampling). Prints the time of each image and writes-the PNGs.+the PNGs. `--ref image.png` (repeatable): an edit, the references sent as Standard detail sends them+(`resize_before_vae=false`; sides multiples of 32), and Prompt+ then looks at them with the edit system prompt.python3 -I tools/api_check.py --port 1234 --images 2 --out /tmp/api [--size 1024x1024] [--cancel-after 10] [--enhance]+ [--ref image.png]"""import argparse, base64, json, os, re, time, urllib.request, urllib.error@@ -32,9 +34,12 @@ def main():p.add_argument("--out", default=".")p.add_argument("--cancel-after", type=float, default=0, help="cancel the last image after this many seconds")p.add_argument("--enhance", action="store_true", help="Prompt+ writes each image's prompt first")+ p.add_argument("--ref", action="append", default=[], help="a reference image: an edit")a = p.parse_args()enhancer = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../crates/qls-core/src/enhancer.rs")- system = re.search(r'pub const T2I_SYSTEM_PROMPT: &str = r#"(.*?)"#;', open(enhancer).read(), re.S).group(1) if a.enhance else ""+ which = "EDIT_SYSTEM_PROMPT" if a.ref else "T2I_SYSTEM_PROMPT"+ system = re.search(rf'pub const {which}: &str = r#"(.*?)"#;', open(enhancer).read(), re.S).group(1) if a.enhance else ""+ refs = [base64.b64encode(open(path, "rb").read()).decode() for path in a.ref]width, height = (int(v) for v in a.size.split("x"))status, caps = call(a.port, "GET", "/sdcpp/v1/capabilities", timeout=10)print(f"capabilities {status}: engine {caps.get('engine', 'sd-server')}, model {caps.get('model', {}).get('name')}")@@ -44,7 +49,7 @@ def main():prompt = a.promptif a.enhance:status, done = call(a.port, "POST", "/sdcpp/v1/text_gen", {- "prompt": a.prompt, "system_prompt": system, "max_tokens": 1024, "stream": False,+ "prompt": a.prompt, "system_prompt": system, "images": refs, "max_tokens": 1024, "stream": False,"sampling": {"temperature": 0.7, "top_p": 0.8, "top_k": 20, "presence_penalty": 1.5, "seed": a.seed + i}}, timeout=300)if status != 200:print(f"text_gen {status}: {done}")@@ -55,6 +60,8 @@ def main():"batch_count": 1, "sample_params": {"sample_method": "euler", "sample_steps": 8,"custom_sigmas": TURBO_SIGMAS, "guidance": {"txt_cfg": 1.0}},"output_format": "png", "embed_image_metadata": False}+ if refs:+ body.update(ref_images=refs, ref_image_args="resize_before_vae=false", image_preprocess=["target=ref,mode=none"])status, accepted = call(a.port, "POST", "/sdcpp/v1/img_gen", body)if status != 202:print(f"img_gen {status}: {accepted}")diff --git a/engine-mac/tools/reference_dit.py b/engine-mac/tools/reference_dit.pyindex 9a317e1..4bf9c27 100644--- a/engine-mac/tools/reference_dit.py+++ b/engine-mac/tools/reference_dit.py@@ -64,8 +64,8 @@ class Weights:module.load_state_dict(state, strict=True)-def layer0_stages(block, x, modulation, rotary_emb, target_token_mask, save):- """QwenImage21TransformerBlock.forward and its processor, unrolled, saving each stage (t2i: one text segment)."""+def layer0_stages(block, x, modulation, rotary_emb, target_token_mask, segments, save):+ """QwenImage21TransformerBlock.forward and its processor, unrolled, saving each stage."""from diffusers.models.transformers.transformer_qwenimage21 import apply_rotary_emb_qwenmod1, mod2 = modulation.chunk(2, dim=-1)normed, gate1 = block._modulate(block.img_norm1(x), mod1, target_token_mask)@@ -79,8 +79,7 @@ def layer0_stages(block, x, modulation, rotary_emb, target_token_mask, save):save("l0_q", q[0].flatten(1))save("l0_k", k[0].flatten(1))save("l0_v", v[0].flatten(1))- out = attn(hidden_states=normed, rotary_emb=rotary_emb,- segments=[(0, int((~target_token_mask).sum()), True)])+ out = attn(hidden_states=normed, rotary_emb=rotary_emb, segments=segments)save("l0_attn_out", out[0])x = x + gate1.tanh() * outsave("l0_res1", x[0])@@ -90,7 +89,7 @@ def layer0_stages(block, x, modulation, rotary_emb, target_token_mask, save):class ReferenceDiT:- """QwenImage21Transformer2DModel.forward, batch 1, no cache, text-to-image, one block module swapped per layer."""+ """QwenImage21Transformer2DModel.forward, batch 1, no cache, one block module swapped per layer."""def __init__(self, path, layers=32, dtype=torch.float32):self.w = w = Weights(path, dtype)@@ -115,13 +114,16 @@ class ReferenceDiT:all_layers = False- def forward(self, encoder, latent, h, w_, timestep, save=None, progress=True):+ def forward(self, encoder, latent, h, w_, timestep, save=None, progress=True, refs=(), slots=None):"""`encoder` [1, T, d], `latent` [1, h * w, 64], `timestep` a [1] tensor (σ); returns the target's velocity- [1, h * w, 64]. `save(name, tensor)` receives every stage when given."""+ [1, h * w, 64]. `save(name, tensor)` receives every stage when given. An edit: `refs` the condition images'+ packed latents with their grids [(latent [1, h_i * w_i, 64], h_i, w_i)], `slots` which encoder rows are their+ slots (the text encoder's `image_pad_mask`)."""d, text, tokens = 4096, encoder.shape[1], h * w_- img_shapes = [(1, h, w_)]- img_mask = torch.tensor([[False] * text + [True] * (tokens // 4)])- hidden_states = self.img_in(latent)+ img_shapes = [(1, rh, rw) for _, rh, rw in refs] + [(1, h, w_)]+ slots = list(slots) if slots is not None else [False] * text+ img_mask = torch.tensor([slots + [True] * (tokens // 4)])+ hidden_states = self.img_in(torch.cat([r for r, _, _ in refs] + [latent], dim=1))encoder_hidden_states = self.txt_in(encoder)repeats = torch.where(img_mask, 4, 1)[0]image_pad_mask = torch.repeat_interleave(img_mask[0], repeats)@@ -146,7 +148,7 @@ class ReferenceDiT:for layer in range(self.layers):self.w.load(self.block, f"transformer_blocks.{layer}.")if layer == 0 and save:- layer0_stages(self.block, joint, modulation, rotary_emb, target_token_mask, save)+ layer0_stages(self.block, joint, modulation, rotary_emb, target_token_mask, segments, save)joint = self.block(hidden_states=joint, modulation=modulation, rotary_emb=rotary_emb, attention_mask=None,target_token_mask=target_token_mask, segments=segments)if save and (self.all_layers or layer in (0, 1) or layer == self.layers - 1):diff --git a/engine-mac/tools/reference_generate.py b/engine-mac/tools/reference_generate.pyindex 2237c83..205c9e6 100644--- a/engine-mac/tools/reference_generate.py+++ b/engine-mac/tools/reference_generate.py@@ -9,8 +9,13 @@ the pieces the other references check, in fp32 on the CPU:Saves the noise, every step's velocity and latent, the decoded image and the final pixels for`kokon-engine-mac check-generate`. 256² takes ~5 min (8 DiT evaluations, each layer's weights unpacked by gguf-py).+**An edit** when the text reference has images (`reference_text_encoder.py --image`): each image as the pipeline hands it+to the VAE (RGBA, `VaeImageProcessor.preprocess` at its own size, which `resize_before_vae=false` keeps), encoded+(`_encode_vae_image`: the mode, (z − mean) / std), packed, and prepended to the target's tokens at every step; the+text reference's `image_pad_mask` places their slots. Saved as `reference<i>`.+python -I engine-mac/tools/reference_generate.py <dit.gguf> <vae.safetensors> <text reference dir> <repo dir> <out>- [--grid 16] [--seed 1] [--dtype float64]+ [--grid 16 | --size WxH] [--seed 1] [--dtype float64]`--dtype float64`: the DiT and the latents in fp64 (the scheduler still rounds the latent to fp32 each step), theVAE in fp32. Needed on unsloth's DiT, where torch's fp32 drifts to 7e-5 of fp64 in one evaluation.@@ -24,6 +29,7 @@ import sysimport numpy as npimport torch+from PIL import Imagefrom diffusers import FlowMatchEulerDiscreteSchedulerfrom diffusers.image_processor import VaeImageProcessorfrom diffusers.pipelines.qwenimage21.pipeline_qwenimage21 import calculate_shift@@ -31,6 +37,7 @@ from diffusers.pipelines.qwenimage21.pipeline_qwenimage21 import calculate_shiftsys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # the sibling referencesfrom reference_dit import ReferenceDiT, enable_float64 # noqa: E402from reference_vae import load_vae # noqa: E402+from reference_vae_encoder import load_full_vae # noqa: E402def main():@@ -41,6 +48,7 @@ def main():parser.add_argument("repo")parser.add_argument("out")parser.add_argument("--grid", type=int, default=16, help="latent grid side (16 = a 256² image)")+ parser.add_argument("--size", help="WxH of the image instead (multiples of 32)")parser.add_argument("--seed", type=int, default=1)parser.add_argument("--dtype", default="float32", choices=["float32", "float64"])args = parser.parse_args()@@ -57,6 +65,25 @@ def main():conditioning = torch.from_numpy(np.fromfile(os.path.join(args.text, "conditioning.f32"), dtype=np.float32))conditioning = conditioning.view(1, -1, 4096).to(dtype)h = w = args.grid+ if args.size:+ w, h = (int(v) // 16 for v in args.size.split("x"))++ # The edit's condition images (prepare_latents with `images`).+ refs, slots, sizes = [], None, text_meta.get("images") or []+ if sizes:+ slots = [bool(v) for v in text_meta["image_pad_mask"]]+ encoder_vae = load_full_vae(args.vae)+ processor = VaeImageProcessor(vae_scale_factor=16, vae_latent_channels=64)+ mean = torch.tensor(encoder_vae.config.latents_mean).view(1, 64, 1, 1, 1)+ std = torch.tensor(encoder_vae.config.latents_std).view(1, 64, 1, 1, 1)+ for i, (rw, rh) in enumerate(sizes):+ image = Image.open(os.path.join(args.text, f"image{i + 1}.png")).convert("RGBA")+ pixels = processor.preprocess(image, width=rw, height=rh).unsqueeze(2)+ latent = (encoder_vae.encode(pixels).latent_dist.mode() - mean) / std+ packed = latent[0, :, 0].reshape(64, -1).transpose(0, 1)[None]+ save(f"reference{i}", packed[0])+ refs.append((packed.to(dtype), rh // 16, rw // 16))+ del encoder_vae# prepare_latents + _pack_latentsnoise = torch.randn((1, 1, 64, h, w), generator=torch.Generator().manual_seed(args.seed), dtype=torch.float32)@@ -75,7 +102,8 @@ def main():dit = ReferenceDiT(args.dit, dtype=dtype)for i, t in enumerate(scheduler.timesteps):- velocity = dit.forward(conditioning, latents, h, w, t.expand(1).to(dtype) / 1000, progress=False)+ velocity = dit.forward(conditioning, latents, h, w, t.expand(1).to(dtype) / 1000, progress=False,+ refs=refs, slots=slots)save(f"velocity{i}", velocity[0])latents = scheduler.step(velocity, t, latents, return_dict=False)[0]save(f"latent{i}", latents[0])@@ -94,7 +122,8 @@ def main():meta = {"prompt": text_meta["prompt"], "grid": [h, w], "seed": args.seed, "mode": pil.mode,"sigmas": scheduler.sigmas.tolist(), "timesteps": (scheduler.timesteps / 1000).tolist(),- "dit": os.path.basename(args.dit), "dtype": args.dtype, "vae": os.path.basename(args.vae)}+ "dit": os.path.basename(args.dit), "dtype": args.dtype, "vae": os.path.basename(args.vae),+ "references": sizes}with open(os.path.join(args.out, "meta.json"), "w") as f:json.dump(meta, f, indent=1)print(f"image {pil.size} {pil.mode}")diff --git a/engine-mac/tools/reference_text_encoder.py b/engine-mac/tools/reference_text_encoder.pyindex daa760c..7d0b9c0 100644--- a/engine-mac/tools/reference_text_encoder.py+++ b/engine-mac/tools/reference_text_encoder.py@@ -76,6 +76,44 @@ class Weights:layer.load_state_dict(state, strict=True)+class VisionPrompt:+ """The processor and the vision tower on a prompt with images: token ids and run()'s `vision` (image rows, merged+ rows, deepstack, 3D positions). `extra` token ids appended after the prompt (text) get positions too."""++ def __init__(self, processor, full_config, mmproj):+ self.processor, self.config, self.mmproj, self.tower = processor, full_config, mmproj, None++ def __call__(self, text, images, save=None, extra=()):+ flat = []+ for image in images: # the pipeline: RGBA composited over white for the vision encoder+ if image.mode == "RGBA":+ white = Image.new("RGB", image.size, (255, 255, 255))+ white.paste(image, mask=image.getchannel("A"))+ image = white+ flat.append(image.convert("RGB"))+ inputs = self.processor(text=[text], images=flat, padding=True, padding_side="left", return_tensors="pt")+ if self.tower is None:+ vconfig = self.config.vision_config+ vconfig._attn_implementation = "eager"+ self.tower = Qwen3VLVisionModel(vconfig).eval()+ self.tower.load_state_dict(vision_state_dict(self.mmproj, vconfig), strict=True)+ out = self.tower(inputs.pixel_values.to(torch.float32), grid_thw=inputs.image_grid_thw, return_dict=True)+ input_ids = torch.cat([inputs.input_ids, torch.tensor([list(extra)], dtype=inputs.input_ids.dtype)], dim=1)+ types = torch.cat([inputs.mm_token_type_ids, torch.zeros(1, len(extra), dtype=inputs.mm_token_type_ids.dtype)], dim=1)+ shim = SimpleNamespace(config=self.config)+ shim.get_vision_position_ids = lambda *a, **k: Qwen3VLModel.get_vision_position_ids(shim, *a, **k)+ position_ids, _ = Qwen3VLModel.get_rope_index(shim, input_ids, types, image_grid_thw=inputs.image_grid_thw,+ attention_mask=torch.ones_like(input_ids))+ if save:+ save("vision_merged", out.pooler_output)+ for j, feature in enumerate(out.deepstack_features):+ save(f"vision_deepstack{j}", feature)+ save("positions", position_ids[:, 0].to(torch.float32))+ ids = input_ids[0].tolist()+ visual = torch.tensor(ids) == self.config.image_token_id+ return ids, (visual, out.pooler_output, out.deepstack_features, position_ids)++def run(weights, config, ids, save=None, vision=None):"""The text model's forward on one unpadded prompt; returns the last layer's output (no final norm). `vision`:(image rows mask, merged rows, deepstack list, position ids [3, 1, T]) for a prompt with images."""@@ -133,38 +171,10 @@ def main():saved[name] = list(array.shape)full_config = AutoConfig.from_pretrained(args.text_encoder)- tower = None+ vision_prompt = VisionPrompt(processor, full_config, args.mmproj) if args.mmproj else Nonedef with_images(text, images, save=None):- """Token ids and run()'s `vision` for `text` (a template with its image markers) and its images."""- nonlocal tower- flat = []- for image in images: # the pipeline: RGBA composited over white for the vision encoder- if image.mode == "RGBA":- white = Image.new("RGB", image.size, (255, 255, 255))- white.paste(image, mask=image.getchannel("A"))- image = white- flat.append(image.convert("RGB"))- inputs = processor(text=[text], images=flat, padding=True, padding_side="left", return_tensors="pt")- if tower is None:- vconfig = full_config.vision_config- vconfig._attn_implementation = "eager"- tower = Qwen3VLVisionModel(vconfig).eval()- tower.load_state_dict(vision_state_dict(args.mmproj, vconfig), strict=True)- out = tower(inputs.pixel_values.to(torch.float32), grid_thw=inputs.image_grid_thw, return_dict=True)- shim = SimpleNamespace(config=full_config)- shim.get_vision_position_ids = lambda *a, **k: Qwen3VLModel.get_vision_position_ids(shim, *a, **k)- position_ids, _ = Qwen3VLModel.get_rope_index(shim, inputs.input_ids, inputs.mm_token_type_ids,- image_grid_thw=inputs.image_grid_thw,- attention_mask=inputs.attention_mask)- if save:- save("vision_merged", out.pooler_output)- for j, feature in enumerate(out.deepstack_features):- save(f"vision_deepstack{j}", feature)- save("positions", position_ids[:, 0].to(torch.float32))- ids = inputs.input_ids[0].tolist()- visual = torch.tensor(ids) == full_config.image_token_id- return ids, (visual, out.pooler_output, out.deepstack_features, position_ids)+ return vision_prompt(text, images, save)vision, images = None, []if args.image:diff --git a/engine-mac/tools/reference_text_generation.py b/engine-mac/tools/reference_text_generation.pyindex 20f567d..5a58d78 100644--- a/engine-mac/tools/reference_text_generation.py+++ b/engine-mac/tools/reference_text_generation.py@@ -6,6 +6,9 @@ engine generated (teacher forcing), with the final RMSNorm and the language modepython -I engine-mac/tools/reference_text_generation.py <encoder.gguf> <processor dir> <text_encoder dir> <dir>kokon-engine-mac check-textgen <encoder.gguf> <dir> # 2. the same tokens again, compared with the reference+With an image (`check-textgen <encoder> <dir> <tokens> <image.png> <mmproj>`, then `--mmproj` here): sd.cpp's template+with the image before the prompt, the vision tower and 3D positions as `reference_text_encoder.py --image`.+The prompt is tokenized by the official processor as sd.cpp's template writes it (system turn, user turn, assistant),and must give the engine's ids. Writes `<dir>/reference_logits.f32` ([generated, vocabulary] fp32: the logits thatpredict each generated token) and `<dir>/reference.json` (its own greedy choice at each position).@@ -17,10 +20,11 @@ import sysimport torchfrom transformers import AutoConfig, AutoProcessor+from PIL import Imagefrom transformers.models.qwen3_vl.modeling_qwen3_vl import Qwen3VLTextRMSNormsys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))-from reference_text_encoder import Weights, run # noqa: E402+from reference_text_encoder import VisionPrompt, Weights, run # noqa: E402def main():@@ -29,6 +33,7 @@ def main():parser.add_argument("processor")parser.add_argument("text_encoder", help="text_encoder/ of the official repository (config.json)")parser.add_argument("dir", help="where check-textgen wrote engine.json")+ parser.add_argument("--mmproj", help="the vision tower's GGUF, when engine.json has an image")args = parser.parse_args()torch.set_grad_enabled(False)with open(os.path.join(args.dir, "engine.json")) as f:@@ -37,17 +42,25 @@ def main():config._attn_implementation = "eager"processor = AutoProcessor.from_pretrained(args.processor)- # sd.cpp's template (src/pipeline/text.cpp), text only.+ # sd.cpp's template (src/pipeline/text.cpp); one image goes before the prompt, unlabelled.system, prompt = engine["system"], engine["prompt"]- text = (f"<|im_start|>system\n{system}<|im_end|>\n" if system else "") + f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n"- ids = processor(text=[text], return_tensors="pt").input_ids[0].tolist()+ marker = "<|vision_start|><|image_pad|><|vision_end|>" if engine.get("image") else ""+ text = (f"<|im_start|>system\n{system}<|im_end|>\n" if system else "") + f"<|im_start|>user\n{marker}{prompt}<|im_end|>\n<|im_start|>assistant\n"+ generated = engine["generated"]+ vision = None+ if marker:+ full_config = AutoConfig.from_pretrained(args.text_encoder)+ sequence, vision = VisionPrompt(processor, full_config, args.mmproj)(text, [Image.open(engine["image"])],+ extra=generated[:-1])+ ids = sequence[:len(sequence) - len(generated) + 1]+ else:+ ids = processor(text=[text], return_tensors="pt").input_ids[0].tolist()+ sequence = ids + generated[:-1] # each position predicts the next tokenif ids != engine["prompt_ids"]:sys.exit(f"prompt ids differ from the engine's: {len(ids)} against {len(engine['prompt_ids'])}")- generated = engine["generated"]- sequence = ids + generated[:-1] # each position predicts the next tokenweights = Weights(args.gguf)- hidden = run(weights, config, sequence)[0]+ hidden = run(weights, config, sequence, vision=vision)[0]norm = Qwen3VLTextRMSNorm(config.hidden_size, eps=config.rms_norm_eps)norm.weight.copy_(weights.get("output_norm.weight"))head = weights.get("output.weight")diff --git a/engine-mac/tools/reference_vae_encoder.py b/engine-mac/tools/reference_vae_encoder.pyindex 2e17a08..33c1159 100644--- a/engine-mac/tools/reference_vae_encoder.py+++ b/engine-mac/tools/reference_vae_encoder.py@@ -47,6 +47,22 @@ def encoder_name(key):raise KeyError(key)+def load_full_vae(path):+ """diffusers' AutoencoderKLQwenImage21, encoder and decoder, with the weights of Kokon's file, fp32, eval mode."""+ from diffusers import AutoencoderKLQwenImage21+ vae = AutoencoderKLQwenImage21() # the defaults are Qwen-Image 2.1's config+ model = vae.state_dict()+ converted = {}+ for key, value in load_file(path).items():+ name = encoder_name(key) or decoder_name(key)+ if name is not None:+ converted[name] = value.float().reshape(model[name].shape)+ missing, unexpected = vae.load_state_dict(converted, strict=False)+ missing = [k for k in missing if k.startswith("encoder.") or k.startswith("quant_conv")]+ assert not missing and not unexpected, (missing[:10], unexpected[:10])+ return vae.eval()++def main():parser = argparse.ArgumentParser()parser.add_argument("vae")@@ -54,23 +70,12 @@ def main():parser.add_argument("--size", default="256x256", help="WxH, multiples of 16")parser.add_argument("--seed", type=int, default=11)args = parser.parse_args()- from diffusers import AutoencoderKLQwenImage21torch.manual_seed(args.seed)torch.set_grad_enabled(False)os.makedirs(args.out, exist_ok=True)width, height = (int(v) for v in args.size.split("x"))- vae = AutoencoderKLQwenImage21() # the defaults are Qwen-Image 2.1's config- model = vae.state_dict()- converted = {}- for key, value in load_file(args.vae).items():- name = encoder_name(key) or decoder_name(key)- if name is not None:- converted[name] = value.float().reshape(model[name].shape)- missing, unexpected = vae.load_state_dict(converted, strict=False)- missing = [k for k in missing if k.startswith("encoder.") or k.startswith("quant_conv")]- assert not missing and not unexpected, (missing[:10], unexpected[:10])- vae.eval()+ vae = load_full_vae(args.vae)image = torch.rand(1, 4, 1, height // 16, width // 16) * 2 - 1image = F.interpolate(image[:, :, 0], size=(height, width), mode="bilinear", align_corners=False)[:, :, None]