Skip to content

armgit / kokon

Public Imported from GitHub
0

Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.

Mac engine as a /sdcpp/v1 server; qls-core selects it on macOS as an experimental opt-in

Gabriele Lozuponeb0cf1319e72f
diff --git a/crates/qls-core/src/config.rs b/crates/qls-core/src/config.rs
index 61a6178..55ae845 100644
--- a/crates/qls-core/src/config.rs
+++ b/crates/qls-core/src/config.rs
@@ -69,6 +69,11 @@ pub enum TextEncoderPlacement {
}
/// Which stable-diffusion.cpp build runs the model. Each build lives in its own folder, so
+/// Folder of Kokon's Mac engine, in the stable-diffusion.cpp folder (beside `engine-metal`), and its executable
+/// (the engine's kernels and resources are the `KokonEngineMac_KokonEngine.bundle` next to it).
+pub const MAC_ENGINE_DIR: &str = "engine-mac";
+pub const MAC_ENGINE_EXE: &str = "kokon-engine-mac";
+
/// switching back and forth needs no re-download.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
@@ -331,6 +336,11 @@ pub struct AppConfig {
/// which avoided grid patterns on clean renders in an A/B on 8 images (2026-10-12).
pub upscale_prescale: bool,
pub extra_server_args: Vec<String>,
+ /// Kokon's own engine for Apple Silicon (`engine-mac/`, `kokon-engine-mac serve`) instead of sd.cpp's Metal build:
+ /// the GPU, the CPU's matrix unit and the Neural Engine together, ~4x faster text to image on an M2 Pro. Experimental
+ /// and off by default (Settings → Performance); text to image only for now. sd.cpp stays the default, the
+ /// fallback, and the engine on Windows and Linux.
+ pub mac_engine: bool,
pub output_dir: PathBuf,
pub keep_server_running: bool,
/// When no desktop window is attached (e.g. MCP only), unload the model after this
@@ -382,6 +392,7 @@ impl Default for AppConfig {
upscale_blend: 1.0,
upscale_prescale: false,
extra_server_args: vec![],
+ mac_engine: false,
output_dir: default_output_dir(),
keep_server_running: false,
idle_unload_minutes: 15,
@@ -705,6 +716,12 @@ impl AppConfig {
/// Why the inference server cannot start: the engine or a model file is missing.
pub fn missing_files_error(&self) -> Option<AppError> {
let exe = self.server_exe();
+ if !exe.is_file() && self.uses_mac_engine() {
+ return Some(
+ AppError::new(ErrorCode::MissingModel, format!("Kokon's Mac engine was not found in {}.", exe.parent().unwrap_or(&exe).display()))
+ .suggest("Turn off Kokon's Mac engine in Settings → Performance to use the standard engine."),
+ );
+ }
if !exe.is_file() {
return Some(
AppError::new(ErrorCode::MissingModel, format!("sd-server was not found in {}.", self.engine_dir().display()))
@@ -776,9 +793,17 @@ impl AppConfig {
}
pub fn server_exe(&self) -> PathBuf {
+ if self.uses_mac_engine() {
+ return self.sdcpp_dir.join(MAC_ENGINE_DIR).join(MAC_ENGINE_EXE);
+ }
self.engine_dir().join(if cfg!(windows) { "sd-server.exe" } else { "sd-server" })
}
+ /// Kokon's Mac engine runs instead of sd-server: chosen in Settings, on the Metal backend only.
+ pub fn uses_mac_engine(&self) -> bool {
+ self.mac_engine && self.gpu_backend == GpuBackend::Metal
+ }
+
/// The configured device when it belongs to the active engine, else its first GPU.
pub fn effective_device(&self) -> String {
let dev = self.device.trim().to_ascii_lowercase();
diff --git a/crates/qls-core/src/engine.rs b/crates/qls-core/src/engine.rs
index 740e4e6..d517ca1 100644
--- a/crates/qls-core/src/engine.rs
+++ b/crates/qls-core/src/engine.rs
@@ -1093,6 +1093,10 @@ impl Engine {
.await
.map_err(AppError::internal)??;
let pipeline::PreparedJob { request: mut sd_req, post } = prepared;
+ if let Some(what) = caps.mac_engine_unsupported(&sd_req) {
+ return Err(AppError::new(ErrorCode::Unsupported, format!("Kokon's Mac engine (experimental) does not do {what} yet."))
+ .suggest("Turn it off in Settings → Performance to use the standard engine for this."));
+ }
// 4b. Prompt enhancer: rewrites the user's prompt, looking at the references
// exactly as the image model gets them.
diff --git a/crates/qls-core/src/process.rs b/crates/qls-core/src/process.rs
index fdd60a1..ec3289f 100644
--- a/crates/qls-core/src/process.rs
+++ b/crates/qls-core/src/process.rs
@@ -19,6 +19,9 @@ use crate::sdcpp::{Capabilities, SdcppClient};
// ---------------------------------------------------------------------------------------
pub fn build_server_args(cfg: &AppConfig, profile: &ModelProfile) -> Vec<String> {
+ if cfg.uses_mac_engine() {
+ return build_mac_engine_args(cfg, profile);
+ }
let p = |p: &Path| p.to_string_lossy().into_owned();
let mut a: Vec<String> = vec![
"--listen-ip".into(),
@@ -119,6 +122,31 @@ pub fn build_server_args(cfg: &AppConfig, profile: &ModelProfile) -> Vec<String>
a
}
+/// `kokon-engine-mac serve`: the same files and port as sd-server. The engine decides its own memory and units, so
+/// none of sd-server's placement or backend flags apply; the vision projector is for edits, not supported yet.
+pub fn build_mac_engine_args(cfg: &AppConfig, profile: &ModelProfile) -> Vec<String> {
+ let p = |p: &Path| p.to_string_lossy().into_owned();
+ vec![
+ "serve".into(),
+ "--listen-ip".into(),
+ "127.0.0.1".into(),
+ "--listen-port".into(),
+ cfg.server_port.to_string(),
+ "--diffusion-model".into(),
+ p(&profile.diffusion_model),
+ "--vae".into(),
+ p(&profile.vae),
+ "--llm".into(),
+ p(&profile.text_encoder),
+ ]
+}
+
+/// A process this module may reuse or stop: sd-server, or Kokon's Mac engine.
+fn is_server_process(name: &str) -> bool {
+ let name = name.to_ascii_lowercase();
+ name.starts_with("sd-server") || name.starts_with(crate::config::MAC_ENGINE_EXE)
+}
+
fn trim_float(v: f32) -> String {
let s = format!("{v:.2}");
s.trim_end_matches('0').trim_end_matches('.').to_string()
@@ -433,7 +461,8 @@ impl ServerManager {
if let Ok(caps) = client.capabilities(Duration::from_secs(3)).await {
let rec = rec.filter(|r| r.port == port);
- if caps.model_matches(&profile.diffusion_model) && caps.supports_img_gen() {
+ // The right kind of server too: switching Kokon's Mac engine on or off restarts ours.
+ if caps.model_matches(&profile.diffusion_model) && caps.supports_img_gen() && caps.is_mac_engine() == cfg.uses_mac_engine() {
let owned = rec.is_some();
if let Some(r) = &rec
&& self.inner.lock().unwrap().tail_stop.is_none()
@@ -728,9 +757,7 @@ pub fn process_alive(pid: u32) -> bool {
let mut sys = System::new();
let pid = Pid::from_u32(pid);
sys.refresh_processes_specifics(ProcessesToUpdate::Some(&[pid]), true, ProcessRefreshKind::nothing());
- sys.process(pid)
- .map(|p| p.name().to_string_lossy().to_ascii_lowercase().starts_with("sd-server"))
- .unwrap_or(false)
+ sys.process(pid).map(|p| is_server_process(&p.name().to_string_lossy())).unwrap_or(false)
}
/// Whether any process with this pid exists (used for client liveness, not sd-server).
@@ -748,7 +775,7 @@ fn kill_pid(pid: u32) {
let p = Pid::from_u32(pid);
sys.refresh_processes_specifics(ProcessesToUpdate::Some(&[p]), true, ProcessRefreshKind::nothing());
if let Some(proc_) = sys.process(p)
- && proc_.name().to_string_lossy().to_ascii_lowercase().starts_with("sd-server")
+ && is_server_process(&proc_.name().to_string_lossy())
{
proc_.kill();
let _ = proc_.wait();
@@ -951,6 +978,65 @@ mod tests {
assert!(!s.contains("--eager-load"), "nothing to preload when weights stream from disk");
}
+ #[test]
+ fn mac_engine_launch() {
+ let mut cfg = AppConfig::default();
+ cfg.gpu_backend = crate::config::GpuBackend::Metal;
+ assert!(!cfg.uses_mac_engine(), "off by default");
+ let sdcpp = build_server_args(&cfg, cfg.profile());
+ cfg.mac_engine = true;
+ assert!(cfg.uses_mac_engine());
+ assert!(cfg.server_exe().ends_with("engine-mac/kokon-engine-mac"), "{}", cfg.server_exe().display());
+ let a = build_server_args(&cfg, cfg.profile());
+ let p = |p: &Path| p.to_string_lossy().into_owned();
+ let profile = cfg.profile();
+ assert_eq!(a, ["serve", "--listen-ip", "127.0.0.1", "--listen-port", &cfg.server_port.to_string(), "--diffusion-model",
+ &p(&profile.diffusion_model), "--vae", &p(&profile.vae), "--llm", &p(&profile.text_encoder)]);
+ assert_ne!(a, sdcpp, "a different launch: switching restarts the server");
+ // Only on the Metal backend: elsewhere sd.cpp stays the engine.
+ cfg.gpu_backend = crate::config::GpuBackend::Cuda;
+ assert!(!cfg.uses_mac_engine());
+ assert!(build_server_args(&cfg, cfg.profile()).contains(&"--diffusion-model".to_string()));
+ assert!(!build_server_args(&cfg, cfg.profile()).contains(&"serve".to_string()));
+ assert!(cfg.server_exe().file_name().unwrap().to_string_lossy().starts_with("sd-server"));
+ }
+
+ #[test]
+ fn server_process_names() {
+ assert!(is_server_process("sd-server") && is_server_process("sd-server.exe") && is_server_process("SD-Server.exe"));
+ assert!(is_server_process("kokon-engine-mac"));
+ assert!(!is_server_process("kokon-service") && !is_server_process("sleep"));
+ }
+
+ /// The lines `kokon-engine-mac serve` prints for a job (engine-mac/Sources/KokonEngine/EngineServer.swift).
+ const MAC_ENGINE_LOG: &str = "[INFO ] engine-mac - generate_image 1024x1024
+[INFO ] engine-mac - sampling using Euler method
+[INFO ] engine-mac - get_learned_condition completed, 37 tokens, taking 2.01s
+[INFO ] engine-mac - generating image: 1/1 - seed 42
+ |======> | 1/8 - 8.35s/it
+ |============> | 2/8 - 7.36s/it
+ |=================================================>| 7/8 - 7.31s/it
+ |==================================================| 8/8 - 7.33s/it
+[INFO ] engine-mac - sampling completed, taking 59.70s
+[INFO ] engine-mac - decoding 1 latents
+[INFO ] engine-mac - memory: footprint 3.10 GB (peak 3.90 GB), resident 9.80 GB (peak 10.20 GB)
+[INFO ] engine-mac - generate_image completed in 65.20s";
+
+ #[test]
+ fn progress_from_mac_engine_log() {
+ let mut p = ProgressParser::default();
+ let updates: Vec<BackendProgress> = MAC_ENGINE_LOG.split(['\n', '\r']).filter_map(|l| p.feed(l)).collect();
+ let stages: Vec<BackendStage> = updates.iter().map(|u| u.stage).collect();
+ assert_eq!(stages.first(), Some(&BackendStage::Preparing));
+ assert!(stages.contains(&BackendStage::EncodingPrompt));
+ assert_eq!(updates.iter().find(|u| u.stage == BackendStage::Generating).unwrap().image, Some((1, 1)));
+ let steps: Vec<(u32, u32, Option<f32>)> = updates.iter().filter_map(|u| u.step.map(|s| (s.0, s.1, u.seconds_per_step))).collect();
+ assert_eq!(steps.iter().map(|s| (s.0, s.1)).collect::<Vec<_>>(), vec![(1, 8), (2, 8), (7, 8), (8, 8)]);
+ assert_eq!(steps[1].2, Some(7.36));
+ assert!(stages.contains(&BackendStage::Decoding));
+ assert_eq!(stages.last(), Some(&BackendStage::Finishing));
+ }
+
#[test]
fn step_bar_parsing() {
assert_eq!(parse_step_bar(" |=====> | 2/20 - 1.99s/it"), Some((2, 20, Some(1.99))));
diff --git a/crates/qls-core/src/sdcpp.rs b/crates/qls-core/src/sdcpp.rs
index da3c8b3..d068efb 100644
--- a/crates/qls-core/src/sdcpp.rs
+++ b/crates/qls-core/src/sdcpp.rs
@@ -203,6 +203,9 @@ pub struct Capabilities {
/// Whether `/sdcpp/v1/text_gen` works with the loaded model (Kokon's engine builds only).
#[serde(default)]
pub text_generation: bool,
+ /// The server's own name when it is not sd-server: `kokon-engine-mac` (Kokon's Mac engine).
+ #[serde(default)]
+ pub engine: String,
}
impl Capabilities {
@@ -212,6 +215,23 @@ impl Capabilities {
pub fn supports_img_gen(&self) -> bool {
self.supported_modes.iter().any(|m| m == "img_gen")
}
+ /// The server is Kokon's Mac engine (`engine-mac/`), not sd-server.
+ pub fn is_mac_engine(&self) -> bool {
+ self.engine == crate::config::MAC_ENGINE_EXE
+ }
+ /// What in `req` Kokon's Mac engine cannot do yet (it runs text to image only), in the user's words; None when it
+ /// can, and always for sd-server.
+ pub fn mac_engine_unsupported(&self, req: &ImgGenRequest) -> Option<&'static str> {
+ if !self.is_mac_engine() {
+ None
+ } else if !req.ref_images.is_empty() || req.init_image.is_some() || req.mask_image.is_some() {
+ Some("edits")
+ } else if req.hires.as_ref().is_some_and(|h| h.enabled) {
+ Some("second passes")
+ } else {
+ None
+ }
+ }
/// True when `name` is a model-backed upscaler the server can load (hires and,
/// if `image` is set, the standalone upscale endpoint).
pub fn has_upscaler(&self, name: &str, image: bool) -> bool {
@@ -878,6 +898,29 @@ mod tests {
assert_eq!(v["images"][0], "AAA");
}
+ #[test]
+ fn mac_engine_capabilities() {
+ // As `kokon-engine-mac serve` answers (EngineServer.capabilities).
+ let caps: Capabilities = serde_json::from_str(r#"{"engine":"kokon-engine-mac","model":{"name":"q.gguf","path":"/m/q.gguf","stem":"q"},
+ "supported_modes":["img_gen"],"features_by_mode":{"img_gen":{"ref_images":false,"lora":false,"cancel_generating":true}},
+ "upscale":false,"upscalers":[],"text_generation":false}"#).unwrap();
+ assert!(caps.is_mac_engine() && caps.supports_img_gen());
+ assert!(caps.img_feature("cancel_generating") && !caps.img_feature("lora") && !caps.img_feature("ref_images"));
+ assert!(caps.model_matches(std::path::Path::new("/m/q.gguf")));
+ let t2i = ImgGenRequest { prompt: "a fox".into(), width: 1024, height: 1024, ..Default::default() };
+ assert_eq!(caps.mac_engine_unsupported(&t2i), None);
+ let edit = ImgGenRequest { ref_images: vec!["png".into()], ..t2i.clone() };
+ assert_eq!(caps.mac_engine_unsupported(&edit), Some("edits"));
+ let inpaint = ImgGenRequest { mask_image: Some("png".into()), ..t2i.clone() };
+ assert_eq!(caps.mac_engine_unsupported(&inpaint), Some("edits"));
+ let hires = ImgGenRequest { hires: Some(HiresParams { enabled: true, ..Default::default() }), ..t2i.clone() };
+ assert_eq!(caps.mac_engine_unsupported(&hires), Some("second passes"));
+ // sd-server: no engine field, nothing refused here.
+ let sdcpp: Capabilities = serde_json::from_str(r#"{"supported_modes":["img_gen"]}"#).unwrap();
+ assert!(!sdcpp.is_mac_engine());
+ assert_eq!(sdcpp.mac_engine_unsupported(&edit), None);
+ }
+
#[test]
fn sse_parser_splits_events_across_chunks() {
// The documented stream, cut at awkward places: inside a line, inside "é", in "\r\n".
diff --git a/crates/qls-core/src/speed.rs b/crates/qls-core/src/speed.rs
index 4806110..ad45aa7 100644
--- a/crates/qls-core/src/speed.rs
+++ b/crates/qls-core/src/speed.rs
@@ -73,7 +73,9 @@ impl SpeedBook {
/// What makes timings comparable: engine, GPU and how the weights are held.
pub fn machine_key(cfg: &AppConfig, gpu: Option<&GpuInfo>) -> String {
let gpu = gpu.map(|g| g.name.as_str()).unwrap_or("unknown GPU");
- format!("{}|{}|{:?}|sage={}", cfg.gpu_backend.device_prefix(), gpu, cfg.text_encoder_placement, cfg.uses_sage_attention())
+ // Kokon's Mac engine has its own speed: its calibration is kept apart from sd.cpp's on the same Mac.
+ let engine = if cfg.uses_mac_engine() { "mac-engine" } else { cfg.gpu_backend.device_prefix() };
+ format!("{}|{}|{:?}|sage={}", engine, gpu, cfg.text_encoder_placement, cfg.uses_sage_attention())
}
/// Whether the uncalibrated model is close enough to show before the first measurement:
@@ -112,6 +114,11 @@ mod tests {
let a = machine_key(&cfg, Some(&gpu));
cfg.gpu_backend = GpuBackend::Vulkan;
assert_ne!(a, machine_key(&cfg, Some(&gpu)), "engines are calibrated separately");
+ cfg.gpu_backend = GpuBackend::Metal;
+ let metal = machine_key(&cfg, Some(&gpu));
+ cfg.mac_engine = true;
+ assert_ne!(metal, machine_key(&cfg, Some(&gpu)), "Kokon's Mac engine apart from sd.cpp's Metal build");
+ cfg.gpu_backend = GpuBackend::Vulkan;
assert!(!trust_uncalibrated(&cfg));
}
}
diff --git a/engine-mac/Package.swift b/engine-mac/Package.swift
index dfda28f..f9a7836 100644
--- a/engine-mac/Package.swift
+++ b/engine-mac/Package.swift
@@ -17,7 +17,8 @@ let package = Package(
cSettings: [.define("ACCELERATE_NEW_LAPACK")],
swiftSettings: [.define("ACCELERATE_NEW_LAPACK")],
linkerSettings: [.linkedFramework("Accelerate"), .linkedFramework("Metal"),
- .linkedFramework("MetalPerformanceShaders"), .linkedFramework("CoreML")]
+ .linkedFramework("MetalPerformanceShaders"), .linkedFramework("CoreML"),
+ .linkedFramework("IOKit")]
),
.executableTarget(name: "kokon-engine-mac", dependencies: ["KokonEngine"]),
.testTarget(name: "KokonEngineTests", dependencies: ["KokonEngine"], resources: [.copy("Fixtures")]),
diff --git a/engine-mac/Sources/KokonEngine/Chip.swift b/engine-mac/Sources/KokonEngine/Chip.swift
new file mode 100644
index 0000000..c8ded3a
--- /dev/null
+++ b/engine-mac/Sources/KokonEngine/Chip.swift
@@ -0,0 +1,82 @@
+import Foundation
+import IOKit
+import Metal
+
+/// **This Mac's chip**, read at run time: the per-chip paths (`Coexecutor`, `QwenDiT`) are chosen from it, each with
+/// the M2 path as its fallback, and `bench` prints it for the report.
+public struct Chip: Sendable, CustomStringConvertible {
+ /// `machdep.cpu.brand_string`, e.g. "Apple M2 Pro".
+ public let name: String
+ /// 1 for M1 … 5 for M5, from the name; nil for a name the engine does not know (a newer chip: treated as the
+ /// newest known one where a path depends on it).
+ public let generation: Int?
+ public let gpuCores: Int, performanceCores: Int, efficiencyCores: Int
+ public let memoryGB: Double
+ /// The newest Apple GPU family the device supports: 7 (M1) … 10 (M5).
+ public let gpuFamily: Int
+ public let supportsMetal4: Bool
+ public let macOS: String
+
+ public static let current = Chip(device: MTLCreateSystemDefaultDevice())
+
+ init(device: MTLDevice?) {
+ func sysctl(_ key: String) -> String {
+ var size = 0
+ guard sysctlbyname(key, nil, &size, nil, 0) == 0, size > 0 else { return "" }
+ var bytes = [CChar](repeating: 0, count: size)
+ guard sysctlbyname(key, &bytes, &size, nil, 0) == 0 else { return "" }
+ return String(cString: bytes)
+ }
+ func number(_ key: String) -> Int {
+ var value: Int64 = 0, size = MemoryLayout<Int64>.size
+ return sysctlbyname(key, &value, &size, nil, 0) == 0 ? Int(value) : 0
+ }
+ name = sysctl("machdep.cpu.brand_string")
+ generation = Chip.generation(of: name)
+ performanceCores = number("hw.perflevel0.physicalcpu")
+ efficiencyCores = number("hw.perflevel1.physicalcpu")
+ memoryGB = Double(number("hw.memsize")) / Double(1 << 30)
+ gpuCores = Chip.gpuCoreCount()
+ let families: [(MTLGPUFamily, Int)] = [(.apple10, 10), (.apple9, 9), (.apple8, 8), (.apple7, 7)]
+ gpuFamily = device.flatMap { d in families.first { d.supportsFamily($0.0) }?.1 } ?? 0
+ if #available(macOS 26, *) { supportsMetal4 = device?.supportsFamily(.metal4) ?? false } else { supportsMetal4 = false }
+ let v = ProcessInfo.processInfo.operatingSystemVersion
+ macOS = "\(v.majorVersion).\(v.minorVersion).\(v.patchVersion)"
+ }
+
+ /// "Apple M4 Pro" → 4.
+ static func generation(of name: String) -> Int? {
+ guard let range = name.range(of: #"\bM(\d+)\b"#, options: .regularExpression) else { return nil }
+ return Int(name[range].dropFirst())
+ }
+
+ /// The GPU's cores, from the IORegistry (`AGXAccelerator`'s `gpu-core-count`); 0 when not found.
+ static func gpuCoreCount() -> Int {
+ let service = IOServiceGetMatchingService(kIOMainPortDefault, IOServiceMatching("AGXAccelerator"))
+ guard service != 0 else { return 0 }
+ defer { IOObjectRelease(service) }
+ let value = IORegistryEntryCreateCFProperty(service, "gpu-core-count" as CFString, kCFAllocatorDefault, 0)?.takeRetainedValue()
+ return (value as? NSNumber)?.intValue ?? 0
+ }
+
+ public var description: String {
+ String(format: "%@, %d GPU cores (Apple%d%@), %d+%d CPU cores, %.0f GB, macOS %@", name, gpuCores, gpuFamily,
+ supportsMetal4 ? ", Metal 4" : "", performanceCores, efficiencyCores, memoryGB, macOS)
+ }
+}
+
+extension Chip {
+ /// Each unit's estimated speed on the DiT's products relative to the M2 Pro (19 GPU cores, 8 performance cores, a
+ /// 16-core Neural Engine), the only chip measured so far, for the first image's shares: the GPU by cores × clock,
+ /// the Neural Engine by Apple's published TOPS (fp16 for M1–M3; the M4's 38 are int8, so ~19 in fp16; the M5's
+ /// assumed the same), the AMX by performance cores; an Ultra has two Neural Engines. Estimates: the balancing loop
+ /// measures the real rates from the first evaluation on and remembers them. nil for a chip without a known name.
+ var unitScale: (gpu: Double, amx: Double, ane: Double)? {
+ guard let g = generation, gpuCores > 0, performanceCores > 0 else { return nil }
+ let generation = min(g, 5)
+ let clock: [Int: Double] = [1: 1.28, 2: 1.40, 3: 1.40, 4: 1.50, 5: 1.55] // GHz, the GPU's highest, approximate
+ let tops: [Int: Double] = [1: 11, 2: 15.8, 3: 18, 4: 19, 5: 19]
+ let engines = name.contains("Ultra") ? 2.0 : 1.0
+ return (Double(gpuCores) * clock[generation]! / (19 * 1.40), Double(performanceCores) / 8, tops[generation]! * engines / 15.8)
+ }
+}
diff --git a/engine-mac/Sources/KokonEngine/EngineServer.swift b/engine-mac/Sources/KokonEngine/EngineServer.swift
new file mode 100644
index 0000000..1b4c0b9
--- /dev/null
+++ b/engine-mac/Sources/KokonEngine/EngineServer.swift
@@ -0,0 +1,293 @@
+import Foundation
+
+/// **The engine as a long-lived server speaking sd-server's `/sdcpp/v1` API**, so qls-core, the app, remote access
+/// and MCP use it as they use sd-server (crates/qls-core/src/sdcpp.rs is the client):
+///
+/// GET /sdcpp/v1/capabilities the model, the modes and limits
+/// POST /sdcpp/v1/img_gen 202 {id, kind, status, created, poll_url}: the job is queued
+/// GET /sdcpp/v1/jobs/{id} {status: queued|generating|completed|failed|cancelled, result, error}
+/// POST /sdcpp/v1/jobs/{id}/cancel 200 {status: cancelled}: a running image stops within a DiT layer
+///
+/// Text to image only for now: requests with reference images, a mask, an init image, LoRAs or a second pass are
+/// refused with 400 and say so. Jobs run one at a time; the text encoder, the DiT and the compiled VAE stay loaded
+/// between them (`QwenPipeline.keepModels`).
+///
+/// Progress goes to stdout as sd-server's log lines, the ones qls-core's `ProgressParser` reads (process.rs):
+/// `generate_image WxH`, `sampling using`, `generating image: i/n - seed s`, step bars `| i/n - x.xxs/it`,
+/// `decoding 1 latents`, `generate_image completed in Xs`.
+public final class EngineServer: @unchecked Sendable {
+ public struct Paths: Sendable {
+ public let diffusionModel, vae, textEncoder: String
+ public init(diffusionModel: String, vae: String, textEncoder: String) {
+ self.diffusionModel = diffusionModel
+ self.vae = vae
+ self.textEncoder = textEncoder
+ }
+ }
+
+ final class Job: @unchecked Sendable {
+ let id: String
+ let created: Int
+ let request: ImageRequest
+ var status = "queued"
+ var started: Int?, completed: Int?
+ var images: [String] = []
+ var error: (code: String, message: String)?
+ init(id: String, request: ImageRequest) {
+ self.id = id
+ self.request = request
+ created = Int(Date().timeIntervalSince1970)
+ }
+ }
+
+ /// The fields of `img_gen` the engine reads (sd-server's names), checked.
+ struct ImageRequest: Sendable {
+ let prompt: String
+ let width, height: Int
+ let seed: Int64
+ let batch: Int
+ let sigmas: [Float]
+
+ /// nil and the reason when the engine cannot do it (yet).
+ static func parse(_ body: Data) -> (ImageRequest?, String?) {
+ guard let json = (try? JSONSerialization.jsonObject(with: body)) as? [String: Any] else {
+ return (nil, "the body is not a JSON object")
+ }
+ func array(_ key: String) -> [Any] { json[key] as? [Any] ?? [] }
+ if !array("ref_images").isEmpty || json["init_image"] is String || json["mask_image"] is String {
+ return (nil, "Kokon's Mac engine does text to image only for now (no reference images, init image or mask)")
+ }
+ if !array("lora").isEmpty { return (nil, "Kokon's Mac engine does not run LoRAs yet") }
+ if let hires = json["hires"] as? [String: Any], hires["enabled"] as? Bool == true {
+ return (nil, "Kokon's Mac engine has no second pass (hires)")
+ }
+ if let format = json["output_format"] as? String, format != "png" { return (nil, "output_format \(format): png only") }
+ let width = json["width"] as? Int ?? 1024, height = json["height"] as? Int ?? 1024
+ guard width % 32 == 0, height % 32 == 0, width >= 64, height >= 64, width <= 4096, height <= 4096 else {
+ return (nil, "\(width)×\(height): each side must be a multiple of 32 from 64 to 4096")
+ }
+ let sample = json["sample_params"] as? [String: Any] ?? [:]
+ let cfg = (sample["guidance"] as? [String: Any])?["txt_cfg"] as? Double ?? 1
+ guard abs(cfg - 1) < 1e-6 else { return (nil, "txt_cfg \(cfg): Qwen-Image 2.1 Turbo runs at CFG 1 only") }
+ var sigmas = (sample["custom_sigmas"] as? [Any] ?? []).compactMap { ($0 as? NSNumber)?.floatValue }
+ let steps = sample["sample_steps"] as? Int ?? QwenPipeline.turboSigmas.count - 1
+ if sigmas.isEmpty {
+ guard steps == QwenPipeline.turboSigmas.count - 1 else {
+ return (nil, "\(steps) steps without custom_sigmas: the engine knows Turbo's 8-step schedule only")
+ }
+ sigmas = QwenPipeline.turboSigmas
+ }
+ guard sigmas.count >= 2, sigmas.allSatisfy({ $0 >= 0 && $0 <= 1 }) else { return (nil, "custom_sigmas \(sigmas) out of [0, 1]") }
+ var seed = (json["seed"] as? NSNumber)?.int64Value ?? 42
+ if seed < 0 { seed = Int64.random(in: 0..<(1 << 31)) }
+ let batch = json["batch_count"] as? Int ?? 1
+ guard (1...EngineServer.maxBatch).contains(batch) else { return (nil, "batch_count \(batch): 1 to \(EngineServer.maxBatch)") }
+ return (ImageRequest(prompt: json["prompt"] as? String ?? "", width: width, height: height, seed: seed,
+ batch: batch, sigmas: sigmas), nil)
+ }
+ }
+
+ static let maxBatch = 4, maxQueue = 8
+ public let paths: Paths
+ let pipeline: QwenPipeline
+ private var jobs: [String: Job] = [:]
+ private var queue: [Job] = []
+ private var running: Job?
+ private let lock = NSLock()
+ private let wake = DispatchSemaphore(value: 0)
+ /// Lines to stdout (qls-core reads them from the log file it redirects stdout to).
+ public var log: @Sendable (String) -> Void = { line in
+ FileHandle.standardOutput.write(Data((line + "\n").utf8))
+ }
+
+ public init(paths: Paths) throws {
+ self.paths = paths
+ for path in [paths.diffusionModel, paths.vae, paths.textEncoder] where !FileManager.default.fileExists(atPath: path) {
+ throw GGUFFile.Failure.open(path)
+ }
+ pipeline = try QwenPipeline(encoder: paths.textEncoder, dit: paths.diffusionModel, vae: paths.vae)
+ pipeline.keepModels = true
+ pipeline.log = { _ in }
+ let worker = Thread { [self] in work() }
+ worker.name = "kokon.jobs"
+ worker.qualityOfService = .userInitiated
+ worker.start()
+ }
+
+ // MARK: - HTTP
+
+ public func handle(_ request: HTTPServer.Request) -> HTTPServer.Response {
+ let parts = request.path.split(separator: "/").map(String.init)
+ guard parts.count >= 3, parts[0] == "sdcpp", parts[1] == "v1" else { return .error(404, "not found: \(request.path)") }
+ switch (request.method, Array(parts.dropFirst(2))) {
+ case ("GET", ["capabilities"]): return .json(200, capabilities())
+ case ("POST", ["img_gen"]): return submit(request.body)
+ case ("GET", let p) where p.count == 2 && p[0] == "jobs":
+ return lock.withLock { jobs[p[1]].map { .json(200, describe($0)) } ?? .error(404, "unknown job \(p[1])") }
+ case ("POST", let p) where p.count == 3 && p[0] == "jobs" && p[2] == "cancel": return cancel(p[1])
+ case (_, ["upscale"]), (_, ["text_gen"]):
+ return .error(404, "Kokon's Mac engine has no \(parts[2]) endpoint yet")
+ default: return .error(404, "not found: \(request.method) \(request.path)")
+ }
+ }
+
+ func capabilities() -> [String: Any] {
+ let url = URL(fileURLWithPath: paths.diffusionModel)
+ let feature: [String: Any] = ["ref_images": false, "init_image": false, "mask_image": false, "lora": false,
+ "hires": false, "custom_sigmas": true, "cancel_generating": true]
+ return [
+ "engine": "kokon-engine-mac",
+ "model": ["name": url.lastPathComponent, "stem": url.deletingPathExtension().lastPathComponent, "path": paths.diffusionModel],
+ "supported_modes": ["img_gen"],
+ "samplers": ["euler"],
+ "schedulers": [String](),
+ "output_formats": ["png"],
+ "limits": ["min_width": 64, "max_width": 4096, "min_height": 64, "max_height": 4096,
+ "max_batch_count": EngineServer.maxBatch, "max_queue_size": EngineServer.maxQueue],
+ "features_by_mode": ["img_gen": feature],
+ "defaults_by_mode": ["img_gen": ["sample_steps": QwenPipeline.turboSigmas.count - 1, "txt_cfg": 1]],
+ "upscalers": [Any](),
+ "upscale": false,
+ "text_generation": false,
+ ]
+ }
+
+ func submit(_ body: Data) -> HTTPServer.Response {
+ let (parsed, reason) = ImageRequest.parse(body)
+ guard let parsed else { return .error(400, reason ?? "bad request") }
+ return lock.withLock {
+ guard queue.count < EngineServer.maxQueue else { return .error(429, "the queue is full") }
+ let job = Job(id: UUID().uuidString.lowercased(), request: parsed)
+ jobs[job.id] = job
+ queue.append(job)
+ wake.signal()
+ return .json(202, ["id": job.id, "kind": "img_gen", "status": job.status, "created": job.created,
+ "poll_url": "/sdcpp/v1/jobs/\(job.id)"])
+ }
+ }
+
+ /// A queued job leaves the queue; a running one is marked and stops at the next DiT layer, its result dropped.
+ func cancel(_ id: String) -> HTTPServer.Response {
+ lock.withLock {
+ guard let job = jobs[id] else { return .error(404, "unknown job \(id)") }
+ if job.status == "queued" || job.status == "generating" {
+ queue.removeAll { $0 === job }
+ job.status = "cancelled"
+ job.completed = Int(Date().timeIntervalSince1970)
+ }
+ return .json(200, describe(job))
+ }
+ }
+
+ /// sd-server's job object. Called with the lock held.
+ func describe(_ job: Job) -> [String: Any] {
+ var out: [String: Any] = ["id": job.id, "kind": "img_gen", "status": job.status, "created": job.created,
+ "queue_position": queue.firstIndex { $0 === job }.map { $0 + 1 } ?? 0]
+ if let s = job.started { out["started"] = s }
+ if let c = job.completed { out["completed"] = c }
+ if job.status == "completed" {
+ out["result"] = ["output_format": "png",
+ "images": job.images.enumerated().map { ["index": $0.offset, "b64_json": $0.element] as [String: Any] }]
+ }
+ if let e = job.error { out["error"] = ["code": e.code, "message": e.message] }
+ return out
+ }
+
+ // MARK: - the jobs
+
+ private func work() {
+ while true {
+ wake.wait()
+ guard let job: Job = lock.withLock({
+ guard !queue.isEmpty else { return nil }
+ let job = queue.removeFirst()
+ job.status = "generating"
+ job.started = Int(Date().timeIntervalSince1970)
+ running = job
+ return job
+ }) else { continue }
+ let outcome: Result<[String], Error> = autoreleasepool { Result { try generate(job) } }
+ lock.withLock {
+ running = nil
+ job.completed = Int(Date().timeIntervalSince1970)
+ guard job.status != "cancelled" else { return }
+ switch outcome {
+ case .success(let images):
+ job.images = images
+ job.status = "completed"
+ case .failure(let error):
+ job.status = "failed"
+ job.error = ("generation_failed", "\(error)")
+ }
+ }
+ if case .failure(let error) = outcome, !(error is QwenDiT.Cancelled) { log("[ERROR ] engine: \(error)") }
+ // Older finished jobs are forgotten (the client fetched their result long ago).
+ lock.withLock {
+ let finished = jobs.values.filter { $0.completed != nil }.sorted { $0.created < $1.created }
+ for old in finished.dropLast(16) { jobs[old.id] = nil }
+ }
+ }
+ }
+
+ private func generate(_ job: Job) throws -> [String] {
+ let r = job.request
+ let started = DispatchTime.now().uptimeNanoseconds
+ func seconds(since t: UInt64) -> Double { Double(DispatchTime.now().uptimeNanoseconds - t) * 1e-9 }
+ log("[INFO ] engine-mac - generate_image \(r.width)x\(r.height)")
+ log("[INFO ] engine-mac - sampling using Euler method")
+ pipeline.cancelled = { [weak self, job] in self?.lock.withLock { job.status == "cancelled" } ?? true }
+ var images: [String] = []
+ for i in 0..<r.batch {
+ let seed = r.seed + Int64(i)
+ var stepStart = DispatchTime.now().uptimeNanoseconds
+ pipeline.events = { [self] event in
+ switch event {
+ case .encoded(let tokens, let s):
+ log(String(format: "[INFO ] engine-mac - get_learned_condition completed, %d tokens, taking %.2fs", tokens, s))
+ log("[INFO ] engine-mac - generating image: \(i + 1)/\(r.batch) - seed \(seed)")
+ stepStart = DispatchTime.now().uptimeNanoseconds
+ case .step(let step, let total, let s):
+ let filled = 50 * step / total
+ let bar = String(repeating: "=", count: max(0, filled - 1)) + (filled < 50 ? ">" : "=")
+ + String(repeating: " ", count: 50 - filled)
+ log(String(format: " |%@| %d/%d - %.2fs/it", bar, step, total, s))
+ case .decoding:
+ let m = MemoryUsage.now()
+ log(String(format: "[INFO ] engine-mac - sampling completed, taking %.2fs (footprint %.2f GB)", seconds(since: stepStart), m.footprint))
+ log("[INFO ] engine-mac - decoding 1 latents")
+ }
+ }
+ let image = try pipeline.textToImage(prompt: r.prompt, width: r.width, height: r.height, seed: UInt64(bitPattern: seed),
+ sigmas: r.sigmas)
+ let png = try QwenPipeline.pngData(QwenPipeline.pixels(image, width: r.width, height: r.height), width: r.width, height: r.height)
+ images.append(png.base64EncodedString())
+ let t = pipeline.timings
+ log(String(format: "[INFO ] engine-mac - image %d/%d: text encoder %.2fs, DiT ready %.2fs, VAE %.2fs, total %.2fs", i + 1, r.batch,
+ t["encode"] ?? 0, t["dit load"] ?? 0, t["vae"] ?? 0, t["total"] ?? 0))
+ }
+ pipeline.events = nil
+ let memory = MemoryUsage.now()
+ log(String(format: "[INFO ] engine-mac - memory: footprint %.2f GB (peak %.2f GB), resident %.2f GB (peak %.2f GB)",
+ memory.footprint, memory.footprintPeak, memory.resident, memory.residentPeak))
+ log(String(format: "[INFO ] engine-mac - generate_image completed in %.2fs", seconds(since: started)))
+ return images
+ }
+}
+
+/// This process's memory, GB: the footprint (what macOS charges it: anonymous and wired memory, not the file pages it
+/// can drop) and the resident size (with the mapped model pages in use), now and at their peak.
+public struct MemoryUsage {
+ public let footprint, footprintPeak, resident, residentPeak: Double
+
+ public static func now() -> MemoryUsage {
+ var info = task_vm_info_data_t()
+ var count = mach_msg_type_number_t(MemoryLayout<task_vm_info_data_t>.size / MemoryLayout<integer_t>.size)
+ let result = withUnsafeMutablePointer(to: &info) {
+ $0.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { task_info(mach_task_self_, task_flavor_t(TASK_VM_INFO), $0, &count) }
+ }
+ guard result == KERN_SUCCESS else { return MemoryUsage(footprint: 0, footprintPeak: 0, resident: 0, residentPeak: 0) }
+ let gb = 1.0 / Double(1 << 30)
+ return MemoryUsage(footprint: Double(info.phys_footprint) * gb, footprintPeak: Double(info.ledger_phys_footprint_peak) * gb,
+ resident: Double(info.resident_size) * gb, residentPeak: Double(info.resident_size_peak) * gb)
+ }
+}
diff --git a/engine-mac/Sources/KokonEngine/HTTPServer.swift b/engine-mac/Sources/KokonEngine/HTTPServer.swift
new file mode 100644
index 0000000..2e5fbcd
--- /dev/null
+++ b/engine-mac/Sources/KokonEngine/HTTPServer.swift
@@ -0,0 +1,150 @@
+import Foundation
+
+/// **A minimal HTTP/1.1 server on a loopback port**, enough for qls-core's client (reqwest): one request per
+/// connection (`Connection: close`), a `Content-Length` body, JSON in and out. POSIX sockets, a thread per
+/// connection: the engine serves one local client, and the slow part of every request is the GPU, not the socket.
+public final class HTTPServer: @unchecked Sendable {
+ public struct Request: Sendable {
+ public let method: String
+ public let path: String
+ public let body: Data
+ }
+
+ public struct Response: Sendable {
+ public var status: Int
+ public var body: Data
+ public var contentType = "application/json"
+
+ public init(status: Int, body: Data, contentType: String = "application/json") {
+ self.status = status
+ self.body = body
+ self.contentType = contentType
+ }
+
+ public static func json(_ status: Int, _ object: Any) -> Response {
+ let data = (try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys])) ?? Data("{}".utf8)
+ return Response(status: status, body: data)
+ }
+
+ /// sd-server's error body: `{"error": message}`.
+ public static func error(_ status: Int, _ message: String) -> Response { json(status, ["error": message]) }
+ }
+
+ public enum Failure: Error, CustomStringConvertible {
+ case socket(String)
+ public var description: String {
+ switch self { case .socket(let s): return "cannot listen: \(s)" }
+ }
+ }
+
+ /// Requests larger than this are refused (reference images later: a few base64 PNGs).
+ static let maxBody = 256 << 20
+ public let port: UInt16
+ private let socket: Int32
+ private let handler: @Sendable (Request) -> Response
+
+ /// Binds `host:port` (port 0: any free one, see `port`).
+ public init(host: String = "127.0.0.1", port: UInt16, handler: @escaping @Sendable (Request) -> Response) throws {
+ self.handler = handler
+ let fd = Darwin.socket(AF_INET, SOCK_STREAM, 0)
+ guard fd >= 0 else { throw Failure.socket("socket: \(String(cString: strerror(errno)))") }
+ var yes: Int32 = 1
+ setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, socklen_t(MemoryLayout<Int32>.size))
+ var address = sockaddr_in()
+ address.sin_family = sa_family_t(AF_INET)
+ address.sin_port = port.bigEndian
+ guard inet_pton(AF_INET, host, &address.sin_addr) == 1 else {
+ close(fd)
+ throw Failure.socket("bad address \(host)")
+ }
+ let bound = withUnsafePointer(to: &address) {
+ $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { bind(fd, $0, socklen_t(MemoryLayout<sockaddr_in>.size)) }
+ }
+ guard bound == 0, listen(fd, 16) == 0 else {
+ let message = String(cString: strerror(errno))
+ close(fd)
+ throw Failure.socket("\(host):\(port): \(message)")
+ }
+ var actual = sockaddr_in(), length = socklen_t(MemoryLayout<sockaddr_in>.size)
+ _ = withUnsafeMutablePointer(to: &actual) {
+ $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { getsockname(fd, $0, &length) }
+ }
+ self.port = UInt16(bigEndian: actual.sin_port)
+ socket = fd
+ }
+
+ /// Accepts connections until the process ends.
+ public func run() -> Never {
+ while true {
+ let client = accept(socket, nil, nil)
+ guard client >= 0 else { continue }
+ var noSigpipe: Int32 = 1
+ setsockopt(client, SOL_SOCKET, SO_NOSIGPIPE, &noSigpipe, socklen_t(MemoryLayout<Int32>.size))
+ let thread = Thread { [self] in serve(client) }
+ thread.name = "kokon.http"
+ thread.start()
+ }
+ }
+
+ private func serve(_ fd: Int32) {
+ defer { close(fd) }
+ let response: Response
+ switch HTTPServer.read(fd) {
+ case .success(let request): response = handler(request)
+ case .failure(let message): response = .error(400, message)
+ }
+ HTTPServer.write(fd, response)
+ }
+
+ enum ReadResult { case success(Request), failure(String) }
+
+ static func read(_ fd: Int32) -> ReadResult {
+ var data = Data()
+ var buffer = [UInt8](repeating: 0, count: 64 * 1024)
+ let separator = Data("\r\n\r\n".utf8)
+ var headerEnd: Range<Data.Index>?
+ while headerEnd == nil {
+ let n = recv(fd, &buffer, buffer.count, 0)
+ guard n > 0 else { return .failure("connection closed before the request ended") }
+ data.append(buffer, count: n)
+ headerEnd = data.range(of: separator)
+ if headerEnd == nil, data.count > 64 * 1024 { return .failure("request headers too long") }
+ }
+ let head = String(decoding: data[..<headerEnd!.lowerBound], as: UTF8.self)
+ let lines = head.components(separatedBy: "\r\n")
+ let parts = lines[0].split(separator: " ")
+ guard parts.count >= 2 else { return .failure("bad request line") }
+ var length = 0
+ for line in lines.dropFirst() {
+ let pair = line.split(separator: ":", maxSplits: 1)
+ if pair.count == 2, pair[0].trimmingCharacters(in: .whitespaces).lowercased() == "content-length" {
+ length = Int(pair[1].trimmingCharacters(in: .whitespaces)) ?? 0
+ }
+ }
+ guard length <= maxBody else { return .failure("request body too large") }
+ var body = Data(data[headerEnd!.upperBound...])
+ while body.count < length {
+ let n = recv(fd, &buffer, min(buffer.count, length - body.count), 0)
+ guard n > 0 else { return .failure("connection closed inside the body") }
+ body.append(buffer, count: n)
+ }
+ // Paths without the query string.
+ let path = String(parts[1].split(separator: "?", maxSplits: 1).first ?? "")
+ return .success(Request(method: String(parts[0]), path: path, body: body))
+ }
+
+ static func write(_ fd: Int32, _ response: Response) {
+ let reason = [200: "OK", 202: "Accepted", 400: "Bad Request", 404: "Not Found", 409: "Conflict",
+ 429: "Too Many Requests", 500: "Internal Server Error"][response.status] ?? "Status"
+ var out = Data("HTTP/1.1 \(response.status) \(reason)\r\nContent-Type: \(response.contentType)\r\nContent-Length: \(response.body.count)\r\nConnection: close\r\n\r\n".utf8)
+ out.append(response.body)
+ out.withUnsafeBytes { raw in
+ var sent = 0
+ while sent < raw.count {
+ let n = send(fd, raw.baseAddress! + sent, raw.count - sent, 0)
+ if n <= 0 { return }
+ sent += n
+ }
+ }
+ }
+}
diff --git a/engine-mac/Sources/KokonEngine/QwenDiT.swift b/engine-mac/Sources/KokonEngine/QwenDiT.swift
index cb9b1e6..0a390dc 100644
--- a/engine-mac/Sources/KokonEngine/QwenDiT.swift
+++ b/engine-mac/Sources/KokonEngine/QwenDiT.swift
@@ -65,6 +65,10 @@ public final class QwenDiT {
private var attentions: [String: GraphAttention] = [:]
/// Seconds per phase of the last evaluation.
public private(set) var timings: [String: Double] = [:]
+ /// Checked after each layer is encoded: true stops the evaluation (`Cancelled`) once the GPU has finished what was
+ /// committed, so a cancel takes effect within a layer (~0.25 s at 1024² on the M2 Pro).
+ public var cancelled: () -> Bool = { false }
+ public struct Cancelled: Error {}
public init(file: GGUFFile, context: MetalContext, sequence: QwenSequence) throws {
self.file = file
@@ -237,6 +241,10 @@ public final class QwenDiT {
try finish(commands); probe("layer\(layer)_out", arena.pointer(x), n * d); commands = try begin()
} else {
commands.commit()
+ if cancelled() {
+ commands.waitUntilCompleted() // the queue runs in order: everything committed is done
+ throw Cancelled()
+ }
commands = try begin()
}
}
diff --git a/engine-mac/Sources/KokonEngine/QwenPipeline.swift b/engine-mac/Sources/KokonEngine/QwenPipeline.swift
index 843a016..103108b 100644
--- a/engine-mac/Sources/KokonEngine/QwenPipeline.swift
+++ b/engine-mac/Sources/KokonEngine/QwenPipeline.swift
@@ -26,6 +26,40 @@ public final class QwenPipeline {
/// once per size, or ahead while idle (`QwenVAE.prepare`).
private var vae: QwenVAE?
+ /// **Keep the text encoder and the DiT between images** (the long-lived server): the files stay mapped and the DiT
+ /// keeps its working memory, its Neural Engine programs and its weights' scales for the last image size. Their GPU
+ /// windows are dropped after each use, so while idle the weights are page cache macOS may reclaim, not wired memory.
+ public var keepModels = false
+ /// With `keepModels`, the text encoder's 5 GB mapping too: only from 32 GB (`KOKON_KEEP_ENCODER=1` / `0` to choose).
+ /// On the M2 Pro 16 GB, everything kept made the server's footprint 9–10.8 GB and its second 1024² image 121 s
+ /// instead of 77 (the encoder's pages evicted and read again, 11.7 s; the VAE 26 s; slower steps), 2026-10-25.
+ /// With `keepModels`, what else stays between images, from 32 GB by default: the encoder's mapping
+ /// (`KOKON_KEEP_ENCODER`), the DiT's working memory before the VAE runs (`KOKON_KEEP_DIT`), the compiled VAE
+ /// (`KOKON_KEEP_VAE`); `=1` / `=0` to choose. M2 Pro 16 GB, 1024², server, 2026-10-25: the steps need 1.7 GB of
+ /// footprint, but the compiled VAE keeps 7–8 GB of GPU memory after its decode (peak 10.8 GB during it, the one-shot
+ /// CLI too). With everything kept the second image took 121–129 s instead of 75 (pages evicted and read again, the
+ /// VAE 23–30 s, slower steps); with the VAE dropped after each decode 80.8 s and 2.3 GB while idle. Rebuilding the
+ /// DiT costs 0.4 s, compiling the VAE again ~0.3 s plus its slower first run.
+ public var keepEncoder = QwenPipeline.keep("KOKON_KEEP_ENCODER")
+ public var keepDiT = QwenPipeline.keep("KOKON_KEEP_DIT")
+ public var keepVAE = QwenPipeline.keep("KOKON_KEEP_VAE")
+ static func keep(_ key: String) -> Bool {
+ switch ProcessInfo.processInfo.environment[key] {
+ case "1": return true
+ case "0": return false
+ default: return Chip.current.memoryGB >= 30
+ }
+ }
+ private var encoderFile: GGUFFile?, encoder: QwenTextEncoder?
+ private var ditFile: GGUFFile?, dit: QwenDiT?
+ /// Checked before each step and after each DiT layer: true stops the image (`QwenDiT.Cancelled`).
+ public var cancelled: () -> Bool = { false }
+ /// What the server reports while an image runs.
+ public enum Event { case encoded(tokens: Int, seconds: Double), step(Int, of: Int, seconds: Double), decoding }
+ public var events: ((Event) -> Void)?
+ /// The co-executor's shares for the next evaluation of the kept DiT (`keepModels`).
+ public var shares: String? { dit?.coexecutor?.shares }
+
public init(encoder: String, dit: String, vae: String) throws {
encoderPath = encoder
ditPath = dit
@@ -58,6 +92,14 @@ public final class QwenPipeline {
var t = DispatchTime.now().uptimeNanoseconds
let (conditioning, rows): ([Float], Int) = try {
if let given { return (given, given.count / QwenDiT.dim) }
+ if keepModels && keepEncoder {
+ if encoder == nil {
+ let file = try GGUFFile(path: encoderPath)
+ (encoderFile, encoder) = (file, try QwenTextEncoder(file: file, context: context))
+ }
+ defer { encoder!.release() }
+ return try encoder!.conditioning(prompt: prompt)
+ }
let file = try GGUFFile(path: encoderPath)
let encoder = try QwenTextEncoder(file: file, context: context)
defer { encoder.release() }
@@ -65,6 +107,7 @@ public final class QwenPipeline {
}()
timings["encode"] = since(t)
log(String(format: "text encoder: %d tokens in %.2f s", rows, timings["encode"]!))
+ events?(.encoded(tokens: rows, seconds: timings["encode"]!))
// 2. Noise, packed: token y·w + x holds the 64 channels of cell (y, x).
let start = noise ?? Noise.philox(seed: seed, count: channels * h * w) // sd.cpp's, value for value
@@ -77,26 +120,43 @@ public final class QwenPipeline {
t = DispatchTime.now().uptimeNanoseconds
let sequence = try QwenSequence.textToImage(text: rows, target: .init(height: h, width: w))
try {
- let file = try GGUFFile(path: ditPath)
+ let file = try ditFile ?? GGUFFile(path: ditPath)
defer { context.release(file) }
- let dit = try QwenDiT(file: file, context: context, sequence: sequence)
+ let dit: QwenDiT
+ if keepModels {
+ ditFile = file
+ if self.dit?.sequence != sequence {
+ self.dit = nil // its working memory goes before the new one's is taken
+ self.dit = try QwenDiT(file: file, context: context, sequence: sequence)
+ }
+ dit = self.dit!
+ } else {
+ dit = try QwenDiT(file: file, context: context, sequence: sequence)
+ }
+ dit.cancelled = cancelled
+ defer { dit.cancelled = { false } }
try dit.prepareModulation([0] + sigmas.dropLast())
timings["dit load"] = since(t)
for i in 0..<(sigmas.count - 1) {
+ if cancelled() { throw QwenDiT.Cancelled() }
let stepStart = DispatchTime.now().uptimeNanoseconds
let velocity = try dit.forward(encoder: conditioning, target: x, sigma: sigmas[i])
let dt = sigmas[i + 1] - sigmas[i]
for j in x.indices { x[j] += dt * velocity[j] }
timings["step \(i + 1)"] = since(stepStart)
log(String(format: "step %d/%d (σ %.4f): %.2f s", i + 1, sigmas.count - 1, sigmas[i], timings["step \(i + 1)"]!))
+ events?(.step(i + 1, of: sigmas.count - 1, seconds: timings["step \(i + 1)"]!))
step?(i, x)
}
}()
// 4. The image.
+ if !keepDiT { dit = nil } // its working memory goes before the VAE's
t = DispatchTime.now().uptimeNanoseconds
+ events?(.decoding)
if vae == nil { vae = try QwenVAE(path: vaePath, context: context) }
let image = try vae!.decode(QwenPipeline.unpack(x, height: h, width: w), height: h, width: w)
+ if !keepVAE { vae = nil }
timings["vae"] = since(t)
timings["total"] = since(started)
log(String(format: "VAE: %.2f s; total %.1f s", timings["vae"]!, timings["total"]!))
@@ -139,17 +199,23 @@ public final class QwenPipeline {
/// Writes interleaved 8-bit RGBA (straight alpha) as a PNG.
public static func writePNG(_ rgba: [UInt8], width: Int, height: Int, to path: String) throws {
+ try pngData(rgba, width: width, height: height).write(to: URL(fileURLWithPath: path))
+ }
+
+ /// Interleaved 8-bit RGBA (straight alpha) encoded as a PNG.
+ public static func pngData(_ rgba: [UInt8], width: Int, height: Int) throws -> Data {
let data = Data(rgba) as CFData
+ let out = NSMutableData()
guard let provider = CGDataProvider(data: data),
let image = CGImage(width: width, height: height, bitsPerComponent: 8, bitsPerPixel: 32, bytesPerRow: width * 4,
space: CGColorSpace(name: CGColorSpace.sRGB)!,
bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.last.rawValue),
provider: provider, decode: nil, shouldInterpolate: false, intent: .defaultIntent),
- let destination = CGImageDestinationCreateWithURL(URL(fileURLWithPath: path) as CFURL,
- UTType.png.identifier as CFString, 1, nil) else {
- throw MetalContext.Failure.buffer("cannot write \(path)")
+ let destination = CGImageDestinationCreateWithData(out as CFMutableData, UTType.png.identifier as CFString, 1, nil) else {
+ throw MetalContext.Failure.buffer("cannot encode a PNG")
}
CGImageDestinationAddImage(destination, image, nil)
- guard CGImageDestinationFinalize(destination) else { throw MetalContext.Failure.buffer("cannot write \(path)") }
+ guard CGImageDestinationFinalize(destination) else { throw MetalContext.Failure.buffer("cannot encode a PNG") }
+ return out as Data
}
}
diff --git a/engine-mac/Sources/kokon-engine-mac/main.swift b/engine-mac/Sources/kokon-engine-mac/main.swift
index 6a26109..d0185c9 100644
--- a/engine-mac/Sources/kokon-engine-mac/main.swift
+++ b/engine-mac/Sources/kokon-engine-mac/main.swift
@@ -19,6 +19,8 @@ kokon-engine-mac: Kokon's Apple Silicon engine (in development)
check-noise <philox.bin> the seed's noise against sd.cpp's generator (tools/philox_reference.cpp)
generate <encoder.gguf> <dit.gguf> <vae.safetensors> "<prompt>" [--size WxH] [--seed N] [--out file.png]
text to image, Turbo's 8 steps (default 1024x1024, seed 42, out.png)
+ serve --diffusion-model <dit.gguf> --vae <vae.safetensors> --llm <encoder.gguf> [--listen-ip 127.0.0.1] [--listen-port 1234]
+ the engine as a server speaking sd-server's /sdcpp/v1 API (text to image)
probe this Mac's matrix throughput: GPU, AMX, Neural Engine, together
probe-attention the target's attention at 1024² on the GPU, the Neural Engine, the CPU
"""
@@ -102,6 +104,28 @@ do {
case "check-noise":
guard arguments.count == 2 else { fail(usage) }
try Checks.noise(dump: arguments[1])
+ case "serve":
+ var (ip, port, dit, vae, llm) = ("127.0.0.1", UInt16(1234), "", "", "")
+ var options = arguments.dropFirst().makeIterator()
+ while let option = options.next() {
+ guard let value = options.next() else { fail("\(option) needs a value\n\n" + usage) }
+ switch option {
+ case "--listen-ip": ip = value
+ case "--listen-port": port = UInt16(value) ?? port
+ case "--diffusion-model": dit = value
+ case "--vae": vae = value
+ case "--llm": llm = value
+ case "--llm_vision": break // edits, later
+ default: fail("unknown option \(option)\n\n" + usage)
+ }
+ }
+ guard !dit.isEmpty, !vae.isEmpty, !llm.isEmpty else { fail("serve needs --diffusion-model, --vae and --llm\n\n" + usage) }
+ setvbuf(stdout, nil, _IOLBF, 0) // qls-core follows the log file stdout goes to
+ print("kokon-engine-mac (experimental), Qwen-Image 2.1 Turbo")
+ let engine = try EngineServer(paths: .init(diffusionModel: dit, vae: vae, textEncoder: llm))
+ let server = try HTTPServer(host: ip, port: port) { engine.handle($0) }
+ print("[INFO ] engine-mac - listening on: http://\(ip):\(server.port)")
+ server.run()
case "probe":
try Probe.run()
case "probe-attention":
diff --git a/engine-mac/tools/api_check.py b/engine-mac/tools/api_check.py
new file mode 100644
index 0000000..3f09f77
--- /dev/null
+++ b/engine-mac/tools/api_check.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""Text to image through a running `kokon-engine-mac serve` (or sd-server) as qls-core sends it: capabilities,
+img_gen, polling jobs/{id}, optionally a cancel. Prints the time of each image and writes the PNGs.
+
+ python3 -I tools/api_check.py --port 1234 --images 2 --out /tmp/api [--size 1024x1024] [--cancel-after 10]
+"""
+import argparse, base64, json, os, time, urllib.request, urllib.error
+
+TURBO_SIGMAS = [1.0, 0.978453, 0.95418, 0.926626, 0.89508, 0.845148, 0.704534, 0.414568, 0.0]
+
+
+def call(port, method, path, body=None, timeout=120):
+ data = json.dumps(body).encode() if body is not None else None
+ request = urllib.request.Request(f"http://127.0.0.1:{port}{path}", data=data, method=method,
+ headers={"Content-Type": "application/json"})
+ try:
+ with urllib.request.urlopen(request, timeout=timeout) as r:
+ return r.status, json.loads(r.read() or b"{}")
+ except urllib.error.HTTPError as e:
+ return e.code, json.loads(e.read() or b"{}")
+
+
+def main():
+ p = argparse.ArgumentParser()
+ p.add_argument("--port", type=int, default=1234)
+ p.add_argument("--images", type=int, default=2)
+ p.add_argument("--size", default="1024x1024")
+ p.add_argument("--prompt", default="a red fox in the snow, morning light, a wooden sign that says KOKON")
+ p.add_argument("--seed", type=int, default=42)
+ p.add_argument("--out", default=".")
+ p.add_argument("--cancel-after", type=float, default=0, help="cancel the last image after this many seconds")
+ a = p.parse_args()
+ width, height = (int(v) for v in a.size.split("x"))
+ status, caps = call(a.port, "GET", "/sdcpp/v1/capabilities", timeout=10)
+ print(f"capabilities {status}: engine {caps.get('engine', 'sd-server')}, model {caps.get('model', {}).get('name')}")
+ os.makedirs(a.out, exist_ok=True)
+ for i in range(a.images):
+ body = {"prompt": a.prompt, "negative_prompt": "", "width": width, "height": height, "seed": a.seed + i,
+ "batch_count": 1, "sample_params": {"sample_method": "euler", "sample_steps": 8,
+ "custom_sigmas": TURBO_SIGMAS, "guidance": {"txt_cfg": 1.0}},
+ "output_format": "png", "embed_image_metadata": False}
+ started = time.time()
+ status, accepted = call(a.port, "POST", "/sdcpp/v1/img_gen", body)
+ if status != 202:
+ print(f"img_gen {status}: {accepted}")
+ return 1
+ cancel = a.cancel_after > 0 and i == a.images - 1
+ while True:
+ time.sleep(0.25)
+ if cancel and time.time() - started > a.cancel_after:
+ t = time.time()
+ status, job = call(a.port, "POST", f"/sdcpp/v1/jobs/{accepted['id']}/cancel", timeout=10)
+ print(f"cancel {status} after {t - started:.1f} s: {job.get('status')} (answered in {time.time() - t:.2f} s)")
+ cancel = False
+ status, job = call(a.port, "GET", f"/sdcpp/v1/jobs/{accepted['id']}", timeout=60)
+ if job.get("status") in ("completed", "failed", "cancelled"):
+ break
+ seconds = time.time() - started
+ if job["status"] != "completed":
+ print(f"image {i + 1}: {job['status']} after {seconds:.1f} s {job.get('error', '')}")
+ continue
+ path = os.path.join(a.out, f"api-{i + 1}.png")
+ with open(path, "wb") as f:
+ f.write(base64.b64decode(job["result"]["images"][0]["b64_json"]))
+ print(f"image {i + 1}: {seconds:.1f} s through the API -> {path}")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/engine-mac/tools/install-dev.sh b/engine-mac/tools/install-dev.sh
new file mode 100755
index 0000000..8792ca0
--- /dev/null
+++ b/engine-mac/tools/install-dev.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+# Builds kokon-engine-mac and puts it where Kokon looks for it when "Kokon's Mac engine" is on (Settings →
+# Performance): <stable-diffusion.cpp folder>/engine-mac/, beside sd.cpp's engine-metal. The kernels and resources
+# are the KokonEngineMac_KokonEngine.bundle next to the executable.
+#
+# bash engine-mac/tools/install-dev.sh [stable-diffusion.cpp folder]
+set -euo pipefail
+here="$(cd "$(dirname "$0")/.." && pwd)"
+target="${1:-$HOME/Library/Application Support/KokonData/sdcpp}/engine-mac"
+(cd "$here" && swift build -c release)
+mkdir -p "$target"
+rm -rf "$target/KokonEngineMac_KokonEngine.bundle"
+cp "$here/.build/release/kokon-engine-mac" "$target/"
+cp -R "$here/.build/release/KokonEngineMac_KokonEngine.bundle" "$target/"
+echo "installed in $target"
diff --git a/src/components/Settings.tsx b/src/components/Settings.tsx
index 6620530..6472115 100644
--- a/src/components/Settings.tsx
+++ b/src/components/Settings.tsx
@@ -583,8 +583,16 @@ export function Settings() {
<>
<div className="field">
<span>Engine</span>
- <span>Metal (Apple Silicon GPU, beta)</span>
+ <span>{cfg.macEngine ? "Kokon's Mac engine (experimental)" : "Metal (Apple Silicon GPU, beta)"}</span>
</div>
+ <div className="field-row">
+ <span>Kokon's Mac engine (experimental)</span>
+ <Switch on={!!cfg.macEngine} onChange={(v) => set({ macEngine: v })} />
+ </div>
+ <span className="muted" style={{ marginTop: -10 }}>
+ Kokon's own engine for Apple Silicon: the GPU, the CPU's matrix unit and the Neural Engine work together, about 4 times faster for new images. Experimental: new images only for now. Edits, LoRAs, Prompt+ and the upscaler need it off.
+ </span>
+ {!cfg.macEngine && (
<div className="field">
<span>Model memory</span>
<Seg<TextEncoderPlacement>
@@ -599,6 +607,7 @@ export function Settings() {
Keep loaded: every model stays in memory, fastest between steps; needs 24 GB or more. Load when needed: each model (text encoder, image model, decoder) is loaded from the SSD when it runs and released after, so the image model has room to work on 16 GB Macs. It adds a few seconds per image. macOS cannot swap memory the GPU is using, so this is what makes 16 GB work.
</span>
</div>
+ )}
<label className="field-row">
<span>Memory kept free for macOS (GiB)</span>
<input className="input" type="number" min={0} max={16} step={0.5} style={{ width: 90 }} value={cfg.vramReserveGib} onChange={(e) => set({ vramReserveGib: Math.max(0, Number(e.target.value) || 0) })} />
diff --git a/src/lib/types.ts b/src/lib/types.ts
index 43a3c9d..a5d559c 100644
--- a/src/lib/types.ts
+++ b/src/lib/types.ts
@@ -298,6 +298,8 @@ export interface AppConfig {
/** Shrink the input before upscaling (enlargements under ~2.8x). */
upscalePrescale: boolean;
extraServerArgs: string[];
+ /** Kokon's own engine for Apple Silicon instead of sd.cpp's Metal build (experimental, off by default; Metal only). */
+ macEngine?: boolean;
outputDir: string;
keepServerRunning: boolean;
idleUnloadMinutes: number;