Skip to content

armgit / kokon

Public Imported from GitHub
0

Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.

Native 2K edits on 16 GB: 1 MP references, larger budget, shaped VAE tiles

Gabriele Lozuponeff42446ccaea
diff --git a/crates/qls-core/src/pipeline.rs b/crates/qls-core/src/pipeline.rs
index a1679d4..627d707 100644
--- a/crates/qls-core/src/pipeline.rs
+++ b/crates/qls-core/src/pipeline.rs
@@ -9,7 +9,7 @@ use image::{GrayImage, Luma, Rgba, RgbaImage};
use crate::error::{AppError, AppResult, ErrorCode};
use crate::geometry::{Rect, Size};
use crate::imaging::{self, LoadedImage};
-use crate::planner::{InpaintAction, Plan, ResolvedMode};
+use crate::planner::{InpaintAction, Plan, ReferenceDetail, ResolvedMode};
use crate::sdcpp::{Guidance, ImgGenRequest, SampleParams, VaeTilingParams, b64};
/// Server-side facts that shape the request but are not part of the plan.
@@ -32,7 +32,11 @@ pub const DEFAULT_VAE_TILE: u32 = 768;
/// when the server already tiles (`--vae-tiling --vae-tile-size N`) nothing is sent —
/// sending `tile_size 0` would silently select 256 px tiles. Only an explicit "off"
/// overrides it.
-pub fn vae_tiling_params(tiling: bool, server_tile: u32) -> Option<VaeTilingParams> {
+/// A per-job tile (`Plan::vae_tile`, large discrete GPUs) replaces the server's size.
+pub fn vae_tiling_params(tiling: bool, server_tile: u32, job_tile: Option<(u32, u32)>) -> Option<VaeTilingParams> {
+ if let (true, Some((w, h))) = (tiling, job_tile) {
+ return Some(VaeTilingParams { enabled: true, tile_size_w: Some(w), tile_size_h: Some(h), target_overlap: Some(crate::planner::VAE_TILE_OVERLAP) });
+ }
match (tiling, server_tile > 0) {
(true, true) | (false, false) => None,
(false, true) => Some(VaeTilingParams { enabled: false, ..Default::default() }),
@@ -167,6 +171,7 @@ pub fn prepare(plan: &Plan, prompt: &str, seed: i64, images: &[LoadedImage], mas
mask_image: None,
ref_images: Vec::new(),
image_preprocess: Vec::new(),
+ ref_image_args: None,
sample_params: SampleParams {
sample_method: Some(plan.sampler.clone()),
scheduler: None,
@@ -175,7 +180,7 @@ pub fn prepare(plan: &Plan, prompt: &str, seed: i64, images: &[LoadedImage], mas
custom_sigmas: plan.custom_sigmas.clone(),
guidance: Guidance { txt_cfg: plan.cfg, img_cfg: None },
},
- vae_tiling_params: vae_tiling_params(plan.vae_tiling, opts.server_vae_tile_size),
+ vae_tiling_params: vae_tiling_params(plan.vae_tiling, opts.server_vae_tile_size, plan.vae_tile),
hires: None,
cache_mode: None,
cache_option: None,
@@ -201,6 +206,11 @@ pub fn prepare(plan: &Plan, prompt: &str, seed: i64, images: &[LoadedImage], mas
req.ref_images.push(png_b64(&to_canvas(img, &r.canvas))?);
}
req.image_preprocess.push("target=ref,mode=none".into());
+ // Standard detail: encode the references at their planned (≤ 1 MP) size; Full keeps
+ // the server's resize to the output's area, as before.
+ if plan.reference_detail == Some(ReferenceDetail::Standard) {
+ req.ref_image_args = Some("resize_before_vae=false".into());
+ }
if let Some(s) = plan.strength {
req.strength = Some(s);
}
@@ -519,7 +529,7 @@ mod tests {
}
use crate::config::RecommendedSettings;
use crate::geometry::ExpansionUnits;
- use crate::planner::{GenerationRequest, Mode, OutpaintSpec, PlanInputs, Quality, ResolutionMode, UpscaleSpec, plan};
+ use crate::planner::{GenerationRequest, Mode, OutpaintSpec, PlanInputs, Quality, ResolutionMode, STANDARD_REFERENCE_AREA, UpscaleSpec, plan};
use crate::sdcpp::b64_decode;
fn env(sizes: Vec<Size>) -> PlanInputs {
@@ -531,10 +541,13 @@ mod tests {
#[test]
fn vae_tiling_never_overrides_the_server_tile_size() {
- assert_eq!(vae_tiling_params(true, 768), None, "server already tiles with its own size");
- assert_eq!(vae_tiling_params(false, 0), None);
- assert_eq!(vae_tiling_params(false, 768), Some(VaeTilingParams { enabled: false, ..Default::default() }));
- let p = vae_tiling_params(true, 0).unwrap();
+ assert_eq!(vae_tiling_params(true, 768, None), None, "server already tiles with its own size");
+ assert_eq!(vae_tiling_params(false, 0, None), None);
+ assert_eq!(vae_tiling_params(false, 768, None), Some(VaeTilingParams { enabled: false, ..Default::default() }));
+ let t = vae_tiling_params(true, 768, Some((1024, 1536))).unwrap();
+ assert_eq!((t.enabled, t.tile_size_w, t.tile_size_h), (true, Some(1024), Some(1536)), "a per-job tile wins");
+ assert_eq!(vae_tiling_params(false, 768, Some((1024, 1536))), Some(VaeTilingParams { enabled: false, ..Default::default() }), "off stays off");
+ let p = vae_tiling_params(true, 0, None).unwrap();
assert_eq!((p.enabled, p.tile_size_w, p.tile_size_h, p.target_overlap), (true, Some(768), Some(768), Some(0.125)));
// Regression: a planned-tiled 2K job sent tile_size 0, i.e. 256 px tiles.
let mut r = req("a cabin");
@@ -544,6 +557,12 @@ mod tests {
let p = plan(&r, &i, &RecommendedSettings::default()).unwrap();
assert!(p.vae_tiling);
let j = prepare(&p, &r.prompt, 1, &[], None, &esrgan_opts()).unwrap();
+ // 16 GB: a tile shaped to the image, never size 0.
+ let v = serde_json::to_value(&j.request).unwrap();
+ assert!(v["vae_tiling_params"]["tile_size_w"].as_u64().unwrap() >= 256 && v["vae_tiling_params"]["tile_size_h"].as_u64().unwrap() >= 256, "{v}");
+ // 12 GB: the server's own tiles, nothing sent.
+ let p = plan(&r, &PlanInputs { vram_total_mib: Some(12282), ..i }, &RecommendedSettings::default()).unwrap();
+ let j = prepare(&p, &r.prompt, 1, &[], None, &esrgan_opts()).unwrap();
assert!(serde_json::to_value(&j.request).unwrap().get("vae_tiling_params").is_none());
}
@@ -566,22 +585,25 @@ mod tests {
#[test]
fn large_edit_is_one_pass_then_enlarged() {
- let img = photo(2560, 1440);
+ let img = photo(3840, 2160);
let mut r = req("make it a dark version");
r.resolution = ResolutionMode::Source;
let p = plan(&r, &env(vec![img.size()]), &RecommendedSettings::default()).unwrap();
let j = prepare(&p, &r.prompt, 3, std::slice::from_ref(&img), None, &esrgan_opts()).unwrap();
assert!(j.request.hires.is_none());
assert_eq!((j.request.width, j.request.height), (p.working.width, p.working.height));
- assert_eq!(decode(&j.request.ref_images[0]).dimensions(), (p.working.width, p.working.height), "reference matches the canvas");
- // The single-pass result is enlarged to the 2560x1440 source size with ESRGAN.
+ let rc = p.references[0].canvas.canvas;
+ assert_eq!(decode(&j.request.ref_images[0]).dimensions(), (rc.width, rc.height), "reference as planned");
+ assert!(rc.area() <= STANDARD_REFERENCE_AREA && p.working.area() > rc.area() * 2, "1 MP reference, larger canvas");
+ assert_eq!(j.request.ref_image_args.as_deref(), Some("resize_before_vae=false"), "encoded at its planned size");
+ // The single-pass result is enlarged to the 3840x2160 source size with ESRGAN.
let content = crop_content(fake_backend(p.working.width, p.working.height), &j.post);
let target = enlarge_target(&j.post).unwrap();
- assert_eq!(target, Size::new(2560, 1440));
+ assert_eq!(target, Size::new(3840, 2160));
let ep = esrgan_plan(Size::new(content.width(), content.height()), target, 4).unwrap();
- assert_eq!(ep.tiles.len(), 1, "fits one upscale call");
+ assert!(!ep.tiles.is_empty(), "the larger canvas may take more than one upscale call");
let out = finish(content, &j.post).unwrap();
- assert_eq!(out.dimensions(), (2560, 1440));
+ assert_eq!(out.dimensions(), (3840, 2160));
}
#[test]
@@ -704,9 +726,11 @@ mod tests {
let img = photo(1920, 1080);
let mut r = req("make it night");
r.resolution = ResolutionMode::Source;
+ r.advanced.reference_detail = Some(ReferenceDetail::Full);
let p = plan(&r, &PlanInputs { image_sizes: vec![img.size()], vram_total_mib: Some(16303), ..Default::default() }, &RecommendedSettings::default()).unwrap();
let j = prepare(&p, &r.prompt, 7, std::slice::from_ref(&img), None, &BackendOptions::default()).unwrap();
assert_eq!((j.request.width, j.request.height), (1920, 1088));
+ assert!(j.request.ref_image_args.is_none(), "Full keeps the server's resize");
assert_eq!(j.request.ref_images.len(), 1);
assert!(j.request.init_image.is_none() && j.request.mask_image.is_none());
let sent = decode(&j.request.ref_images[0]);
diff --git a/crates/qls-core/src/planner.rs b/crates/qls-core/src/planner.rs
index bd2cd6a..98a9da9 100644
--- a/crates/qls-core/src/planner.rs
+++ b/crates/qls-core/src/planner.rs
@@ -200,8 +200,26 @@ pub struct AdvancedOptions {
pub vae_tiling: Option<bool>,
/// Keep the HDR gain map of a HEIC source (Ultra HDR JPEG output); None = on.
pub hdr: Option<bool>,
+ /// How large edit/combine sources are given to the model; None = standard (1 MP).
+ pub reference_detail: Option<ReferenceDetail>,
}
+/// Size of the source images the model sees in edits and combines. They share the token
+/// budget with the image being drawn, so a smaller reference leaves room for a larger output.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
+#[serde(rename_all = "snake_case")]
+pub enum ReferenceDetail {
+ /// At most 1 MP per reference (diffusers' Qwen edit pipelines use ~1 MP too): the rest
+ /// of the budget goes to the output, which renders larger and in fewer total tokens.
+ #[default]
+ Standard,
+ /// Up to the output's own area: finer source detail, but a single edit gets half the budget.
+ Full,
+}
+
+/// Largest reference area with [`ReferenceDetail::Standard`].
+pub const STANDARD_REFERENCE_AREA: u64 = 1024 * 1024;
+
fn default_true() -> bool {
true
}
@@ -318,6 +336,13 @@ pub struct Plan {
/// Exact noise schedule (Turbo's); empty when no diffusion runs.
#[serde(default)]
pub custom_sigmas: Vec<f32>,
+ /// Reference size policy of edits and combines (None for the other modes and in records
+ /// of older versions, which always encoded references at the output's area).
+ #[serde(default)]
+ pub reference_detail: Option<ReferenceDetail>,
+ /// Decode tile (width, height in pixels) sent with the job; None = the server's own.
+ #[serde(default)]
+ pub vae_tile: Option<(u32, u32)>,
}
/// Facts about inputs and the environment, measured before planning.
@@ -521,6 +546,77 @@ pub fn token_budget(vram_total_mib: Option<u64>) -> u64 {
tokens.clamp(4_096.0, 24_576.0) as u64
}
+/// Extra tokens for discrete GPUs of 15 GiB and up: 16 GB cards run native 2K edits
+/// (2752×1536 = 16 512 tokens) with a standard 1 MP reference (1344×768 = 4 032), like
+/// Unsloth Studio's 2720×1536 + 1 MP on the same RTX 5070 Ti. sd-server's graph cut
+/// (`--max-vram -1`) splits the graph instead of failing if the activations do not fit.
+pub const DISCRETE_LARGE_GPU_BONUS: u64 = 20_736 - 16_384;
+
+/// Token budget the planner uses: [`token_budget`], plus [`DISCRETE_LARGE_GPU_BONUS`] on
+/// discrete GPUs of 15 GiB and up. Macs (unified memory) and smaller cards are not measured
+/// at larger sizes yet and keep the base budget.
+pub fn planning_token_budget(vram_total_mib: Option<u64>, unified_memory: bool) -> u64 {
+ let base = token_budget(vram_total_mib);
+ match vram_total_mib {
+ Some(mib) if !unified_memory && mib as f64 / 1024.0 >= 15.0 => base + DISCRETE_LARGE_GPU_BONUS,
+ _ => base,
+ }
+}
+
+/// Largest VAE decode tile on discrete GPUs of 15 GiB and up: an untiled decode of ~1.5 MP
+/// already fills a 16 GB card with the DiT resident, so 1 MP leaves margin. sd.cpp retries a
+/// decode that runs out of memory with smaller tiles itself, so too large only costs time.
+pub const LARGE_GPU_VAE_TILE_AREA: u64 = 1024 * 1024;
+/// Overlap the server is launched with (`--vae-tile-overlap`), also sent with per-job tiles.
+pub const VAE_TILE_OVERLAP: f32 = 0.125;
+/// Pixels per latent of Qwen-Image 2.1's VAE.
+const VAE_SCALE: u32 = 16;
+/// Shortest side of a shaped decode tile, in latents (512 px).
+const MIN_VAE_TILE_SIDE: u32 = 32;
+
+/// Tiles sd.cpp makes along one axis of `dim` latents with `tile`-latent tiles
+/// (`sd_tiling_plan_axis` in runtime/tiling.cpp, non-circular).
+fn vae_axis_tiles(dim: u32, tile: u32, overlap: f32) -> u32 {
+ if tile >= dim {
+ return 1;
+ }
+ let extent = (dim - tile) as i64;
+ let tile = tile as i64;
+ let target = extent as f32 / ((1.0 - overlap) * tile as f32);
+ let min_i = 1 + (extent - 1) / (tile - 1).max(1);
+ let max_i = (2 * extent / tile).max(1);
+ let lower = (target.floor() as i64).clamp(min_i, max_i.max(min_i));
+ let upper = (target.ceil() as i64).clamp(min_i, max_i.max(min_i));
+ let err = |n: i64| (1.0 / n as f32 - 1.0 / target).abs();
+ (if err(upper) < err(lower) { upper } else { lower }) as u32 + 1
+}
+
+/// Decode tile (width, height in pixels) for a `size` image whose tiles may cover at most
+/// `max_area`: the grid with the least decoded area, overlap included. The server's square
+/// 768 px tiles cut 2752×1536 into 4×3 tiles covering 67 % more than the image; shaped
+/// tiles (at least 512 px a side) cover it in 5 full-height 608 px columns (+10 %).
+pub fn vae_decode_tile(size: Size, max_area: u64) -> (u32, u32) {
+ let (lw, lh) = (size.width / VAE_SCALE, size.height / VAE_SCALE);
+ // Tile length for `n` tiles along an axis (sd.cpp's relative size > 1).
+ let len = |dim: u32, n: u32| if n <= 1 { dim } else { ((dim as f32 / (n as f32 * (1.0 - VAE_TILE_OVERLAP) + VAE_TILE_OVERLAP)).floor() as u32).max(4) };
+ let mut best: Option<(u64, u32, (u32, u32))> = None;
+ for nx in 1..=12 {
+ for ny in 1..=12 {
+ let (tw, th) = (len(lw, nx), len(lh, ny));
+ // Thin strips decode least but leave little context at the seams.
+ if (tw * th * VAE_SCALE * VAE_SCALE) as u64 > max_area || tw.min(th) < MIN_VAE_TILE_SIDE.min(lw).min(lh) {
+ continue;
+ }
+ let count = vae_axis_tiles(lw, tw, VAE_TILE_OVERLAP) * vae_axis_tiles(lh, th, VAE_TILE_OVERLAP);
+ let cost = count as u64 * (tw * th) as u64;
+ if best.map(|(c, n, _)| (cost, count) < (c, n)).unwrap_or(true) {
+ best = Some((cost, count, (tw * VAE_SCALE, th * VAE_SCALE)));
+ }
+ }
+ }
+ best.map(|b| b.2).unwrap_or((768, 768))
+}
+
/// Apple Silicon: the VRAM-equivalent to plan with on a Mac with `ram_mib` of unified
/// memory, so presets and working sizes follow the same rules as a discrete GPU.
///
@@ -739,7 +835,7 @@ pub fn plan(req: &GenerationRequest, inputs: &PlanInputs, rec: &RecommendedSetti
let quality = req.quality;
// Turbo: its fixed 8-step schedule without CFG (model card).
let cfg = 1.0;
- let budget_tokens = token_budget(inputs.vram_total_mib);
+ let budget_tokens = planning_token_budget(inputs.vram_total_mib, inputs.unified_memory);
let budget_area = budget_tokens * PIXELS_PER_TOKEN;
let tier_area = resolution_area(req.resolution, quality, rec);
let adv = &req.advanced;
@@ -762,9 +858,14 @@ pub fn plan(req: &GenerationRequest, inputs: &PlanInputs, rec: &RecommendedSetti
}
};
- // Edit/combine: largest area `a <= wanted` with `a + Σ min(ref_i, a)` within the budget.
+ // Reference cap of edits and combines (besides the output's own area, see `ref_canvas`).
+ let ref_cap = match adv.reference_detail.unwrap_or_default() {
+ ReferenceDetail::Standard => STANDARD_REFERENCE_AREA,
+ ReferenceDetail::Full => u64::MAX,
+ };
+ // Edit/combine: largest area `a <= wanted` with `a + Σ min(ref_i, a, ref_cap)` within the budget.
let cap_for_sources = |wanted: u64, refs: &[Size], notes: &mut Vec<String>| -> u64 {
- let fits = |a: u64| a + refs.iter().map(|r| r.area().min(a)).sum::<u64>() <= budget_area;
+ let fits = |a: u64| a + refs.iter().map(|r| r.area().min(a).min(ref_cap)).sum::<u64>() <= budget_area;
if fits(wanted) {
return wanted;
}
@@ -856,8 +957,11 @@ pub fn plan(req: &GenerationRequest, inputs: &PlanInputs, rec: &RecommendedSetti
sources_reduced = area < wanted;
let full = plan_working_canvas(target, &limits(area, rec));
working = full;
+ // Within 10 % of the cap (1 MP outputs, after alignment) the source keeps the
+ // output's own canvas, as with Full.
+ let ref_area = if working.canvas.area() <= ref_cap.saturating_add(ref_cap / 10) { working.canvas.area() } else { ref_cap };
for (i, s) in inputs.image_sizes.iter().enumerate() {
- let c = if i == 0 && *s == target { working } else { ref_canvas(*s, working.canvas.area(), rec) };
+ let c = if i == 0 && *s == target && ref_area == working.canvas.area() { working } else { ref_canvas(*s, ref_area, rec) };
references.push(RefPlan { index: i, source: *s, canvas: c });
}
output = if req.preserve_dimensions || tier || chosen_aspect.is_some() { target } else { full.content };
@@ -942,6 +1046,11 @@ pub fn plan(req: &GenerationRequest, inputs: &PlanInputs, rec: &RecommendedSetti
optimized = true;
notes.push("Memory optimization enabled for this size.".into());
}
+ // Large discrete GPUs: decode tiles shaped to the image (fewer, larger tiles); others keep
+ // the server's square tiles.
+ let large_gpu = !inputs.unified_memory && inputs.vram_total_mib.map(|m| m as f64 / 1024.0 >= 15.0).unwrap_or(false);
+ let vae_tile = (vae_tiling && large_gpu && !quick_upscale && working.canvas.area() > LARGE_GPU_VAE_TILE_AREA)
+ .then(|| vae_decode_tile(working.canvas, LARGE_GPU_VAE_TILE_AREA));
if (working.content.area() as f64) < output.area() as f64 * 0.98 && matches!(mode, ResolvedMode::Generate | ResolvedMode::Edit | ResolvedMode::MultiReference) {
notes.push(format!(
"Rendered at {}×{}, then enlarged to {}×{} with {}.",
@@ -998,6 +1107,8 @@ pub fn plan(req: &GenerationRequest, inputs: &PlanInputs, rec: &RecommendedSetti
upscale_tile: if inputs.upscale_tile > 0 { inputs.upscale_tile } else { LARGE_UPSCALE_TILE },
upscale,
custom_sigmas: if quick_upscale { Vec::new() } else { TURBO_SIGMAS.to_vec() },
+ vae_tile,
+ reference_detail: matches!(mode, ResolvedMode::Edit | ResolvedMode::MultiReference).then(|| adv.reference_detail.unwrap_or_default()),
})
}
@@ -1147,7 +1258,11 @@ mod tests {
let a = p.content.width as f64 / p.content.height as f64;
assert!((a / (1920.0 / 1080.0) - 1.0).abs() < 0.01);
assert_eq!(p.references.len(), 1);
- assert_eq!(p.references[0].canvas.canvas, p.working, "ref matches canvas so the backend never resizes it");
+ // Standard detail (default): the source is given at ~1 MP, same shape.
+ let rc = p.references[0].canvas;
+ assert!(rc.canvas.area() <= STANDARD_REFERENCE_AREA, "{:?}", rc.canvas);
+ assert!((rc.content.width as f64 / rc.content.height as f64 / (1920.0 / 1080.0) - 1.0).abs() < 0.02);
+ assert_eq!(p.reference_detail, Some(ReferenceDetail::Standard));
assert!(p.strength.is_none());
assert_eq!(p.cfg, 1.0, "official: edits run without CFG too");
}
@@ -1177,12 +1292,21 @@ mod tests {
fn huge_photo_is_reduced_for_vram_and_restored() {
let mut r = req("make it autumn");
r.resolution = ResolutionMode::Source;
+ let budget = (planning_token_budget(Some(16303), false) * 256) as f64 * 1.05;
let p = plan(&r, &inputs(&[(4032, 3024)]), &rec()).unwrap();
assert!(p.optimized_for_vram);
- assert_eq!(p.references[0].canvas.canvas, p.working);
- assert!((p.working.area() * 2) as f64 <= (token_budget(Some(16303)) * 256) as f64 * 1.05, "target + reference share the budget");
+ // Standard detail: a 1 MP reference leaves ~3 MP for the image being drawn.
+ assert!(p.references[0].canvas.canvas.area() <= STANDARD_REFERENCE_AREA);
+ assert!(p.working.area() > 2_900_000, "{:?}", p.working);
+ assert!((p.working.area() + p.references[0].canvas.canvas.area()) as f64 <= budget, "target + reference share the budget");
assert_eq!(p.output, Size::new(4032, 3024));
assert!(p.notes.iter().any(|n| n.contains("enlarged to 4032×3024")));
+ // Full detail: the reference takes the output's canvas, half the budget each.
+ r.advanced.reference_detail = Some(ReferenceDetail::Full);
+ let p = plan(&r, &inputs(&[(4032, 3024)]), &rec()).unwrap();
+ assert_eq!(p.references[0].canvas.canvas, p.working);
+ assert!((p.working.area() * 2) as f64 <= budget);
+ assert_eq!(p.reference_detail, Some(ReferenceDetail::Full));
}
fn env(upscaler: bool) -> PlanInputs {
@@ -1201,7 +1325,7 @@ mod tests {
let p = plan(&r, &env(true), &rec()).unwrap();
assert!(!p.notes.iter().any(|n| n.contains("Two passes")), "{res:?} {aspect} {q:?}");
assert!(p.working.is_aligned(32));
- assert!(p.working.area() as f64 <= (token_budget(Some(16303)) * 256) as f64 * 1.02, "{:?}", p.working);
+ assert!(p.working.area() as f64 <= (planning_token_budget(Some(16303), false) * 256) as f64 * 1.02, "{:?}", p.working);
}
}
}
@@ -1223,7 +1347,7 @@ mod tests {
r.height = Some(2160);
let p = plan(&r, &env(true), &rec()).unwrap();
assert_eq!(p.output, Size::new(3840, 2160), "exact requested size is delivered");
- assert!(p.working.area() as f64 <= (token_budget(Some(16303)) * 256) as f64 * 1.02, "alignment padding only");
+ assert!(p.working.area() as f64 <= (planning_token_budget(Some(16303), false) * 256) as f64 * 1.02, "alignment padding only");
assert!((p.content.width as f64 / p.content.height as f64 / (3840.0 / 2160.0) - 1.0).abs() < 0.01);
assert!(p.notes.iter().any(|n| n.contains("enlarged to 3840×2160 with 4xNomos2_hq_mosr")), "{:?}", p.notes);
}
@@ -1250,21 +1374,27 @@ mod tests {
let mut r = req("remove the people");
r.quality = Quality::Quality;
let i = PlanInputs { image_sizes: vec![Size::new(5712, 4284)], ..env(true) };
+ // Standard detail on 16 GB: 1 MP reference and the full native 2K area (~4.2 MP) drawn.
+ let p = plan(&r, &i, &rec()).unwrap();
+ assert!((4_100_000..=4_300_000).contains(&p.working.area()), "{:?}", p.working);
+ assert!(p.references[0].canvas.canvas.area() <= STANDARD_REFERENCE_AREA);
+ assert_eq!(p.output, Size::new(5712, 4284));
+ r.advanced.reference_detail = Some(ReferenceDetail::Full);
let p = plan(&r, &i, &rec()).unwrap();
assert_eq!((p.steps, p.cfg), (8, 1.0));
- // Native 2K plus a 2K reference exceeds 16 GB: both get half the budget (~2 MP each).
+ // Full: native 2K plus a 2K reference exceeds 16 GB, so both get half the budget (~2.65 MP each).
// Each side rounds to 32 px, so allow the same 5 % slack as many_references_share_the_budget.
- assert!((1_850_000..=2_202_010).contains(&p.working.area()), "{:?}", p.working);
- assert_eq!(p.references[0].canvas.canvas, p.working, "the model sees the source at ~2 MP");
+ assert!((2_450_000..=2_800_000).contains(&p.working.area()), "{:?}", p.working);
+ assert_eq!(p.references[0].canvas.canvas, p.working, "the model sees the source at the canvas size");
assert!(p.notes.iter().any(|n| n.contains("Resolution reduced")));
- assert!((p.working.area() + p.references[0].canvas.canvas.area()) as f64 <= (token_budget(Some(16303)) * PIXELS_PER_TOKEN) as f64 * 1.05);
+ assert!((p.working.area() + p.references[0].canvas.canvas.area()) as f64 <= (planning_token_budget(Some(16303), false) * PIXELS_PER_TOKEN) as f64 * 1.05);
assert_eq!(p.output, Size::new(5712, 4284));
}
#[test]
fn tiers_set_the_output_size_of_edits_and_combines() {
// Regression: 1K/1.5K/2K were capped by the smallest source, so 800x800 refs always gave 800x800.
- let budget = (token_budget(Some(16303)) * PIXELS_PER_TOKEN) as f64 * 1.05;
+ let budget = (planning_token_budget(Some(16303), false) * PIXELS_PER_TOKEN) as f64 * 1.05;
let mut r = req("put both people in a kitchen");
r.resolution = ResolutionMode::K15;
let p = plan(&r, &inputs(&[(800, 800), (800, 800)]), &rec()).unwrap();
@@ -1313,7 +1443,7 @@ mod tests {
let p = plan(&r, &i, &rec()).unwrap();
assert!(p.optimized_for_vram);
let total: u64 = p.working.area() + p.references.iter().map(|r| r.canvas.canvas.area()).sum::<u64>();
- assert!(total as f64 <= (token_budget(Some(16303)) * 256) as f64 * 1.05, "{total}");
+ assert!(total as f64 <= (planning_token_budget(Some(16303), false) * 256) as f64 * 1.05, "{total}");
}
#[test]
@@ -1506,6 +1636,47 @@ mod tests {
assert!(token_budget(Some(8192)) < token_budget(Some(16303)));
assert!(token_budget(Some(24576)) > token_budget(Some(16303)));
assert_eq!(token_budget(None), 12_288);
+ // Discrete 16 GB cards: native 2K plus a 1 MP reference; Macs and 12 GB cards unchanged.
+ assert_eq!(planning_token_budget(Some(16303), false), 20_736);
+ assert!(planning_token_budget(Some(16303), false) * PIXELS_PER_TOKEN >= 2752 * 1536 + 1344 * 768);
+ assert_eq!(planning_token_budget(Some(16303), true), 16_384);
+ assert_eq!(planning_token_budget(Some(12282), false), token_budget(Some(12282)));
+ assert_eq!(planning_token_budget(None, false), 12_288);
+ }
+
+ #[test]
+ fn vae_tiles_follow_sdcpp_and_fit_the_image() {
+ // sd.cpp's own split of 2752×1536 (172×96 latents) with 768 px (48) tiles: 4×3.
+ assert_eq!((vae_axis_tiles(172, 48, 0.125), vae_axis_tiles(96, 48, 0.125)), (4, 3));
+ assert_eq!(vae_axis_tiles(96, 96, 0.125), 1);
+ let (w, h) = vae_decode_tile(Size::new(2752, 1536), LARGE_GPU_VAE_TILE_AREA);
+ assert!((w * h) as u64 <= LARGE_GPU_VAE_TILE_AREA, "{w}x{h}");
+ let decoded = |w: u32, h: u32| vae_axis_tiles(172, w / 16, 0.125) * vae_axis_tiles(96, h / 16, 0.125) * (w / 16) * (h / 16);
+ assert!(decoded(w, h) * 10 < decoded(768, 768) * 8, "at least 20 % less decoding than 768² tiles: {w}x{h}");
+ assert_eq!((w % 16, h % 16), (0, 0));
+ assert_eq!((w, h), (608, 1536), "5 full-height columns");
+ assert!(w.min(h) >= 512, "no thin strips");
+ // Every size gets a tile within the area.
+ for s in [Size::new(2048, 2048), Size::new(2368, 1792), Size::new(1536, 2752), Size::new(1376, 768)] {
+ let (w, h) = vae_decode_tile(s, LARGE_GPU_VAE_TILE_AREA);
+ assert!((w * h) as u64 <= LARGE_GPU_VAE_TILE_AREA && w <= s.width && h <= s.height, "{s:?}: {w}x{h}");
+ }
+ }
+
+ #[test]
+ fn shaped_vae_tiles_only_on_large_discrete_gpus() {
+ let mut r = req("a lighthouse at dusk");
+ r.quality = Quality::Quality;
+ r.aspect_ratio = Some("16:9".into());
+ let big = PlanInputs { vram_total_mib: Some(16303), server_vae_tiling: true, ..inputs(&[]) };
+ let p = plan(&r, &big, &rec()).unwrap();
+ assert!(p.vae_tile.is_some(), "16 GB card");
+ let p = plan(&r, &PlanInputs { unified_memory: true, ..big.clone() }, &rec()).unwrap();
+ assert!(p.vae_tile.is_none(), "Mac: the server's tiles");
+ let p = plan(&r, &PlanInputs { vram_total_mib: Some(12282), ..big.clone() }, &rec()).unwrap();
+ assert!(p.vae_tile.is_none(), "12 GB card: the server's tiles");
+ r.advanced.vae_tiling = Some(false);
+ assert!(plan(&r, &big, &rec()).unwrap().vae_tile.is_none(), "tiling off");
}
#[test]
diff --git a/crates/qls-core/src/sdcpp.rs b/crates/qls-core/src/sdcpp.rs
index 5c55105..6f46f5e 100644
--- a/crates/qls-core/src/sdcpp.rs
+++ b/crates/qls-core/src/sdcpp.rs
@@ -94,6 +94,10 @@ pub struct ImgGenRequest {
/// disabled (`mode=none`) to guarantee no crop/stretch happens server-side.
#[serde(skip_serializing_if = "Vec::is_empty")]
pub image_preprocess: Vec<String>,
+ /// e.g. "resize_before_vae=false": for Qwen-Image 2.1 the server otherwise resizes every
+ /// reference to the output's area before encoding (pipeline/image.cpp), smaller ones too.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub ref_image_args: Option<String>,
pub sample_params: SampleParams,
#[serde(skip_serializing_if = "Option::is_none")]
pub vae_tiling_params: Option<VaeTilingParams>,
@@ -455,6 +459,7 @@ mod tests {
mask_image: None,
ref_images: vec!["AAA".into(), "BBB".into()],
image_preprocess: vec!["target=ref,mode=none".into()],
+ ref_image_args: None,
sample_params: SampleParams {
sample_method: Some("euler".into()),
scheduler: None,
diff --git a/src/components/Advanced.tsx b/src/components/Advanced.tsx
index 1f75852..28cd84d 100644
--- a/src/components/Advanced.tsx
+++ b/src/components/Advanced.tsx
@@ -1,6 +1,6 @@
import { Dices, RotateCcw } from "lucide-react";
import { updateComposer, useStore } from "../lib/store";
-import type { Advanced, OutputFormat, Plan } from "../lib/types";
+import type { Advanced, OutputFormat, Plan, ReferenceDetail } from "../lib/types";
import { Seg, Switch } from "./ui";
import { UpscalerSettings } from "./Upscalers";
@@ -77,6 +77,22 @@ export function AdvancedPanel({ plan }: { plan: Plan | null }) {
</label>
)}
+ {plan?.reference_detail && (
+ <div className="field">
+ <span title="The source images share the GPU budget with the image being drawn. 1 MP leaves more of it for the result, which renders larger and faster; Full gives the model finer source detail but halves the result's size. New chats start at 1 MP.">
+ Reference detail
+ </span>
+ <Seg<ReferenceDetail>
+ value={a.reference_detail ?? "standard"}
+ options={[
+ { value: "standard", label: "1 MP" },
+ { value: "full", label: "Full" },
+ ]}
+ onChange={(v) => setA({ reference_detail: v === "standard" ? null : v })}
+ />
+ </div>
+ )}
+
<NumField label="Images per run" value={a.batch_count} placeholder="1" min={1} max={4} onChange={(v) => setA({ batch_count: v })} />
<div className="field">
diff --git a/src/lib/logic.test.ts b/src/lib/logic.test.ts
index 8e90d14..3bd82b8 100644
--- a/src/lib/logic.test.ts
+++ b/src/lib/logic.test.ts
@@ -27,6 +27,10 @@ describe("advanced overrides", () => {
expect(activeOverrides({ sampler: "heun" })).toEqual(["Sampler heun"]);
expect(activeOverrides({ output_format: "webp", hdr: true, batch_count: 2 })).toEqual([]);
expect(activeOverrides({})).toEqual([]);
+ expect(activeOverrides({ reference_detail: "full" })).toEqual(["Reference full"]);
+ expect(persistentAdvanced({ reference_detail: "full", output_format: "webp" })).toEqual({ output_format: "webp" });
+ // Every new chat starts at the standard 1 MP reference, other settings carry over.
+ expect(emptyComposer({ advanced: { reference_detail: "full", batch_count: 2 } }).advanced).toEqual({ batch_count: 2 });
});
it("keeps preferences but never persists sampling overrides", () => {
diff --git a/src/lib/logic.ts b/src/lib/logic.ts
index cb43b6b..a269fd1 100644
--- a/src/lib/logic.ts
+++ b/src/lib/logic.ts
@@ -52,9 +52,16 @@ export const emptyComposer = (prefs?: Partial<Composer>): Composer => ({
customWidth: prefs?.customWidth ?? 1024,
customHeight: prefs?.customHeight ?? 1024,
seed: null,
- advanced: prefs?.advanced ?? {},
+ advanced: newChatAdvanced(prefs?.advanced),
});
+/** Advanced settings a new chat starts with: everything kept except Reference detail, which
+ * every chat starts at the standard 1 MP. */
+function newChatAdvanced(a: Advanced | undefined): Advanced {
+ const { reference_detail: _, ...rest } = a ?? {};
+ return rest;
+}
+
let counter = 0;
export const newKey = () => `k${Date.now().toString(36)}${(counter++).toString(36)}`;
@@ -116,14 +123,14 @@ export function needsPrompt(c: Composer): boolean {
/** Advanced fields that change how the model samples. They last for the session only: a value
* typed once must not silently override the presets after a restart. */
-const SAMPLING_OVERRIDES = ["sampler", "strength"];
+const SAMPLING_OVERRIDES = ["sampler", "strength", "reference_detail"];
/** Fields of older versions that Turbo's fixed schedule no longer uses (may sit in saved
* preferences, projects and gallery records): dropped everywhere. */
const REMOVED_ADVANCED = ["steps", "cfg", "scheduler", "flow_shift", "refine"];
/** Labels of the sampling overrides currently set, e.g. ["Sampler heun", "Strength 0.8"]. */
export function activeOverrides(a: Advanced): string[] {
- const label: Record<string, string> = { sampler: "Sampler", strength: "Strength" };
+ const label: Record<string, string> = { sampler: "Sampler", strength: "Strength", reference_detail: "Reference" };
return Object.entries(cleanAdvanced(a))
.filter(([k]) => SAMPLING_OVERRIDES.includes(k))
.map(([k, v]) => `${label[k]} ${v}`);
diff --git a/src/lib/types.ts b/src/lib/types.ts
index a5b4475..8d2b313 100644
--- a/src/lib/types.ts
+++ b/src/lib/types.ts
@@ -29,8 +29,13 @@ export interface Advanced {
vae_tiling?: boolean | null;
/** Keep a HEIC source's HDR gain map (Ultra HDR JPEG); null = on. */
hdr?: boolean | null;
+ /** Size of edit/combine sources given to the model; null = standard (1 MP). */
+ reference_detail?: ReferenceDetail | null;
}
+/** "standard": sources at most 1 MP, so the output renders larger; "full": up to the output's size. */
+export type ReferenceDetail = "standard" | "full";
+
export interface GenerationRequest {
prompt: string;
negative_prompt?: string | null;
@@ -82,6 +87,8 @@ export interface Plan {
/** Upscaler tile size in px (0 in plans of older versions). */
upscale_tile?: number;
upscale: { method: UpscaleMethod; source: Size } | null;
+ /** Edits and combines only (absent in older records). */
+ reference_detail?: ReferenceDetail | null;
}
export interface AppErrorBody { code: string; message: string; suggestion?: string; detail?: string }