Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.
TypeScript ai-photo-editorcomfyui-alternativegenerative-fillimage-editingimage-generationinpaintinglinuxlocal-aimacosmcpmcp-servernano-banana-alternativeofflinephoto-editorprivacyqwen-imageqwen-image-2-1stable-diffusion-cppvulkanwindows
perf(inpaint): give the model image 1 at 1 MP and the guide at its planned size
Gabriele Lozuponeac4b5276b6bd
diff --git a/crates/qls-core/src/pipeline.rs b/crates/qls-core/src/pipeline.rsindex 627d707..0d67c3a 100644--- a/crates/qls-core/src/pipeline.rs+++ b/crates/qls-core/src/pipeline.rs@@ -7,7 +7,7 @@use image::{GrayImage, Luma, Rgba, RgbaImage};use crate::error::{AppError, AppResult, ErrorCode};-use crate::geometry::{Rect, Size};+use crate::geometry::{Rect, Size, WorkingCanvas};use crate::imaging::{self, LoadedImage};use crate::planner::{InpaintAction, Plan, ReferenceDetail, ResolvedMode};use crate::sdcpp::{Guidance, ImgGenRequest, SampleParams, VaeTilingParams, b64};@@ -223,7 +223,16 @@ pub fn prepare(plan: &Plan, prompt: &str, seed: i64, images: &[LoadedImage], maslet user_mask = imaging::resize_mask(user_mask, src_size.width, src_size.height);let region = plan.inpaint_region.unwrap_or(Rect::new(0, 0, src_size.width, src_size.height));let action = plan.inpaint_action.unwrap_or_default();- let wc = plan.references[0].canvas;+ // The working canvas (image being painted), from the plan; image 1 for the model+ // may be smaller (references[0], capped at 1 MP with Standard detail).+ let wc = WorkingCanvas {+ source: region.size(),+ content: Size::new(plan.content.width, plan.content.height),+ canvas: plan.working,+ content_offset: (plan.content.x, plan.content.y),+ scale: plan.content.width as f64 / region.width.max(1) as f64,+ };+ let rc = plan.references[0].canvas;let crop = imaging::crop(src, region);let content = imaging::resize(&crop, wc.content.width, wc.content.height);drop(crop);@@ -259,8 +268,14 @@ pub fn prepare(plan: &Plan, prompt: &str, seed: i64, images: &[LoadedImage], maslet image1_b64 = png_b64(&pad(&image1))?;req.init_image = Some(image1_b64.clone());- req.ref_images.push(image1_b64);+ let same = rc.canvas == wc.canvas && rc.content == wc.content && rc.content_offset == wc.content_offset;+ req.ref_images.push(if same { image1_b64 } else { png_b64(&imaging::pad_edge(&imaging::resize(&image1, rc.content.width, rc.content.height), rc.canvas, rc.content_offset))? });req.ref_images.push(png_b64(&imaging::pad_edge(&guide, gc.canvas, gc.content_offset))?);+ // Standard detail: the references are encoded at their planned sizes (image 1 ≤ 1 MP,+ // the guide a quarter of the canvas); Full lets the server resize both to the output's area.+ if plan.reference_detail == Some(ReferenceDetail::Standard) {+ req.ref_image_args = Some("resize_before_vae=false".into());+ }req.prompt = inpaint_prompt(action, prompt, color_name);req.mask_image = Some(gray_png_b64(&m_canvas)?);req.strength = plan.strength;@@ -838,6 +853,48 @@ mod tests {assert!(j.request.prompt.contains("add a red hat inside"), "{}", j.request.prompt);}+ #[test]+ fn large_inpaints_give_the_model_a_1mp_image_1() {+ let mut img = photo(3000, 2000);+ let mut m = GrayImage::new(3000, 2000);+ for y in 800..1000 {+ for x in 1200..1500 {+ m.put_pixel(x, y, Luma([255]));+ img.pixels.put_pixel(x, y, Rgba([255, 255, 255, 255]));+ }+ }+ let job = |detail: ReferenceDetail| {+ let mut r = req("a red hat");+ r.mask = Some("input:m".into());+ r.quality = crate::planner::Quality::Quality;+ r.advanced.reference_detail = Some(detail);+ let inputs = PlanInputs {+ image_sizes: vec![img.size()],+ mask: Some((Size::new(3000, 2000), imaging::mask_bbox(&m, 127), imaging::mask_coverage(&m, 127))),+ vram_total_mib: Some(32607),+ ..Default::default()+ };+ let p = plan(&r, &inputs, &RecommendedSettings::default()).unwrap();+ (prepare(&p, &r.prompt, 1, std::slice::from_ref(&img), Some(&m), &BackendOptions::default()).unwrap(), p)+ };+ let area = |b64: &str| {+ let d = decode(b64);+ d.width() as u64 * d.height() as u64+ };+ let (std, p) = job(ReferenceDetail::Standard);+ let canvas = p.working.area();+ assert!(canvas > 2_000_000, "2K inpaint: {:?}", p.working);+ assert_eq!(area(std.request.init_image.as_ref().unwrap()), canvas, "the painted image keeps the working size");+ assert!(area(&std.request.ref_images[0]) <= STANDARD_REFERENCE_AREA, "image 1 capped at 1 MP");+ assert!(area(&std.request.ref_images[1]) <= canvas / 4 + canvas / 20, "guide a quarter of the canvas");+ assert_eq!(std.request.ref_image_args.as_deref(), Some("resize_before_vae=false"), "encoded at those sizes");+ assert_eq!(p.reference_detail, Some(ReferenceDetail::Standard));+ // Full: as before 0.9.4 (image 1 = the painted image, the server resizes the references).+ let (full, _) = job(ReferenceDetail::Full);+ assert_eq!(full.request.init_image.as_ref(), Some(&full.request.ref_images[0]));+ assert!(full.request.ref_image_args.is_none());+ }+#[test]fn remove_blurs_content_out_of_image_1() {let center = |j: &PreparedJob| {diff --git a/crates/qls-core/src/planner.rs b/crates/qls-core/src/planner.rsindex bfa9fb9..971e16b 100644--- a/crates/qls-core/src/planner.rs+++ b/crates/qls-core/src/planner.rs@@ -991,8 +991,12 @@ pub fn plan(req: &GenerationRequest, inputs: &PlanInputs, rec: &RecommendedSetti((bbox.width as f64 * sx).ceil() as u32).max(1),((bbox.height as f64 * sy).ceil() as u32).max(1),);- // References: the region itself, plus a smaller copy with the area marked.- let area = cap_for_refs(tier_area.unwrap_or(src.area()), 1.0 + INPAINT_GUIDE_AREA, &mut notes);+ // References: the region itself (at most the reference cap, as in edits), plus a+ // smaller copy with the area marked. Full: both at the output's area (the server's+ // own resize), as before 0.9.4.+ let wanted = tier_area.unwrap_or(src.area());+ let image1_share = if wanted > ref_cap.saturating_add(ref_cap / 10) { ref_cap as f64 / wanted as f64 } else { 1.0 };+ let area = cap_for_refs(wanted, image1_share + INPAINT_GUIDE_AREA, &mut notes);// At least the working size of context around the mask, so the model sees the scene.let min_ctx = (area as f64).sqrt() as u32;let region = geometry::inpaint_region(src, bbox, min_ctx, 0.55);@@ -1003,7 +1007,9 @@ pub fn plan(req: &GenerationRequest, inputs: &PlanInputs, rec: &RecommendedSetti}inpaint_region = Some(region);output = src;- references.push(RefPlan { index: 0, source: region.size(), canvas: working });+ // The image being painted stays at the working size; only the model's view of it shrinks.+ let image1 = if working.canvas.area() <= ref_cap.saturating_add(ref_cap / 10) { working } else { ref_canvas(region.size(), ref_cap, rec) };+ references.push(RefPlan { index: 0, source: region.size(), canvas: image1 });let guide_area = (working.canvas.area() as f64 * INPAINT_GUIDE_AREA) as u64;references.push(RefPlan { index: 0, source: region.size(), canvas: ref_canvas(region.size(), guide_area, rec) });strength = Some(adv.strength.unwrap_or(1.0).clamp(0.05, 1.0));@@ -1108,7 +1114,7 @@ pub fn plan(req: &GenerationRequest, inputs: &PlanInputs, rec: &RecommendedSettiupscale,custom_sigmas: if quick_upscale { Vec::new() } else { TURBO_SIGMAS.to_vec() },vae_tile,- reference_detail: matches!(mode, ResolvedMode::Edit | ResolvedMode::MultiReference).then(|| adv.reference_detail.unwrap_or_default()),+ reference_detail: matches!(mode, ResolvedMode::Edit | ResolvedMode::MultiReference | ResolvedMode::Inpaint).then(|| adv.reference_detail.unwrap_or_default()),})}