Skip to content

armgit / kokon

Public Imported from GitHub
0

Edit photos by describing the change, on your own GPU. A free, open-source alternative to Nano Banana and ChatGPT image editing: photos stay on your computer, no daily limit. One installer for Windows, Mac and Linux sets up the best open model (Qwen-Image 2.1). MCP server for AI agents.

perf(inpaint): give the model image 1 at 1 MP and the guide at its planned size

Gabriele Lozuponeac4b5276b6bd
diff --git a/crates/qls-core/src/pipeline.rs b/crates/qls-core/src/pipeline.rs
index 627d707..0d67c3a 100644
--- a/crates/qls-core/src/pipeline.rs
+++ b/crates/qls-core/src/pipeline.rs
@@ -7,7 +7,7 @@
use image::{GrayImage, Luma, Rgba, RgbaImage};
use crate::error::{AppError, AppResult, ErrorCode};
-use crate::geometry::{Rect, Size};
+use crate::geometry::{Rect, Size, WorkingCanvas};
use crate::imaging::{self, LoadedImage};
use crate::planner::{InpaintAction, Plan, ReferenceDetail, ResolvedMode};
use crate::sdcpp::{Guidance, ImgGenRequest, SampleParams, VaeTilingParams, b64};
@@ -223,7 +223,16 @@ pub fn prepare(plan: &Plan, prompt: &str, seed: i64, images: &[LoadedImage], mas
let user_mask = imaging::resize_mask(user_mask, src_size.width, src_size.height);
let region = plan.inpaint_region.unwrap_or(Rect::new(0, 0, src_size.width, src_size.height));
let action = plan.inpaint_action.unwrap_or_default();
- let wc = plan.references[0].canvas;
+ // The working canvas (image being painted), from the plan; image 1 for the model
+ // may be smaller (references[0], capped at 1 MP with Standard detail).
+ let wc = WorkingCanvas {
+ source: region.size(),
+ content: Size::new(plan.content.width, plan.content.height),
+ canvas: plan.working,
+ content_offset: (plan.content.x, plan.content.y),
+ scale: plan.content.width as f64 / region.width.max(1) as f64,
+ };
+ let rc = plan.references[0].canvas;
let crop = imaging::crop(src, region);
let content = imaging::resize(&crop, wc.content.width, wc.content.height);
drop(crop);
@@ -259,8 +268,14 @@ pub fn prepare(plan: &Plan, prompt: &str, seed: i64, images: &[LoadedImage], mas
let image1_b64 = png_b64(&pad(&image1))?;
req.init_image = Some(image1_b64.clone());
- req.ref_images.push(image1_b64);
+ let same = rc.canvas == wc.canvas && rc.content == wc.content && rc.content_offset == wc.content_offset;
+ req.ref_images.push(if same { image1_b64 } else { png_b64(&imaging::pad_edge(&imaging::resize(&image1, rc.content.width, rc.content.height), rc.canvas, rc.content_offset))? });
req.ref_images.push(png_b64(&imaging::pad_edge(&guide, gc.canvas, gc.content_offset))?);
+ // Standard detail: the references are encoded at their planned sizes (image 1 ≤ 1 MP,
+ // the guide a quarter of the canvas); Full lets the server resize both to the output's area.
+ if plan.reference_detail == Some(ReferenceDetail::Standard) {
+ req.ref_image_args = Some("resize_before_vae=false".into());
+ }
req.prompt = inpaint_prompt(action, prompt, color_name);
req.mask_image = Some(gray_png_b64(&m_canvas)?);
req.strength = plan.strength;
@@ -838,6 +853,48 @@ mod tests {
assert!(j.request.prompt.contains("add a red hat inside"), "{}", j.request.prompt);
}
+ #[test]
+ fn large_inpaints_give_the_model_a_1mp_image_1() {
+ let mut img = photo(3000, 2000);
+ let mut m = GrayImage::new(3000, 2000);
+ for y in 800..1000 {
+ for x in 1200..1500 {
+ m.put_pixel(x, y, Luma([255]));
+ img.pixels.put_pixel(x, y, Rgba([255, 255, 255, 255]));
+ }
+ }
+ let job = |detail: ReferenceDetail| {
+ let mut r = req("a red hat");
+ r.mask = Some("input:m".into());
+ r.quality = crate::planner::Quality::Quality;
+ r.advanced.reference_detail = Some(detail);
+ let inputs = PlanInputs {
+ image_sizes: vec![img.size()],
+ mask: Some((Size::new(3000, 2000), imaging::mask_bbox(&m, 127), imaging::mask_coverage(&m, 127))),
+ vram_total_mib: Some(32607),
+ ..Default::default()
+ };
+ let p = plan(&r, &inputs, &RecommendedSettings::default()).unwrap();
+ (prepare(&p, &r.prompt, 1, std::slice::from_ref(&img), Some(&m), &BackendOptions::default()).unwrap(), p)
+ };
+ let area = |b64: &str| {
+ let d = decode(b64);
+ d.width() as u64 * d.height() as u64
+ };
+ let (std, p) = job(ReferenceDetail::Standard);
+ let canvas = p.working.area();
+ assert!(canvas > 2_000_000, "2K inpaint: {:?}", p.working);
+ assert_eq!(area(std.request.init_image.as_ref().unwrap()), canvas, "the painted image keeps the working size");
+ assert!(area(&std.request.ref_images[0]) <= STANDARD_REFERENCE_AREA, "image 1 capped at 1 MP");
+ assert!(area(&std.request.ref_images[1]) <= canvas / 4 + canvas / 20, "guide a quarter of the canvas");
+ assert_eq!(std.request.ref_image_args.as_deref(), Some("resize_before_vae=false"), "encoded at those sizes");
+ assert_eq!(p.reference_detail, Some(ReferenceDetail::Standard));
+ // Full: as before 0.9.4 (image 1 = the painted image, the server resizes the references).
+ let (full, _) = job(ReferenceDetail::Full);
+ assert_eq!(full.request.init_image.as_ref(), Some(&full.request.ref_images[0]));
+ assert!(full.request.ref_image_args.is_none());
+ }
+
#[test]
fn remove_blurs_content_out_of_image_1() {
let center = |j: &PreparedJob| {
diff --git a/crates/qls-core/src/planner.rs b/crates/qls-core/src/planner.rs
index bfa9fb9..971e16b 100644
--- a/crates/qls-core/src/planner.rs
+++ b/crates/qls-core/src/planner.rs
@@ -991,8 +991,12 @@ pub fn plan(req: &GenerationRequest, inputs: &PlanInputs, rec: &RecommendedSetti
((bbox.width as f64 * sx).ceil() as u32).max(1),
((bbox.height as f64 * sy).ceil() as u32).max(1),
);
- // References: the region itself, plus a smaller copy with the area marked.
- let area = cap_for_refs(tier_area.unwrap_or(src.area()), 1.0 + INPAINT_GUIDE_AREA, &mut notes);
+ // References: the region itself (at most the reference cap, as in edits), plus a
+ // smaller copy with the area marked. Full: both at the output's area (the server's
+ // own resize), as before 0.9.4.
+ let wanted = tier_area.unwrap_or(src.area());
+ let image1_share = if wanted > ref_cap.saturating_add(ref_cap / 10) { ref_cap as f64 / wanted as f64 } else { 1.0 };
+ let area = cap_for_refs(wanted, image1_share + INPAINT_GUIDE_AREA, &mut notes);
// At least the working size of context around the mask, so the model sees the scene.
let min_ctx = (area as f64).sqrt() as u32;
let region = geometry::inpaint_region(src, bbox, min_ctx, 0.55);
@@ -1003,7 +1007,9 @@ pub fn plan(req: &GenerationRequest, inputs: &PlanInputs, rec: &RecommendedSetti
}
inpaint_region = Some(region);
output = src;
- references.push(RefPlan { index: 0, source: region.size(), canvas: working });
+ // The image being painted stays at the working size; only the model's view of it shrinks.
+ let image1 = if working.canvas.area() <= ref_cap.saturating_add(ref_cap / 10) { working } else { ref_canvas(region.size(), ref_cap, rec) };
+ references.push(RefPlan { index: 0, source: region.size(), canvas: image1 });
let guide_area = (working.canvas.area() as f64 * INPAINT_GUIDE_AREA) as u64;
references.push(RefPlan { index: 0, source: region.size(), canvas: ref_canvas(region.size(), guide_area, rec) });
strength = Some(adv.strength.unwrap_or(1.0).clamp(0.05, 1.0));
@@ -1108,7 +1114,7 @@ pub fn plan(req: &GenerationRequest, inputs: &PlanInputs, rec: &RecommendedSetti
upscale,
custom_sigmas: if quick_upscale { Vec::new() } else { TURBO_SIGMAS.to_vec() },
vae_tile,
- reference_detail: matches!(mode, ResolvedMode::Edit | ResolvedMode::MultiReference).then(|| adv.reference_detail.unwrap_or_default()),
+ reference_detail: matches!(mode, ResolvedMode::Edit | ResolvedMode::MultiReference | ResolvedMode::Inpaint).then(|| adv.reference_detail.unwrap_or_default()),
})
}