Python
added flux 2 lanpaint, add outpainting for flux & qwen
DeepBeepMeep88feb42ec04b
diff --git a/defaults/i2v_2_2_Enhanced_Lightning_v2.json b/defaults/i2v_2_2_Enhanced_Lightning_v2.jsonindex 3840293..d27d045 100644--- a/defaults/i2v_2_2_Enhanced_Lightning_v2.json+++ b/defaults/i2v_2_2_Enhanced_Lightning_v2.json@@ -11,7 +11,8 @@"https://huggingface.co/DeepBeepMeep/Wan2.2/resolve/main/wan22EnhancedLightning_v2I2VFP8LOW.safetensors"],"author": "https://civitai.com/models/2053259",- "profile_dirs": [""]+ "profile_dirs": [""],+ "video_prompt_enhancer_instructions": "You are an AI vision model. Analyze the provided image and the user's goal.\nProduce a second-by-second description of how the scene evolves over time.\n\nOutput rules:\nEach line must begin with \"(at X seconds:\" where X is the second number.\nAfter the timestamp, provide a dense, comma-separated visual description.\nKeep everything inside a single set of parentheses per line.\nMaintain a consistent cinematic style: shot type, lighting, subject details, camera movement, and actions.\nDo not include commentary, explanations, or meta-text.\nOnly describe visual, physical, or camera changes.\nStay consistent with the input image while expanding logically to meet the goal.\n\nOutput Format Example:\n(at 0 seconds: 4K, realistic, close-up shot, clean single subject, daylight, natural lighting, bedroom near a small window with light coming in from the left, a young college aged woman with blonde hair, blue eyes dressed in a blue pajama top, sitting up, she looks to be annoyed, the camera pushes-in on the character's face, she pulls the blanket off of her)\n\n(at 1 seconds: [next description])\n\n(at 2 seconds: [next description])\n\nContinue until the goal is fully represented."},"guidance_phases": 2,"switch_threshold" : 900,@@ -21,4 +22,4 @@"num_inference_steps": 4,"multi_prompts_gen_type": 2,"prompt": "(at 0 seconds: wide frontal shot of a man standing in front of an open fridge, cinematic lighting, subtle ambient kitchen reflections, the fridge contents visible, camera static).\n(at 1 second: medium shot from the front as he opens the fridge fully, reaches for a can, slight zoom-in to emphasize the action, cinematic framing).\n(at 2 seconds: camera shifts to a side medium shot, tracking him as he lifts the can to his mouth, fluid movement, maintaining lighting and reflections).\n(at 3 seconds: camera starts a smooth 360-degree orbit around the man, following him as he drinks from the can, motion fluid, background slightly blurred for cinematic effect).\n(at 4 seconds: close-up on his face and upper body while drinking, orbit continues subtly, fridge reflections accentuating realism, cinematic polish).\n(at 5 seconds: final wide shot as he lowers the can, camera completes orbit to original angle, showcasing the kitchen space, lighting, and dynamic movement)."-}\ No newline at end of file+}diff --git a/defaults/i2v_2_2_Enhanced_Lightning_v2_svi2pro.json b/defaults/i2v_2_2_Enhanced_Lightning_v2_svi2pro.jsonindex 60d0d76..fc407b6 100644--- a/defaults/i2v_2_2_Enhanced_Lightning_v2_svi2pro.json+++ b/defaults/i2v_2_2_Enhanced_Lightning_v2_svi2pro.json@@ -14,7 +14,8 @@"author": "https://civitai.com/models/2053259","loras": "i2v_2_2_svi2pro","loras_multipliers": "i2v_2_2_svi2pro",- "profile_dirs": [""]+ "profile_dirs": [""],+ "video_prompt_enhancer_instructions": "You are an AI vision model. Analyze the provided image and the user's goal.\nProduce a second-by-second description of how the scene evolves over time.\n\nOutput rules:\nEach line must begin with \"(at X seconds:\" where X is the second number.\nAfter the timestamp, provide a dense, comma-separated visual description.\nKeep everything inside a single set of parentheses per line.\nMaintain a consistent cinematic style: shot type, lighting, subject details, camera movement, and actions.\nDo not include commentary, explanations, or meta-text.\nOnly describe visual, physical, or camera changes.\nStay consistent with the input image while expanding logically to meet the goal.\n\nOutput Format Example:\n(at 0 seconds: 4K, realistic, close-up shot, clean single subject, daylight, natural lighting, bedroom near a small window with light coming in from the left, a young college aged woman with blonde hair, blue eyes dressed in a blue pajama top, sitting up, she looks to be annoyed, the camera pushes-in on the character's face, she pulls the blanket off of her)\n\n(at 1 seconds: [next description])\n\n(at 2 seconds: [next description])\n\nContinue until the goal is fully represented."},"guidance_phases": 2,"switch_threshold" : 900,@@ -23,4 +24,4 @@"flow_shift" : 5,"num_inference_steps": 8,"prompt": "(at 0 seconds:a man open his fridge, takes a beer and drink it)"-}\ No newline at end of file+}diff --git a/defaults/pi_flux2_nvfp4.json b/defaults/pi_flux2_nvfp4.jsonindex 366f81e..b95033b 100644--- a/defaults/pi_flux2_nvfp4.json+++ b/defaults/pi_flux2_nvfp4.json@@ -1,6 +1,6 @@{"model": {- "name": "pi-FLUX.2 Dev NVFP 32B",+ "name": "pi-FLUX.2 Dev NVFP4 32B","architecture": "pi_flux2","description": "pi-FLUX.2 using the NVFP4 Flux 2 Dev checkpoint.","URLs": "flux2_dev_nvfp4",diff --git a/models/flux/flux_handler.py b/models/flux/flux_handler.pyindex beceafb..471285a 100644--- a/models/flux/flux_handler.py+++ b/models/flux/flux_handler.py@@ -1,5 +1,6 @@import osimport torch+import gradio as grfrom PIL import Imagefrom shared.utils import files_locator as flfrom shared.utils.hf import build_hf_url@@ -64,7 +65,8 @@ class family_handler():@staticmethoddef query_model_def(base_model_type, model_def):flux_model = "flux-dev" if base_model_type == "flux" else base_model_type.replace("_", "-")- pi_flux2 = flux_model == "pi-flux2"+ flux_dev = base_model_type == "flux"+ pi_flux2 = base_model_type == "pi_flux2"flux2_klein_4b = base_model_type == "flux2_klein_4b"flux2_klein_9b = base_model_type == "flux2_klein_9b"flux2_klein = flux2_klein_4b or flux2_klein_9b@@ -81,7 +83,35 @@ class family_handler():"image_outputs" : True,"no_negative_prompt" : flux2 or not (flux_chroma or flux_chroma_radiance),"flux-model": flux_model,+ "flux2": flux2,}+ supports_inpaint = flux_kontext or flux_schnell or flux_dev or flux2+ if supports_inpaint:+ extra_model_def["inpaint_support"] = True+ if not pi_flux2:+ lanpaint_choices = [+ ("LanPaint (2 steps): ~2x slower, easy task", 2),+ ("LanPaint (5 steps): ~5x slower, medium task", 3),+ ("LanPaint (10 steps): ~10x slower, hard task", 4),+ ("LanPaint (15 steps): ~15x slower, very hard task", 5),+ ]+ if flux_dev or flux_schnell:+ choices = lanpaint_choices+ default_mode = 2+ extra_model_def["inpaint_video_prompt_type"] = "VA"+ extra_model_def["image_video_prompt_type"] = ""+ else:+ choices = [("Masked Denoising : Inpainted area may reuse some content that has been masked", 0)] + lanpaint_choices+ default_mode = 0+ extra_model_def["model_modes"] = {+ "choices": choices,+ "default": default_mode,+ "label": "Inpainting Method",+ "image_modes": [2],+ }+ extra_model_def["inpaint_color"] = "FF0000"+ extra_model_def["video_guide_outpainting"] = [1,2]+if flux2:if flux2_klein:if flux2_klein_4b:@@ -137,7 +167,6 @@ class family_handler():}if flux_kontext or flux_kontext_dreamomni2 or flux2:- extra_model_def["inpaint_support"] = flux_kontextextra_model_def["image_ref_choices"] = {"choices": [("None", ""),@@ -179,12 +208,9 @@ class family_handler():"visible": True,}- # extra_model_def["guide_inpaint_color"] = 0- # extra_model_def["video_guide_outpainting"] = [1,2]if pi_flux2:extra_model_def["piflow"] = True- extra_model_def["inpaint_support"] = Trueextra_model_def["fit_into_canvas_image_refs"] = 0@@ -429,3 +455,27 @@ class family_handler():})+ @staticmethod+ def validate_generative_settings(base_model_type, model_def, inputs):+ model_mode = inputs.get("model_mode")+ model_mode_int = None+ if model_mode is not None:+ try:+ model_mode_int = int(model_mode)+ except (TypeError, ValueError):+ model_mode_int = None+ if model_mode_int in (2, 3, 4, 5):+ denoising_strength = inputs.get("denoising_strength", 1)+ masking_strength = inputs.get("masking_strength", 1)+ if denoising_strength != 1 or masking_strength != 1:+ gr.Info("LanPaint forces Denoising Strength and Masking Strength to 1; non-1 values will be ignored.")+++ @staticmethod+ def custom_prompt_preprocess(prompt, video_guide_outpainting, model_mode, **kwargs):+ if model_mode == 0:+ # from wgp import get_outpainting_dims+ if len(video_guide_outpainting) and not video_guide_outpainting.startswith("#") and video_guide_outpainting != "0 0 0 0":+ if not prompt.endswith("."): prompt += "."+ prompt += "Remove the red paddings on the sides and show what's behind them."+ return promptdiff --git a/models/flux/flux_main.py b/models/flux/flux_main.pyindex ae5e530..cf338a0 100644--- a/models/flux/flux_main.py+++ b/models/flux/flux_main.py@@ -270,6 +270,14 @@ class model_factory:return Nonedevice="cuda"flux2 = self.is_flux2+ model_mode = bbargs.get("model_mode", None)+ model_mode_int = None+ if model_mode is not None:+ try:+ model_mode_int = int(model_mode)+ except (TypeError, ValueError):+ model_mode_int = None+ lanpaint_enabled = model_mode_int in (2, 3, 4, 5)if flux2:guide_scale = 1.0if self.guidance_max_phases < 1: guide_scale = 1@@ -448,6 +456,10 @@ class model_factory:joint_pass=joint_pass,denoising_strength=denoising_strength,masking_strength=masking_strength,+ model_mode=model_mode,+ height=height,+ width=width,+ vae_scale_factor=8,)if x==None: return None# decode latents to pixel space@@ -456,10 +468,17 @@ class model_factory:with torch.autocast(device_type=device, dtype=torch.bfloat16):x = self.vae.decode(x)- if image_mask is not None and masking_strength == 1 and not flux2:- img_msk_rebuilt = inp["img_msk_rebuilt"]- img= input_frames.squeeze(1).unsqueeze(0) # convert_image_to_tensor(image_guide)- x = img * (1 - img_msk_rebuilt) + x.to(img) * img_msk_rebuilt+ img_msk_rebuilt = inp.get("img_msk_rebuilt") if isinstance(inp, dict) else None+ if img_msk_rebuilt is not None and (lanpaint_enabled or (masking_strength == 1 and not flux2)):+ img = None+ if input_frames is not None:+ img = input_frames.squeeze(1).unsqueeze(0)+ elif input_ref_images is not None and len(input_ref_images) > 0:+ img = convert_image_to_tensor(+ input_ref_images[0].resize((width, height), resample=Image.Resampling.LANCZOS)+ ).unsqueeze(0)+ if img is not None:+ x = img * (1 - img_msk_rebuilt) + x.to(img) * img_msk_rebuiltx = x.clamp(-1, 1)x = x.transpose(0, 1)diff --git a/models/flux/sampling.py b/models/flux/sampling.pyindex ff6f88f..56bf328 100644--- a/models/flux/sampling.py+++ b/models/flux/sampling.py@@ -541,6 +541,10 @@ def denoise(img_msk_rebuilt = None,denoising_strength = 1,masking_strength = 1,+ model_mode = None,+ height: int | None = None,+ width: int | None = None,+ vae_scale_factor: int = 8,preview_meta = None,original_image_latents = None,# pi-Flow settings@@ -566,8 +570,59 @@ def denoise(piflow_sigmas = piflow_sigmas_dst = piflow_m_vals = Nonemorph, first_step = False, 0+ model_mode_int = None+ if model_mode is not None:+ try:+ model_mode_int = int(model_mode)+ except (TypeError, ValueError):+ model_mode_int = None+ lanpaint_proc = None+ lanpaint_mask = None+ true_cfg_scale = 1.0 if real_guidance_scale is None else real_guidance_scaleif img_msk_latents is not None:- if original_image_latents is None: original_image_latents= img_cond_seq.clone()+ if not is_piflow and model_mode_int in (2, 3, 4, 5):+ if img_cond_seq is not None:+ if img_cond_seq_ids is not None and img_cond_seq_ids.shape[-1] == 4:+ first_t = img_cond_seq_ids[:, :1, 0]+ first_mask = img_cond_seq_ids[..., 0] == first_t+ if original_image_latents is None:+ original_image_latents = img_cond_seq[first_mask].view(+ img_cond_seq.shape[0], -1, img_cond_seq.shape[-1]+ ).clone()+ keep_mask = ~first_mask+ if keep_mask.any():+ img_cond_seq = img_cond_seq[keep_mask].view(+ img_cond_seq.shape[0], -1, img_cond_seq.shape[-1]+ )+ img_cond_seq_ids = img_cond_seq_ids[keep_mask].view(+ img_cond_seq_ids.shape[0], -1, img_cond_seq_ids.shape[-1]+ )+ else:+ img_cond_seq = None+ img_cond_seq_ids = None+ else:+ base_len = img.shape[1]+ if original_image_latents is None:+ original_image_latents = img_cond_seq[:, :base_len].clone()+ if img_cond_seq.shape[1] <= base_len:+ img_cond_seq = None+ img_cond_seq_ids = None+ else:+ img_cond_seq = img_cond_seq[:, base_len:]+ if img_cond_seq_ids is not None:+ img_cond_seq_ids = img_cond_seq_ids[:, base_len:]+ from shared.inpainting.lanpaint import LanPaint+ lanpaint_steps = {2: 2, 3: 5, 4: 10, 5: 15}.get(model_mode_int, 5)+ lanpaint_proc = LanPaint(NSteps=lanpaint_steps)+ denoising_strength = 1.0+ masking_strength = 1.0+ if img_msk_latents.shape[-1] != img.shape[-1]:+ lanpaint_mask = img_msk_latents.expand(img.shape[0], img.shape[1], img.shape[2]).contiguous()+ else:+ lanpaint_mask = img_msk_latents+ lanpaint_proc is not None+ if original_image_latents is None and img_cond_seq is not None:+ original_image_latents = img_cond_seq.clone()randn = torch.randn_like(original_image_latents)if denoising_strength < 1.:first_step = int(len(timesteps[:-1]) * (1. - denoising_strength))@@ -612,53 +667,86 @@ def denoise(img = original_image_latents * (1.0 - latent_noise_factor) + img * latent_noise_factort_vec = torch.full((img.shape[0],), t_curr, dtype=img.dtype, device=img.device)- img_input = img- img_input_ids = img_ids- if img_cond is not None:- img_input = torch.cat((img, img_cond), dim=-1)- if img_cond_seq is not None:- img_input = torch.cat((img_input, img_cond_seq), dim=1)- img_input_ids = torch.cat((img_input_ids, img_cond_seq_ids), dim=1)- if not joint_pass or real_guidance_scale == 1:- pred = model(- img=img_input,- img_ids=img_input_ids,- txt_list=[txt],- txt_ids_list=[txt_ids],- y_list=[vec],- timesteps=t_vec,- guidance=guidance_vec,- **kwargs- )[0]- if pred == None: return None- if real_guidance_scale> 1:- neg_pred = model(++ def run_model(latents, cfg_scale):+ img_input = latents+ img_input_ids = img_ids+ if img_cond is not None:+ img_input = torch.cat((img_input, img_cond), dim=-1)+ if img_cond_seq is not None:+ img_input = torch.cat((img_input, img_cond_seq), dim=1)+ img_input_ids = torch.cat((img_input_ids, img_cond_seq_ids), dim=1)+ if not joint_pass or cfg_scale == 1:+ noise_pred = model(img=img_input,img_ids=img_input_ids,- txt_list=[neg_txt],- txt_ids_list=[neg_txt_ids],- y_list=[neg_vec],+ txt_list=[txt],+ txt_ids_list=[txt_ids],+ y_list=[vec],timesteps=t_vec,guidance=guidance_vec,**kwargs)[0]- if neg_pred == None: return None- else:- pred, neg_pred = model(- img=img_input,- img_ids=img_input_ids,- txt_list=[txt, neg_txt],- txt_ids_list=[txt_ids, neg_txt_ids],- y_list=[vec, neg_vec],- timesteps=t_vec,- guidance=guidance_vec,- **kwargs+ if noise_pred == None:+ return None, None+ neg_noise_pred = None+ if cfg_scale > 1:+ neg_noise_pred = model(+ img=img_input,+ img_ids=img_input_ids,+ txt_list=[neg_txt],+ txt_ids_list=[neg_txt_ids],+ y_list=[neg_vec],+ timesteps=t_vec,+ guidance=guidance_vec,+ **kwargs+ )[0]+ if neg_noise_pred == None:+ return None, None+ else:+ noise_pred, neg_noise_pred = model(+ img=img_input,+ img_ids=img_input_ids,+ txt_list=[txt, neg_txt],+ txt_ids_list=[txt_ids, neg_txt_ids],+ y_list=[vec, neg_vec],+ timesteps=t_vec,+ guidance=guidance_vec,+ **kwargs+ )+ if noise_pred == None:+ return None, None+ return noise_pred, neg_noise_pred++ def cfg_predictions(noise_pred, neg_noise_pred, cfg_scale, t):+ if cfg_scale > 1:+ return neg_noise_pred + cfg_scale * (noise_pred - neg_noise_pred)+ return noise_pred++ if lanpaint_proc is not None and height is not None and width is not None and i < updated_num_steps - 1:+ img = lanpaint_proc(+ run_model,+ cfg_predictions,+ true_cfg_scale,+ 1.0,+ img,+ original_image_latents,+ randn,+ t_vec,+ lanpaint_mask,+ height=height,+ width=width,+ vae_scale_factor=vae_scale_factor,)- if pred == None: return None+ if img is None:+ return None++ pred, neg_pred = run_model(img, true_cfg_scale)+ if pred == None: return Noneif is_piflow and isinstance(pred, dict):- if real_guidance_scale > 1:- pred = {k: neg_pred[k] + real_guidance_scale * (pred[k] - neg_pred[k]) for k in pred}+ if true_cfg_scale > 1:+ pred = {k: neg_pred[k] + true_cfg_scale * (pred[k] - neg_pred[k]) for k in pred}patch_size = getattr(model, "piflow_patch_size", 2)img_packed = rearrange(@@ -700,8 +788,8 @@ def denoise(img_packed = _pack_latent_piflux2(img_unpacked, patch_size=patch_size)img = rearrange(img_packed, "b c h w -> b (h w) c").to(img.dtype)else:- if real_guidance_scale > 1:- pred = neg_pred + real_guidance_scale * (pred - neg_pred)+ if true_cfg_scale > 1:+ pred = cfg_predictions(pred, neg_pred, true_cfg_scale, t_vec)step_size = t_prev - t_currif final_step_size_scale is not None and i == len(timesteps) - 2:diff --git a/models/qwen/pipeline_qwenimage.py b/models/qwen/pipeline_qwenimage.pyindex 4ad13ce..655d13f 100644--- a/models/qwen/pipeline_qwenimage.py+++ b/models/qwen/pipeline_qwenimage.py@@ -659,8 +659,14 @@ class QwenImagePipeline(): #DiffusionPipelinereturning a tuple, the first element is a list with the generated images."""- lora_inpaint = image_mask is not None and model_mode == 1- lanpaint_enabled = image_mask is not None and model_mode == 2+ model_mode_int = None+ if model_mode is not None:+ try:+ model_mode_int = int(model_mode)+ except (TypeError, ValueError):+ model_mode_int = None+ lora_inpaint = image_mask is not None and model_mode_int == 1+ lanpaint_enabled = image_mask is not None and model_mode_int in (2, 3, 4, 5)kwargs = {'pipeline': pipeline, 'callback': callback}if callback != None:@@ -819,6 +825,8 @@ class QwenImagePipeline(): #DiffusionPipeline(1, vae_height // self.vae_scale_factor // 2, vae_width // self.vae_scale_factor // 2)for vae_width, vae_height in vae_image_sizes]+ if lanpaint_enabled:+ condition_shapes = condition_shapes[1:]img_shapes = [output_shapes + condition_shapes] * effective_batch_sizeelse:output_shape = (1, height // self.vae_scale_factor // 2, width // self.vae_scale_factor // 2)@@ -860,9 +868,16 @@ class QwenImagePipeline(): #DiffusionPipelineif image_mask_latents is not None:original_image_latents = image_latents[:, :latents.shape[1]].clone()if lanpaint_enabled:+ if image_latents.shape[1]==latents.shape[1]:+ image_latents = None+ else:+ image_latents = image_latents[:, latents.shape[1]:]+from shared.inpainting.lanpaint import LanPaint- lanpaint_proc = LanPaint()+ lanpaint_steps = {2: 2, 3: 5, 4: 10, 5: 15}.get(model_mode_int, 5)+ lanpaint_proc = LanPaint(NSteps=lanpaint_steps)denoising_strength = 1.+ masking_strength = 1.randn = torch.randn_like(original_image_latents)if denoising_strength < 1.:first_step = int(len(timesteps) * (1. - denoising_strength))@@ -963,7 +978,7 @@ class QwenImagePipeline(): #DiffusionPipelinereturn noise_pred- if lanpaint_proc is not None and i<=updated_num_steps-1:+ if lanpaint_proc is not None and i < updated_num_steps - 1:latents = lanpaint_proc(denoise, cfg_predictions, true_cfg_scale, 1., latents, original_image_latents, randn, t/1000, image_mask_latents, height=height , width= width, vae_scale_factor= 8)if latents is None: return None@@ -976,8 +991,9 @@ class QwenImagePipeline(): #DiffusionPipelinenoise_pred = Noneif image_mask_latents is not None and i < masked_steps:- if lanpaint_proc is not None:- latents = original_image_latents * (1-image_mask_latents) + image_mask_latents * latents+ if lanpaint_proc is None :+ pass+ # latents = original_image_latents * (1-image_mask_latents) + image_mask_latents * latentselse:next_t = timesteps[i+1] if i<len(timesteps)-1 else 0latent_noise_factor = next_t / 1000@@ -1026,15 +1042,15 @@ class QwenImagePipeline(): #DiffusionPipeline)latents_to_decode = latents_to_decode / latents_std + latents_meanoutput_image = self.vae.decode(latents_to_decode, return_dict=False)[0][:, :, 0]-- if (- num_layers == 1- and image_mask_rebuilt is not None- and not lora_inpaint- and self.vae.upsampling_set is None- and masking_strength == 1- ):- output_image = vae_images[0].squeeze(2) * (1 - image_mask_rebuilt) + output_image.to(vae_images[0] ) * image_mask_rebuilt+ # looks worse+ # if (+ # num_layers == 1+ # and image_mask_rebuilt is not None+ # and not lora_inpaint+ # and self.vae.upsampling_set is None+ # and masking_strength == 10+ # ):+ # output_image = vae_images[0].squeeze(2) * (1 - image_mask_rebuilt) + output_image.to(vae_images[0] ) * image_mask_rebuiltreturn output_imagediff --git a/models/qwen/qwen_handler.py b/models/qwen/qwen_handler.pyindex 665d39c..dbbecd5 100644--- a/models/qwen/qwen_handler.py+++ b/models/qwen/qwen_handler.py@@ -35,6 +35,23 @@ class family_handler():extra_model_def["vae_upsampler"] = [1]extra_model_def["sample_solvers"] = [("Default", "default")]+ if base_model_type in ["qwen_image_20B"]:+ extra_model_def["inpaint_support"] = True+ extra_model_def["inpaint_video_prompt_type"] = "VA"+ extra_model_def["image_video_prompt_type"] = ""+ extra_model_def["video_guide_outpainting"] = [2]+ extra_model_def["model_modes"] = {+ "choices": [+ ("LanPaint (2 steps): ~2x slower, easy task", 2),+ ("LanPaint (5 steps): ~5x slower, medium task", 3),+ ("LanPaint (10 steps): ~10x slower, hard task", 4),+ ("LanPaint (15 steps): ~15x slower, very hard task", 5),+ ],+ "default": 2,+ "label" : "Inpainting Method",+ "image_modes" : [2],+ }+if base_model_type in ["qwen_image_edit_20B", "qwen_image_edit_plus_20B", "qwen_image_edit_plus2_20B"]:extra_model_def["inpaint_support"] = Trueif base_model_type in ["qwen_image_edit_plus_20B", "qwen_image_edit_plus2_20B"]:@@ -54,12 +71,16 @@ class family_handler():"choices": [("Lora Inpainting: Inpainted area completely unrelated to masked content", 1),("Masked Denoising : Inpainted area may reuse some content that has been masked", 0),- ("LanPaint : High Quality Inpainting but up x5 slower", 2),+ ("LanPaint (2 steps): ~2x slower, easy task", 2),+ ("LanPaint (5 steps): ~5x slower, medium task", 3),+ ("LanPaint (10 steps): ~10x slower, hard task", 4),+ ("LanPaint (15 steps): ~15x slower, very hard task", 5),],"default": 1,"label" : "Inpainting Method","image_modes" : [2],}+ extra_model_def["inpaint_color"] = "FF0000"if base_model_type in ["qwen_image_edit_plus_20B", "qwen_image_edit_plus2_20B"]:extra_model_def["guide_preprocessing"] = {@@ -187,21 +208,37 @@ class family_handler():@staticmethoddef validate_generative_settings(base_model_type, model_def, inputs):- if base_model_type in ["qwen_image_edit_20B", "qwen_image_edit_plus_20B", "qwen_image_edit_plus2_20B"]:+ if base_model_type in ["qwen_image_20B", "qwen_image_edit_20B", "qwen_image_edit_plus_20B", "qwen_image_edit_plus2_20B"]:model_mode = inputs["model_mode"]- denoising_strength= inputs["denoising_strength"]- video_guide_outpainting= inputs["video_guide_outpainting"]- from wgp import get_outpainting_dims- outpainting_dims = get_outpainting_dims(video_guide_outpainting)-- if denoising_strength < 1 and model_mode != 0:+ denoising_strength = inputs["denoising_strength"]+ masking_strength = inputs["masking_strength"]+ model_mode_int = None+ if model_mode is not None:+ try:+ model_mode_int = int(model_mode)+ except (TypeError, ValueError):+ model_mode_int = None++ if model_mode_int in (2, 3, 4, 5):+ if denoising_strength != 1 or masking_strength != 1:+ gr.Info("LanPaint forces Denoising Strength and Masking Strength to 1; non-1 values will be ignored.")+ elif denoising_strength < 1 and model_mode_int != 0:gr.Info("Denoising Strength will be ignored if Masked Denoising is not used")- if outpainting_dims is not None and model_mode != 1 :- return "Outpainting is supported only with Lora Inpainting"+if base_model_type in ["qwen_image_layered_20B"]:if inputs.get("image_guide") is None:return "Qwen Image Layered requires a Control Image."-++ @staticmethod+ def custom_prompt_preprocess(prompt, video_guide_outpainting, model_mode, **kwargs):+ if model_mode == 0:+ # from wgp import get_outpainting_dims+ if len(video_guide_outpainting) and not video_guide_outpainting.startswith("#") and video_guide_outpainting != "0 0 0 0":+ if not prompt.endswith("."): prompt += "."+ prompt += "Remove the red paddings on the sides and show what's behind them."+ return prompt++@staticmethoddef get_rgb_factors(base_model_type ):from shared.RGB_factors import get_rgb_factorsdiff --git a/models/wan/ovi_fusion_engine.py b/models/wan/ovi_fusion_engine.pyindex de06197..d24e97d 100644--- a/models/wan/ovi_fusion_engine.py+++ b/models/wan/ovi_fusion_engine.py@@ -129,7 +129,7 @@ class OviFusionEngine:- @torch.inference_mode()+ @torch.no_grad()def generate(self,input_prompt,image_start=None,diff --git a/models/z_image/z_image_transformer2d.py b/models/z_image/z_image_transformer2d.pyindex 6738ca0..d55513b 100644--- a/models/z_image/z_image_transformer2d.py+++ b/models/z_image/z_image_transformer2d.py@@ -386,6 +386,23 @@ class RopeEmbedder:class ZImageTransformer2DModel(nn.Module):+ def preprocess_loras(self, model_type, sd):+ first = next(iter(sd), None)+ if first is None:+ return sd++ if ".default." not in first and ".lora." not in first:+ return sd++ new_sd = {}+ for k, v in sd.items():+ if ".default." in k:+ k = k.replace(".default.", ".")+ if ".lora." in k:+ k = k.replace(".lora.", ".lora_")+ new_sd[k] = v+ return new_sd+def __init__(self,# Control-specific parameters (optional)diff --git a/shared/utils/utils.py b/shared/utils/utils.pyindex 7a31c57..5e08ff9 100644--- a/shared/utils/utils.py+++ b/shared/utils/utils.py@@ -99,6 +99,25 @@ def truncate_for_filesystem(s, max_bytes=None):def get_default_workers():return os.cpu_count()/ 2+def to_rgb_tensor(value, device="cpu", dtype=torch.float):+ if isinstance(value, torch.Tensor):+ tensor = value.to(device=device, dtype=dtype)+ else:+ if isinstance(value, (list, tuple, np.ndarray)):+ vals = value+ else:+ vals = [value, value, value]+ tensor = torch.tensor(vals, device=device, dtype=dtype)+ if tensor.numel() == 1:+ tensor = tensor.repeat(3)+ elif tensor.numel() != 3:+ tensor = tensor.flatten()+ if tensor.numel() < 3:+ tensor = tensor.repeat(3)[:3]+ else:+ tensor = tensor[:3]+ return tensor.view(3, 1, 1)+def process_images_multithread(image_processor, items, process_type, wrap_in_list = True, max_workers: int = os.cpu_count()/ 2, in_place = False) :if not items:return []@@ -374,7 +393,7 @@ def resize_and_remove_background(img_list, budget_width, budget_height, rm_backgreturn output_list, output_mask_listdef fit_image_into_canvas(ref_img, image_size, canvas_tf_bg =127.5, device ="cpu", full_frame = False, outpainting_dims = None, return_mask = False, return_image = False):- inpaint_color = canvas_tf_bg / 127.5 - 1+ inpaint_color = to_rgb_tensor(canvas_tf_bg, device=device, dtype=torch.float) / 127.5 - 1ref_width, ref_height = ref_img.sizeif (ref_height, ref_width) == image_size and outpainting_dims == None:@@ -403,10 +422,10 @@ def fit_image_into_canvas(ref_img, image_size, canvas_tf_bg =127.5, device ="cpuref_img = ref_img.resize((new_width, new_height), resample=Image.Resampling.LANCZOS)ref_img = TF.to_tensor(ref_img).sub_(0.5).div_(0.5).unsqueeze(1)if outpainting_dims != None:- canvas = torch.full((3, 1, final_height, final_width), inpaint_color, dtype= torch.float, device=device) # [-1, 1]+ canvas = inpaint_color.expand(3, 1, final_height, final_width).clone()canvas[:, :, margin_top + top:margin_top + top + new_height, margin_left + left:margin_left + left + new_width] = ref_imgelse:- canvas = torch.full((3, 1, canvas_height, canvas_width), inpaint_color, dtype= torch.float, device=device) # [-1, 1]+ canvas = inpaint_color.expand(3, 1, canvas_height, canvas_width).clone()canvas[:, :, top:top + new_height, left:left + new_width] = ref_imgref_img = canvascanvas = None@@ -425,7 +444,7 @@ def fit_image_into_canvas(ref_img, image_size, canvas_tf_bg =127.5, device ="cpudef prepare_video_guide_and_mask( video_guides, video_masks, pre_video_guide, image_size, current_video_length = 81, latent_size = 4, any_mask = False, any_guide_padding = False, guide_inpaint_color = 127.5, keep_video_guide_frames = [], inject_frames = [], outpainting_dims = None, device ="cpu"):src_videos, src_masks = [], []- inpaint_color_compressed = guide_inpaint_color/127.5 - 1+ inpaint_color_compressed = to_rgb_tensor(guide_inpaint_color, device=device, dtype=torch.float) / 127.5 - 1prepend_count = pre_video_guide.shape[1] if pre_video_guide is not None else 0for guide_no, (cur_video_guide, cur_video_mask) in enumerate(zip(video_guides, video_masks)):src_video, src_mask = cur_video_guide, cur_video_mask@@ -436,9 +455,10 @@ def prepare_video_guide_and_mask( video_guides, video_masks, pre_video_guide, imif any_guide_padding:if src_video is None:- src_video = torch.full( (3, current_video_length, *image_size ), inpaint_color_compressed, dtype = torch.float, device= device)+ src_video = inpaint_color_compressed.expand(3, current_video_length, *image_size).clone()elif src_video.shape[1] < current_video_length:- src_video = torch.cat([src_video, torch.full( (3, current_video_length - src_video.shape[1], *src_video.shape[-2:] ), inpaint_color_compressed, dtype = src_video.dtype, device= src_video.device) ], dim=1)+ pad = inpaint_color_compressed.to(src_video.device).expand(3, current_video_length - src_video.shape[1], *src_video.shape[-2:]).clone()+ src_video = torch.cat([src_video, pad], dim=1)elif src_video is not None:new_num_frames = (src_video.shape[1] - 1) // latent_size * latent_size + 1if new_num_frames < src_video.shape[1]:@@ -457,7 +477,7 @@ def prepare_video_guide_and_mask( video_guides, video_masks, pre_video_guide, imfor k, keep in enumerate(keep_video_guide_frames):if not keep:pos = prepend_count + k- src_video[:, pos:pos+1] = inpaint_color_compressed+ src_video[:, pos:pos+1] = inpaint_color_compressed.to(src_video.device)if any_mask: src_mask[:, pos:pos+1] = 1for k, frame in enumerate(inject_frames):diff --git a/wgp.py b/wgp.pyindex 27f61fe..dc8f7ec 100644--- a/wgp.py+++ b/wgp.py@@ -33,7 +33,7 @@ import numpy as npimport importlibfrom shared.utils import notification_soundfrom shared.utils.loras_mutipliers import preparse_loras_multipliers, parse_loras_multipliers-from shared.utils.utils import convert_tensor_to_image, save_image, get_video_info, get_file_creation_date, convert_image_to_video, calculate_new_dimensions, convert_image_to_tensor, calculate_dimensions_and_resize_image, rescale_and_crop, get_video_frame, resize_and_remove_background, rgb_bw_to_rgba_mask+from shared.utils.utils import convert_tensor_to_image, save_image, get_video_info, get_file_creation_date, convert_image_to_video, calculate_new_dimensions, convert_image_to_tensor, calculate_dimensions_and_resize_image, rescale_and_crop, get_video_frame, resize_and_remove_background, rgb_bw_to_rgba_mask, to_rgb_tensorfrom shared.utils.utils import calculate_new_dimensions, get_outpainting_frame_location, get_outpainting_full_area_dimensionsfrom shared.utils.utils import has_video_file_extension, has_image_file_extension, has_audio_file_extensionfrom shared.utils.audio_video import extract_audio_tracks, combine_video_with_audio_tracks, combine_and_concatenate_video_with_audio_tracks, cleanup_temp_audio_files, save_video, save_image@@ -360,7 +360,7 @@ def process_prompt_and_add_tasks(state, current_gallery_tab, model_choice):inputs["state"] = stateinputs["model_type"] = model_type- inputs.pop("lset_name")+ inputs.pop("lset_name", None)if inputs == None:gr.Warning("Internal state error: Could not retrieve inputs for the model.")queue = gen.get("queue", [])@@ -4234,7 +4234,8 @@ def get_preprocessor(process_type, inpaint_color):}anno_ins = lambda img: FlowVisAnnotator(cfg_dict).forward(img)elif process_type=="inpaint":- anno_ins = lambda img : len(img) * [inpaint_color]+ color = tuple(int(v) for v in inpaint_color.view(-1).tolist())+ anno_ins = lambda img : len(img) * [color]elif process_type == None or process_type in ["raw", "identity"]:anno_ins = lambda img : imgelse:@@ -4316,7 +4317,9 @@ def preprocess_video_with_mask(input_video_path, input_mask_path, height, width,box = [xmin, ymin, xmax, ymax]box = [int(x) for x in box]return box- inpaint_color = int(inpaint_color)+ inpaint_color = parse_guide_inpaint_color(inpaint_color)+ inpaint_color = to_rgb_tensor(inpaint_color, device="cpu", dtype=torch.uint8)+ inpaint_color_np = tuple(int(v) for v in inpaint_color.view(-1).tolist())pad_frames = 0if start_frame < 0:pad_frames= -start_frame@@ -4447,6 +4450,10 @@ def preprocess_video_with_mask(input_video_path, input_mask_path, height, width,masked_frames = []masks = []for frame_no, (processed_img, processed_img_outside, mask) in enumerate(zip(proc_list, proc_list_outside, proc_mask)):+ if isinstance(processed_img, (list, tuple)):+ processed_img = np.full((height, width, 3), processed_img, dtype=np.uint8)+ if isinstance(processed_img_outside, (list, tuple)):+ processed_img_outside = np.full((height, width, 3), processed_img_outside, dtype=np.uint8)if any_mask :masked_frame = np.where(mask[..., None], processed_img, processed_img_outside)if process_outside_mask != None:@@ -4462,15 +4469,16 @@ def preprocess_video_with_mask(input_video_path, input_mask_path, height, width,else:masked_frame = processed_img- if isinstance(masked_frame, int):- masked_frame= np.full( (height, width, 3), inpaint_color, dtype= np.uint8)+ if isinstance(masked_frame, (int, float, np.integer)) or (isinstance(masked_frame, (list, tuple)) and len(masked_frame) == 3):+ masked_frame= np.full( (height, width, 3), inpaint_color_np, dtype= np.uint8)masked_frame = torch.from_numpy(masked_frame)if masked_frame.shape[-1] == 1:masked_frame = masked_frame.repeat(1,1,3).to(torch.uint8)if outpainting_dims != None:- full_frame= torch.full( (final_height, final_width, masked_frame.shape[-1]), inpaint_color, dtype= torch.uint8, device= masked_frame.device)+ color = inpaint_color.to(masked_frame.device).view(1, 1, 3)+ full_frame = color.expand(final_height, final_width, masked_frame.shape[-1]).clone()full_frame[margin_top:margin_top+height, margin_left:margin_left+width] = masked_framemasked_frame = full_frame@@ -5032,6 +5040,28 @@ def enhance_prompt(state, prompt, prompt_enhancer, multi_images_gen_type, overridef get_outpainting_dims(video_guide_outpainting):return None if video_guide_outpainting== None or len(video_guide_outpainting) == 0 or video_guide_outpainting == "0 0 0 0" or video_guide_outpainting.startswith("#") else [int(v) for v in video_guide_outpainting.split(" ")]+def parse_guide_inpaint_color(value):+ if isinstance(value, str):+ cleaned = value.strip()+ hex_value = cleaned[1:] if cleaned.startswith("#") else cleaned+ if len(hex_value) == 6 and all(c in "0123456789abcdefABCDEF" for c in hex_value):+ return tuple(int(hex_value[i:i+2], 16) for i in (0, 2, 4))+ if cleaned.lower().startswith("rgb"):+ cleaned = cleaned[3:]+ for ch in "()[]{}":+ cleaned = cleaned.replace(ch, "")+ cleaned = cleaned.replace(",", " ")+ parts = [p for p in cleaned.split() if p]+ if len(parts) == 3:+ try:+ return tuple(max(0, min(255, int(round(float(p))))) for p in parts)+ except ValueError:+ return 127.5+ return 127.5+ if isinstance(value, (list, tuple)) and len(value) == 3:+ return tuple(max(0, min(255, int(round(float(p))))) for p in value)+ return value+def truncate_audio(generated_audio, trim_video_frames_beginning, trim_video_frames_end, video_fps, audio_sampling_rate):samples_per_frame = audio_sampling_rate / video_fpsstart = int(trim_video_frames_beginning * samples_per_frame)@@ -5474,6 +5504,9 @@ def generate_video(# VAE Tilingdevice_mem_capacity = torch.cuda.get_device_properties(None).total_memory / 1048576guide_inpaint_color = model_def.get("guide_inpaint_color", 127.5)+ if image_mode==2:+ guide_inpaint_color = model_def.get("inpaint_color", guide_inpaint_color)+ guide_inpaint_color = parse_guide_inpaint_color(guide_inpaint_color)extract_guide_from_window_start = model_def.get("extract_guide_from_window_start", False)hunyuan_custom = "hunyuan_video_custom" in model_filenamehunyuan_custom_edit = hunyuan_custom and "edit" in model_filename@@ -5703,6 +5736,7 @@ def generate_video(prompts = enhanced_promptsabort = gen.get("abort", False)+while not abort:enable_RIFLEx = RIFLEx_setting == 0 and current_video_length > (6* get_model_fps(base_model_type)+1) or RIFLEx_setting == 1prompt = prompts[window_no] if window_no < len(prompts) else prompts[-1]@@ -5728,6 +5762,8 @@ def generate_video(return_latent_slice = slice(- max(1, (reuse_frames + discard_last_frames ) // latent_size) , None if discard_last_frames == 0 else -(discard_last_frames // latent_size) )refresh_preview = {"image_guide" : image_guide, "image_mask" : image_mask} if image_mode >= 1 else {}+ if hasattr(model_handler, "custom_prompt_preprocess"):+ prompt = model_handler.custom_prompt_preprocess(**locals())image_start_tensor = image_end_tensor = Noneif window_no == 1 and (video_source is not None or image_start is not None):if image_start is not None:@@ -6257,6 +6293,7 @@ def generate_video(if film_grain_intensity> 0:from postprocessing.film_grain import add_film_grainsample = add_film_grain(sample, film_grain_intensity, film_grain_saturation)+ mmaudio_enabled, mmaudio_mode, mmaudio_persistence, mmaudio_model_name, mmaudio_model_path = get_mmaudio_settings(server_config)if audio_only or is_image:output_video_frames = Noneoutput_frame_count = None@@ -6286,7 +6323,6 @@ def generate_video(else:file_name = f"{time_flag}_seed{seed}_{sanitize_file_name(truncate_for_filesystem(save_prompt)).strip()}.{extension}"video_path = os.path.join(save_path, file_name)- mmaudio_enabled, mmaudio_mode, mmaudio_persistence, mmaudio_model_name, mmaudio_model_path = get_mmaudio_settings(server_config)if BGRA_frames is not None:from models.wan.alpha.utils import write_zip_file@@ -7930,7 +7966,7 @@ def switch_image_mode(state):if model_cache is None:inpaint_cache[model_type] = model_cache ={}video_prompt_inpaint_mode = model_def.get("inpaint_video_prompt_type", "VAG")- video_prompt_image_mode = "KI"+ video_prompt_image_mode = model_def.get("image_video_prompt_type", "KI")old_video_prompt_type = video_prompt_typeif image_mode == 1:model_cache[2] = video_prompt_type@@ -8980,7 +9016,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Noninput_video_strength = gr.Slider(0, 1, value=ui_get("input_video_strength", 1.0), step=0.01, label=input_video_strength_label, visible = len(input_video_strength_label) and any_letters(image_prompt_type_value, "SVL"), show_reset_button= False)image_end_row, image_end, image_end_extra = get_image_gallery(label= get_image_end_label(ui_get("multi_prompts_gen_type")), value = ui_defaults.get("image_end", None), visible= any_letters(image_prompt_type_value, "SVL") and ("E" in image_prompt_type_value) )if model_mode_choices is None or image_mode_value not in model_modes_visibility:- model_mode = gr.Dropdown(value=None, label="model mode", visible=False, allow_custom_value= True)+ model_mode = gr.Dropdown(value=None, label="model mode", visible=False) #, allow_custom_value= Falseelse:model_mode_value = get_default_value(model_mode_choices["choices"], ui_get("model_mode", None), model_mode_choices["default"] )model_mode = gr.Dropdown(choices=model_mode_choices["choices"], value=model_mode_value, label=model_mode_choices["label"], visible=True)