Skip to content
Python

added flux 2 lanpaint, add outpainting for flux & qwen

DeepBeepMeep88feb42ec04b
diff --git a/defaults/i2v_2_2_Enhanced_Lightning_v2.json b/defaults/i2v_2_2_Enhanced_Lightning_v2.json
index 3840293..d27d045 100644
--- a/defaults/i2v_2_2_Enhanced_Lightning_v2.json
+++ b/defaults/i2v_2_2_Enhanced_Lightning_v2.json
@@ -11,7 +11,8 @@
"https://huggingface.co/DeepBeepMeep/Wan2.2/resolve/main/wan22EnhancedLightning_v2I2VFP8LOW.safetensors"
],
"author": "https://civitai.com/models/2053259",
- "profile_dirs": [""]
+ "profile_dirs": [""],
+ "video_prompt_enhancer_instructions": "You are an AI vision model. Analyze the provided image and the user's goal.\nProduce a second-by-second description of how the scene evolves over time.\n\nOutput rules:\nEach line must begin with \"(at X seconds:\" where X is the second number.\nAfter the timestamp, provide a dense, comma-separated visual description.\nKeep everything inside a single set of parentheses per line.\nMaintain a consistent cinematic style: shot type, lighting, subject details, camera movement, and actions.\nDo not include commentary, explanations, or meta-text.\nOnly describe visual, physical, or camera changes.\nStay consistent with the input image while expanding logically to meet the goal.\n\nOutput Format Example:\n(at 0 seconds: 4K, realistic, close-up shot, clean single subject, daylight, natural lighting, bedroom near a small window with light coming in from the left, a young college aged woman with blonde hair, blue eyes dressed in a blue pajama top, sitting up, she looks to be annoyed, the camera pushes-in on the character's face, she pulls the blanket off of her)\n\n(at 1 seconds: [next description])\n\n(at 2 seconds: [next description])\n\nContinue until the goal is fully represented."
},
"guidance_phases": 2,
"switch_threshold" : 900,
@@ -21,4 +22,4 @@
"num_inference_steps": 4,
"multi_prompts_gen_type": 2,
"prompt": "(at 0 seconds: wide frontal shot of a man standing in front of an open fridge, cinematic lighting, subtle ambient kitchen reflections, the fridge contents visible, camera static).\n(at 1 second: medium shot from the front as he opens the fridge fully, reaches for a can, slight zoom-in to emphasize the action, cinematic framing).\n(at 2 seconds: camera shifts to a side medium shot, tracking him as he lifts the can to his mouth, fluid movement, maintaining lighting and reflections).\n(at 3 seconds: camera starts a smooth 360-degree orbit around the man, following him as he drinks from the can, motion fluid, background slightly blurred for cinematic effect).\n(at 4 seconds: close-up on his face and upper body while drinking, orbit continues subtly, fridge reflections accentuating realism, cinematic polish).\n(at 5 seconds: final wide shot as he lowers the can, camera completes orbit to original angle, showcasing the kitchen space, lighting, and dynamic movement)."
-}
\ No newline at end of file
+}
diff --git a/defaults/i2v_2_2_Enhanced_Lightning_v2_svi2pro.json b/defaults/i2v_2_2_Enhanced_Lightning_v2_svi2pro.json
index 60d0d76..fc407b6 100644
--- a/defaults/i2v_2_2_Enhanced_Lightning_v2_svi2pro.json
+++ b/defaults/i2v_2_2_Enhanced_Lightning_v2_svi2pro.json
@@ -14,7 +14,8 @@
"author": "https://civitai.com/models/2053259",
"loras": "i2v_2_2_svi2pro",
"loras_multipliers": "i2v_2_2_svi2pro",
- "profile_dirs": [""]
+ "profile_dirs": [""],
+ "video_prompt_enhancer_instructions": "You are an AI vision model. Analyze the provided image and the user's goal.\nProduce a second-by-second description of how the scene evolves over time.\n\nOutput rules:\nEach line must begin with \"(at X seconds:\" where X is the second number.\nAfter the timestamp, provide a dense, comma-separated visual description.\nKeep everything inside a single set of parentheses per line.\nMaintain a consistent cinematic style: shot type, lighting, subject details, camera movement, and actions.\nDo not include commentary, explanations, or meta-text.\nOnly describe visual, physical, or camera changes.\nStay consistent with the input image while expanding logically to meet the goal.\n\nOutput Format Example:\n(at 0 seconds: 4K, realistic, close-up shot, clean single subject, daylight, natural lighting, bedroom near a small window with light coming in from the left, a young college aged woman with blonde hair, blue eyes dressed in a blue pajama top, sitting up, she looks to be annoyed, the camera pushes-in on the character's face, she pulls the blanket off of her)\n\n(at 1 seconds: [next description])\n\n(at 2 seconds: [next description])\n\nContinue until the goal is fully represented."
},
"guidance_phases": 2,
"switch_threshold" : 900,
@@ -23,4 +24,4 @@
"flow_shift" : 5,
"num_inference_steps": 8,
"prompt": "(at 0 seconds:a man open his fridge, takes a beer and drink it)"
-}
\ No newline at end of file
+}
diff --git a/defaults/pi_flux2_nvfp4.json b/defaults/pi_flux2_nvfp4.json
index 366f81e..b95033b 100644
--- a/defaults/pi_flux2_nvfp4.json
+++ b/defaults/pi_flux2_nvfp4.json
@@ -1,6 +1,6 @@
{
"model": {
- "name": "pi-FLUX.2 Dev NVFP 32B",
+ "name": "pi-FLUX.2 Dev NVFP4 32B",
"architecture": "pi_flux2",
"description": "pi-FLUX.2 using the NVFP4 Flux 2 Dev checkpoint.",
"URLs": "flux2_dev_nvfp4",
diff --git a/models/flux/flux_handler.py b/models/flux/flux_handler.py
index beceafb..471285a 100644
--- a/models/flux/flux_handler.py
+++ b/models/flux/flux_handler.py
@@ -1,5 +1,6 @@
import os
import torch
+import gradio as gr
from PIL import Image
from shared.utils import files_locator as fl
from shared.utils.hf import build_hf_url
@@ -64,7 +65,8 @@ class family_handler():
@staticmethod
def query_model_def(base_model_type, model_def):
flux_model = "flux-dev" if base_model_type == "flux" else base_model_type.replace("_", "-")
- pi_flux2 = flux_model == "pi-flux2"
+ flux_dev = base_model_type == "flux"
+ pi_flux2 = base_model_type == "pi_flux2"
flux2_klein_4b = base_model_type == "flux2_klein_4b"
flux2_klein_9b = base_model_type == "flux2_klein_9b"
flux2_klein = flux2_klein_4b or flux2_klein_9b
@@ -81,7 +83,35 @@ class family_handler():
"image_outputs" : True,
"no_negative_prompt" : flux2 or not (flux_chroma or flux_chroma_radiance),
"flux-model": flux_model,
+ "flux2": flux2,
}
+ supports_inpaint = flux_kontext or flux_schnell or flux_dev or flux2
+ if supports_inpaint:
+ extra_model_def["inpaint_support"] = True
+ if not pi_flux2:
+ lanpaint_choices = [
+ ("LanPaint (2 steps): ~2x slower, easy task", 2),
+ ("LanPaint (5 steps): ~5x slower, medium task", 3),
+ ("LanPaint (10 steps): ~10x slower, hard task", 4),
+ ("LanPaint (15 steps): ~15x slower, very hard task", 5),
+ ]
+ if flux_dev or flux_schnell:
+ choices = lanpaint_choices
+ default_mode = 2
+ extra_model_def["inpaint_video_prompt_type"] = "VA"
+ extra_model_def["image_video_prompt_type"] = ""
+ else:
+ choices = [("Masked Denoising : Inpainted area may reuse some content that has been masked", 0)] + lanpaint_choices
+ default_mode = 0
+ extra_model_def["model_modes"] = {
+ "choices": choices,
+ "default": default_mode,
+ "label": "Inpainting Method",
+ "image_modes": [2],
+ }
+ extra_model_def["inpaint_color"] = "FF0000"
+ extra_model_def["video_guide_outpainting"] = [1,2]
+
if flux2:
if flux2_klein:
if flux2_klein_4b:
@@ -137,7 +167,6 @@ class family_handler():
}
if flux_kontext or flux_kontext_dreamomni2 or flux2:
- extra_model_def["inpaint_support"] = flux_kontext
extra_model_def["image_ref_choices"] = {
"choices": [
("None", ""),
@@ -179,12 +208,9 @@ class family_handler():
"visible": True,
}
- # extra_model_def["guide_inpaint_color"] = 0
- # extra_model_def["video_guide_outpainting"] = [1,2]
if pi_flux2:
extra_model_def["piflow"] = True
- extra_model_def["inpaint_support"] = True
extra_model_def["fit_into_canvas_image_refs"] = 0
@@ -429,3 +455,27 @@ class family_handler():
})
+ @staticmethod
+ def validate_generative_settings(base_model_type, model_def, inputs):
+ model_mode = inputs.get("model_mode")
+ model_mode_int = None
+ if model_mode is not None:
+ try:
+ model_mode_int = int(model_mode)
+ except (TypeError, ValueError):
+ model_mode_int = None
+ if model_mode_int in (2, 3, 4, 5):
+ denoising_strength = inputs.get("denoising_strength", 1)
+ masking_strength = inputs.get("masking_strength", 1)
+ if denoising_strength != 1 or masking_strength != 1:
+ gr.Info("LanPaint forces Denoising Strength and Masking Strength to 1; non-1 values will be ignored.")
+
+
+ @staticmethod
+ def custom_prompt_preprocess(prompt, video_guide_outpainting, model_mode, **kwargs):
+ if model_mode == 0:
+ # from wgp import get_outpainting_dims
+ if len(video_guide_outpainting) and not video_guide_outpainting.startswith("#") and video_guide_outpainting != "0 0 0 0":
+ if not prompt.endswith("."): prompt += "."
+ prompt += "Remove the red paddings on the sides and show what's behind them."
+ return prompt
diff --git a/models/flux/flux_main.py b/models/flux/flux_main.py
index ae5e530..cf338a0 100644
--- a/models/flux/flux_main.py
+++ b/models/flux/flux_main.py
@@ -270,6 +270,14 @@ class model_factory:
return None
device="cuda"
flux2 = self.is_flux2
+ model_mode = bbargs.get("model_mode", None)
+ model_mode_int = None
+ if model_mode is not None:
+ try:
+ model_mode_int = int(model_mode)
+ except (TypeError, ValueError):
+ model_mode_int = None
+ lanpaint_enabled = model_mode_int in (2, 3, 4, 5)
if flux2:
guide_scale = 1.0
if self.guidance_max_phases < 1: guide_scale = 1
@@ -448,6 +456,10 @@ class model_factory:
joint_pass=joint_pass,
denoising_strength=denoising_strength,
masking_strength=masking_strength,
+ model_mode=model_mode,
+ height=height,
+ width=width,
+ vae_scale_factor=8,
)
if x==None: return None
# decode latents to pixel space
@@ -456,10 +468,17 @@ class model_factory:
with torch.autocast(device_type=device, dtype=torch.bfloat16):
x = self.vae.decode(x)
- if image_mask is not None and masking_strength == 1 and not flux2:
- img_msk_rebuilt = inp["img_msk_rebuilt"]
- img= input_frames.squeeze(1).unsqueeze(0) # convert_image_to_tensor(image_guide)
- x = img * (1 - img_msk_rebuilt) + x.to(img) * img_msk_rebuilt
+ img_msk_rebuilt = inp.get("img_msk_rebuilt") if isinstance(inp, dict) else None
+ if img_msk_rebuilt is not None and (lanpaint_enabled or (masking_strength == 1 and not flux2)):
+ img = None
+ if input_frames is not None:
+ img = input_frames.squeeze(1).unsqueeze(0)
+ elif input_ref_images is not None and len(input_ref_images) > 0:
+ img = convert_image_to_tensor(
+ input_ref_images[0].resize((width, height), resample=Image.Resampling.LANCZOS)
+ ).unsqueeze(0)
+ if img is not None:
+ x = img * (1 - img_msk_rebuilt) + x.to(img) * img_msk_rebuilt
x = x.clamp(-1, 1)
x = x.transpose(0, 1)
diff --git a/models/flux/sampling.py b/models/flux/sampling.py
index ff6f88f..56bf328 100644
--- a/models/flux/sampling.py
+++ b/models/flux/sampling.py
@@ -541,6 +541,10 @@ def denoise(
img_msk_rebuilt = None,
denoising_strength = 1,
masking_strength = 1,
+ model_mode = None,
+ height: int | None = None,
+ width: int | None = None,
+ vae_scale_factor: int = 8,
preview_meta = None,
original_image_latents = None,
# pi-Flow settings
@@ -566,8 +570,59 @@ def denoise(
piflow_sigmas = piflow_sigmas_dst = piflow_m_vals = None
morph, first_step = False, 0
+ model_mode_int = None
+ if model_mode is not None:
+ try:
+ model_mode_int = int(model_mode)
+ except (TypeError, ValueError):
+ model_mode_int = None
+ lanpaint_proc = None
+ lanpaint_mask = None
+ true_cfg_scale = 1.0 if real_guidance_scale is None else real_guidance_scale
if img_msk_latents is not None:
- if original_image_latents is None: original_image_latents= img_cond_seq.clone()
+ if not is_piflow and model_mode_int in (2, 3, 4, 5):
+ if img_cond_seq is not None:
+ if img_cond_seq_ids is not None and img_cond_seq_ids.shape[-1] == 4:
+ first_t = img_cond_seq_ids[:, :1, 0]
+ first_mask = img_cond_seq_ids[..., 0] == first_t
+ if original_image_latents is None:
+ original_image_latents = img_cond_seq[first_mask].view(
+ img_cond_seq.shape[0], -1, img_cond_seq.shape[-1]
+ ).clone()
+ keep_mask = ~first_mask
+ if keep_mask.any():
+ img_cond_seq = img_cond_seq[keep_mask].view(
+ img_cond_seq.shape[0], -1, img_cond_seq.shape[-1]
+ )
+ img_cond_seq_ids = img_cond_seq_ids[keep_mask].view(
+ img_cond_seq_ids.shape[0], -1, img_cond_seq_ids.shape[-1]
+ )
+ else:
+ img_cond_seq = None
+ img_cond_seq_ids = None
+ else:
+ base_len = img.shape[1]
+ if original_image_latents is None:
+ original_image_latents = img_cond_seq[:, :base_len].clone()
+ if img_cond_seq.shape[1] <= base_len:
+ img_cond_seq = None
+ img_cond_seq_ids = None
+ else:
+ img_cond_seq = img_cond_seq[:, base_len:]
+ if img_cond_seq_ids is not None:
+ img_cond_seq_ids = img_cond_seq_ids[:, base_len:]
+ from shared.inpainting.lanpaint import LanPaint
+ lanpaint_steps = {2: 2, 3: 5, 4: 10, 5: 15}.get(model_mode_int, 5)
+ lanpaint_proc = LanPaint(NSteps=lanpaint_steps)
+ denoising_strength = 1.0
+ masking_strength = 1.0
+ if img_msk_latents.shape[-1] != img.shape[-1]:
+ lanpaint_mask = img_msk_latents.expand(img.shape[0], img.shape[1], img.shape[2]).contiguous()
+ else:
+ lanpaint_mask = img_msk_latents
+ lanpaint_proc is not None
+ if original_image_latents is None and img_cond_seq is not None:
+ original_image_latents = img_cond_seq.clone()
randn = torch.randn_like(original_image_latents)
if denoising_strength < 1.:
first_step = int(len(timesteps[:-1]) * (1. - denoising_strength))
@@ -612,53 +667,86 @@ def denoise(
img = original_image_latents * (1.0 - latent_noise_factor) + img * latent_noise_factor
t_vec = torch.full((img.shape[0],), t_curr, dtype=img.dtype, device=img.device)
- img_input = img
- img_input_ids = img_ids
- if img_cond is not None:
- img_input = torch.cat((img, img_cond), dim=-1)
- if img_cond_seq is not None:
- img_input = torch.cat((img_input, img_cond_seq), dim=1)
- img_input_ids = torch.cat((img_input_ids, img_cond_seq_ids), dim=1)
- if not joint_pass or real_guidance_scale == 1:
- pred = model(
- img=img_input,
- img_ids=img_input_ids,
- txt_list=[txt],
- txt_ids_list=[txt_ids],
- y_list=[vec],
- timesteps=t_vec,
- guidance=guidance_vec,
- **kwargs
- )[0]
- if pred == None: return None
- if real_guidance_scale> 1:
- neg_pred = model(
+
+ def run_model(latents, cfg_scale):
+ img_input = latents
+ img_input_ids = img_ids
+ if img_cond is not None:
+ img_input = torch.cat((img_input, img_cond), dim=-1)
+ if img_cond_seq is not None:
+ img_input = torch.cat((img_input, img_cond_seq), dim=1)
+ img_input_ids = torch.cat((img_input_ids, img_cond_seq_ids), dim=1)
+ if not joint_pass or cfg_scale == 1:
+ noise_pred = model(
img=img_input,
img_ids=img_input_ids,
- txt_list=[neg_txt],
- txt_ids_list=[neg_txt_ids],
- y_list=[neg_vec],
+ txt_list=[txt],
+ txt_ids_list=[txt_ids],
+ y_list=[vec],
timesteps=t_vec,
guidance=guidance_vec,
**kwargs
)[0]
- if neg_pred == None: return None
- else:
- pred, neg_pred = model(
- img=img_input,
- img_ids=img_input_ids,
- txt_list=[txt, neg_txt],
- txt_ids_list=[txt_ids, neg_txt_ids],
- y_list=[vec, neg_vec],
- timesteps=t_vec,
- guidance=guidance_vec,
- **kwargs
+ if noise_pred == None:
+ return None, None
+ neg_noise_pred = None
+ if cfg_scale > 1:
+ neg_noise_pred = model(
+ img=img_input,
+ img_ids=img_input_ids,
+ txt_list=[neg_txt],
+ txt_ids_list=[neg_txt_ids],
+ y_list=[neg_vec],
+ timesteps=t_vec,
+ guidance=guidance_vec,
+ **kwargs
+ )[0]
+ if neg_noise_pred == None:
+ return None, None
+ else:
+ noise_pred, neg_noise_pred = model(
+ img=img_input,
+ img_ids=img_input_ids,
+ txt_list=[txt, neg_txt],
+ txt_ids_list=[txt_ids, neg_txt_ids],
+ y_list=[vec, neg_vec],
+ timesteps=t_vec,
+ guidance=guidance_vec,
+ **kwargs
+ )
+ if noise_pred == None:
+ return None, None
+ return noise_pred, neg_noise_pred
+
+ def cfg_predictions(noise_pred, neg_noise_pred, cfg_scale, t):
+ if cfg_scale > 1:
+ return neg_noise_pred + cfg_scale * (noise_pred - neg_noise_pred)
+ return noise_pred
+
+ if lanpaint_proc is not None and height is not None and width is not None and i < updated_num_steps - 1:
+ img = lanpaint_proc(
+ run_model,
+ cfg_predictions,
+ true_cfg_scale,
+ 1.0,
+ img,
+ original_image_latents,
+ randn,
+ t_vec,
+ lanpaint_mask,
+ height=height,
+ width=width,
+ vae_scale_factor=vae_scale_factor,
)
- if pred == None: return None
+ if img is None:
+ return None
+
+ pred, neg_pred = run_model(img, true_cfg_scale)
+ if pred == None: return None
if is_piflow and isinstance(pred, dict):
- if real_guidance_scale > 1:
- pred = {k: neg_pred[k] + real_guidance_scale * (pred[k] - neg_pred[k]) for k in pred}
+ if true_cfg_scale > 1:
+ pred = {k: neg_pred[k] + true_cfg_scale * (pred[k] - neg_pred[k]) for k in pred}
patch_size = getattr(model, "piflow_patch_size", 2)
img_packed = rearrange(
@@ -700,8 +788,8 @@ def denoise(
img_packed = _pack_latent_piflux2(img_unpacked, patch_size=patch_size)
img = rearrange(img_packed, "b c h w -> b (h w) c").to(img.dtype)
else:
- if real_guidance_scale > 1:
- pred = neg_pred + real_guidance_scale * (pred - neg_pred)
+ if true_cfg_scale > 1:
+ pred = cfg_predictions(pred, neg_pred, true_cfg_scale, t_vec)
step_size = t_prev - t_curr
if final_step_size_scale is not None and i == len(timesteps) - 2:
diff --git a/models/qwen/pipeline_qwenimage.py b/models/qwen/pipeline_qwenimage.py
index 4ad13ce..655d13f 100644
--- a/models/qwen/pipeline_qwenimage.py
+++ b/models/qwen/pipeline_qwenimage.py
@@ -659,8 +659,14 @@ class QwenImagePipeline(): #DiffusionPipeline
returning a tuple, the first element is a list with the generated images.
"""
- lora_inpaint = image_mask is not None and model_mode == 1
- lanpaint_enabled = image_mask is not None and model_mode == 2
+ model_mode_int = None
+ if model_mode is not None:
+ try:
+ model_mode_int = int(model_mode)
+ except (TypeError, ValueError):
+ model_mode_int = None
+ lora_inpaint = image_mask is not None and model_mode_int == 1
+ lanpaint_enabled = image_mask is not None and model_mode_int in (2, 3, 4, 5)
kwargs = {'pipeline': pipeline, 'callback': callback}
if callback != None:
@@ -819,6 +825,8 @@ class QwenImagePipeline(): #DiffusionPipeline
(1, vae_height // self.vae_scale_factor // 2, vae_width // self.vae_scale_factor // 2)
for vae_width, vae_height in vae_image_sizes
]
+ if lanpaint_enabled:
+ condition_shapes = condition_shapes[1:]
img_shapes = [output_shapes + condition_shapes] * effective_batch_size
else:
output_shape = (1, height // self.vae_scale_factor // 2, width // self.vae_scale_factor // 2)
@@ -860,9 +868,16 @@ class QwenImagePipeline(): #DiffusionPipeline
if image_mask_latents is not None:
original_image_latents = image_latents[:, :latents.shape[1]].clone()
if lanpaint_enabled:
+ if image_latents.shape[1]==latents.shape[1]:
+ image_latents = None
+ else:
+ image_latents = image_latents[:, latents.shape[1]:]
+
from shared.inpainting.lanpaint import LanPaint
- lanpaint_proc = LanPaint()
+ lanpaint_steps = {2: 2, 3: 5, 4: 10, 5: 15}.get(model_mode_int, 5)
+ lanpaint_proc = LanPaint(NSteps=lanpaint_steps)
denoising_strength = 1.
+ masking_strength = 1.
randn = torch.randn_like(original_image_latents)
if denoising_strength < 1.:
first_step = int(len(timesteps) * (1. - denoising_strength))
@@ -963,7 +978,7 @@ class QwenImagePipeline(): #DiffusionPipeline
return noise_pred
- if lanpaint_proc is not None and i<=updated_num_steps-1:
+ if lanpaint_proc is not None and i < updated_num_steps - 1:
latents = lanpaint_proc(denoise, cfg_predictions, true_cfg_scale, 1., latents, original_image_latents, randn, t/1000, image_mask_latents, height=height , width= width, vae_scale_factor= 8)
if latents is None: return None
@@ -976,8 +991,9 @@ class QwenImagePipeline(): #DiffusionPipeline
noise_pred = None
if image_mask_latents is not None and i < masked_steps:
- if lanpaint_proc is not None:
- latents = original_image_latents * (1-image_mask_latents) + image_mask_latents * latents
+ if lanpaint_proc is None :
+ pass
+ # latents = original_image_latents * (1-image_mask_latents) + image_mask_latents * latents
else:
next_t = timesteps[i+1] if i<len(timesteps)-1 else 0
latent_noise_factor = next_t / 1000
@@ -1026,15 +1042,15 @@ class QwenImagePipeline(): #DiffusionPipeline
)
latents_to_decode = latents_to_decode / latents_std + latents_mean
output_image = self.vae.decode(latents_to_decode, return_dict=False)[0][:, :, 0]
-
- if (
- num_layers == 1
- and image_mask_rebuilt is not None
- and not lora_inpaint
- and self.vae.upsampling_set is None
- and masking_strength == 1
- ):
- output_image = vae_images[0].squeeze(2) * (1 - image_mask_rebuilt) + output_image.to(vae_images[0] ) * image_mask_rebuilt
+ # looks worse
+ # if (
+ # num_layers == 1
+ # and image_mask_rebuilt is not None
+ # and not lora_inpaint
+ # and self.vae.upsampling_set is None
+ # and masking_strength == 10
+ # ):
+ # output_image = vae_images[0].squeeze(2) * (1 - image_mask_rebuilt) + output_image.to(vae_images[0] ) * image_mask_rebuilt
return output_image
diff --git a/models/qwen/qwen_handler.py b/models/qwen/qwen_handler.py
index 665d39c..dbbecd5 100644
--- a/models/qwen/qwen_handler.py
+++ b/models/qwen/qwen_handler.py
@@ -35,6 +35,23 @@ class family_handler():
extra_model_def["vae_upsampler"] = [1]
extra_model_def["sample_solvers"] = [("Default", "default")]
+ if base_model_type in ["qwen_image_20B"]:
+ extra_model_def["inpaint_support"] = True
+ extra_model_def["inpaint_video_prompt_type"] = "VA"
+ extra_model_def["image_video_prompt_type"] = ""
+ extra_model_def["video_guide_outpainting"] = [2]
+ extra_model_def["model_modes"] = {
+ "choices": [
+ ("LanPaint (2 steps): ~2x slower, easy task", 2),
+ ("LanPaint (5 steps): ~5x slower, medium task", 3),
+ ("LanPaint (10 steps): ~10x slower, hard task", 4),
+ ("LanPaint (15 steps): ~15x slower, very hard task", 5),
+ ],
+ "default": 2,
+ "label" : "Inpainting Method",
+ "image_modes" : [2],
+ }
+
if base_model_type in ["qwen_image_edit_20B", "qwen_image_edit_plus_20B", "qwen_image_edit_plus2_20B"]:
extra_model_def["inpaint_support"] = True
if base_model_type in ["qwen_image_edit_plus_20B", "qwen_image_edit_plus2_20B"]:
@@ -54,12 +71,16 @@ class family_handler():
"choices": [
("Lora Inpainting: Inpainted area completely unrelated to masked content", 1),
("Masked Denoising : Inpainted area may reuse some content that has been masked", 0),
- ("LanPaint : High Quality Inpainting but up x5 slower", 2),
+ ("LanPaint (2 steps): ~2x slower, easy task", 2),
+ ("LanPaint (5 steps): ~5x slower, medium task", 3),
+ ("LanPaint (10 steps): ~10x slower, hard task", 4),
+ ("LanPaint (15 steps): ~15x slower, very hard task", 5),
],
"default": 1,
"label" : "Inpainting Method",
"image_modes" : [2],
}
+ extra_model_def["inpaint_color"] = "FF0000"
if base_model_type in ["qwen_image_edit_plus_20B", "qwen_image_edit_plus2_20B"]:
extra_model_def["guide_preprocessing"] = {
@@ -187,21 +208,37 @@ class family_handler():
@staticmethod
def validate_generative_settings(base_model_type, model_def, inputs):
- if base_model_type in ["qwen_image_edit_20B", "qwen_image_edit_plus_20B", "qwen_image_edit_plus2_20B"]:
+ if base_model_type in ["qwen_image_20B", "qwen_image_edit_20B", "qwen_image_edit_plus_20B", "qwen_image_edit_plus2_20B"]:
model_mode = inputs["model_mode"]
- denoising_strength= inputs["denoising_strength"]
- video_guide_outpainting= inputs["video_guide_outpainting"]
- from wgp import get_outpainting_dims
- outpainting_dims = get_outpainting_dims(video_guide_outpainting)
-
- if denoising_strength < 1 and model_mode != 0:
+ denoising_strength = inputs["denoising_strength"]
+ masking_strength = inputs["masking_strength"]
+ model_mode_int = None
+ if model_mode is not None:
+ try:
+ model_mode_int = int(model_mode)
+ except (TypeError, ValueError):
+ model_mode_int = None
+
+ if model_mode_int in (2, 3, 4, 5):
+ if denoising_strength != 1 or masking_strength != 1:
+ gr.Info("LanPaint forces Denoising Strength and Masking Strength to 1; non-1 values will be ignored.")
+ elif denoising_strength < 1 and model_mode_int != 0:
gr.Info("Denoising Strength will be ignored if Masked Denoising is not used")
- if outpainting_dims is not None and model_mode != 1 :
- return "Outpainting is supported only with Lora Inpainting"
+
if base_model_type in ["qwen_image_layered_20B"]:
if inputs.get("image_guide") is None:
return "Qwen Image Layered requires a Control Image."
-
+
+ @staticmethod
+ def custom_prompt_preprocess(prompt, video_guide_outpainting, model_mode, **kwargs):
+ if model_mode == 0:
+ # from wgp import get_outpainting_dims
+ if len(video_guide_outpainting) and not video_guide_outpainting.startswith("#") and video_guide_outpainting != "0 0 0 0":
+ if not prompt.endswith("."): prompt += "."
+ prompt += "Remove the red paddings on the sides and show what's behind them."
+ return prompt
+
+
@staticmethod
def get_rgb_factors(base_model_type ):
from shared.RGB_factors import get_rgb_factors
diff --git a/models/wan/ovi_fusion_engine.py b/models/wan/ovi_fusion_engine.py
index de06197..d24e97d 100644
--- a/models/wan/ovi_fusion_engine.py
+++ b/models/wan/ovi_fusion_engine.py
@@ -129,7 +129,7 @@ class OviFusionEngine:
- @torch.inference_mode()
+ @torch.no_grad()
def generate(self,
input_prompt,
image_start=None,
diff --git a/models/z_image/z_image_transformer2d.py b/models/z_image/z_image_transformer2d.py
index 6738ca0..d55513b 100644
--- a/models/z_image/z_image_transformer2d.py
+++ b/models/z_image/z_image_transformer2d.py
@@ -386,6 +386,23 @@ class RopeEmbedder:
class ZImageTransformer2DModel(nn.Module):
+ def preprocess_loras(self, model_type, sd):
+ first = next(iter(sd), None)
+ if first is None:
+ return sd
+
+ if ".default." not in first and ".lora." not in first:
+ return sd
+
+ new_sd = {}
+ for k, v in sd.items():
+ if ".default." in k:
+ k = k.replace(".default.", ".")
+ if ".lora." in k:
+ k = k.replace(".lora.", ".lora_")
+ new_sd[k] = v
+ return new_sd
+
def __init__(
self,
# Control-specific parameters (optional)
diff --git a/shared/utils/utils.py b/shared/utils/utils.py
index 7a31c57..5e08ff9 100644
--- a/shared/utils/utils.py
+++ b/shared/utils/utils.py
@@ -99,6 +99,25 @@ def truncate_for_filesystem(s, max_bytes=None):
def get_default_workers():
return os.cpu_count()/ 2
+def to_rgb_tensor(value, device="cpu", dtype=torch.float):
+ if isinstance(value, torch.Tensor):
+ tensor = value.to(device=device, dtype=dtype)
+ else:
+ if isinstance(value, (list, tuple, np.ndarray)):
+ vals = value
+ else:
+ vals = [value, value, value]
+ tensor = torch.tensor(vals, device=device, dtype=dtype)
+ if tensor.numel() == 1:
+ tensor = tensor.repeat(3)
+ elif tensor.numel() != 3:
+ tensor = tensor.flatten()
+ if tensor.numel() < 3:
+ tensor = tensor.repeat(3)[:3]
+ else:
+ tensor = tensor[:3]
+ return tensor.view(3, 1, 1)
+
def process_images_multithread(image_processor, items, process_type, wrap_in_list = True, max_workers: int = os.cpu_count()/ 2, in_place = False) :
if not items:
return []
@@ -374,7 +393,7 @@ def resize_and_remove_background(img_list, budget_width, budget_height, rm_backg
return output_list, output_mask_list
def fit_image_into_canvas(ref_img, image_size, canvas_tf_bg =127.5, device ="cpu", full_frame = False, outpainting_dims = None, return_mask = False, return_image = False):
- inpaint_color = canvas_tf_bg / 127.5 - 1
+ inpaint_color = to_rgb_tensor(canvas_tf_bg, device=device, dtype=torch.float) / 127.5 - 1
ref_width, ref_height = ref_img.size
if (ref_height, ref_width) == image_size and outpainting_dims == None:
@@ -403,10 +422,10 @@ def fit_image_into_canvas(ref_img, image_size, canvas_tf_bg =127.5, device ="cpu
ref_img = ref_img.resize((new_width, new_height), resample=Image.Resampling.LANCZOS)
ref_img = TF.to_tensor(ref_img).sub_(0.5).div_(0.5).unsqueeze(1)
if outpainting_dims != None:
- canvas = torch.full((3, 1, final_height, final_width), inpaint_color, dtype= torch.float, device=device) # [-1, 1]
+ canvas = inpaint_color.expand(3, 1, final_height, final_width).clone()
canvas[:, :, margin_top + top:margin_top + top + new_height, margin_left + left:margin_left + left + new_width] = ref_img
else:
- canvas = torch.full((3, 1, canvas_height, canvas_width), inpaint_color, dtype= torch.float, device=device) # [-1, 1]
+ canvas = inpaint_color.expand(3, 1, canvas_height, canvas_width).clone()
canvas[:, :, top:top + new_height, left:left + new_width] = ref_img
ref_img = canvas
canvas = None
@@ -425,7 +444,7 @@ def fit_image_into_canvas(ref_img, image_size, canvas_tf_bg =127.5, device ="cpu
def prepare_video_guide_and_mask( video_guides, video_masks, pre_video_guide, image_size, current_video_length = 81, latent_size = 4, any_mask = False, any_guide_padding = False, guide_inpaint_color = 127.5, keep_video_guide_frames = [], inject_frames = [], outpainting_dims = None, device ="cpu"):
src_videos, src_masks = [], []
- inpaint_color_compressed = guide_inpaint_color/127.5 - 1
+ inpaint_color_compressed = to_rgb_tensor(guide_inpaint_color, device=device, dtype=torch.float) / 127.5 - 1
prepend_count = pre_video_guide.shape[1] if pre_video_guide is not None else 0
for guide_no, (cur_video_guide, cur_video_mask) in enumerate(zip(video_guides, video_masks)):
src_video, src_mask = cur_video_guide, cur_video_mask
@@ -436,9 +455,10 @@ def prepare_video_guide_and_mask( video_guides, video_masks, pre_video_guide, im
if any_guide_padding:
if src_video is None:
- src_video = torch.full( (3, current_video_length, *image_size ), inpaint_color_compressed, dtype = torch.float, device= device)
+ src_video = inpaint_color_compressed.expand(3, current_video_length, *image_size).clone()
elif src_video.shape[1] < current_video_length:
- src_video = torch.cat([src_video, torch.full( (3, current_video_length - src_video.shape[1], *src_video.shape[-2:] ), inpaint_color_compressed, dtype = src_video.dtype, device= src_video.device) ], dim=1)
+ pad = inpaint_color_compressed.to(src_video.device).expand(3, current_video_length - src_video.shape[1], *src_video.shape[-2:]).clone()
+ src_video = torch.cat([src_video, pad], dim=1)
elif src_video is not None:
new_num_frames = (src_video.shape[1] - 1) // latent_size * latent_size + 1
if new_num_frames < src_video.shape[1]:
@@ -457,7 +477,7 @@ def prepare_video_guide_and_mask( video_guides, video_masks, pre_video_guide, im
for k, keep in enumerate(keep_video_guide_frames):
if not keep:
pos = prepend_count + k
- src_video[:, pos:pos+1] = inpaint_color_compressed
+ src_video[:, pos:pos+1] = inpaint_color_compressed.to(src_video.device)
if any_mask: src_mask[:, pos:pos+1] = 1
for k, frame in enumerate(inject_frames):
diff --git a/wgp.py b/wgp.py
index 27f61fe..dc8f7ec 100644
--- a/wgp.py
+++ b/wgp.py
@@ -33,7 +33,7 @@ import numpy as np
import importlib
from shared.utils import notification_sound
from shared.utils.loras_mutipliers import preparse_loras_multipliers, parse_loras_multipliers
-from shared.utils.utils import convert_tensor_to_image, save_image, get_video_info, get_file_creation_date, convert_image_to_video, calculate_new_dimensions, convert_image_to_tensor, calculate_dimensions_and_resize_image, rescale_and_crop, get_video_frame, resize_and_remove_background, rgb_bw_to_rgba_mask
+from shared.utils.utils import convert_tensor_to_image, save_image, get_video_info, get_file_creation_date, convert_image_to_video, calculate_new_dimensions, convert_image_to_tensor, calculate_dimensions_and_resize_image, rescale_and_crop, get_video_frame, resize_and_remove_background, rgb_bw_to_rgba_mask, to_rgb_tensor
from shared.utils.utils import calculate_new_dimensions, get_outpainting_frame_location, get_outpainting_full_area_dimensions
from shared.utils.utils import has_video_file_extension, has_image_file_extension, has_audio_file_extension
from shared.utils.audio_video import extract_audio_tracks, combine_video_with_audio_tracks, combine_and_concatenate_video_with_audio_tracks, cleanup_temp_audio_files, save_video, save_image
@@ -360,7 +360,7 @@ def process_prompt_and_add_tasks(state, current_gallery_tab, model_choice):
inputs["state"] = state
inputs["model_type"] = model_type
- inputs.pop("lset_name")
+ inputs.pop("lset_name", None)
if inputs == None:
gr.Warning("Internal state error: Could not retrieve inputs for the model.")
queue = gen.get("queue", [])
@@ -4234,7 +4234,8 @@ def get_preprocessor(process_type, inpaint_color):
}
anno_ins = lambda img: FlowVisAnnotator(cfg_dict).forward(img)
elif process_type=="inpaint":
- anno_ins = lambda img : len(img) * [inpaint_color]
+ color = tuple(int(v) for v in inpaint_color.view(-1).tolist())
+ anno_ins = lambda img : len(img) * [color]
elif process_type == None or process_type in ["raw", "identity"]:
anno_ins = lambda img : img
else:
@@ -4316,7 +4317,9 @@ def preprocess_video_with_mask(input_video_path, input_mask_path, height, width,
box = [xmin, ymin, xmax, ymax]
box = [int(x) for x in box]
return box
- inpaint_color = int(inpaint_color)
+ inpaint_color = parse_guide_inpaint_color(inpaint_color)
+ inpaint_color = to_rgb_tensor(inpaint_color, device="cpu", dtype=torch.uint8)
+ inpaint_color_np = tuple(int(v) for v in inpaint_color.view(-1).tolist())
pad_frames = 0
if start_frame < 0:
pad_frames= -start_frame
@@ -4447,6 +4450,10 @@ def preprocess_video_with_mask(input_video_path, input_mask_path, height, width,
masked_frames = []
masks = []
for frame_no, (processed_img, processed_img_outside, mask) in enumerate(zip(proc_list, proc_list_outside, proc_mask)):
+ if isinstance(processed_img, (list, tuple)):
+ processed_img = np.full((height, width, 3), processed_img, dtype=np.uint8)
+ if isinstance(processed_img_outside, (list, tuple)):
+ processed_img_outside = np.full((height, width, 3), processed_img_outside, dtype=np.uint8)
if any_mask :
masked_frame = np.where(mask[..., None], processed_img, processed_img_outside)
if process_outside_mask != None:
@@ -4462,15 +4469,16 @@ def preprocess_video_with_mask(input_video_path, input_mask_path, height, width,
else:
masked_frame = processed_img
- if isinstance(masked_frame, int):
- masked_frame= np.full( (height, width, 3), inpaint_color, dtype= np.uint8)
+ if isinstance(masked_frame, (int, float, np.integer)) or (isinstance(masked_frame, (list, tuple)) and len(masked_frame) == 3):
+ masked_frame= np.full( (height, width, 3), inpaint_color_np, dtype= np.uint8)
masked_frame = torch.from_numpy(masked_frame)
if masked_frame.shape[-1] == 1:
masked_frame = masked_frame.repeat(1,1,3).to(torch.uint8)
if outpainting_dims != None:
- full_frame= torch.full( (final_height, final_width, masked_frame.shape[-1]), inpaint_color, dtype= torch.uint8, device= masked_frame.device)
+ color = inpaint_color.to(masked_frame.device).view(1, 1, 3)
+ full_frame = color.expand(final_height, final_width, masked_frame.shape[-1]).clone()
full_frame[margin_top:margin_top+height, margin_left:margin_left+width] = masked_frame
masked_frame = full_frame
@@ -5032,6 +5040,28 @@ def enhance_prompt(state, prompt, prompt_enhancer, multi_images_gen_type, overri
def get_outpainting_dims(video_guide_outpainting):
return None if video_guide_outpainting== None or len(video_guide_outpainting) == 0 or video_guide_outpainting == "0 0 0 0" or video_guide_outpainting.startswith("#") else [int(v) for v in video_guide_outpainting.split(" ")]
+def parse_guide_inpaint_color(value):
+ if isinstance(value, str):
+ cleaned = value.strip()
+ hex_value = cleaned[1:] if cleaned.startswith("#") else cleaned
+ if len(hex_value) == 6 and all(c in "0123456789abcdefABCDEF" for c in hex_value):
+ return tuple(int(hex_value[i:i+2], 16) for i in (0, 2, 4))
+ if cleaned.lower().startswith("rgb"):
+ cleaned = cleaned[3:]
+ for ch in "()[]{}":
+ cleaned = cleaned.replace(ch, "")
+ cleaned = cleaned.replace(",", " ")
+ parts = [p for p in cleaned.split() if p]
+ if len(parts) == 3:
+ try:
+ return tuple(max(0, min(255, int(round(float(p))))) for p in parts)
+ except ValueError:
+ return 127.5
+ return 127.5
+ if isinstance(value, (list, tuple)) and len(value) == 3:
+ return tuple(max(0, min(255, int(round(float(p))))) for p in value)
+ return value
+
def truncate_audio(generated_audio, trim_video_frames_beginning, trim_video_frames_end, video_fps, audio_sampling_rate):
samples_per_frame = audio_sampling_rate / video_fps
start = int(trim_video_frames_beginning * samples_per_frame)
@@ -5474,6 +5504,9 @@ def generate_video(
# VAE Tiling
device_mem_capacity = torch.cuda.get_device_properties(None).total_memory / 1048576
guide_inpaint_color = model_def.get("guide_inpaint_color", 127.5)
+ if image_mode==2:
+ guide_inpaint_color = model_def.get("inpaint_color", guide_inpaint_color)
+ guide_inpaint_color = parse_guide_inpaint_color(guide_inpaint_color)
extract_guide_from_window_start = model_def.get("extract_guide_from_window_start", False)
hunyuan_custom = "hunyuan_video_custom" in model_filename
hunyuan_custom_edit = hunyuan_custom and "edit" in model_filename
@@ -5703,6 +5736,7 @@ def generate_video(
prompts = enhanced_prompts
abort = gen.get("abort", False)
+
while not abort:
enable_RIFLEx = RIFLEx_setting == 0 and current_video_length > (6* get_model_fps(base_model_type)+1) or RIFLEx_setting == 1
prompt = prompts[window_no] if window_no < len(prompts) else prompts[-1]
@@ -5728,6 +5762,8 @@ def generate_video(
return_latent_slice = slice(- max(1, (reuse_frames + discard_last_frames ) // latent_size) , None if discard_last_frames == 0 else -(discard_last_frames // latent_size) )
refresh_preview = {"image_guide" : image_guide, "image_mask" : image_mask} if image_mode >= 1 else {}
+ if hasattr(model_handler, "custom_prompt_preprocess"):
+ prompt = model_handler.custom_prompt_preprocess(**locals())
image_start_tensor = image_end_tensor = None
if window_no == 1 and (video_source is not None or image_start is not None):
if image_start is not None:
@@ -6257,6 +6293,7 @@ def generate_video(
if film_grain_intensity> 0:
from postprocessing.film_grain import add_film_grain
sample = add_film_grain(sample, film_grain_intensity, film_grain_saturation)
+ mmaudio_enabled, mmaudio_mode, mmaudio_persistence, mmaudio_model_name, mmaudio_model_path = get_mmaudio_settings(server_config)
if audio_only or is_image:
output_video_frames = None
output_frame_count = None
@@ -6286,7 +6323,6 @@ def generate_video(
else:
file_name = f"{time_flag}_seed{seed}_{sanitize_file_name(truncate_for_filesystem(save_prompt)).strip()}.{extension}"
video_path = os.path.join(save_path, file_name)
- mmaudio_enabled, mmaudio_mode, mmaudio_persistence, mmaudio_model_name, mmaudio_model_path = get_mmaudio_settings(server_config)
if BGRA_frames is not None:
from models.wan.alpha.utils import write_zip_file
@@ -7930,7 +7966,7 @@ def switch_image_mode(state):
if model_cache is None:
inpaint_cache[model_type] = model_cache ={}
video_prompt_inpaint_mode = model_def.get("inpaint_video_prompt_type", "VAG")
- video_prompt_image_mode = "KI"
+ video_prompt_image_mode = model_def.get("image_video_prompt_type", "KI")
old_video_prompt_type = video_prompt_type
if image_mode == 1:
model_cache[2] = video_prompt_type
@@ -8980,7 +9016,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Non
input_video_strength = gr.Slider(0, 1, value=ui_get("input_video_strength", 1.0), step=0.01, label=input_video_strength_label, visible = len(input_video_strength_label) and any_letters(image_prompt_type_value, "SVL"), show_reset_button= False)
image_end_row, image_end, image_end_extra = get_image_gallery(label= get_image_end_label(ui_get("multi_prompts_gen_type")), value = ui_defaults.get("image_end", None), visible= any_letters(image_prompt_type_value, "SVL") and ("E" in image_prompt_type_value) )
if model_mode_choices is None or image_mode_value not in model_modes_visibility:
- model_mode = gr.Dropdown(value=None, label="model mode", visible=False, allow_custom_value= True)
+ model_mode = gr.Dropdown(value=None, label="model mode", visible=False) #, allow_custom_value= False
else:
model_mode_value = get_default_value(model_mode_choices["choices"], ui_get("model_mode", None), model_mode_choices["default"] )
model_mode = gr.Dropdown(choices=model_mode_choices["choices"], value=model_mode_value, label=model_mode_choices["label"], visible=True)