Python
add NAG support for Z-Image
DeepBeepMeep7a09f617eb25
diff --git a/README.md b/README.mdindex 8e36747..84770cd 100644--- a/README.md+++ b/README.md@@ -69,7 +69,8 @@ Processing Settings can be useful to do some quick gen / testing if you don't ne*update 9.82*: added Settings headless processing, output file customization, refactored Task edition and queue processing\*update 9.83*: Qwen Edit+ upgraded: no more any zoom out at 1080p, enabled mask, enabled image refs with inpainting\*update 9.84*: added Wan-Move support\-*update 9.85*: added Z-Image Control net v2+*update 9.85*: added Z-Image Control net v2\+*update 9.86*: added NAG support for Z-Image### December 4 2025: WanGP v9.74, The Alpha & the Omega ... and the Dancerdiff --git a/models/wan/wan_handler.py b/models/wan/wan_handler.pyindex b3f7606..0e9f955 100644--- a/models/wan/wan_handler.py+++ b/models/wan/wan_handler.py@@ -580,6 +580,7 @@ class family_handler():extra_model_def["background_removal_color"] = [128, 128, 128]if base_model_type in ["fantasy"] or multitalk:extra_model_def["audio_guidance"] = True+ extra_model_def["NAG"] = vace_class or t2v or i2vif test_oneframe_overlap(base_model_type):extra_model_def["sliding_window_defaults"] = { "overlap_min" : 1, "overlap_max" : 1, "overlap_step": 0, "overlap_default": 1}diff --git a/models/z_image/pipeline_z_image.py b/models/z_image/pipeline_z_image.pyindex 765e44f..1a9c1c5 100644--- a/models/z_image/pipeline_z_image.py+++ b/models/z_image/pipeline_z_image.py@@ -25,6 +25,7 @@ from diffusers.pipelines.pipeline_utils import DiffusionPipelinefrom diffusers.schedulers import FlowMatchEulerDiscreteSchedulerfrom diffusers.utils import logging, replace_example_docstringfrom diffusers.utils.torch_utils import randn_tensor+from mmgp import offloadfrom .z_image_transformer2d import ZImageTransformer2DModelfrom .pipeline_output import ZImagePipelineOutputfrom shared.utils.utils import get_outpainting_frame_location, resize_lanczos, calculate_new_dimensions, convert_image_to_tensor, fit_image_into_canvas@@ -331,6 +332,9 @@ class ZImagePipeline(DiffusionPipeline, FromSingleFileMixin):inpaint_mask: Optional[torch.Tensor] = None,control_context_scale: float = 0.75,input_ref_images = None,+ NAG_scale: float = 1.0,+ NAG_tau: float = 3.5,+ NAG_alpha: float = 0.5,):r"""Function invoked when calling the pipeline for generation.@@ -425,6 +429,11 @@ class ZImagePipeline(DiffusionPipeline, FromSingleFileMixin):)self._guidance_scale = guidance_scale+ kwargs = {}+ NAG = None+ if NAG_scale > 1:+ NAG = { "scale": NAG_scale, "tau": NAG_tau, "alpha": NAG_alpha, }+dtype = (self.text_encoder.dtypeif prompt_embeds is None@@ -463,7 +472,7 @@ class ZImagePipeline(DiffusionPipeline, FromSingleFileMixin):) = self.encode_prompt(prompt=prompt,negative_prompt=negative_prompt,- do_classifier_free_guidance=self.do_classifier_free_guidance,+ do_classifier_free_guidance=self.do_classifier_free_guidance or NAG is not None,prompt_embeds=prompt_embeds,negative_prompt_embeds=negative_prompt_embeds,dtype=dtype,@@ -472,6 +481,11 @@ class ZImagePipeline(DiffusionPipeline, FromSingleFileMixin):max_sequence_length=max_sequence_length,lora_scale=lora_scale,)+ neg_embeds_list = negative_prompt_embeds if isinstance(negative_prompt_embeds, list) else []+ if NAG is not None and len(neg_embeds_list) > 0:+ NAG["neg_feats"] = neg_embeds_list[0]+ kwargs["NAG"] = NAG+batch_size = num_images_per_promptnum_images_per_prompt = 1if batch_size > 1:@@ -479,8 +493,6 @@ class ZImagePipeline(DiffusionPipeline, FromSingleFileMixin):if len(negative_prompt_embeds) > 0:negative_prompt_embeds = negative_prompt_embeds * batch_size- # if prompt_embeds is not None:- # prompt_embeds = prompt_embeds.expand(num_images_per_prompt, -1, -1)# 4. Prepare latent variablesnum_channels_latents = self.transformer.in_channels@@ -612,6 +624,7 @@ class ZImagePipeline(DiffusionPipeline, FromSingleFileMixin):control_context_scale=control_context_scale,callback=callback,pipeline=self,+ **kwargs,)if model_out_list is None:@@ -642,6 +655,7 @@ class ZImagePipeline(DiffusionPipeline, FromSingleFileMixin):control_context_scale=control_context_scale,callback=callback,pipeline=self,+ **kwargs,)if model_out_list is None:diff --git a/models/z_image/z_image_handler.py b/models/z_image/z_image_handler.pyindex da3e1e6..2eea21a 100644--- a/models/z_image/z_image_handler.py+++ b/models/z_image/z_image_handler.py@@ -18,7 +18,6 @@ class family_handler:"guidance_max_phases": 0,"fit_into_canvas_image_refs": 0,"profiles_dir": [],- "no_negative_prompt": True,}if base_model_type in ["z_image_control", "z_image_control2"]:@@ -40,14 +39,14 @@ class family_handler:"selection":[ "", "A", "NA"],"visible": False,}- extra_model_def["image_ref_choices"] = {- "choices":[("No Reference Image",""), ("Image is a Reference Image", "KI")],- "default": "",- "letters_filter": "KI",- "label": "Reference Image for Inpainting",- "visible": False,- }-+ # extra_model_def["image_ref_choices"] = {+ # "choices":[("No Reference Image",""), ("Image is a Reference Image", "KI")],+ # "default": "",+ # "letters_filter": "KI",+ # "label": "Reference Image for Inpainting",+ # "visible": False,+ # }+ extra_model_def["NAG"] = base_model_type in ["z_image"]return extra_model_def@staticmethod@@ -152,6 +151,9 @@ class family_handler:{"guidance_scale": 0.0,"num_inference_steps": ui_defaults.get("num_inference_steps", 9),+ "NAG_scale": ui_defaults.get("NAG_scale", 1.0),+ "NAG_tau": ui_defaults.get("NAG_tau", 3.5),+ "NAG_alpha": ui_defaults.get("NAG_alpha", 0.5),})diff --git a/models/z_image/z_image_main.py b/models/z_image/z_image_main.pyindex ac0d45b..0368313 100644--- a/models/z_image/z_image_main.py+++ b/models/z_image/z_image_main.py@@ -196,6 +196,9 @@ class model_factory:input_masks=None,context_scale: float = [0],input_ref_images = None,+ NAG_scale: float = 1.0,+ NAG_tau: float = 3.5,+ NAG_alpha: float = 0.5,**kwargs,):generator = torch.Generator(device="cuda" if torch.cuda.is_available() else "cpu")@@ -237,7 +240,10 @@ class model_factory:control_image=input_frames,inpaint_mask=input_masks,control_context_scale=None if context_scale is None else context_scale[0],- input_ref_images= input_ref_images ,+ input_ref_images= input_ref_images,+ NAG_scale=NAG_scale,+ NAG_tau=NAG_tau,+ NAG_alpha=NAG_alpha,)if images is None:diff --git a/models/z_image/z_image_transformer2d.py b/models/z_image/z_image_transformer2d.pyindex cf12605..6397f79 100644--- a/models/z_image/z_image_transformer2d.py+++ b/models/z_image/z_image_transformer2d.py@@ -114,7 +114,7 @@ class Attention(nn.Module):self.norm_q = RMSNorm(self.head_dim, eps=1e-5) if qk_norm else Noneself.norm_k = RMSNorm(self.head_dim, eps=1e-5) if qk_norm else None- def forward(self, h_list: list, freqs_cis: torch.Tensor) -> torch.Tensor:+ def forward(self, h_list: list, freqs_cis: torch.Tensor, NAG= None) -> torch.Tensor:"""Compute self-attention with RoPE. h_list is cleared to free memory early."""h = h_list.pop()query = self.to_q(h)@@ -134,10 +134,48 @@ class Attention(nn.Module):query = apply_rotary_emb_inplace(q_list, freqs_cis)k_list = [key]; del keykey = apply_rotary_emb_inplace(k_list, freqs_cis)- # Compute attentiondtype = query.dtype- out = pay_attention([query, key, value]); del query, key, value- return self.to_out(out.flatten(2, 3).to(dtype))++ # NAG for joint-attention models: transformer duplicates batch into [pos, neg] halves.++ if NAG is not None:+ nag_scale = NAG["scale"]+ nag_alpha = NAG["alpha"]+ nag_tau = NAG["tau"]+ cap_embed_len = NAG["cap_embed_len"]++ x_list = [query[:, :-cap_embed_len], key[:, :-cap_embed_len], value[:, :-cap_embed_len] ]+ x_pos = pay_attention(x_list).flatten(2, 3).to(dtype)+ query[:, -2 *cap_embed_len:-cap_embed_len] = query[:, -cap_embed_len:]+ key[:, -2 *cap_embed_len:-cap_embed_len] = key[:, -cap_embed_len:]+ value[:, -2 *cap_embed_len:-cap_embed_len] = value[:, -cap_embed_len:]+ x_list = [query[:, :-cap_embed_len], key[:, :-cap_embed_len], value[:, :-cap_embed_len] ]+ del query, key, value+ x_neg = pay_attention(x_list).flatten(2, 3).to(dtype)++ x_neg_tail = x_neg[:, -cap_embed_len:].clone()+ x_guidance = x_neg+ x_guidance.mul_(1 - nag_scale)+ x_guidance.add_(x_pos, alpha=nag_scale)+ norm_positive = torch.norm(x_pos, p=1, dim=-1, keepdim=True)+ norm_guidance = torch.norm(x_guidance, p=1, dim=-1, keepdim=True)+ scale = norm_guidance / norm_positive+ scale = torch.nan_to_num(scale, 10)+ factor = (1 / (norm_guidance + 1e-7) * norm_positive * nag_tau).to(x_guidance.dtype)+ x_guidance = torch.where(scale > nag_tau, x_guidance * factor, x_guidance).to(dtype)+ del norm_positive, norm_guidance, scale, factor++ x_guidance.mul_(nag_alpha)+ x_guidance.add_(x_pos, alpha=(1 - nag_alpha))+ x_pos = None+ out = torch.cat([x_guidance, x_neg_tail], dim=1)+ x_pos = x_neg = x_guidance = None+ else:+ x_list = [query, key, value]+ del query, key, value+ out = pay_attention(x_list).flatten(2, 3).to(dtype);++ return self.to_out(out)class ZImageTransformerBlock(nn.Module):@@ -189,6 +227,7 @@ class ZImageTransformerBlock(nn.Module):attn_mask: torch.Tensor,freqs_cis: torch.Tensor,adaln_input: Optional[torch.Tensor] = None,+ NAG = None,):if self.modulation:scale_msa, gate_msa, scale_mlp, gate_mlp = self.adaLN_modulation(adaln_input).unsqueeze(1).chunk(4, dim=2)@@ -196,7 +235,7 @@ class ZImageTransformerBlock(nn.Module):scale_msa.add_(1.0)normed = self.attention_norm1(x)normed.mul_(scale_msa)- attn_out = self.attention_norm2(self.attention([normed], freqs_cis))+ attn_out = self.attention_norm2(self.attention([normed], freqs_cis, NAG=NAG))attn_out.mul_(gate_msa.tanh_())x.add_(attn_out); attn_out = None# In-place modulation for FFN block (chunked)@@ -208,7 +247,7 @@ class ZImageTransformerBlock(nn.Module):ffn_out.mul_(gate_mlp.tanh_())x.add_(ffn_out); ffn_out = Noneelse:- x.add_(self.attention_norm2(self.attention([self.attention_norm1(x)], freqs_cis)))+ x.add_(self.attention_norm2(self.attention([self.attention_norm1(x)], freqs_cis, NAG=NAG)))normed = self.ffn_norm1(x)self._apply_ffn_chunked(normed)x.add_(self.ffn_norm2(normed)); normed = None@@ -276,7 +315,7 @@ class BaseZImageTransformerBlock(ZImageTransformerBlock):hints_processed = self.control()(hints, **hints_kwargs)hidden_states = super().forward(hidden_states, **kwargs)if hints_processed is not None:- hidden_states.add_(hints_processed, alpha=context_scale); hints_processed = None+ hidden_states[:, :hints_processed.shape[1]].add_(hints_processed, alpha=context_scale)return hidden_states@@ -912,6 +951,7 @@ class ZImageTransformer2DModel(nn.Module):control_context_scale=1.0,callback=None,pipeline=None,+ NAG =None,):"""Forward pass with list-based processing (outer loop over layers, inner loop over samples)."""assert patch_size in self.all_patch_size@@ -932,6 +972,8 @@ class ZImageTransformer2DModel(nn.Module):cap_seqlen = Nonex_freqs_cis = Nonecap_freqs_cis = None+ cap_ori_len_ref = None+ cap_pad_mask_ref = Nonefor i,(x,cap_feats) in enumerate(zip(x_list, cap_feats_list)):bsz = x.shape[0]@@ -944,6 +986,8 @@ class ZImageTransformer2DModel(nn.Module):x_freqs_cis = self.rope_embedder(torch.cat(x_pos_ids[:1], dim=0))cap_freqs_cis = self.rope_embedder(torch.cat(cap_pos_ids[:1], dim=0))x_size = x_i_size+ cap_ori_len_ref = len(cap_feats)+ cap_pad_mask_ref = cap_inner_pad_mask[0]# Embed x# x_patches =x_embedded = x_embedder(torch.stack(x_patches))@@ -1016,6 +1060,28 @@ class ZImageTransformer2DModel(nn.Module):# Context refinercap_attn_mask = torch.ones((1, cap_seqlen), dtype=torch.bool, device=device)cap_freqs_i = cap_freqs_cis.unsqueeze(0)++ # NAG: prepare negative caption embedding once (it is static w.r.t. timestep).+ NAG_index = -1+ nag_enabled = NAG is not None+ if nag_enabled:+ neg_feats = NAG["neg_feats"]+ NAG_index = 0+ if len(neg_feats) < cap_ori_len_ref:+ pad_len = cap_ori_len_ref - len(neg_feats)+ neg_feats = torch.cat([neg_feats, neg_feats[-1:].repeat(pad_len, 1)], dim=0)+ elif len(neg_feats) > cap_ori_len_ref:+ neg_feats = neg_feats[:cap_ori_len_ref]+ pad_len = cap_seqlen - len(neg_feats)+ if pad_len > 0:+ neg_feats = torch.cat([neg_feats, neg_feats[-1:].repeat(pad_len, 1)], dim=0)+ neg_cap_embedded = self.cap_embedder(neg_feats.unsqueeze(0)).type_as(cap_embedded_list[0])+ neg_cap_embedded[cap_pad_mask_ref.unsqueeze(0)] = self.cap_pad_token+ for layer in self.context_refiner:+ neg_cap_embedded = layer(neg_cap_embedded, attn_mask=cap_attn_mask, freqs_cis=cap_freqs_i)+ NAG["cap_embed_len"] = neg_cap_embedded.shape[1]+ neg_feats = None+for layer in self.context_refiner:for i, cap_i in enumerate(cap_embedded_list):cap_embedded_list[i] = layer( cap_i, attn_mask=cap_attn_mask, freqs_cis=cap_freqs_i)@@ -1024,10 +1090,13 @@ class ZImageTransformer2DModel(nn.Module):# Create unified (x + cap) for each sampleunified_list = []for i,(embedded_x, cap_embedded) in enumerate(zip(embedded_x_list, cap_embedded_list)):- unified_list.append(torch.cat([embedded_x, cap_embedded], dim=1))+ unified_list.append(torch.cat([embedded_x, cap_embedded] + ([neg_cap_embedded.expand(len(cap_embedded), -1, -1)] if i==NAG_index else []), dim=1))embedded_x_list[i] = None- unified_freqs_i = torch.cat([x_freqs_i, cap_freqs_i], dim=1)- unified_seqlen = x_seqlen + cap_seqlen+ neg_cap_embedded = None+ unified_freqs_i = torch.cat([x_freqs_i, cap_freqs_i] + ([cap_freqs_i] if nag_enabled else []) , dim=1)+ unified_seqlen = x_seqlen + cap_seqlen+ control_seqlen = unified_seqlen+ if nag_enabled: unified_seqlen += cap_seqlenunified_attn_mask = torch.ones((1, unified_seqlen), dtype=torch.bool, device=device)hints_list = []hints_kwargs_list = []@@ -1036,21 +1105,21 @@ class ZImageTransformer2DModel(nn.Module):if any_control:cap_embedded_ref = cap_embedded_list[0]control_kwargs = dict(- attn_mask=unified_attn_mask,- freqs_cis=unified_freqs_i,+ attn_mask=unified_attn_mask[:, :control_seqlen],+ freqs_cis=unified_freqs_i[:, :control_seqlen],adaln_input=adaln_input.expand(num_noise_samples, -1),)if control_V2:# Control v2.0for unified_batch, ctrl_ctx_tensor, ctrl_seqlens in zip(unified_list, ctrl_ctx_tensor_list, ctrl_seqlens_list):- hints_kwargs, hints_tuple = self.prepare_forward_control_2_0_layers( unified_batch, cap_embedded_ref , ctrl_ctx_tensor, ctrl_seqlens, control_kwargs )+ hints_kwargs, hints_tuple = self.prepare_forward_control_2_0_layers( unified_batch[:, :control_seqlen], cap_embedded_ref , ctrl_ctx_tensor, ctrl_seqlens, control_kwargs )hints_kwargs_list.append(hints_kwargs)hints_list.append([hints_tuple])unified_batch = ctrl_ctx_tensor = ctrl_seqlens = Noneelse:# Control v1.0for unified_batch, ctrl_ctx_tensor in zip(unified_list, control_context_list):- hints_kwargs, hints_tuple = self.prepare_forward_control_1_0( unified_batch, cap_embedded_ref, ctrl_ctx_tensor, control_kwargs, t=adaln_input, patch_size=patch_size, f_patch_size=f_patch_size)+ hints_kwargs, hints_tuple = self.prepare_forward_control_1_0( unified_batch[:, :control_seqlen], cap_embedded_ref, ctrl_ctx_tensor, control_kwargs, t=adaln_input, patch_size=patch_size, f_patch_size=f_patch_size)hints_kwargs_list.append(hints_kwargs)hints_list.append([hints_tuple])unified_batch = ctrl_ctx_tensor = None@@ -1070,6 +1139,7 @@ class ZImageTransformer2DModel(nn.Module):kwargs = {}if hints is not None:kwargs.update(dict(hints= hints, hints_kwargs = hints_kwargs, context_scale = control_context_scale))+ if NAG_index == i: kwargs["NAG"]= NAGunified_list[i] = layer(unified_i, attn_mask=unified_attn_mask, freqs_cis=unified_freqs_i, adaln_input=adaln_input, **kwargs)unified_i = hints = kwargs = hints_kwargs = Nonediff --git a/wgp.py b/wgp.pyindex 784a958..02d8e66 100644--- a/wgp.py+++ b/wgp.py@@ -83,7 +83,7 @@ global_queue_ref = []AUTOSAVE_FILENAME = "queue.zip"PROMPT_VARS_MAX = 10target_mmgp_version = "3.6.9"-WanGP_version = "9.85"+WanGP_version = "9.86"settings_version = 2.41max_source_video_frames = 3000prompt_enhancer_image_caption_model, prompt_enhancer_image_caption_processor, prompt_enhancer_llm_model, prompt_enhancer_llm_tokenizer = None, None, None, None@@ -7048,7 +7048,7 @@ def prepare_inputs_dict(target, inputs, model_type = None, model_filename = Noneif not model_def.get("adaptive_projected_guidance", False):pop += ["apg_switch"]- if not model_family == "wan" or diffusion_forcing:+ if not model_def.get("NAG", False):pop +=["NAG_scale", "NAG_tau", "NAG_alpha" ]for k in pop:@@ -8973,12 +8973,12 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Noncontrol_net_weight_alt = gr.Slider(0.0, 2.0, value=ui_get("control_net_weight_alt"), step=0.01, label=control_net_weight_alt_name + " Weight", visible=len(control_net_weight_alt_name) >0, show_reset_button= False)with gr.Row(visible = not (hunyuan_t2v or hunyuan_i2v or no_negative_prompt)) as negative_prompt_row:negative_prompt = gr.Textbox(label="Negative Prompt (ignored if no Guidance that is if CFG = 1)", value=ui_get("negative_prompt") )- with gr.Column(visible = vace or t2v or test_class_i2v(model_type)) as NAG_col:+ with gr.Column(visible = model_def.get("NAG", False)) as NAG_col:gr.Markdown("<B>NAG enforces Negative Prompt even if no Guidance is set (CFG = 1), set NAG Scale to > 1 to enable it</B>")with gr.Row():- NAG_scale = gr.Slider(1.0, 20.0, value=ui_get("NAG_scale"), step=0.1, label="NAG Scale", visible = True, show_reset_button= False)- NAG_tau = gr.Slider(1.0, 5.0, value=ui_get("NAG_tau"), step=0.1, label="NAG Tau", visible = True, show_reset_button= False)- NAG_alpha = gr.Slider(0.0, 2.0, value=ui_get("NAG_alpha"), step=0.1, label="NAG Alpha", visible = True, show_reset_button= False)+ NAG_scale = gr.Slider(1.0, 20.0, value=ui_get("NAG_scale"), step=0.01, label="NAG Scale", visible = True, show_reset_button= False)+ NAG_tau = gr.Slider(1.0, 5.0, value=ui_get("NAG_tau"), step=0.01, label="NAG Tau", visible = True, show_reset_button= False)+ NAG_alpha = gr.Slider(0.0, 2.0, value=ui_get("NAG_alpha"), step=0.01, label="NAG Alpha", visible = True, show_reset_button= False)with gr.Row():repeat_generation = gr.Slider(1, 25.0, value=ui_get("repeat_generation"), step=1, label=f"Num. of Generated {'Audio Files' if audio_only else 'Videos'} per Prompt", visible = not image_outputs, show_reset_button= False)multi_images_gen_type = gr.Dropdown( value=ui_get("multi_images_gen_type"),