Skip to content
Python

add NAG support for Z-Image

DeepBeepMeep7a09f617eb25
diff --git a/README.md b/README.md
index 8e36747..84770cd 100644
--- a/README.md
+++ b/README.md
@@ -69,7 +69,8 @@ Processing Settings can be useful to do some quick gen / testing if you don't ne
*update 9.82*: added Settings headless processing, output file customization, refactored Task edition and queue processing\
*update 9.83*: Qwen Edit+ upgraded: no more any zoom out at 1080p, enabled mask, enabled image refs with inpainting\
*update 9.84*: added Wan-Move support\
-*update 9.85*: added Z-Image Control net v2
+*update 9.85*: added Z-Image Control net v2\
+*update 9.86*: added NAG support for Z-Image
### December 4 2025: WanGP v9.74, The Alpha & the Omega ... and the Dancer
diff --git a/models/wan/wan_handler.py b/models/wan/wan_handler.py
index b3f7606..0e9f955 100644
--- a/models/wan/wan_handler.py
+++ b/models/wan/wan_handler.py
@@ -580,6 +580,7 @@ class family_handler():
extra_model_def["background_removal_color"] = [128, 128, 128]
if base_model_type in ["fantasy"] or multitalk:
extra_model_def["audio_guidance"] = True
+ extra_model_def["NAG"] = vace_class or t2v or i2v
if test_oneframe_overlap(base_model_type):
extra_model_def["sliding_window_defaults"] = { "overlap_min" : 1, "overlap_max" : 1, "overlap_step": 0, "overlap_default": 1}
diff --git a/models/z_image/pipeline_z_image.py b/models/z_image/pipeline_z_image.py
index 765e44f..1a9c1c5 100644
--- a/models/z_image/pipeline_z_image.py
+++ b/models/z_image/pipeline_z_image.py
@@ -25,6 +25,7 @@ from diffusers.pipelines.pipeline_utils import DiffusionPipeline
from diffusers.schedulers import FlowMatchEulerDiscreteScheduler
from diffusers.utils import logging, replace_example_docstring
from diffusers.utils.torch_utils import randn_tensor
+from mmgp import offload
from .z_image_transformer2d import ZImageTransformer2DModel
from .pipeline_output import ZImagePipelineOutput
from shared.utils.utils import get_outpainting_frame_location, resize_lanczos, calculate_new_dimensions, convert_image_to_tensor, fit_image_into_canvas
@@ -331,6 +332,9 @@ class ZImagePipeline(DiffusionPipeline, FromSingleFileMixin):
inpaint_mask: Optional[torch.Tensor] = None,
control_context_scale: float = 0.75,
input_ref_images = None,
+ NAG_scale: float = 1.0,
+ NAG_tau: float = 3.5,
+ NAG_alpha: float = 0.5,
):
r"""
Function invoked when calling the pipeline for generation.
@@ -425,6 +429,11 @@ class ZImagePipeline(DiffusionPipeline, FromSingleFileMixin):
)
self._guidance_scale = guidance_scale
+ kwargs = {}
+ NAG = None
+ if NAG_scale > 1:
+ NAG = { "scale": NAG_scale, "tau": NAG_tau, "alpha": NAG_alpha, }
+
dtype = (
self.text_encoder.dtype
if prompt_embeds is None
@@ -463,7 +472,7 @@ class ZImagePipeline(DiffusionPipeline, FromSingleFileMixin):
) = self.encode_prompt(
prompt=prompt,
negative_prompt=negative_prompt,
- do_classifier_free_guidance=self.do_classifier_free_guidance,
+ do_classifier_free_guidance=self.do_classifier_free_guidance or NAG is not None,
prompt_embeds=prompt_embeds,
negative_prompt_embeds=negative_prompt_embeds,
dtype=dtype,
@@ -472,6 +481,11 @@ class ZImagePipeline(DiffusionPipeline, FromSingleFileMixin):
max_sequence_length=max_sequence_length,
lora_scale=lora_scale,
)
+ neg_embeds_list = negative_prompt_embeds if isinstance(negative_prompt_embeds, list) else []
+ if NAG is not None and len(neg_embeds_list) > 0:
+ NAG["neg_feats"] = neg_embeds_list[0]
+ kwargs["NAG"] = NAG
+
batch_size = num_images_per_prompt
num_images_per_prompt = 1
if batch_size > 1:
@@ -479,8 +493,6 @@ class ZImagePipeline(DiffusionPipeline, FromSingleFileMixin):
if len(negative_prompt_embeds) > 0:
negative_prompt_embeds = negative_prompt_embeds * batch_size
- # if prompt_embeds is not None:
- # prompt_embeds = prompt_embeds.expand(num_images_per_prompt, -1, -1)
# 4. Prepare latent variables
num_channels_latents = self.transformer.in_channels
@@ -612,6 +624,7 @@ class ZImagePipeline(DiffusionPipeline, FromSingleFileMixin):
control_context_scale=control_context_scale,
callback=callback,
pipeline=self,
+ **kwargs,
)
if model_out_list is None:
@@ -642,6 +655,7 @@ class ZImagePipeline(DiffusionPipeline, FromSingleFileMixin):
control_context_scale=control_context_scale,
callback=callback,
pipeline=self,
+ **kwargs,
)
if model_out_list is None:
diff --git a/models/z_image/z_image_handler.py b/models/z_image/z_image_handler.py
index da3e1e6..2eea21a 100644
--- a/models/z_image/z_image_handler.py
+++ b/models/z_image/z_image_handler.py
@@ -18,7 +18,6 @@ class family_handler:
"guidance_max_phases": 0,
"fit_into_canvas_image_refs": 0,
"profiles_dir": [],
- "no_negative_prompt": True,
}
if base_model_type in ["z_image_control", "z_image_control2"]:
@@ -40,14 +39,14 @@ class family_handler:
"selection":[ "", "A", "NA"],
"visible": False,
}
- extra_model_def["image_ref_choices"] = {
- "choices":[("No Reference Image",""), ("Image is a Reference Image", "KI")],
- "default": "",
- "letters_filter": "KI",
- "label": "Reference Image for Inpainting",
- "visible": False,
- }
-
+ # extra_model_def["image_ref_choices"] = {
+ # "choices":[("No Reference Image",""), ("Image is a Reference Image", "KI")],
+ # "default": "",
+ # "letters_filter": "KI",
+ # "label": "Reference Image for Inpainting",
+ # "visible": False,
+ # }
+ extra_model_def["NAG"] = base_model_type in ["z_image"]
return extra_model_def
@staticmethod
@@ -152,6 +151,9 @@ class family_handler:
{
"guidance_scale": 0.0,
"num_inference_steps": ui_defaults.get("num_inference_steps", 9),
+ "NAG_scale": ui_defaults.get("NAG_scale", 1.0),
+ "NAG_tau": ui_defaults.get("NAG_tau", 3.5),
+ "NAG_alpha": ui_defaults.get("NAG_alpha", 0.5),
}
)
diff --git a/models/z_image/z_image_main.py b/models/z_image/z_image_main.py
index ac0d45b..0368313 100644
--- a/models/z_image/z_image_main.py
+++ b/models/z_image/z_image_main.py
@@ -196,6 +196,9 @@ class model_factory:
input_masks=None,
context_scale: float = [0],
input_ref_images = None,
+ NAG_scale: float = 1.0,
+ NAG_tau: float = 3.5,
+ NAG_alpha: float = 0.5,
**kwargs,
):
generator = torch.Generator(device="cuda" if torch.cuda.is_available() else "cpu")
@@ -237,7 +240,10 @@ class model_factory:
control_image=input_frames,
inpaint_mask=input_masks,
control_context_scale=None if context_scale is None else context_scale[0],
- input_ref_images= input_ref_images ,
+ input_ref_images= input_ref_images,
+ NAG_scale=NAG_scale,
+ NAG_tau=NAG_tau,
+ NAG_alpha=NAG_alpha,
)
if images is None:
diff --git a/models/z_image/z_image_transformer2d.py b/models/z_image/z_image_transformer2d.py
index cf12605..6397f79 100644
--- a/models/z_image/z_image_transformer2d.py
+++ b/models/z_image/z_image_transformer2d.py
@@ -114,7 +114,7 @@ class Attention(nn.Module):
self.norm_q = RMSNorm(self.head_dim, eps=1e-5) if qk_norm else None
self.norm_k = RMSNorm(self.head_dim, eps=1e-5) if qk_norm else None
- def forward(self, h_list: list, freqs_cis: torch.Tensor) -> torch.Tensor:
+ def forward(self, h_list: list, freqs_cis: torch.Tensor, NAG= None) -> torch.Tensor:
"""Compute self-attention with RoPE. h_list is cleared to free memory early."""
h = h_list.pop()
query = self.to_q(h)
@@ -134,10 +134,48 @@ class Attention(nn.Module):
query = apply_rotary_emb_inplace(q_list, freqs_cis)
k_list = [key]; del key
key = apply_rotary_emb_inplace(k_list, freqs_cis)
- # Compute attention
dtype = query.dtype
- out = pay_attention([query, key, value]); del query, key, value
- return self.to_out(out.flatten(2, 3).to(dtype))
+
+ # NAG for joint-attention models: transformer duplicates batch into [pos, neg] halves.
+
+ if NAG is not None:
+ nag_scale = NAG["scale"]
+ nag_alpha = NAG["alpha"]
+ nag_tau = NAG["tau"]
+ cap_embed_len = NAG["cap_embed_len"]
+
+ x_list = [query[:, :-cap_embed_len], key[:, :-cap_embed_len], value[:, :-cap_embed_len] ]
+ x_pos = pay_attention(x_list).flatten(2, 3).to(dtype)
+ query[:, -2 *cap_embed_len:-cap_embed_len] = query[:, -cap_embed_len:]
+ key[:, -2 *cap_embed_len:-cap_embed_len] = key[:, -cap_embed_len:]
+ value[:, -2 *cap_embed_len:-cap_embed_len] = value[:, -cap_embed_len:]
+ x_list = [query[:, :-cap_embed_len], key[:, :-cap_embed_len], value[:, :-cap_embed_len] ]
+ del query, key, value
+ x_neg = pay_attention(x_list).flatten(2, 3).to(dtype)
+
+ x_neg_tail = x_neg[:, -cap_embed_len:].clone()
+ x_guidance = x_neg
+ x_guidance.mul_(1 - nag_scale)
+ x_guidance.add_(x_pos, alpha=nag_scale)
+ norm_positive = torch.norm(x_pos, p=1, dim=-1, keepdim=True)
+ norm_guidance = torch.norm(x_guidance, p=1, dim=-1, keepdim=True)
+ scale = norm_guidance / norm_positive
+ scale = torch.nan_to_num(scale, 10)
+ factor = (1 / (norm_guidance + 1e-7) * norm_positive * nag_tau).to(x_guidance.dtype)
+ x_guidance = torch.where(scale > nag_tau, x_guidance * factor, x_guidance).to(dtype)
+ del norm_positive, norm_guidance, scale, factor
+
+ x_guidance.mul_(nag_alpha)
+ x_guidance.add_(x_pos, alpha=(1 - nag_alpha))
+ x_pos = None
+ out = torch.cat([x_guidance, x_neg_tail], dim=1)
+ x_pos = x_neg = x_guidance = None
+ else:
+ x_list = [query, key, value]
+ del query, key, value
+ out = pay_attention(x_list).flatten(2, 3).to(dtype);
+
+ return self.to_out(out)
class ZImageTransformerBlock(nn.Module):
@@ -189,6 +227,7 @@ class ZImageTransformerBlock(nn.Module):
attn_mask: torch.Tensor,
freqs_cis: torch.Tensor,
adaln_input: Optional[torch.Tensor] = None,
+ NAG = None,
):
if self.modulation:
scale_msa, gate_msa, scale_mlp, gate_mlp = self.adaLN_modulation(adaln_input).unsqueeze(1).chunk(4, dim=2)
@@ -196,7 +235,7 @@ class ZImageTransformerBlock(nn.Module):
scale_msa.add_(1.0)
normed = self.attention_norm1(x)
normed.mul_(scale_msa)
- attn_out = self.attention_norm2(self.attention([normed], freqs_cis))
+ attn_out = self.attention_norm2(self.attention([normed], freqs_cis, NAG=NAG))
attn_out.mul_(gate_msa.tanh_())
x.add_(attn_out); attn_out = None
# In-place modulation for FFN block (chunked)
@@ -208,7 +247,7 @@ class ZImageTransformerBlock(nn.Module):
ffn_out.mul_(gate_mlp.tanh_())
x.add_(ffn_out); ffn_out = None
else:
- x.add_(self.attention_norm2(self.attention([self.attention_norm1(x)], freqs_cis)))
+ x.add_(self.attention_norm2(self.attention([self.attention_norm1(x)], freqs_cis, NAG=NAG)))
normed = self.ffn_norm1(x)
self._apply_ffn_chunked(normed)
x.add_(self.ffn_norm2(normed)); normed = None
@@ -276,7 +315,7 @@ class BaseZImageTransformerBlock(ZImageTransformerBlock):
hints_processed = self.control()(hints, **hints_kwargs)
hidden_states = super().forward(hidden_states, **kwargs)
if hints_processed is not None:
- hidden_states.add_(hints_processed, alpha=context_scale); hints_processed = None
+ hidden_states[:, :hints_processed.shape[1]].add_(hints_processed, alpha=context_scale)
return hidden_states
@@ -912,6 +951,7 @@ class ZImageTransformer2DModel(nn.Module):
control_context_scale=1.0,
callback=None,
pipeline=None,
+ NAG =None,
):
"""Forward pass with list-based processing (outer loop over layers, inner loop over samples)."""
assert patch_size in self.all_patch_size
@@ -932,6 +972,8 @@ class ZImageTransformer2DModel(nn.Module):
cap_seqlen = None
x_freqs_cis = None
cap_freqs_cis = None
+ cap_ori_len_ref = None
+ cap_pad_mask_ref = None
for i,(x,cap_feats) in enumerate(zip(x_list, cap_feats_list)):
bsz = x.shape[0]
@@ -944,6 +986,8 @@ class ZImageTransformer2DModel(nn.Module):
x_freqs_cis = self.rope_embedder(torch.cat(x_pos_ids[:1], dim=0))
cap_freqs_cis = self.rope_embedder(torch.cat(cap_pos_ids[:1], dim=0))
x_size = x_i_size
+ cap_ori_len_ref = len(cap_feats)
+ cap_pad_mask_ref = cap_inner_pad_mask[0]
# Embed x
# x_patches =
x_embedded = x_embedder(torch.stack(x_patches))
@@ -1016,6 +1060,28 @@ class ZImageTransformer2DModel(nn.Module):
# Context refiner
cap_attn_mask = torch.ones((1, cap_seqlen), dtype=torch.bool, device=device)
cap_freqs_i = cap_freqs_cis.unsqueeze(0)
+
+ # NAG: prepare negative caption embedding once (it is static w.r.t. timestep).
+ NAG_index = -1
+ nag_enabled = NAG is not None
+ if nag_enabled:
+ neg_feats = NAG["neg_feats"]
+ NAG_index = 0
+ if len(neg_feats) < cap_ori_len_ref:
+ pad_len = cap_ori_len_ref - len(neg_feats)
+ neg_feats = torch.cat([neg_feats, neg_feats[-1:].repeat(pad_len, 1)], dim=0)
+ elif len(neg_feats) > cap_ori_len_ref:
+ neg_feats = neg_feats[:cap_ori_len_ref]
+ pad_len = cap_seqlen - len(neg_feats)
+ if pad_len > 0:
+ neg_feats = torch.cat([neg_feats, neg_feats[-1:].repeat(pad_len, 1)], dim=0)
+ neg_cap_embedded = self.cap_embedder(neg_feats.unsqueeze(0)).type_as(cap_embedded_list[0])
+ neg_cap_embedded[cap_pad_mask_ref.unsqueeze(0)] = self.cap_pad_token
+ for layer in self.context_refiner:
+ neg_cap_embedded = layer(neg_cap_embedded, attn_mask=cap_attn_mask, freqs_cis=cap_freqs_i)
+ NAG["cap_embed_len"] = neg_cap_embedded.shape[1]
+ neg_feats = None
+
for layer in self.context_refiner:
for i, cap_i in enumerate(cap_embedded_list):
cap_embedded_list[i] = layer( cap_i, attn_mask=cap_attn_mask, freqs_cis=cap_freqs_i)
@@ -1024,10 +1090,13 @@ class ZImageTransformer2DModel(nn.Module):
# Create unified (x + cap) for each sample
unified_list = []
for i,(embedded_x, cap_embedded) in enumerate(zip(embedded_x_list, cap_embedded_list)):
- unified_list.append(torch.cat([embedded_x, cap_embedded], dim=1))
+ unified_list.append(torch.cat([embedded_x, cap_embedded] + ([neg_cap_embedded.expand(len(cap_embedded), -1, -1)] if i==NAG_index else []), dim=1))
embedded_x_list[i] = None
- unified_freqs_i = torch.cat([x_freqs_i, cap_freqs_i], dim=1)
- unified_seqlen = x_seqlen + cap_seqlen
+ neg_cap_embedded = None
+ unified_freqs_i = torch.cat([x_freqs_i, cap_freqs_i] + ([cap_freqs_i] if nag_enabled else []) , dim=1)
+ unified_seqlen = x_seqlen + cap_seqlen
+ control_seqlen = unified_seqlen
+ if nag_enabled: unified_seqlen += cap_seqlen
unified_attn_mask = torch.ones((1, unified_seqlen), dtype=torch.bool, device=device)
hints_list = []
hints_kwargs_list = []
@@ -1036,21 +1105,21 @@ class ZImageTransformer2DModel(nn.Module):
if any_control:
cap_embedded_ref = cap_embedded_list[0]
control_kwargs = dict(
- attn_mask=unified_attn_mask,
- freqs_cis=unified_freqs_i,
+ attn_mask=unified_attn_mask[:, :control_seqlen],
+ freqs_cis=unified_freqs_i[:, :control_seqlen],
adaln_input=adaln_input.expand(num_noise_samples, -1),
)
if control_V2:
# Control v2.0
for unified_batch, ctrl_ctx_tensor, ctrl_seqlens in zip(unified_list, ctrl_ctx_tensor_list, ctrl_seqlens_list):
- hints_kwargs, hints_tuple = self.prepare_forward_control_2_0_layers( unified_batch, cap_embedded_ref , ctrl_ctx_tensor, ctrl_seqlens, control_kwargs )
+ hints_kwargs, hints_tuple = self.prepare_forward_control_2_0_layers( unified_batch[:, :control_seqlen], cap_embedded_ref , ctrl_ctx_tensor, ctrl_seqlens, control_kwargs )
hints_kwargs_list.append(hints_kwargs)
hints_list.append([hints_tuple])
unified_batch = ctrl_ctx_tensor = ctrl_seqlens = None
else:
# Control v1.0
for unified_batch, ctrl_ctx_tensor in zip(unified_list, control_context_list):
- hints_kwargs, hints_tuple = self.prepare_forward_control_1_0( unified_batch, cap_embedded_ref, ctrl_ctx_tensor, control_kwargs, t=adaln_input, patch_size=patch_size, f_patch_size=f_patch_size)
+ hints_kwargs, hints_tuple = self.prepare_forward_control_1_0( unified_batch[:, :control_seqlen], cap_embedded_ref, ctrl_ctx_tensor, control_kwargs, t=adaln_input, patch_size=patch_size, f_patch_size=f_patch_size)
hints_kwargs_list.append(hints_kwargs)
hints_list.append([hints_tuple])
unified_batch = ctrl_ctx_tensor = None
@@ -1070,6 +1139,7 @@ class ZImageTransformer2DModel(nn.Module):
kwargs = {}
if hints is not None:
kwargs.update(dict(hints= hints, hints_kwargs = hints_kwargs, context_scale = control_context_scale))
+ if NAG_index == i: kwargs["NAG"]= NAG
unified_list[i] = layer(unified_i, attn_mask=unified_attn_mask, freqs_cis=unified_freqs_i, adaln_input=adaln_input, **kwargs)
unified_i = hints = kwargs = hints_kwargs = None
diff --git a/wgp.py b/wgp.py
index 784a958..02d8e66 100644
--- a/wgp.py
+++ b/wgp.py
@@ -83,7 +83,7 @@ global_queue_ref = []
AUTOSAVE_FILENAME = "queue.zip"
PROMPT_VARS_MAX = 10
target_mmgp_version = "3.6.9"
-WanGP_version = "9.85"
+WanGP_version = "9.86"
settings_version = 2.41
max_source_video_frames = 3000
prompt_enhancer_image_caption_model, prompt_enhancer_image_caption_processor, prompt_enhancer_llm_model, prompt_enhancer_llm_tokenizer = None, None, None, None
@@ -7048,7 +7048,7 @@ def prepare_inputs_dict(target, inputs, model_type = None, model_filename = None
if not model_def.get("adaptive_projected_guidance", False):
pop += ["apg_switch"]
- if not model_family == "wan" or diffusion_forcing:
+ if not model_def.get("NAG", False):
pop +=["NAG_scale", "NAG_tau", "NAG_alpha" ]
for k in pop:
@@ -8973,12 +8973,12 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Non
control_net_weight_alt = gr.Slider(0.0, 2.0, value=ui_get("control_net_weight_alt"), step=0.01, label=control_net_weight_alt_name + " Weight", visible=len(control_net_weight_alt_name) >0, show_reset_button= False)
with gr.Row(visible = not (hunyuan_t2v or hunyuan_i2v or no_negative_prompt)) as negative_prompt_row:
negative_prompt = gr.Textbox(label="Negative Prompt (ignored if no Guidance that is if CFG = 1)", value=ui_get("negative_prompt") )
- with gr.Column(visible = vace or t2v or test_class_i2v(model_type)) as NAG_col:
+ with gr.Column(visible = model_def.get("NAG", False)) as NAG_col:
gr.Markdown("<B>NAG enforces Negative Prompt even if no Guidance is set (CFG = 1), set NAG Scale to > 1 to enable it</B>")
with gr.Row():
- NAG_scale = gr.Slider(1.0, 20.0, value=ui_get("NAG_scale"), step=0.1, label="NAG Scale", visible = True, show_reset_button= False)
- NAG_tau = gr.Slider(1.0, 5.0, value=ui_get("NAG_tau"), step=0.1, label="NAG Tau", visible = True, show_reset_button= False)
- NAG_alpha = gr.Slider(0.0, 2.0, value=ui_get("NAG_alpha"), step=0.1, label="NAG Alpha", visible = True, show_reset_button= False)
+ NAG_scale = gr.Slider(1.0, 20.0, value=ui_get("NAG_scale"), step=0.01, label="NAG Scale", visible = True, show_reset_button= False)
+ NAG_tau = gr.Slider(1.0, 5.0, value=ui_get("NAG_tau"), step=0.01, label="NAG Tau", visible = True, show_reset_button= False)
+ NAG_alpha = gr.Slider(0.0, 2.0, value=ui_get("NAG_alpha"), step=0.01, label="NAG Alpha", visible = True, show_reset_button= False)
with gr.Row():
repeat_generation = gr.Slider(1, 25.0, value=ui_get("repeat_generation"), step=1, label=f"Num. of Generated {'Audio Files' if audio_only else 'Videos'} per Prompt", visible = not image_outputs, show_reset_button= False)
multi_images_gen_type = gr.Dropdown( value=ui_get("multi_images_gen_type"),