Skip to content
Python

added pi-flux 2 support

DeepBeepMeepe86b92dac246
diff --git a/README.md b/README.md
index f07b22a..838ead4 100644
--- a/README.md
+++ b/README.md
@@ -36,6 +36,15 @@ WanGP supports the Wan (and derived models), Hunyuan Video and LTV Video models
-----
## 🔥 Latest Updates :
+### December 19 2025: WanGP v9.9, Early Christmas
+
+- **SCAIL Preview**: enjoy this *Wan Animate*, *Steady Dancer* contender that can support multiple people. Thanks to its 3D positioning, it can take into account which parts of the body are hidden and which are not.
+
+WanGP version has the following perks: 3D pose Preprocessing entirely rewritten to be fast, and compatible with any pytorch version, very Low VRAM requirements for multicharacters, experimental long gen mode / sliding windows (SCAIL Preview doesnt support officialy long gen yet)
+
+- **pi-Flux 2**: you don't Flux 2 because you find it too slow ? You won't be able to use this excuse anymore: pi-Flux 2 is *4 steps distills* of the best image generator. It supports both image edition and text to image generation.
+
+- **loras accelerator**: *loras accelerator* for *Qwen* and *Wan i2v* have been added (activable using the *Profile settings* as usual)
### December 14 2025: WanGP v9.86, Simple Pleasures...
diff --git a/defaults/pi_flux2.json b/defaults/pi_flux2.json
new file mode 100644
index 0000000..f224c71
--- /dev/null
+++ b/defaults/pi_flux2.json
@@ -0,0 +1,17 @@
+{
+ "model": {
+ "name": "pi-FLUX.2 Dev 32B",
+ "architecture": "pi_flux2",
+ "modules": [ [ "https://huggingface.co/DeepBeepMeep/Flux2/resolve/main/pi_flux2_heads_bf16.safetensors"] ],
+ "description": "pi-FLUX.2 is a 4-step distilled version of FLUX.2-dev using the pi-Flow method with Gaussian mixture velocity field prediction. Supports text-to-image and image editing.",
+ "URLs": "flux2_dev",
+ "preload_URLs": ["https://huggingface.co/DeepBeepMeep/Flux2/resolve/main/gmflux2_k8_piid_4step_lora.safetensors"],
+ "image_outputs": true,
+ "no_negative_prompt": true
+ },
+ "prompt": "a beautiful sunset over the ocean with vibrant colors",
+ "resolution": "1024x1024",
+ "batch_size": 1,
+ "embedded_guidance_scale": 4,
+ "sampling_steps": 4
+}
diff --git a/models/flux/flux_handler.py b/models/flux/flux_handler.py
index 9e7b173..9488a47 100644
--- a/models/flux/flux_handler.py
+++ b/models/flux/flux_handler.py
@@ -4,7 +4,7 @@ from PIL import Image
from shared.utils import files_locator as fl
def test_flux2(base_model_type):
- return base_model_type in ["flux2_dev"]
+ return base_model_type in ["flux2_dev", "pi_flux2"]
def get_flux_text_encoder_filename(text_encoder_quantization, base_model_type):
@@ -23,6 +23,7 @@ class family_handler():
return [
"flux",
"flux2_dev",
+ "pi_flux2",
"flux_chroma",
"flux_chroma_radiance",
"flux_dev_kontext",
@@ -44,18 +45,20 @@ class family_handler():
"flux_chroma_radiance": "flux",
"flux_dev_kontext_dreamomni2": "flux",
"flux2_dev": "flux",
+ "pi_flux2": "flux",
}
- models_comp_map = {
- "flux": ["flux2_dev", "flux_chroma", "flux_chroma_radiance", "flux_dev_kontext", "flux_dev_umo", "flux_dev_uso", "flux_schnell", "flux_dev_kontext_dreamomni2" ]
+ models_comp_map = {
+ "flux": ["flux2_dev", "pi_flux2", "flux_chroma", "flux_chroma_radiance", "flux_dev_kontext", "flux_dev_umo", "flux_dev_uso", "flux_schnell", "flux_dev_kontext_dreamomni2" ]
}
return models_eqv_map, models_comp_map
@staticmethod
def query_model_def(base_model_type, model_def):
flux_model = "flux-dev" if base_model_type == "flux" else base_model_type.replace("_", "-")
- flux2 = flux_model.startswith("flux2")
- flux_schnell = flux_model == "flux-schnell"
- flux_chroma = flux_model == "flux-chroma"
+ pi_flux2 = flux_model == "pi-flux2"
+ flux2 = flux_model.startswith("flux2") or pi_flux2
+ flux_schnell = flux_model == "flux-schnell"
+ flux_chroma = flux_model == "flux-chroma"
flux_chroma_radiance = flux_model == "flux-chroma-radiance"
flux_uso = flux_model == "flux-dev-uso"
flux_umo = flux_model == "flux-dev-umo"
@@ -109,7 +112,7 @@ class family_handler():
}
if flux2:
- extra_model_def["group"] ="flux2"
+ extra_model_def["group"] ="flux2"
extra_model_def["no_background_removal"] = True
# extra_model_def["inpaint_support"] = True
extra_model_def["mask_preprocessing"] = {
@@ -131,6 +134,10 @@ class family_handler():
# extra_model_def["guide_inpaint_color"] = 0
# extra_model_def["video_guide_outpainting"] = [1,2]
+ if pi_flux2:
+ extra_model_def["piflow"] = True
+ extra_model_def["inpaint_support"] = True
+
extra_model_def["fit_into_canvas_image_refs"] = 0
return extra_model_def
diff --git a/models/flux/flux_main.py b/models/flux/flux_main.py
index 7d2f761..a11520d 100644
--- a/models/flux/flux_main.py
+++ b/models/flux/flux_main.py
@@ -6,7 +6,7 @@ from glob import iglob
from mmgp import offload as offload
import torch
from shared.utils.utils import calculate_new_dimensions
-from .sampling import denoise, get_schedule, get_schedule_flux2, prepare_kontext, prepare_prompt, prepare_multi_ip, unpack, resizeinput, patches_to_image, build_mask
+from .sampling import denoise, get_schedule, get_schedule_flux2, get_schedule_piflux2, prepare_kontext, prepare_prompt, prepare_multi_ip, unpack, resizeinput, patches_to_image, build_mask
from .modules.layers import get_linear_split_map
from transformers import SiglipVisionModel, SiglipImageProcessor
import torchvision.transforms.functional as TVF
@@ -94,16 +94,16 @@ class model_factory:
self.dtype = dtype
torch_device = "cpu"
self.model_def = model_def
+ self.guidance_max_phases = model_def.get("guidance_max_phases", 0)
self.name = model_def.get("flux-model", "flux-dev")
- self.guidance_max_phases = model_def.get("guidance_max_phases", 0)
- self.is_flux2 = self.name.startswith("flux2")
+ self.is_piflux2 = self.name == "pi-flux2"
+ self.is_flux2 = self.name.startswith("flux2") or self.is_piflux2
# model_filename = ["c:/temp/flux1-schnell.safetensors"]
source = model_def.get("source", None)
self.clip = self.t5 = self.vision_encoder = self.mistal = None
if self.is_flux2:
- self.model = load_flow_model(self.name, model_filename[0] if source is None else source, torch_device)
-
+ self.model = load_flow_model(self.name, model_filename if source is None else source, torch_device)
from .modules.text_encoder_mistral import Mistral3SmallEmbedder
self.mistral = Mistral3SmallEmbedder( model_spec = text_encoder_filename)
@@ -275,7 +275,10 @@ class model_factory:
inp.update({"img_cond_seq": cond_latents, "img_cond_seq_ids": cond_ids})
noise_patch_size = 2
- timesteps = get_schedule_flux2(sampling_steps, inp["img"].shape[1])
+ if self.is_piflux2:
+ timesteps = get_schedule_piflux2(sampling_steps, inp["img"].shape[1])
+ else:
+ timesteps = get_schedule_flux2(sampling_steps, inp["img"].shape[1])
unpack_latent = lambda x : self.vae.pre_decode(torch.cat(scatter_ids(x, inp["img_ids"])).squeeze(2))
ref_style_imgs = []
image_mask = None
@@ -386,6 +389,7 @@ class model_factory:
timesteps=timesteps,
guidance=embedded_guidance_scale,
real_guidance_scale=guide_scale,
+ final_step_size_scale=0.5 if self.is_piflux2 else None,
callback=callback,
pipeline=self,
loras_slists=loras_slists,
@@ -411,9 +415,22 @@ class model_factory:
return x
def get_loras_transformer(self, get_model_recursive_prop, model_type, model_mode, video_prompt_type, **kwargs):
- if model_type != "flux_dev_kontext_dreamomni2": return [], []
+ def resolve_preload_lora(lora_ref: str) -> str:
+ resolved = fl.locate_file(lora_ref, error_if_none=False)
+ if resolved is None:
+ resolved = fl.locate_file(os.path.basename(lora_ref))
+ return resolved
preloadURLs = get_model_recursive_prop(model_type, "preload_URLs")
- if len(preloadURLs) < 2: return [], []
+ if self.is_piflux2:
+ if len(preloadURLs) < 1:
+ return [], []
+ return [resolve_preload_lora(preloadURLs[0])], [1]
+
+ if model_type != "flux_dev_kontext_dreamomni2":
+ return [], []
+
+ if len(preloadURLs) < 2:
+ return [], []
edit = "K" in video_prompt_type
- return [ fl.locate_file(os.path.basename(preloadURLs[0 if edit else 1]))] , [1]
+ return [resolve_preload_lora(preloadURLs[0 if edit else 1])], [1]
diff --git a/models/flux/model.py b/models/flux/model.py
index 79ba5ee..3157bf0 100644
--- a/models/flux/model.py
+++ b/models/flux/model.py
@@ -1,4 +1,5 @@
from dataclasses import dataclass
+import math
import torch
from torch import Tensor, nn
@@ -51,6 +52,7 @@ class FluxParams:
double_mlp_ratio: float | None = None
double_linear1_mlp_ratio: float | None = None
flux2: bool = False
+ piflow: bool = False
class Flux(nn.Module):
"""
@@ -92,6 +94,7 @@ class Flux(nn.Module):
self.chroma = params.chroma
self.is_flux2 = getattr(params, "flux2", False)
self.radiance = getattr(params, "radiance", False)
+ self.piflow = getattr(params, "piflow", False)
if params.hidden_size % params.num_heads != 0:
raise ValueError(
f"Hidden size {params.hidden_size} must be divisible by num_heads {params.num_heads}"
@@ -183,6 +186,20 @@ class Flux(nn.Module):
modulation_bias=not self.is_flux2,
)
+ if self.piflow:
+ # Pi-Flow prediction heads for Gaussian mixture velocity field.
+ self.proj_out_means = nn.Linear(self.hidden_size, 1024, bias=True)
+ self.proj_out_logweights = nn.Linear(self.hidden_size, 32, bias=True)
+ self.proj_out_logstds = nn.Sequential(
+ nn.Identity(),
+ nn.Linear(self.hidden_size, 1024, bias=True),
+ nn.SiLU(),
+ nn.Linear(1024, 1, bias=True),
+ )
+ self.num_gaussians = self.proj_out_means.out_features // self.out_channels
+ self.logweights_channels = self.proj_out_logweights.out_features // self.num_gaussians
+ self.piflow_patch_size = int(math.sqrt(self.logweights_channels))
+
def _apply_final_layer(self, tokens: Tensor, vec):
final_layer = self.final_layer
normed = final_layer.norm_final(tokens)
@@ -198,6 +215,54 @@ class Flux(nn.Module):
base_tokens = final_layer.linear(modulated)
return modulated, base_tokens
+ def _apply_piflow_final_layer(self, tokens: Tensor, vec: Tensor, img_ids: Tensor, img_len: int):
+ if img_ids is None:
+ raise RuntimeError("pi-Flow requires image ids to reshape outputs.")
+ final_layer = self.final_layer
+ normed = final_layer.norm_final(tokens)
+ shift, scale = final_layer.adaLN_modulation(vec).chunk(2, dim=1)
+ modulated = torch.addcmul(shift[:, None, :], 1 + scale[:, None, :], normed)
+
+ proj_dtype = self.proj_out_means.weight.dtype
+ modulated = modulated.to(proj_dtype)
+ means = self.proj_out_means(modulated)
+ logweights = self.proj_out_logweights(modulated)
+ logstds = self.proj_out_logstds(vec.detach().to(self.proj_out_logstds[-1].weight.dtype))
+
+ base_img_ids = img_ids[:, :img_len]
+ h_len = int(base_img_ids[..., 1].max().item() + 1)
+ w_len = int(base_img_ids[..., 2].max().item() + 1)
+ if h_len * w_len != img_len:
+ raise RuntimeError("pi-Flow token length does not match latent grid.")
+
+ patch_size = self.piflow_patch_size
+ if patch_size * patch_size != self.logweights_channels:
+ raise RuntimeError("pi-Flow logweights channels mismatch.")
+
+ bsz = means.shape[0]
+ k = self.num_gaussians
+ c = self.out_channels
+ c_unpacked = c // (patch_size * patch_size)
+
+ means = means.view(
+ bsz, h_len, w_len, k, c_unpacked, patch_size, patch_size
+ ).permute(
+ 0, 3, 4, 1, 5, 2, 6
+ ).reshape(
+ bsz, k, c_unpacked, h_len * patch_size, w_len * patch_size
+ )
+
+ logweights = logweights.view(
+ bsz, h_len, w_len, k, 1, patch_size, patch_size
+ ).permute(
+ 0, 3, 4, 1, 5, 2, 6
+ ).reshape(
+ bsz, k, 1, h_len * patch_size, w_len * patch_size
+ ).log_softmax(dim=1)
+
+ logstds = logstds.reshape(bsz, 1, 1, 1, 1)
+ return modulated, {"means": means, "logweights": logweights, "logstds": logstds}
+
def preprocess_loras(self, model_type, sd):
new_sd = {}
if len(sd) == 0: return sd
@@ -250,7 +315,7 @@ class Flux(nn.Module):
"img_in", "txt_in", "final_layer.linear", "time_in.in_layer", "time_in.out_layer" ]
double_src = ["norm1.linear", "norm1_context.linear", "attn.norm_q", "attn.norm_k", "ff.net.0.proj", "ff.net.2", "ff_context.net.0.proj", "ff_context.net.2", "attn.to_out.0" ,"attn.to_add_out", "attn.to_out", ".attn.to_", ".attn.add_q_proj.", ".attn.add_k_proj.", ".attn.add_v_proj.", ".ff_context.linear_out.", ".ff_context.linear_in.", ".ff.linear_out.", ".ff.linear_in." ]
- double_tgt = ["img_mod.lin", "txt_mod.lin", "img_attn.norm.query_norm", "img_attn.norm.key_norm", "img_mlp.0", "img_mlp.2", "txt_mlp.0", "txt_mlp.2", "img_attn.proj", "txt_attn.proj", "img_attn.proj", ".img_attn.", ".txt_attn.q.", ".txt_attn.k.", ".txt_attn.v.", ".txt_mlp.0.", ".txt_mlp.2.", ".img_mlp.0.", ".img_mlp.2." ]
+ double_tgt = ["img_mod.lin", "txt_mod.lin", "img_attn.norm.query_norm", "img_attn.norm.key_norm", "img_mlp.0", "img_mlp.2", "txt_mlp.0", "txt_mlp.2", "img_attn.proj", "txt_attn.proj", "img_attn.proj", ".img_attn.", ".txt_attn.q.", ".txt_attn.k.", ".txt_attn.v.", ".txt_mlp.2.", ".txt_mlp.0.", ".img_mlp.2.", ".img_mlp.0." ]
single_src = ["norm.linear", "attn.norm_q", "attn.norm_k", "proj_out",".attn.to_q.", ".attn.to_k.", ".attn.to_v.", ".proj_mlp.", ".attn.to_out."]
single_tgt = ["modulation.lin","norm.query_norm", "norm.key_norm", "linear2", ".linear1_attn_q.", ".linear1_attn_k.", ".linear1_attn_v.", ".linear1_mlp.", ".linear2."]
@@ -424,7 +489,12 @@ class Flux(nn.Module):
img_list[i] = hidden_seq = None
else:
vec = final_vecs[i]
- modulated, pred_tokens = self._apply_final_layer(hidden_seq, vec)
+ if self.piflow:
+ modulated, pred_tokens = self._apply_piflow_final_layer(
+ hidden_seq, vec, img_ids, img_len
+ )
+ else:
+ modulated, pred_tokens = self._apply_final_layer(hidden_seq, vec)
img_list[i] = hidden_seq = vec = modulated = None
out_list.append(pred_tokens)
return out_list
diff --git a/models/flux/sampling.py b/models/flux/sampling.py
index 26c3d8d..313ca0a 100644
--- a/models/flux/sampling.py
+++ b/models/flux/sampling.py
@@ -282,6 +282,197 @@ def get_schedule_flux2(num_steps: int, image_seq_len: int) -> list[float]:
return timesteps.tolist()
+def get_schedule_piflux2(num_steps: int, image_seq_len: int) -> list[float]:
+ """
+ pi-FLUX.2 FlowMapSDE schedule with shift=3.2 and final_step_size_scale=0.5.
+ """
+ if num_steps <= 0:
+ return [0.0]
+ shift = 3.2
+ final_step_size_scale = 0.5
+ end = (final_step_size_scale - 1.0) / (num_steps + final_step_size_scale - 1.0)
+ step = (end - 1.0) / num_steps
+ raw_timesteps = 1.0 + step * torch.arange(num_steps, dtype=torch.float32)
+ raw_timesteps = raw_timesteps.clamp(min=0)
+ sigmas = shift * raw_timesteps / (1 + (shift - 1) * raw_timesteps)
+ sigmas = torch.cat([sigmas, torch.zeros(1, dtype=sigmas.dtype)])
+ return sigmas.tolist()
+
+
+def _flow_map_sde_warp_t(raw_t: Tensor, shift: float) -> Tensor:
+ return shift * raw_t / (1 + (shift - 1) * raw_t)
+
+
+def _flow_map_sde_unwarp_t(sigma_t: Tensor, shift: float) -> Tensor:
+ return sigma_t / (shift + (1 - shift) * sigma_t)
+
+
+def _flow_map_sde_calculate_sigmas_dst(
+ sigmas: Tensor, h: float = 0.0, eps: float = 1e-6
+) -> tuple[Tensor, Tensor]:
+ sigmas_src = sigmas[:-1]
+ sigmas_to = sigmas[1:]
+ alphas_src = 1 - sigmas_src
+ alphas_to = 1 - sigmas_to
+
+ if h <= 0.0:
+ m_vals = torch.ones_like(sigmas_src)
+ else:
+ h2 = h * h
+ m_vals = (sigmas_to * alphas_src / (sigmas_src * alphas_to).clamp(min=eps)) ** h2
+
+ sigmas_to_mul_m = sigmas_to * m_vals
+ sigmas_dst = sigmas_to_mul_m / (alphas_to + sigmas_to_mul_m).clamp(min=eps)
+ return sigmas_dst, m_vals
+
+
+def _gmflow_posterior_mean(
+ sigma_t_src: Tensor,
+ sigma_t: Tensor,
+ x_t_src: Tensor,
+ x_t: Tensor,
+ gm_means: Tensor,
+ gm_vars: Tensor,
+ gm_logweights: Tensor,
+ *,
+ eps: float = 1e-6,
+ gm_dim: int = -4,
+ channel_dim: int = -3,
+) -> Tensor:
+ sigma_t_src = sigma_t_src.clamp(min=eps)
+ sigma_t = sigma_t.clamp(min=eps)
+
+ alpha_t_src = 1 - sigma_t_src
+ alpha_t = 1 - sigma_t
+
+ alpha_over_sigma_t_src = alpha_t_src / sigma_t_src
+ alpha_over_sigma_t = alpha_t / sigma_t
+
+ zeta = alpha_over_sigma_t.square() - alpha_over_sigma_t_src.square()
+ nu = alpha_over_sigma_t * x_t / sigma_t - alpha_over_sigma_t_src * x_t_src / sigma_t_src
+
+ nu = nu.unsqueeze(gm_dim)
+ zeta = zeta.unsqueeze(gm_dim)
+ denom = (gm_vars * zeta + 1).clamp(min=eps)
+
+ out_means = (gm_vars * nu + gm_means) / denom
+ logweights_delta = (gm_means * (nu - 0.5 * zeta * gm_means)).sum(dim=channel_dim, keepdim=True) / denom
+ out_weights = (gm_logweights + logweights_delta).softmax(dim=gm_dim)
+
+ return (out_means * out_weights).sum(dim=gm_dim)
+
+
+class _GMFlowPolicy:
+ def __init__(
+ self,
+ denoising_output: dict[str, Tensor],
+ x_t_src: Tensor,
+ sigma_t_src: Tensor,
+ eps: float = 1e-4,
+ ) -> None:
+ self.x_t_src = x_t_src
+ self.ndim = x_t_src.dim()
+ self.eps = eps
+ self.sigma_t_src = sigma_t_src.reshape(
+ *sigma_t_src.size(), *((self.ndim - sigma_t_src.dim()) * [1])
+ )
+ self.denoising_output_x_0 = self._u_to_x_0(denoising_output, self.x_t_src, self.sigma_t_src)
+
+ @staticmethod
+ def _u_to_x_0(denoising_output: dict[str, Tensor], x_t: Tensor, sigma_t: Tensor) -> dict[str, Tensor]:
+ x_t = x_t.unsqueeze(1)
+ sigma_t = sigma_t.unsqueeze(1)
+ means_x_0 = x_t - sigma_t * denoising_output["means"]
+ gm_vars = (denoising_output["logstds"] * 2).exp() * sigma_t.square()
+ return {"means": means_x_0, "gm_vars": gm_vars, "logweights": denoising_output["logweights"]}
+
+ def pi(self, x_t: Tensor, sigma_t: Tensor) -> Tensor:
+ sigma_t = sigma_t.reshape(*sigma_t.size(), *((self.ndim - sigma_t.dim()) * [1]))
+ means = self.denoising_output_x_0["means"]
+ gm_vars = self.denoising_output_x_0["gm_vars"]
+ logweights = self.denoising_output_x_0["logweights"]
+
+ if (sigma_t == self.sigma_t_src).all() and (x_t == self.x_t_src).all():
+ x_0 = (logweights.softmax(dim=1) * means).sum(dim=1)
+ else:
+ x_0 = _gmflow_posterior_mean(
+ self.sigma_t_src,
+ sigma_t,
+ self.x_t_src,
+ x_t,
+ means,
+ gm_vars,
+ logweights,
+ eps=self.eps,
+ )
+ return (x_t - x_0) / sigma_t.clamp(min=self.eps)
+
+ def temperature_(self, temperature: float) -> None:
+ if temperature >= 1.0:
+ return
+ temperature = max(temperature, self.eps)
+ gm = self.denoising_output_x_0
+ gm["logweights"] = (gm["logweights"] / temperature).log_softmax(dim=1)
+ gm["gm_vars"] = gm["gm_vars"] * temperature
+
+
+def _policy_rollout(
+ x_t_start: Tensor,
+ sigma_t_start: Tensor,
+ sigma_t_end: Tensor,
+ total_substeps: int,
+ policy: _GMFlowPolicy,
+ *,
+ shift: float = 3.2,
+) -> Tensor:
+ num_batches = x_t_start.size(0)
+ ndim = x_t_start.dim()
+ sigma_t_start = sigma_t_start.reshape(num_batches, *((ndim - 1) * [1]))
+ sigma_t_end = sigma_t_end.reshape(num_batches, *((ndim - 1) * [1]))
+
+ raw_t_start = _flow_map_sde_unwarp_t(sigma_t_start, shift)
+ raw_t_end = _flow_map_sde_unwarp_t(sigma_t_end, shift)
+ delta_raw_t = raw_t_start - raw_t_end
+ num_substeps = (delta_raw_t * total_substeps).round().to(torch.long).clamp(min=1)
+ substep_size = delta_raw_t / num_substeps
+ max_num_substeps = num_substeps.max()
+
+ raw_t = raw_t_start
+ sigma_t = sigma_t_start
+ x_t = x_t_start
+
+ for substep_id in range(max_num_substeps.item()):
+ u = policy.pi(x_t, sigma_t)
+ raw_t_minus = (raw_t - substep_size).clamp(min=0)
+ sigma_t_minus = _flow_map_sde_warp_t(raw_t_minus, shift)
+ x_t_minus = x_t + u * (sigma_t_minus - sigma_t)
+ active_mask = num_substeps > substep_id
+ x_t = torch.where(active_mask, x_t_minus, x_t)
+ sigma_t = torch.where(active_mask, sigma_t_minus, sigma_t)
+ raw_t = torch.where(active_mask, raw_t_minus, raw_t)
+
+ return x_t
+
+
+def _unpack_latent_piflux2(x: Tensor, patch_size: int = 2) -> Tensor:
+ bsz, packed_channels, h, w = x.shape
+ channels = packed_channels // (patch_size * patch_size)
+ x = x.view(bsz, channels, patch_size, patch_size, h, w)
+ x = x.permute(0, 1, 4, 2, 5, 3).reshape(bsz, channels, h * patch_size, w * patch_size)
+ return x
+
+
+def _pack_latent_piflux2(x: Tensor, patch_size: int = 2) -> Tensor:
+ bsz, channels, h, w = x.shape
+ h_packed = h // patch_size
+ w_packed = w // patch_size
+ x = x.view(bsz, channels, h_packed, patch_size, w_packed, patch_size)
+ x = x.permute(0, 1, 3, 5, 2, 4).reshape(
+ bsz, channels * patch_size * patch_size, h_packed, w_packed
+ )
+ return x
+
+
def compute_empirical_mu(image_seq_len: int, num_steps: int) -> float:
a1, b1 = 8.73809524e-05, 1.89833333
a2, b2 = 0.00016927, 0.45666666
@@ -330,6 +521,7 @@ def denoise(
timesteps: list[float],
guidance: float = 4.0,
real_guidance_scale = None,
+ final_step_size_scale: float | None = None,
# extra img tokens (channel-wise)
neg_txt: Tensor = None,
neg_txt_ids: Tensor= None,
@@ -351,6 +543,10 @@ def denoise(
masking_strength = 1,
preview_meta = None,
original_image_latents = None,
+ # pi-Flow settings
+ piflow_substeps: int = 128,
+ piflow_generator: torch.Generator | None = None,
+ piflow_gm_temperature: float | None = None,
):
kwargs = {
@@ -365,6 +561,9 @@ def denoise(
callback(-1, None, True)
original_timesteps = timesteps
+ is_piflow = getattr(model, "piflow", False)
+ piflow_spatial_h = piflow_spatial_w = None
+ piflow_sigmas = piflow_sigmas_dst = piflow_m_vals = None
morph, first_step = False, 0
if img_msk_latents is not None:
@@ -381,6 +580,11 @@ def denoise(
timesteps = timesteps[first_step:]
+ if is_piflow:
+ base_img_ids = img_ids[:, :img.shape[1]]
+ piflow_spatial_h = int(base_img_ids[..., 1].max().item() + 1)
+ piflow_spatial_w = int(base_img_ids[..., 2].max().item() + 1)
+
updated_num_steps= len(timesteps) -1
if callback != None:
from shared.utils.loras_mutipliers import update_loras_slists
@@ -389,6 +593,15 @@ def denoise(
from mmgp import offload
# this is ignored for schnell
guidance_vec = torch.full((img.shape[0],), guidance, device=img.device, dtype=img.dtype)
+ if is_piflow and len(timesteps) > 1:
+ piflow_sigmas = torch.tensor(timesteps, device=img.device, dtype=torch.float32)
+ piflow_sigmas_dst, piflow_m_vals = _flow_map_sde_calculate_sigmas_dst(
+ piflow_sigmas, h=0.0
+ )
+ if piflow_gm_temperature is None:
+ nfe = len(timesteps) - 1
+ piflow_gm_temperature = min(max(0.1 * (nfe - 1), 0.0), 1.0)
+
for i, (t_curr, t_prev) in enumerate(zip(timesteps[:-1], timesteps[1:])):
offload.set_step_no_for_lora(model, first_step + i)
if pipeline._interrupt:
@@ -443,10 +656,57 @@ def denoise(
)
if pred == None: return None
- if real_guidance_scale > 1:
- pred = neg_pred + real_guidance_scale * (pred - neg_pred)
+ if is_piflow and isinstance(pred, dict):
+ if real_guidance_scale > 1:
+ pred = {k: neg_pred[k] + real_guidance_scale * (pred[k] - neg_pred[k]) for k in pred}
+
+ patch_size = getattr(model, "piflow_patch_size", 2)
+ img_packed = rearrange(
+ img, "b (h w) c -> b c h w", h=piflow_spatial_h, w=piflow_spatial_w
+ )
+ img_unpacked = _unpack_latent_piflux2(img_packed, patch_size=patch_size).float()
+ pred = {k: v.float() for k, v in pred.items()}
+
+ sigma_t_src = piflow_sigmas[i].expand(img.shape[0])
+ sigma_t_dst = piflow_sigmas_dst[i].expand(img.shape[0])
+ policy = _GMFlowPolicy(pred, img_unpacked, sigma_t_src)
+ if (
+ piflow_gm_temperature is not None
+ and i != len(timesteps) - 2
+ and piflow_gm_temperature < 1.0
+ ):
+ policy.temperature_(piflow_gm_temperature)
+ img_unpacked = _policy_rollout(
+ img_unpacked,
+ sigma_t_src,
+ sigma_t_dst,
+ total_substeps=piflow_substeps,
+ policy=policy,
+ shift=3.2,
+ )
+
+ sigma_t_to = piflow_sigmas[i + 1]
+ m = piflow_m_vals[i]
+ alpha_t_to = 1 - sigma_t_to
+ if not torch.allclose(m, torch.ones_like(m)):
+ if piflow_generator is not None and img_unpacked.device.type == "cpu":
+ noise = torch.randn_like(img_unpacked, generator=piflow_generator)
+ else:
+ noise = torch.randn(img_unpacked.shape, device=img_unpacked.device, dtype=img_unpacked.dtype)
+ img_unpacked = (alpha_t_to + sigma_t_to * m) * img_unpacked + sigma_t_to * (
+ 1 - m.square()
+ ).clamp(min=0).sqrt() * noise
+
+ img_packed = _pack_latent_piflux2(img_unpacked, patch_size=patch_size)
+ img = rearrange(img_packed, "b c h w -> b (h w) c").to(img.dtype)
+ else:
+ if real_guidance_scale > 1:
+ pred = neg_pred + real_guidance_scale * (pred - neg_pred)
- img += (t_prev - t_curr) * pred
+ step_size = t_prev - t_curr
+ if final_step_size_scale is not None and i == len(timesteps) - 2:
+ step_size = step_size * final_step_size_scale
+ img += step_size * pred
if img_msk_latents is not None and i < masked_steps:
latent_noise_factor = t_prev
diff --git a/models/flux/util.py b/models/flux/util.py
index 2fc4b98..2befddf 100644
--- a/models/flux/util.py
+++ b/models/flux/util.py
@@ -330,6 +330,43 @@ configs = {
shift_factor=0.0,
),
),
+ "pi-flux2": ModelSpec(
+ repo_id="",
+ repo_flow="",
+ repo_ae="ckpts/flux2_vae.safetensors",
+ params=FluxParams(
+ in_channels=128,
+ out_channels=128,
+ vec_in_dim=1,
+ context_in_dim=15360,
+ hidden_size=6144,
+ mlp_ratio=3.0,
+ single_linear1_mlp_ratio=6.0,
+ single_mlp_hidden_ratio=3.0,
+ double_mlp_ratio=3.0,
+ double_linear1_mlp_ratio=6.0,
+ num_heads=48,
+ depth=8,
+ depth_single_blocks=48,
+ axes_dim=[32, 32, 32, 32],
+ theta=2000,
+ qkv_bias=False,
+ guidance_embed=True,
+ flux2=True,
+ piflow=True,
+ ),
+ ae_params=AutoEncoderParams(
+ resolution=1024,
+ in_channels=3,
+ ch=128,
+ out_ch=3,
+ ch_mult=[1, 2, 4, 4],
+ num_res_blocks=2,
+ z_channels=32,
+ scale_factor=0.5,
+ shift_factor=0.0,
+ ),
+ ),
"flux-dev": ModelSpec(
repo_id="",
repo_flow="",
@@ -897,7 +934,7 @@ def load_flow_model(name: str, model_filename, device: str | torch.device = "cud
# print(f"Loading checkpoint: {ckpt_path}")
from mmgp import offload
- offload.load_model_data(model, model_filename )
+ offload.load_model_data(model, model_filename)
# # load_sft doesn't support torch.device
# sd = load_sft(ckpt_path, device=str(device))
diff --git a/models/wan/any2video.py b/models/wan/any2video.py
index 6b53603..9235f60 100644
--- a/models/wan/any2video.py
+++ b/models/wan/any2video.py
@@ -537,7 +537,7 @@ class WanAny2V:
ref_images_count = inner_latent_frames = 0
trim_frames = 0
last_latent_preview = False
- extended_overlapped_latents = clip_image_start = clip_image_end = image_mask_latents = latent_slice = freqs = None
+ extended_overlapped_latents = clip_image_start = clip_image_end = image_mask_latents = latent_slice = freqs = post_freqs = None
# SCAIL uses a fixed ref latent frame that should not be noised.
no_noise_latents_injection = infinitetalk or scail
timestep_injection = False
diff --git a/models/wan/wan_handler.py b/models/wan/wan_handler.py
index ff78a81..b205cc9 100644
--- a/models/wan/wan_handler.py
+++ b/models/wan/wan_handler.py
@@ -699,19 +699,18 @@ class family_handler():
if video_guide_processed.numel() == 0:
gr.Info("Unable to detect a Person")
return None, None, None, None
- # SCAIL doesn't use augmented poses
return video_guide_processed, None, video_mask, None
else:
- # Steadydancer/wanmove: Use 2D pose alignment
+ # Steadydancer
from .steadydancer.pose_align import PoseAligner
aligner = PoseAligner()
- outputs = aligner.align(frames, ref_image, ref_video_mask=mask_frames, align_frame=0, max_frames=None, augment=True, include_composite=False, cpu_resize_workers=max_workers, expand_scale=expand_scale)
+ outputs = aligner.align(frames, ref_image, ref_video_mask=None, align_frame=0, max_frames=None, augment=True, include_composite=False, cpu_resize_workers=max_workers, expand_scale=expand_scale)
video_guide_processed, video_guide_processed2 = outputs["pose_only"], outputs["pose_aug"]
if video_guide_processed.numel() == 0:
return None, None, None, None
- return video_guide_processed, video_guide_processed2, video_mask_processed, video_mask_processed2
+ return video_guide_processed, video_guide_processed2, None, None
@staticmethod