Python
wan-move suppor
DeepBeepMeepaeb80f36ece7
diff --git a/README.md b/README.mdindex cca37a9..c32aa4a 100644--- a/README.md+++ b/README.md@@ -37,7 +37,7 @@ WanGP supports the Wan (and derived models), Hunyuan Video and LTV Video models## 🔥 Latest Updates :-### December 10 2025: WanGP v9.83, Simple Pleasures...+### December 11 2025: WanGP v9.84, Simple Pleasures...These two features are going to change the life of many people:- **Pause Button**: ever had a urge to use your GPU for a very important task that can't wait (a game for instance ?), here comes your new friend the *Pause* button. Not only it will suspend the current gen in progress but it will free most of the VRAM used by WanGP (please note that the RAM by WanGP used wont be released). When you are done just click the *Resume* button to restart exactly from where you stopped.@@ -61,11 +61,13 @@ Processing Settings can be useful to do some quick gen / testing if you don't ne- **Hunyuan Video 1.5 i2v distilled** : for those in need of their daily dose of new models, added *Hunyuan Video 1.5 i2v Distilled* (official release) + Lora Accelerator extracted from it (to be used in future finetunes). Also added *Magcache* support (optimized for 20 steps) for Hunyuan Video 1.5.+- **Wan-Move** : Another model specialized to control motion using a *Start Image* and *Trajectories*. According to the author's paper it is the best one. *Motion Designer* has been upgraded to generate also trajectories for *Wan-Move*.*update 9.81*: added Hunyuan Video 1.5 i2v distilled + magcache*update 9.82*: added Settings headless processing, output file customization, refactored Task edition and queue processing-*update 9.83*: Qwen Edit+ upgraded: no more any zoom at 1080p, enabled mask, enabled image refs with inpainting+*update 9.83*: Qwen Edit+ upgraded: no more any zoom out at 1080p, enabled mask, enabled image refs with inpainting+*update 9.84*: added Wan-Move support### December 4 2025: WanGP v9.74, The Alpha & the Omega ... and the Dancerdiff --git a/models/wan/any2video.py b/models/wan/any2video.pyindex c87f4f9..e81303e 100644--- a/models/wan/any2video.py+++ b/models/wan/any2video.py@@ -36,6 +36,7 @@ from shared.utils.basic_flowmatch import FlowMatchSchedulerfrom shared.utils.lcm_scheduler import LCMSchedulerfrom shared.utils.utils import get_outpainting_frame_location, resize_lanczos, calculate_new_dimensions, convert_image_to_tensor, fit_image_into_canvasfrom .multitalk.multitalk_utils import MomentumBuffer, adaptive_projected_guidance, match_and_blend_colors, match_and_blend_colors_with_mask+from .wanmove.trajectory import replace_feature, create_pos_feature_mapfrom shared.utils.audio_video import save_videofrom mmgp import safetensors2from shared.utils import files_locator as fl@@ -372,6 +373,7 @@ class WanAny2V:input_video = None,image_start = None,image_end = None,+ input_custom = None,denoising_strength = 1.0,masking_strength = 1.0,target_camera=None,@@ -432,7 +434,7 @@ class WanAny2V:face_arc_embeds = None,control_scale_alt = 1.,motion_amplitude = 1.,- full_prefix_video = None,+ window_start_frame_no = 0,**bbargs):@@ -529,6 +531,7 @@ class WanAny2V:chrono_edit = model_type in ["chrono_edit"]mocha = model_type in ["mocha"]steadydancer = model_type in ["steadydancer"]+ wanmove = model_type in ["wanmove"]start_step_no = 0ref_images_count = inner_latent_frames = 0trim_frames = 0@@ -644,6 +647,19 @@ class WanAny2V:lat_y = Nonekwargs.update({ 'y': y})+ # Wan-Move+ if wanmove:+ track = np.load(input_custom)+ if track.ndim == 4: track = track.squeeze(0)+ if track.max() <= 1:+ track = np.round(track * [width, height]).astype(np.int64)+ control_video_pos= 0 if "T" in video_prompt_type else window_start_frame_no+ track = torch.from_numpy(track[control_video_pos:control_video_pos+frame_num]).to(self.device)+ track_feats, track_pos = create_pos_feature_map(track, None, [4, 8, 8], height, width, 16, device=y.device)+ track_feats = None #track_feats.permute(3, 0, 1, 2)+ y_cond = kwargs.pop("y")+ y_uncond = y_cond.clone()+ y_cond[4:20] = replace_feature(y[4:20].unsqueeze(0), track_pos.unsqueeze(0))[0]# Steady Dancerif steadydancer:@@ -1051,6 +1067,12 @@ class WanAny2V:# "face_pixel_values": [face_pixel_values, None]"face_pixel_values": [face_pixel_values, face_pixel_values] # seems to look better this way}+ elif wanmove:+ gen_args = {+ "x" : [latent_model_input, latent_model_input],+ "context" : [context, context_null],+ "y" : [y_cond, y_uncond],+ }elif lynx:gen_args = {"x" : [latent_model_input, latent_model_input],@@ -1244,7 +1266,7 @@ class WanAny2V:else:videos = videos[0] # return only first videoif any_vae2: videos2 = videos2[0] # return only first video- if color_correction_strength > 0 and not ( window_no == 1 and (full_prefix_video is None or full_prefix_video.shape[1]<= 1) ):+ if color_correction_strength > 0 and (window_start_frame_no + prefix_frames_count) >1:if vace and False:# videos = match_and_blend_colors_with_mask(videos.unsqueeze(0), input_frames[0].unsqueeze(0), input_masks[0][:1].unsqueeze(0), color_correction_strength,copy_mode= "progressive_blend").squeeze(0)videos = match_and_blend_colors_with_mask(videos.unsqueeze(0), input_frames[0].unsqueeze(0), input_masks[0][:1].unsqueeze(0), color_correction_strength,copy_mode= "reference").squeeze(0)diff --git a/models/wan/configs/wanmove.json b/models/wan/configs/wanmove.jsonnew file mode 100644index 0000000..f5a12b2--- /dev/null+++ b/models/wan/configs/wanmove.json@@ -0,0 +1,14 @@+{+ "_class_name": "WanModel",+ "_diffusers_version": "0.30.0",+ "dim": 5120,+ "eps": 1e-06,+ "ffn_dim": 13824,+ "freq_dim": 256,+ "in_dim": 36,+ "model_type": "i2v",+ "num_heads": 40,+ "num_layers": 40,+ "out_dim": 16,+ "text_len": 512+}diff --git a/models/wan/modules/model.py b/models/wan/modules/model.pyindex 5c4f86b..ba81dce 100644--- a/models/wan/modules/model.py+++ b/models/wan/modules/model.py@@ -1471,8 +1471,12 @@ class WanModel(ModelMixin, ConfigMixin):steadydancer = steadydancer_condition is not Noneif steadydancer: # steady dancerx_noise_clone = x_list[0].clone()+ if isinstance(y, list):+ y_list = y+ else:+ y_list = [y] * len(x_list)- for i, (is_source, x) in enumerate(zip(is_source_x, x_list)):+ for i, (is_source, x, y) in enumerate(zip(is_source_x, x_list, y_list)):if is_source:x_list[i] = x_list[0].clone()last_x_idx = i@@ -1488,7 +1492,7 @@ class WanModel(ModelMixin, ConfigMixin):x = self.patch_embedding(x).to(modulation_dtype)grid_sizes = x.shape[2:]x_list[i] = x- y = None+ y = y_list = Noneif steadydancer: # steady dancer# Spatial Structure Adaptive Extractor.diff --git a/models/wan/wan_handler.py b/models/wan/wan_handler.pyindex 31fde9e..b3f7606 100644--- a/models/wan/wan_handler.py+++ b/models/wan/wan_handler.py@@ -11,7 +11,7 @@ def test_vace(base_model_type):return base_model_type in ["vace_14B", "vace_14B_2_2", "vace_1.3B", "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_14B", "vace_ditto_14B"]def test_class_i2v(base_model_type):- return base_model_type in ["i2v", "i2v_2_2", "fun_inp_1.3B", "fun_inp", "flf2v_720p", "fantasy", "multitalk", "infinitetalk", "i2v_2_2_multitalk", "animate", "chrono_edit", "steadydancer" ]+ return base_model_type in ["i2v", "i2v_2_2", "fun_inp_1.3B", "fun_inp", "flf2v_720p", "fantasy", "multitalk", "infinitetalk", "i2v_2_2_multitalk", "animate", "chrono_edit", "steadydancer", "wanmove" ]def test_class_t2v(base_model_type):return base_model_type in ["t2v", "t2v_2_2", "alpha", "lynx"]@@ -43,7 +43,7 @@ class family_handler():return ["multitalk", "infinitetalk", "fantasy", "vace_14B", "vace_14B_2_2", "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_14B","t2v_1.3B", "standin", "lynx_lite", "lynx", "t2v", "t2v_2_2", "vace_1.3B", "vace_ditto_14B", "phantom_1.3B", "phantom_14B","recam_1.3B", "animate", "alpha", "alpha_lynx", "chrono_edit",- "i2v", "i2v_2_2", "i2v_2_2_multitalk", "ti2v_2_2", "lucy_edit", "flf2v_720p", "fun_inp_1.3B", "fun_inp", "mocha", "steadydancer"]+ "i2v", "i2v_2_2", "i2v_2_2_multitalk", "ti2v_2_2", "lucy_edit", "flf2v_720p", "fun_inp_1.3B", "fun_inp", "mocha", "steadydancer", "wanmove"]@staticmethod@@ -301,7 +301,11 @@ class family_handler():}extra_model_def["v2i_switch_supported"] = True-++ if base_model_type in ["wanmove"]:+ extra_model_def["custom_guide"] = { "label": "Trajectory File", "required": True, "file_types": [".npy"]}+ extra_model_def["i2v_trajectory"] = True+if base_model_type in ["steadydancer"]:extra_model_def["guide_custom_choices"] = {"choices":[diff --git a/models/wan/wanmove/trajectory.py b/models/wan/wanmove/trajectory.pynew file mode 100644index 0000000..029a695--- /dev/null+++ b/models/wan/wanmove/trajectory.py@@ -0,0 +1,485 @@+import os+os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"+import sys+sys.path.append(os.path.abspath('.'))+import math+import random+from io import BytesIO++import imageio.v3 as iio+import numpy as np++import torch+import decord+from decord import VideoReader+from PIL import Image, ImageDraw+from torchvision import transforms++++SKIP_ZERO = False++def get_pos_emb(+ pos_k: torch.Tensor,+ pos_emb_dim: int,+ theta_func: callable = lambda i, d: torch.pow(10000, torch.mul(2, torch.div(i.to(torch.float32), d))),+ device: torch.device = torch.device("cuda" if torch.cuda.is_available() else "cpu"),+ dtype: torch.dtype = torch.float32,+) -> torch.Tensor:+ """+ Generate batch position embeddings.++ Args:+ pos_k (torch.Tensor): A 1D tensor containing positions for which to generate embeddings.+ pos_emb_dim (int): The dimension of position embeddings.+ theta_func (callable): Function to compute thetas based on position and embedding dimensions.+ device (torch.device): Device to store the position embeddings.+ dtype (torch.dtype): Desired data type for computations.++ Returns:+ torch.Tensor: The position embeddings with shape (batch_size, pos_emb_dim).+ """+ assert pos_emb_dim % 2 == 0, "The dimension of position embeddings must be even."+ pos_k = pos_k.to(device, dtype)+ if SKIP_ZERO:+ pos_k = pos_k + 1+ batch_size = pos_k.size(0)++ denominator = torch.arange(0, pos_emb_dim // 2, device=device, dtype=dtype)+ # Expand denominator to match the shape needed for broadcasting+ denominator_expanded = denominator.view(1, -1).expand(batch_size, -1)++ thetas = theta_func(denominator_expanded, pos_emb_dim)++ # Ensure pos_k is in the correct shape for broadcasting+ pos_k_expanded = pos_k.view(-1, 1).to(dtype)+ sin_thetas = torch.sin(torch.div(pos_k_expanded, thetas))+ cos_thetas = torch.cos(torch.div(pos_k_expanded, thetas))++ # Concatenate sine and cosine embeddings along the last dimension+ pos_emb = torch.cat([sin_thetas, cos_thetas], dim=-1)++ return pos_emb++def create_pos_feature_map(+ pred_tracks: torch.Tensor, # [T, N, 2]+ pred_visibility: torch.Tensor, # [T, N]+ downsample_ratios: list[int],+ height: int,+ width: int,+ pos_emb_dim: int,+ track_num: int = -1,+ t_down_strategy: str = "sample",+ device: torch.device = torch.device("cuda" if torch.cuda.is_available() else "cpu"),+ dtype: torch.dtype = torch.float32,+):+ """+ Create a feature map from the predicted tracks.++ Args:+ - pred_tracks: torch.Tensor, the predicted tracks, [T, N, 2]+ - pred_visibility: torch.Tensor, the predicted visibility, [T, N]+ - downsample_ratios: list[int], the ratios for downsampling time, height, and width+ - height: int, the height of the feature map+ - width: int, the width of the feature map+ - pos_emb_dim: int, the dimension of the position embeddings+ - track_num: int, the number of tracks to use+ - t_down_strategy: str, the strategy for downsampling time dimension+ - device: torch.device, the device+ - dtype: torch.dtype, the data type++ Returns:+ - feature_map: torch.Tensor, the feature map, [T', H', W', pos_emb_dim]+ - track_pos: torch.Tensor, the position embeddings, [N, T', 2], 2 = height, width+ """++ assert t_down_strategy in ["sample", "average"], "Invalid strategy for downsampling time dimension."++ t, n, _ = pred_tracks.shape+ t_down, h_down, w_down = downsample_ratios+ feature_map = torch.zeros((t-1) // t_down + 1, height // h_down, width // w_down, pos_emb_dim, device=device, dtype=dtype)+ track_pos = - torch.ones(n, (t-1) // t_down + 1, 2, dtype=torch.long)++ if track_num == -1:+ track_num = n++ tracks_idx = torch.randperm(n)[:track_num]+ tracks = pred_tracks[:, tracks_idx]+ visibility = pred_visibility[:, tracks_idx] if pred_visibility is not None else None+ tracks_embs = get_pos_emb(torch.randperm(n)[:track_num], pos_emb_dim, device=device, dtype=dtype)++ cover_count = 0+ if visibility is None:+ cur_visibility = [True] * track_num++ for t_idx in range(0, t, t_down):+ if t_down_strategy == "sample" or t_idx == 0:+ cur_tracks = tracks[t_idx] # [N, 2]+ if visibility is not None:+ cur_visibility = visibility[t_idx] # [N]+ else:+ cur_tracks = tracks[t_idx:t_idx+t_down].mean(dim=0)+ cur_visibility = torch.any(visibility[t_idx:t_idx+t_down], dim=0)++ for i in range(track_num):+ if not cur_visibility[i] or cur_tracks[i][0] < 0 or cur_tracks[i][1] < 0 or cur_tracks[i][0] >= width or cur_tracks[i][1] >= height:+ continue+ x, y = cur_tracks[i]+ x, y = int(x // w_down), int(y // h_down)+ feature_map[t_idx // t_down, y, x] += tracks_embs[i]+ track_pos[i, t_idx // t_down, 0], track_pos[i, t_idx // t_down, 1] = y, x++ return feature_map, track_pos+++def replace_feature(+ vae_feature: torch.Tensor, # [B, C', T', H', W']+ track_pos: torch.Tensor, # [B, N, T', 2]+) -> torch.Tensor:+ b, _, t, h, w = vae_feature.shape+ assert b == track_pos.shape[0], "Batch size mismatch."+ n = track_pos.shape[1]++ # Shuffle the trajectory order+ track_pos = track_pos[:, torch.randperm(n), :, :]++ # Extract coordinates at time steps ≥ 1 and generate a valid mask+ current_pos = track_pos[:, :, 1:, :] # [B, N, T-1, 2]+ mask = (current_pos[..., 0] >= 0) & (current_pos[..., 1] >= 0) # [B, N, T-1]++ # Get all valid indices+ valid_indices = mask.nonzero(as_tuple=False) # [num_valid, 3]+ num_valid = valid_indices.shape[0]++ if num_valid == 0:+ return vae_feature++ # Decompose valid indices into each dimension+ batch_idx = valid_indices[:, 0]+ track_idx = valid_indices[:, 1]+ t_rel = valid_indices[:, 2]+ t_target = t_rel + 1 # Convert to original time step indices++ # Extract target position coordinates+ h_target = current_pos[batch_idx, track_idx, t_rel, 0].long() # Ensure integer indices+ w_target = current_pos[batch_idx, track_idx, t_rel, 1].long()++ # Extract source position coordinates (t=0)+ h_source = track_pos[batch_idx, track_idx, 0, 0].long()+ w_source = track_pos[batch_idx, track_idx, 0, 1].long()++ # Get source features and assign to target positions+ src_features = vae_feature[batch_idx, :, 0, h_source, w_source]+ vae_feature[batch_idx, :, t_target, h_target, w_target] = src_features++ return vae_feature++def get_video_track_video(+ model,+ video_tensor: torch.Tensor, # [T, C, H, W]+ downsample_ratios: list[int],+ pos_emb_dim: int,+ grid_size: int = 32,+ track_num: int = -1,+ t_down_strategy: str = "sample",+ device: torch.device = torch.device("cuda" if torch.cuda.is_available() else "cpu"),+ dtype: torch.dtype = torch.float32,+) -> tuple[torch.Tensor, torch.Tensor]:+ """+ Get the track video from the video tensor.++ Args:+ - model: torch.nn.Module, the model for tracking, CoTracker+ - video_tensor: torch.Tensor, the video tensor, [T, C, H, W]+ - downsample_ratios: list[int], the ratios for downsampling time, height, and width+ - height: int, the height of the feature map+ - width: int, the width of the feature map+ - pos_emb_dim: int, the dimension of the position embeddings+ - grid_size: int, the size of the grid+ - track_num: int, the number of tracks to use+ - t_down_strategy: str, the strategy for downsampling time dimension+ - device: torch.device, the device+ - dtype: torch.dtype, the data type++ Returns:+ - track_video: torch.Tensor, the track video, [pos_emb_dim, T', H', W']+ - track_pos: torch.Tensor, the position embeddings, [N, T', 2], 2 = height, width+ - pred_tracks: the predicted point trajectories+ - pred_visibility: visibility of the predicted point trajectories+ """++ t, c, height, width = video_tensor.shape+ with (+ torch.autocast(device_type=device.type, dtype=dtype),+ torch.no_grad(),+ ):+ pred_tracks, pred_visibility = model(+ video_tensor.unsqueeze(0),+ grid_size=grid_size,+ backward_tracking=False,+ )++ track_video, track_pos = create_pos_feature_map(+ pred_tracks[0], pred_visibility[0], downsample_ratios, height, width, pos_emb_dim, track_num, t_down_strategy, device, dtype+ )++ return track_video.permute(3, 0, 1, 2), track_pos, pred_tracks, pred_visibility+++# === user input tracks ===++def resize_tracks(+ img_tracks: torch.Tensor, # [T, N, height, width]+ target_frame_num: int,+ t_strategy: str = "sample",+ device: torch.device = torch.device("cuda" if torch.cuda.is_available() else "cpu"),+ dtype: torch.dtype = torch.float32,+) -> tuple[torch.Tensor, torch.Tensor]:+ """+ Resize tracks to a specified number of frames.++ Args:+ - img_tracks: torch.Tensor, the tracks, [T, N, height, width]+ - target_frame_num: int, the number of frames to resize to+ - t_strategy: str, the strategy for downsampling time dimension+ - device: torch.device, the device+ - dtype: torch.dtype, the data type++ Returns:+ - resized_tracks: torch.Tensor, the resized tracks, [target_frame_num, N, 2]+ - resized_visibility: torch.Tensor, the resized visibility, [target_frame_num, N]+ """++ assert t_strategy in ["sample", "average"], "Invalid strategy for downsampling time dimension."+ assert t_strategy in ["sample"], "only support sample strategy."++ def get_xy_from_hw(hw_tensor: torch.Tensor) -> torch.Tensor:+ """+ Get the x and y coordinates from the height and width tensor.++ Args:+ - hw_tensor: torch.Tensor, the tensor of height and width, [N, height, width]++ Returns:+ - xy_tensor: torch.Tensor, the tensor of x and y coordinates, [N, 2]+ """++ h, w = hw_tensor.shape[-2:]+ _, y, x = torch.nonzero(hw_tensor, as_tuple=True)+ xy_tensor = torch.stack((x, y), dim=-1)+ assert xy_tensor.shape[0] == hw_tensor.shape[0], "The number of points should be the same."+ return xy_tensor++ def get_average_xy_from_batch_hw(hw_tensor: torch.Tensor) -> torch.Tensor:+ for b in range(hw_tensor.shape[0]):+ xy_tensor = get_xy_from_hw(hw_tensor[b])+ if b == 0:+ xy_tensors = xy_tensor+ else:+ xy_tensors += xy_tensor+ xy_tensors /= hw_tensor.shape[0]+ return xy_tensors++ # Get the number of frames in the input tracks+ num_frames, num_tracks, _, _ = img_tracks.shape++ new_tracks = torch.zeros(target_frame_num, num_tracks, 2, device=device, dtype=dtype)+ new_visibility = torch.ones(target_frame_num, num_tracks, device=device, dtype=torch.bool)++ new_tracks[0] = get_xy_from_hw(img_tracks[0])+ # -1 for removing the first frame+ num_frames -= 1+ target_frame_num -= 1++ new_frame_idx = 1+ if target_frame_num <= num_frames:+ t_down = num_frames / target_frame_num+ frame_idxs = [int((i - 1) * t_down + 1) for i in range(1, target_frame_num + 1)]+ for i, frame_idx in enumerate(frame_idxs):+ if t_strategy == "sample":+ new_tracks[new_frame_idx] = get_xy_from_hw(img_tracks[frame_idx])+ else:+ next_frame_idx = frame_idxs[i + 1] if i + 1 < len(frame_idxs) else num_frames + 1 # +1 as compensation for the -1+ new_tracks[new_frame_idx] = get_average_xy_from_batch_hw(img_tracks[frame_idx:next_frame_idx])++ new_frame_idx += 1+ else:+ t_repeat = target_frame_num / num_frames+ target_frame_idxs = [int((i - 1) * t_repeat + 1) for i in range(1, num_frames + 1)]+ for i, target_frame_idx in enumerate(target_frame_idxs):+ next_target_frame_idx = target_frame_idxs[i + 1] if i + 1 < len(target_frame_idxs) else target_frame_num + 1+ if t_strategy == "sample":+ new_tracks[target_frame_idx:next_target_frame_idx] = get_xy_from_hw(img_tracks[new_frame_idx])+ else:+ if target_frame_idx == next_target_frame_idx:+ new_tracks[target_frame_idx] = get_xy_from_hw(img_tracks[new_frame_idx])+ else:+ next_new_frame_idx = new_frame_idx + 1 if new_frame_idx + 1 < num_frames else new_frame_idx+ for j in range(target_frame_idx, next_target_frame_idx):+ new_tracks[j] = (1 - (next_target_frame_idx - j) / (next_target_frame_idx - target_frame_idx)) * get_xy_from_hw(img_tracks[new_frame_idx]) + (next_target_frame_idx - j) / (next_target_frame_idx - target_frame_idx) * get_xy_from_hw(img_tracks[next_new_frame_idx])++ new_frame_idx += 1++ # print(new_tracks[1])+ return new_tracks, new_visibility++def generate_custom_feature_map(+ img_tracks: torch.Tensor, # [T, N, height, width]+ target_frame_num: int,+ downsample_ratios: list[int],+ pos_emb_dim: int,+ t_down_strategy: str = "sample",+ device: torch.device = torch.device("cuda" if torch.cuda.is_available() else "cpu"),+ dtype: torch.dtype = torch.float32,+) -> tuple[torch.Tensor, torch.Tensor]:+ """+ Generate a custom feature map from the tracks.++ Args:+ - img_tracks: torch.Tensor, the tracks, [T, N, height, width]+ - target_frame_num: int, the number of frames to resize to+ - downsample_ratios: List[int], the ratios for downsampling time, height, and width+ - pos_emb_dim: int, the dimension of the position embeddings+ - t_down_strategy: str, the strategy for downsampling time dimension+ - device: torch.device, the device+ - dtype: torch.dtype, the data type++ Returns:+ - feature_map: torch.Tensor, the feature map, [T', H', W', pos_emb_dim]+ """++ height, width = img_tracks.shape[-2:]+ resized_tracks, resized_visibility = resize_tracks(img_tracks, target_frame_num, t_down_strategy, device, dtype)+ feature_map, track_pos = create_pos_feature_map(+ resized_tracks,+ resized_visibility,+ downsample_ratios,+ height,+ width,+ pos_emb_dim,+ track_num=-1,+ t_down_strategy=t_down_strategy,+ device=device,+ dtype=dtype+ )++ return feature_map, track_pos++++# ---------------------------+# Visualize functions+# --------------------------+++def draw_overall_gradient_polyline_on_image(image, line_width, points, start_color):+ """+ - image (Image): target image to draw on.+ - line_width (int): initial line width.+ - points (list of tuples): list of points forming the polyline, each point is (x, y).+ - start_color (tuple): starting color of the line (R, G, B).++ Return:+ - Image: original image with the gradient polyline drawn.+ """++ def get_distance(p1, p2):+ return ((p2[0] - p1[0]) ** 2 + (p2[1] - p1[1]) ** 2) ** 0.5++ # Create a new image with the same size as the original+ new_image = Image.new('RGBA', image.size)+ draw = ImageDraw.Draw(new_image, 'RGBA')+ points = points[::-1]++ # Compute total length+ total_length = sum(get_distance(points[i], points[i+1]) for i in range(len(points)-1))++ # Accumulated length+ accumulated_length = 0++ # Draw the gradient polyline+ for start_point, end_point in zip(points[:-1], points[1:]):+ segment_length = get_distance(start_point, end_point)+ steps = int(segment_length)++ for i in range(steps):+ # Current accumulated length+ current_length = accumulated_length + (i / steps) * segment_length++ # Alpha from fully opaque to fully transparent+ alpha = int(255 * (1 - current_length / total_length))+ color = (*start_color, alpha)++ # Interpolated coordinates+ x = int(start_point[0] + (end_point[0] - start_point[0]) * i / steps)+ y = int(start_point[1] + (end_point[1] - start_point[1]) * i / steps)++ # Dynamic line width, decreasing from initial width to 1+ dynamic_line_width = int(line_width * (1 - (current_length / total_length)))+ dynamic_line_width = max(dynamic_line_width, 1) # minimum width is 1 to avoid 0++ draw.line([(x, y), (x + 1, y)], fill=color, width=dynamic_line_width)++ accumulated_length += segment_length++ return new_image++def add_weighted(rgb, track):+ rgb = np.array(rgb) # [H, W, C] "RGB"+ track = np.array(track) # [H, W, C] "RGBA"++ # Compute weights from the alpha channel+ alpha = track[:, :, 3] / 255.0++ # Expand alpha to 3 channels to match RGB+ alpha = np.stack([alpha] * 3, axis=-1)++ # Blend the two images+ blend_img = track[:, :, :3] * alpha + rgb * (1 - alpha)++ return Image.fromarray(blend_img.astype(np.uint8))++def draw_tracks_on_video(video, tracks, visibility=None, track_frame=24):+ color_map = [+ (102, 153, 255),+ (0, 255, 255),+ (255, 255, 0),+ (255, 102, 204),+ (0, 255, 0)+ ]+ circle_size = 12+ line_width = 16++ video = video[0].permute(0, 2, 3, 1).byte().detach().cpu().numpy() # (81, 480, 832, 3), uint8+ tracks = tracks[0].long().detach().cpu().numpy()+ if visibility is not None:+ visibility = visibility[0].detach().cpu().numpy()+ # print(video.shape, tracks.shape)++ output_frames = []+ # Process the video+ for t in range(video.shape[0]):+ # Extract current frame+ frame = video[t]+ frame = Image.fromarray(frame).convert("RGB")++ # Draw tracks+ for n in range(tracks.shape[1]):+ if visibility is not None and visibility[t, n] == 0:+ continue++ # Track coordinate at current frame+ track_coord = tracks[t, n]+ tracks_coord = tracks[max(t-track_frame, 0):t+1, n]++ # Draw a circle+ draw = ImageDraw.Draw(frame)+ draw.ellipse((track_coord[0] - circle_size, track_coord[1] - circle_size, track_coord[0] + circle_size, track_coord[1] + circle_size), fill=color_map[n % len(color_map)])+ # Draw the polyline+ track_image = draw_overall_gradient_polyline_on_image(frame, line_width, tracks_coord, color_map[n % len(color_map)])+ frame = add_weighted(frame, track_image)++ # Save current frame+ output_frames.append(frame.convert("RGB"))++ return output_framesdiff --git a/plugins/wan2gp-motion-designer/assets/app.js b/plugins/wan2gp-motion-designer/assets/app.jsindex b5fc26c..db48360 100644--- a/plugins/wan2gp-motion-designer/assets/app.js+++ b/plugins/wan2gp-motion-designer/assets/app.js@@ -5,6 +5,7 @@const RENDER_MODES = {CUT_DRAG: "cut_drag",CLASSIC: "classic",+ TRAJECTORY: "trajectory",};const MIME_CANDIDATES = ["video/webm;codecs=vp9",@@ -92,6 +93,7 @@updateModeToggleUI();applyModeToggleVisibility();updateClassicOutlineControls();+ updateTrajectoryModeUI();initExternalBridge();setupHeightObserver();requestAnimationFrame(render);@@ -145,6 +147,7 @@dom.modeSwitcher = document.getElementById("modeSwitcher");dom.modeCutDragBtn = document.getElementById("modeCutDragBtn");dom.modeClassicBtn = document.getElementById("modeClassicBtn");+ dom.modeTrajectoryBtn = document.getElementById("modeTrajectoryBtn");dom.canvasFooter = document.getElementById("canvasFooter");dom.downloadMaskBtn = document.getElementById("downloadMaskBtn");dom.sendToWangpBtn = document.getElementById("sendToWangpBtn");@@ -290,6 +293,9 @@if (dom.modeClassicBtn) {dom.modeClassicBtn.addEventListener("click", () => setRenderMode(RENDER_MODES.CLASSIC));}+ if (dom.modeTrajectoryBtn) {+ dom.modeTrajectoryBtn.addEventListener("click", () => setRenderMode(RENDER_MODES.TRAJECTORY));+ }dom.canvas.addEventListener("pointerdown", onPointerDown);dom.canvas.addEventListener("pointermove", onPointerMove);@@ -328,16 +334,69 @@}function setRenderMode(mode) {- const normalized = mode === RENDER_MODES.CLASSIC ? RENDER_MODES.CLASSIC : RENDER_MODES.CUT_DRAG;+ let normalized;+ if (mode === RENDER_MODES.CLASSIC) {+ normalized = RENDER_MODES.CLASSIC;+ } else if (mode === RENDER_MODES.TRAJECTORY) {+ normalized = RENDER_MODES.TRAJECTORY;+ } else {+ normalized = RENDER_MODES.CUT_DRAG;+ }if (state.renderMode === normalized) {updateModeToggleUI();updateClassicOutlineControls();+ updateTrajectoryModeUI();return;}+ const previousMode = state.renderMode;state.renderMode = normalized;+ // Handle mode switching edge cases+ handleModeSwitchCleanup(previousMode, normalized);updateModeToggleUI();updateClassicOutlineControls();updateBackgroundToggleUI();+ updateTrajectoryModeUI();+ }++ function handleModeSwitchCleanup(fromMode, toMode) {+ const wasTrajectory = fromMode === RENDER_MODES.TRAJECTORY;+ const isTrajectory = toMode === RENDER_MODES.TRAJECTORY;+ if (wasTrajectory && !isTrajectory) {+ // Switching FROM trajectory mode: remove ALL trajectory-only layers+ const trajectoryLayers = state.layers.filter((l) => isTrajectoryOnlyLayer(l));+ if (trajectoryLayers.length > 0) {+ state.layers = state.layers.filter((l) => !isTrajectoryOnlyLayer(l));+ state.activeLayerId = state.layers[0]?.id || null;+ recomputeBackgroundFill();+ refreshLayerSelect();+ updateBadge();+ updateActionAvailability();+ if (state.baseImage) {+ setStatus("Switched mode. Trajectory layers removed.", "info");+ }+ }+ // Add a new empty layer for shape creation if none exist+ if (state.layers.length === 0 && state.baseImage) {+ addLayer(state.shapeMode);+ }+ } else if (!wasTrajectory && isTrajectory) {+ // Switching TO trajectory mode: remove ALL non-trajectory layers+ const shapeLayers = state.layers.filter((l) => !isTrajectoryOnlyLayer(l));+ if (shapeLayers.length > 0) {+ state.layers = state.layers.filter((l) => isTrajectoryOnlyLayer(l));+ state.activeLayerId = state.layers[0]?.id || null;+ recomputeBackgroundFill();+ refreshLayerSelect();+ updateBadge();+ updateActionAvailability();+ if (state.baseImage) {+ setStatus("Trajectory mode. Shape layers cleared.", "info");+ }+ }+ if (state.baseImage && state.layers.length === 0) {+ setStatus("Trajectory mode. Click to place trajectory points.", "info");+ }+ }}function updateModeToggleUI() {@@ -345,10 +404,16 @@return;}const isClassic = state.renderMode === RENDER_MODES.CLASSIC;- dom.modeCutDragBtn.classList.toggle("active", !isClassic);+ const isCutDrag = state.renderMode === RENDER_MODES.CUT_DRAG;+ const isTrajectory = state.renderMode === RENDER_MODES.TRAJECTORY;+ dom.modeCutDragBtn.classList.toggle("active", isCutDrag);dom.modeClassicBtn.classList.toggle("active", isClassic);- dom.modeCutDragBtn.setAttribute("aria-pressed", (!isClassic).toString());+ dom.modeCutDragBtn.setAttribute("aria-pressed", isCutDrag.toString());dom.modeClassicBtn.setAttribute("aria-pressed", isClassic.toString());+ if (dom.modeTrajectoryBtn) {+ dom.modeTrajectoryBtn.classList.toggle("active", isTrajectory);+ dom.modeTrajectoryBtn.setAttribute("aria-pressed", isTrajectory.toString());+ }}function setModeToggleVisibility(visible) {@@ -364,6 +429,32 @@updateBackgroundToggleUI();}+ function updateTrajectoryModeUI() {+ const isTrajectory = state.renderMode === RENDER_MODES.TRAJECTORY;+ // Hide shape selector in trajectory mode (points only, no shapes)+ const shapeSelector = document.querySelector(".shape-selector");+ if (shapeSelector) {+ shapeSelector.classList.toggle("is-hidden", isTrajectory);+ }+ // Hide scale and rotation controls in trajectory mode (not applicable to points)+ const scaleStartLabel = dom.scaleStartInput?.closest(".multi-field");+ const rotationStartLabel = dom.rotationStartInput?.closest(".multi-field");+ if (scaleStartLabel) {+ scaleStartLabel.classList.toggle("is-hidden", isTrajectory);+ }+ if (rotationStartLabel) {+ rotationStartLabel.classList.toggle("is-hidden", isTrajectory);+ }+ // Update Send button label for trajectory mode+ if (dom.sendToWangpBtn) {+ dom.sendToWangpBtn.textContent = isTrajectory ? "Export Trajectories" : "Send to Video Generator";+ }+ // Update Preview button label for trajectory mode+ if (dom.previewModeBtn && !state.previewMode) {+ dom.previewModeBtn.textContent = isTrajectory ? "Preview Trajectories" : "Preview Mask";+ }+ }+function initExternalBridge() {if (typeof window === "undefined") {return;@@ -505,6 +596,26 @@return layer;}+ function addTrajectoryLayer(initialPoint) {+ // Creates a trajectory-only layer for trajectory mode (no shape/polygon)+ const layer = createLayer(`Trajectory ${state.layers.length + 1}`, state.layers.length, "trajectory");+ // Mark polygon as closed so we skip polygon creation and go directly to trajectory+ layer.polygonClosed = true;+ layer.shapeType = "trajectory";+ // Add initial trajectory point+ if (initialPoint) {+ layer.path.push({ x: initialPoint.x, y: initialPoint.y });+ }+ state.layers.push(layer);+ state.activeLayerId = layer.id;+ updateLayerPathCache(layer);+ refreshLayerSelect();+ updateBindings();+ updateBadge();+ updateActionAvailability();+ return layer;+ }+function removeLayer() {if (!state.activeLayerId) {return;@@ -1229,6 +1340,30 @@stage = "none";selectionChanged = true;}+ // In trajectory mode, handle differently+ if (state.renderMode === RENDER_MODES.TRAJECTORY) {+ // If no layer exists, or existing layer is an empty non-trajectory layer, create a trajectory layer+ const isEmptyRegularLayer = layer && !isTrajectoryOnlyLayer(layer) && isLayerEmpty(layer);+ if (!layer || isEmptyRegularLayer) {+ // Remove the empty regular layer if it exists+ if (isEmptyRegularLayer) {+ state.layers = state.layers.filter((l) => l.id !== layer.id);+ }+ const newLayer = addTrajectoryLayer(pt);+ setActiveLayer(newLayer.id);+ layer = newLayer;+ stage = getLayerStage(layer);+ selectionChanged = false;+ evt.preventDefault();+ return;+ }+ // Existing trajectory layer - add point to it+ if (isTrajectoryOnlyLayer(layer) && !layer.pathLocked) {+ addTrajectoryPoint(pt);+ evt.preventDefault();+ return;+ }+ }if (!layer) {const newLayer = addLayer(state.shapeMode);setActiveLayer(newLayer.id);@@ -1767,6 +1902,10 @@if (!layer) {return false;}+ // Trajectory-only layers are ready if they have at least one trajectory point+ if (layer.shapeType === "trajectory") {+ return Array.isArray(layer.path) && layer.path.length >= 1;+ }if (!layer.objectCut || !layer.polygonClosed) {return false;}@@ -1774,6 +1913,10 @@return true;}+ function isTrajectoryOnlyLayer(layer) {+ return layer && layer.shapeType === "trajectory";+ }+function downloadBackgroundImage() {if (!state.backgroundDataUrl) {setStatus("Background image not ready yet.", "warn");@@ -1797,6 +1940,11 @@}function handleSendToWangp() {+ // Handle trajectory mode export separately+ if (state.renderMode === RENDER_MODES.TRAJECTORY) {+ exportTrajectoryData();+ return;+ }if (state.renderMode === RENDER_MODES.CUT_DRAG) {state.transfer.pending = true;state.transfer.mask = null;@@ -1812,6 +1960,133 @@}}+ function exportTrajectoryData() {+ const readyLayers = state.layers.filter((layer) => isTrajectoryOnlyLayer(layer) && layerReadyForExport(layer));+ if (readyLayers.length === 0) {+ setStatus("Add at least one trajectory point before exporting.", "warn");+ return;+ }+ setStatus("Exporting trajectory data...", "info");+ const totalFrames = Math.max(1, Math.round(state.scene.totalFrames));+ const width = state.resolution.width || 1;+ const height = state.resolution.height || 1;+ const numLayers = readyLayers.length;+ // Build trajectories array: [T, N, 2] where T=frames, N=trajectory count, 2=X,Y (normalized to [0,1])+ // Frames outside the layer's [startFrame, endFrame] range get [-1, -1]+ const trajectories = [];+ for (let frame = 0; frame < totalFrames; frame++) {+ const frameData = [];+ readyLayers.forEach((layer) => {+ const startFrame = layer.startFrame ?? 0;+ const endFrame = layer.endFrame ?? totalFrames;+ // Check if object exists at this frame+ if (frame < startFrame || frame > endFrame) {+ frameData.push([-1, -1]);+ return;+ }+ // Compute progress within the layer's active range+ const rangeFrames = endFrame - startFrame;+ const progress = rangeFrames === 0 ? 0 : (frame - startFrame) / rangeFrames;+ const position = computeTrajectoryPosition(layer, progress);+ if (position) {+ // Normalize coordinates to [0, 1] range+ frameData.push([position.x / width, position.y / height]);+ } else {+ // Use first point if no position computed+ const firstPt = layer.path[0] || { x: 0, y: 0 };+ frameData.push([firstPt.x / width, firstPt.y / height]);+ }+ });+ trajectories.push(frameData);+ }+ const metadata = {+ renderMode: "trajectory",+ width: state.resolution.width,+ height: state.resolution.height,+ fps: state.scene.fps,+ totalFrames: totalFrames,+ trajectoryCount: numLayers,+ };+ // Get background image for image_start+ const backgroundImage = getTrajectoryBackgroundImage();+ // Send trajectory data to WanGP+ const success = sendTrajectoryPayload(trajectories, metadata, backgroundImage);+ if (success) {+ setStatus("Trajectory data sent to WanGP.", "success");+ }+ }++ function getTrajectoryBackgroundImage() {+ if (!state.baseImage) {+ return null;+ }+ // Export the base canvas as data URL+ try {+ return state.baseCanvas.toDataURL("image/png");+ } catch (err) {+ console.warn("Failed to export background image", err);+ return null;+ }+ }++ function computeTrajectoryPosition(layer, progress) {+ if (!layer || !Array.isArray(layer.path) || layer.path.length === 0) {+ return null;+ }+ // If only one point, return it directly+ if (layer.path.length === 1) {+ return { x: layer.path[0].x, y: layer.path[0].y };+ }+ // Use the cached render path for interpolation+ const renderPath = getRenderPath(layer);+ if (!renderPath || renderPath.length === 0) {+ return { x: layer.path[0].x, y: layer.path[0].y };+ }+ if (renderPath.length === 1) {+ return { x: renderPath[0].x, y: renderPath[0].y };+ }+ // Interpolate along the render path based on progress+ const meta = layer.pathMeta || computePathMeta(renderPath);+ if (!meta || meta.total === 0) {+ return { x: renderPath[0].x, y: renderPath[0].y };+ }+ // Apply speed mode if configured+ const adjustedProgress = getSpeedProfileProgress(layer, progress);+ const targetDist = adjustedProgress * meta.total;+ // meta.lengths is cumulative: [0, dist0to1, dist0to2, ...]+ // Find the segment where targetDist falls+ for (let i = 1; i < renderPath.length; i++) {+ const cumulativeDist = meta.lengths[i];+ if (targetDist <= cumulativeDist) {+ const prevCumulativeDist = meta.lengths[i - 1];+ const segLen = cumulativeDist - prevCumulativeDist;+ const segProgress = segLen > 0 ? (targetDist - prevCumulativeDist) / segLen : 0;+ const p0 = renderPath[i - 1];+ const p1 = renderPath[i];+ return {+ x: p0.x + (p1.x - p0.x) * segProgress,+ y: p0.y + (p1.y - p0.y) * segProgress,+ };+ }+ }+ // Return last point if we've exceeded the path+ return { x: renderPath[renderPath.length - 1].x, y: renderPath[renderPath.length - 1].y };+ }++ function sendTrajectoryPayload(trajectories, metadata, backgroundImage) {+ try {+ window.parent?.postMessage(+ { type: EVENT_TYPE, trajectoryData: trajectories, metadata, backgroundImage, isTrajectoryExport: true },+ "*",+ );+ return true;+ } catch (err) {+ console.error("Unable to send trajectory data to WanGP", err);+ setStatus("Failed to send trajectory data to WanGP.", "error");+ return false;+ }+ }+function startExport(mode, variant = "mask") {const readyLayers = state.layers.filter((layer) => layerReadyForExport(layer));if (readyLayers.length === 0) {@@ -2317,13 +2592,21 @@const { fillBackground = false, backgroundColor = "#000", fillColor = "white" } = options;const currentRenderMode = state.export.running ? state.export.renderMode : state.renderMode;const outlineMode = currentRenderMode === RENDER_MODES.CLASSIC;+ const isTrajectoryMode = currentRenderMode === RENDER_MODES.TRAJECTORY;const outlineWidth = outlineMode ? getClassicOutlineWidth() : 0;ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);ctx.fillStyle = backgroundColor;ctx.fillRect(0, 0, ctx.canvas.width, ctx.canvas.height);state.layers.filter((layer) => layerReadyForExport(layer))- .forEach((layer) => {+ .forEach((layer, index) => {+ // Handle trajectory-only layers differently+ if (isTrajectoryOnlyLayer(layer)) {+ if (isTrajectoryMode) {+ drawTrajectoryMarker(ctx, layer, progress, index, fillColor);+ }+ return;+ }const transform = computeTransform(layer, progress);if (!transform || layer.localPolygon.length === 0) {return;@@ -2349,6 +2632,22 @@});}+ function drawTrajectoryMarker(ctx, layer, progress, index, fillColor) {+ const position = computeTrajectoryPosition(layer, progress);+ if (!position) {+ return;+ }+ const markerRadius = 5;+ const color = layer.color || COLOR_POOL[index % COLOR_POOL.length];+ ctx.save();+ // Draw filled circle with the layer color for visibility+ ctx.beginPath();+ ctx.arc(position.x, position.y, markerRadius, 0, Math.PI * 2);+ ctx.fillStyle = color;+ ctx.fill();+ ctx.restore();+ }+function drawGuideFrame(ctx, progress) {ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);if (!state.baseImage) {@@ -2370,11 +2669,16 @@function render() {const ctx = dom.ctx;if (state.previewMode) {- drawMaskFrame(ctx, state.animation.playhead, {- fillBackground: true,- backgroundColor: "#04060b",- fillColor: "white",- });+ // In trajectory mode, show base image with animated markers+ if (state.renderMode === RENDER_MODES.TRAJECTORY) {+ drawTrajectoryPreview(ctx, state.animation.playhead);+ } else {+ drawMaskFrame(ctx, state.animation.playhead, {+ fillBackground: true,+ backgroundColor: "#04060b",+ fillColor: "white",+ });+ }} else {ctx.clearRect(0, 0, dom.canvas.width, dom.canvas.height);if (state.baseImage) {@@ -2396,6 +2700,28 @@requestAnimationFrame(render);}+ function drawTrajectoryPreview(ctx, progress) {+ // Black background like other preview modes+ ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);+ ctx.fillStyle = "#04060b";+ ctx.fillRect(0, 0, ctx.canvas.width, ctx.canvas.height);+ // Draw white dots at current position for each trajectory+ state.layers+ .filter((layer) => isTrajectoryOnlyLayer(layer) && layerReadyForExport(layer))+ .forEach((layer) => {+ const position = computeTrajectoryPosition(layer, progress);+ if (!position) {+ return;+ }+ ctx.save();+ ctx.beginPath();+ ctx.arc(position.x, position.y, 5, 0, Math.PI * 2);+ ctx.fillStyle = "white";+ ctx.fill();+ ctx.restore();+ });+ }+function drawPolygonOverlay(ctx) {state.layers.forEach((layer) => {const points = getPolygonDisplayPoints(layer);@@ -2436,15 +2762,20 @@}function drawTrajectoryOverlay(ctx) {- state.layers.forEach((layer) => {+ const isTrajectoryMode = state.renderMode === RENDER_MODES.TRAJECTORY;+ const isPlaying = state.animation.playing;+ const playhead = state.animation.playhead;++ state.layers.forEach((layer, layerIndex) => {const points = getRenderPath(layer);if (!points || points.length === 0) {return;}const showPath = points.length > 1;+ const isActive = layer.id === state.activeLayerId;ctx.save();ctx.strokeStyle = TRAJECTORY_EDGE_COLOR;- ctx.globalAlpha = layer.id === state.activeLayerId ? 0.95 : 0.5;+ ctx.globalAlpha = isActive ? 0.95 : 0.5;ctx.lineWidth = 2;if (showPath) {ctx.beginPath();@@ -2454,10 +2785,10 @@}ctx.stroke();}- // Draw anchor indicator+ // Draw anchor indicator (start point)ctx.beginPath();ctx.arc(points[0].x, points[0].y, 5, 0, Math.PI * 2);- ctx.fillStyle = layer.id === state.activeLayerId ? "#57f0b7" : "#1a2b3d";+ ctx.fillStyle = isActive ? "#57f0b7" : "#1a2b3d";ctx.fill();ctx.strokeStyle = TRAJECTORY_EDGE_COLOR;ctx.stroke();@@ -2470,6 +2801,27 @@ctx.strokeStyle = "#ffbe7a";ctx.stroke();});+ // Draw animated position marker when playing (in trajectory mode)+ if (isTrajectoryMode && (isPlaying || playhead > 0) && isTrajectoryOnlyLayer(layer) && layerReadyForExport(layer)) {+ const position = computeTrajectoryPosition(layer, playhead);+ if (position) {+ const color = layer.color || COLOR_POOL[layerIndex % COLOR_POOL.length];+ // Draw animated marker with glow+ ctx.globalAlpha = 1;+ ctx.shadowColor = color;+ ctx.shadowBlur = 10;+ ctx.beginPath();+ ctx.arc(position.x, position.y, 8, 0, Math.PI * 2);+ ctx.fillStyle = color;+ ctx.fill();+ // Inner dot+ ctx.shadowBlur = 0;+ ctx.beginPath();+ ctx.arc(position.x, position.y, 3, 0, Math.PI * 2);+ ctx.fillStyle = "#ffffff";+ ctx.fill();+ }+ }ctx.restore();});}diff --git a/plugins/wan2gp-motion-designer/assets/motion_designer_iframe_template.html b/plugins/wan2gp-motion-designer/assets/motion_designer_iframe_template.htmlindex 47aa558..1178d4d 100644--- a/plugins/wan2gp-motion-designer/assets/motion_designer_iframe_template.html+++ b/plugins/wan2gp-motion-designer/assets/motion_designer_iframe_template.html@@ -23,6 +23,9 @@<button id="modeCutDragBtn" type="button" class="mode-option active" data-mode="cut_drag">i2v TTM</button>+ <button id="modeTrajectoryBtn" type="button" class="mode-option" data-mode="trajectory">+ Wan-Move+ </button></div></div><div class="header-status">diff --git a/plugins/wan2gp-motion-designer/plugin.py b/plugins/wan2gp-motion-designer/plugin.pyindex 3a2312a..7ccc695 100644--- a/plugins/wan2gp-motion-designer/plugin.py+++ b/plugins/wan2gp-motion-designer/plugin.py@@ -7,6 +7,7 @@ from pathlib import Pathimport ffmpegimport gradio as gr+import numpy as npfrom PIL import Image, ImageOpsfrom shared.utils.plugins import WAN2GPPlugin@@ -88,11 +89,47 @@ class MotionDesignerPlugin(WAN2GPPlugin):visible=False,elem_id="motion_designer_mode_sync",)+ trajectory_payload = gr.Textbox(+ label="Trajectory Payload",+ visible=False,+ elem_id="motion_designer_trajectory_payload",+ )+ trajectory_metadata = gr.Textbox(+ label="Trajectory Metadata",+ visible=False,+ elem_id="motion_designer_trajectory_meta",+ )+ trajectory_background = gr.Textbox(+ label="Trajectory Background",+ visible=False,+ elem_id="motion_designer_trajectory_background",+ )trigger = gr.Button("Apply Motion Designer data",visible=False,elem_id="motion_designer_apply_trigger",)+ trajectory_trigger = gr.Button(+ "Apply Trajectory data",+ visible=False,+ elem_id="motion_designer_trajectory_trigger",+ )++ trajectory_trigger.click(+ fn=self._apply_trajectory,+ inputs=[+ self.state,+ trajectory_payload,+ trajectory_metadata,+ trajectory_background,+ ],+ outputs=[self.refresh_form_trigger],+ show_progress="hidden",+ ).then(+ fn=self.goto_video_tab,+ inputs=[self.state],+ outputs=[self.main_tabs],+ )trigger.click(fn=self._apply_mask,@@ -126,7 +163,7 @@ class MotionDesignerPlugin(WAN2GPPlugin):return;}if (window.motionDesignerSetRenderMode) {- window.motionDesignerSetRenderMode(normalized === "classic" ? "classic" : "cut_drag");+ window.motionDesignerSetRenderMode(normalized);} else {console.warn("[MotionDesignerPlugin] motionDesignerSetRenderMode not ready yet.");}@@ -142,6 +179,8 @@ class MotionDesignerPlugin(WAN2GPPlugin):mode = "cut_drag"elif model_def.get("vace_class", False):mode = "classic"+ elif model_def.get("i2v_trajectory", False):+ mode = "trajectory"else:return gr.update()return f"{mode}|{time.time():.6f}"@@ -265,6 +304,61 @@ class MotionDesignerPlugin(WAN2GPPlugin):gr.Info("Motion Designer data transferred to the Video Generator.")return time.time()+ def _apply_trajectory(+ self,+ state,+ trajectory_json: str | None,+ metadata_json: str | None,+ background_data_url: str | None,+ ):+ if not trajectory_json:+ raise gr.Error("No trajectory data received from Motion Designer.")++ try:+ trajectories = json.loads(trajectory_json)+ if not isinstance(trajectories, list) or len(trajectories) == 0:+ raise gr.Error("Invalid trajectory data: expected non-empty array.")+ except json.JSONDecodeError as exc:+ raise gr.Error("Unable to parse trajectory data.") from exc++ metadata: dict[str, object] = {}+ if metadata_json:+ try:+ metadata = json.loads(metadata_json)+ if not isinstance(metadata, dict):+ metadata = {}+ except json.JSONDecodeError:+ metadata = {}++ # Convert to numpy array with shape [T, N, 2]+ # T = number of frames, N = number of trajectories, 2 = (x, y) coordinates+ trajectory_array = np.array(trajectories, dtype=np.float32)++ # Validate shape+ if len(trajectory_array.shape) != 3 or trajectory_array.shape[2] != 2:+ raise gr.Error(f"Invalid trajectory shape: expected [T, N, 2], got {trajectory_array.shape}")++ # Save to .npy file+ output_dir = Path("mask_outputs")+ output_dir.mkdir(parents=True, exist_ok=True)+ timestamp = time.strftime("%Y%m%d_%H%M%S")+ file_path = output_dir / f"motion_designer_trajectory_{timestamp}.npy"+ np.save(file_path, trajectory_array)++ print(f"[MotionDesignerPlugin] Trajectory saved: {file_path} (shape: {trajectory_array.shape})")++ # Update UI settings with custom_guide path+ ui_settings = self.get_current_model_settings(state)+ ui_settings["custom_guide"] = str(file_path.absolute())++ # Decode and set background image as image_start+ background_image = self._decode_background_image(background_data_url)+ if background_image is not None:+ ui_settings["image_start"] = [background_image]++ gr.Info(f"Trajectory data saved ({trajectory_array.shape[0]} frames, {trajectory_array.shape[1]} trajectories).")+ return time.time()+def _decode_background_image(self, data_url: str | None):if not data_url:return None@@ -477,7 +571,14 @@ class MotionDesignerPlugin(WAN2GPPlugin):if (!normalized) {return;}- const target = normalized === "classic" ? "classic" : "cut_drag";+ let target;+ if (normalized === "classic") {+ target = "classic";+ } else if (normalized === "trajectory") {+ target = "trajectory";+ } else {+ target = "cut_drag";+ }console.log("[MotionDesignerPlugin] Mode sync triggered:", target);motionDesignerSendControlMessage("setMode", target);};@@ -576,6 +677,27 @@ class MotionDesignerPlugin(WAN2GPPlugin):}console.debug("[MotionDesignerPlugin] Received iframe payload", event.data);const appRoot = motionDesignerRoot();++ // Handle trajectory export separately+ if (event.data.isTrajectoryExport) {+ const trajectoryInput = appRoot.querySelector("#motion_designer_trajectory_payload textarea, #motion_designer_trajectory_payload input");+ const trajectoryMetaInput = appRoot.querySelector("#motion_designer_trajectory_meta textarea, #motion_designer_trajectory_meta input");+ const trajectoryBgInput = appRoot.querySelector("#motion_designer_trajectory_background textarea, #motion_designer_trajectory_background input");+ const trajectoryButton = appRoot.querySelector("#motion_designer_trajectory_trigger button, #motion_designer_trajectory_trigger");+ if (!trajectoryInput || !trajectoryMetaInput || !trajectoryBgInput || !trajectoryButton) {+ console.warn("[MotionDesignerPlugin] Trajectory bridge components missing in Gradio DOM.");+ return;+ }+ const trajectoryData = event.data.trajectoryData || [];+ const trajectoryMetadata = event.data.metadata || {};+ const backgroundImage = event.data.backgroundImage || "";+ motionDesignerDispatchInput(trajectoryInput, JSON.stringify(trajectoryData));+ motionDesignerDispatchInput(trajectoryMetaInput, JSON.stringify(trajectoryMetadata));+ motionDesignerDispatchInput(trajectoryBgInput, backgroundImage);+ trajectoryButton.click();+ return;+ }+const maskInput = appRoot.querySelector("#motion_designer_mask_payload textarea, #motion_designer_mask_payload input");const metaInput = appRoot.querySelector("#motion_designer_meta_payload textarea, #motion_designer_meta_payload input");const bgInput = appRoot.querySelector("#motion_designer_background_payload textarea, #motion_designer_background_payload input");diff --git a/wgp.py b/wgp.pyindex fb4f388..19e043a 100644--- a/wgp.py+++ b/wgp.py@@ -83,14 +83,14 @@ global_queue_ref = []AUTOSAVE_FILENAME = "queue.zip"PROMPT_VARS_MAX = 10target_mmgp_version = "3.6.9"-WanGP_version = "9.83"+WanGP_version = "9.84"settings_version = 2.41max_source_video_frames = 3000prompt_enhancer_image_caption_model, prompt_enhancer_image_caption_processor, prompt_enhancer_llm_model, prompt_enhancer_llm_tokenizer = None, None, None, Noneimage_names_list = ["image_start", "image_end", "image_refs"]# All media attachment keys for queue save/loadATTACHMENT_KEYS = ["image_start", "image_end", "image_refs", "image_guide", "image_mask",- "video_guide", "video_mask", "video_source", "audio_guide", "audio_guide2", "audio_source"]+ "video_guide", "video_mask", "video_source", "audio_guide", "audio_guide2", "audio_source", "custom_guide"]from importlib.metadata import versionmmgp_version = version("mmgp")@@ -445,7 +445,7 @@ def process_prompt_and_add_tasks(state, current_gallery_tab, model_choice):override_inputs, prompts, image_start, image_end = validate_settings(state, model_type, False, inputs)if override_inputs is None:- ret()+ return ret()multi_prompts_gen_type = inputs["multi_prompts_gen_type"]@@ -588,6 +588,7 @@ def validate_settings(state, model_type, single_prompt, inputs):image_guide = inputs["image_guide"]video_mask = inputs["video_mask"]image_mask = inputs["image_mask"]+ custom_guide = inputs["custom_guide"]speakers_locations = inputs["speakers_locations"]video_source = inputs["video_source"]frames_positions = inputs["frames_positions"]@@ -692,6 +693,13 @@ def validate_settings(state, model_type, single_prompt, inputs):if image_outputs:image_prompt_type = image_prompt_type.replace("V", "").replace("L", "")+ custom_guide_def = model_def.get("custom_guide", None)+ if custom_guide_def is not None:+ if custom_guide is None and custom_guide_def.get("required", False):+ gr.Info(f"You must provide a {custom_guide_def.get('label', 'Custom Guide')}")+ return ret()+ else:+ custom_guide = Noneif "V" in image_prompt_type:if video_source == None:@@ -891,6 +899,7 @@ def validate_settings(state, model_type, single_prompt, inputs):"image_guide": image_guide,"video_mask": video_mask,"image_mask": image_mask,+ "custom_guide": custom_guide,"video_source": video_source,"frames_positions": frames_positions,"keep_frames_video_source": keep_frames_video_source,@@ -4932,6 +4941,7 @@ def generate_video(mask_expand,audio_guide,audio_guide2,+ custom_guide,audio_source,audio_prompt_type,speakers_locations,@@ -5653,6 +5663,7 @@ def generate_video(input_masks2 = src_mask2,input_video= pre_video_guide,input_faces = src_faces,+ input_custom = custom_guide,denoising_strength=denoising_strength,masking_strength=masking_strength,prefix_frames_count = source_video_overlap_frames_count if window_no <= 1 else reuse_frames,@@ -5725,7 +5736,7 @@ def generate_video(exaggeration=exaggeration,pace=pace,temperature=temperature,- full_prefix_video = prefix_video,+ window_start_frame_no = window_start_frame,)except Exception as e:if len(control_audio_tracks) > 0 or len(source_audio_tracks) > 0:@@ -6910,7 +6921,7 @@ def prepare_inputs_dict(target, inputs, model_type = None, model_filename = Noneif "lset_name" in inputs:inputs.pop("lset_name")- unsaved_params = ["image_start", "image_end", "image_refs", "video_guide", "image_guide", "video_source", "video_mask", "image_mask", "audio_guide", "audio_guide2", "audio_source"]+ unsaved_params = ATTACHMENT_KEYSfor k in unsaved_params:inputs.pop(k)inputs["type"] = get_model_record(get_model_name(model_type))@@ -7611,6 +7622,7 @@ def save_inputs(mask_expand,audio_guide,audio_guide2,+ custom_guide,audio_source,audio_prompt_type,speakers_locations,@@ -8780,6 +8792,13 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Nonany_audio_guide2 = any_multi_speakersaudio_guide = gr.Audio(value= ui_defaults.get("audio_guide", None), type="filepath", label= model_def.get("audio_guide_label","Voice to follow"), show_download_button= True, visible= any_audio_voices_support and "A" in audio_prompt_type_value )audio_guide2 = gr.Audio(value= ui_defaults.get("audio_guide2", None), type="filepath", label=model_def.get("audio_guide2_label","Voice to follow #2"), show_download_button= True, visible= any_audio_voices_support and "B" in audio_prompt_type_value )+ custom_guide_def = model_def.get("custom_guide", None)+ any_custom_guide= custom_guide_def is not None+ with gr.Row(visible = any_custom_guide) as custom_guide_row:+ if custom_guide_def is None:+ custom_guide = gr.File(value= None, type="filepath", label= "Custom Guide", height=41, visible= False )+ else:+ custom_guide = gr.File(value= ui_defaults.get("custom_guide", None), type="filepath", label= custom_guide_def.get("label","Custom Guide"), height=41, visible= True, file_types = custom_guide_def.get("file_types", ["*.*"]) )remove_background_sound = gr.Checkbox(label= "Remove Background Music" if audio_only else "Video Motion ignores Background Music (to get a better LipSync)", value="V" in audio_prompt_type_value, visible = any_audio_voices_support and any_letters(audio_prompt_type_value, "ABX") and not image_outputs)with gr.Row(visible = any_audio_voices_support and ("B" in audio_prompt_type_value or "X" in audio_prompt_type_value) and not image_outputs ) as speakers_locations_row:speakers_locations = gr.Text( ui_get("speakers_locations"), label="Speakers Locations separated by a Space. Each Location = Left:Right or a BBox Left:Top:Right:Bottom", visible= True)@@ -9212,7 +9231,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Non],value=filter_letters(video_prompt_type_value, "T"),label="Control Video / Control Audio / Positioned Frames Temporal Alignment when any Video to continue",- visible = vace or ltxv or t2v or infinitetalk+ visible = any_control_image or any_control_video or any_audio_guide or any_audio_guide2 or any_custom_guide)@@ -9410,7 +9429,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Nonextra_inputs = prompt_vars + [wizard_prompt, wizard_variables_var, wizard_prompt_activated_var, video_prompt_column, image_prompt_column, image_prompt_type_group, image_prompt_type_radio, image_prompt_type_endcheckbox,prompt_column_advanced, prompt_column_wizard_vars, prompt_column_wizard, lset_name, save_lset_prompt_drop, advanced_row, speed_tab, audio_tab, mmaudio_col, quality_tab,- sliding_window_tab, misc_tab, prompt_enhancer_row, inference_steps_row, skip_layer_guidance_row, audio_guide_row, RIFLEx_setting_col,+ sliding_window_tab, misc_tab, prompt_enhancer_row, inference_steps_row, skip_layer_guidance_row, audio_guide_row, custom_guide_row, RIFLEx_setting_col,video_prompt_type_video_guide, video_prompt_type_video_guide_alt, video_prompt_type_video_mask, video_prompt_type_image_refs, video_prompt_type_video_custom_dropbox, video_prompt_type_video_custom_checkbox,apg_col, audio_prompt_type_sources, audio_prompt_type_remux, audio_prompt_type_remux_row, force_fps_col,video_guide_outpainting_col,video_guide_outpainting_top, video_guide_outpainting_bottom, video_guide_outpainting_left, video_guide_outpainting_right,