SkyReels-V2: Infinite-length Film Generative model
Python
support teacache (#35)
linchunze1028f5379c8a
diff --git a/README.md b/README.mdindex 6da9680..f01ff22 100644--- a/README.md+++ b/README.md@@ -191,13 +191,16 @@ python3 generate_video_df.py \--overlap_history 17 \--prompt "A graceful white swan with a curved neck and delicate feathers swimming in a serene lake at dawn, its reflection perfectly mirrored in the still water as mist rises from the surface, with the swan occasionally dipping its head into the water to feed." \--addnoise_condition 20 \- --offload+ --offload \+ --teacache \+ --use_ret_steps \+ --teacache_thresh 0.3```asynchronous generation for 30s video```shellmodel_id=Skywork/SkyReels-V2-DF-14B-540P-# synchronous inference+# asynchronous inferencepython3 generate_video_df.py \--model_id ${model_id} \--resolution 540P \@@ -232,7 +235,10 @@ python3 generate_video.py \--shift 8.0 \--fps 24 \--prompt "A serene lake surrounded by towering mountains, with a few swans gracefully gliding across the water and sunlight dancing on the surface." \- --offload+ --offload \+ --teacache \+ --use_ret_steps \+ --teacache_thresh 0.3```> **Note**:> - When using an **image-to-video (I2V)** model, you must provide an input image using the `--image ${image_path}` parameter. The `--guidance_scale 5.0` and `--shift 3.0` is recommended for I2V model.@@ -269,7 +275,10 @@ Below are the key parameters you can customize for video generation:| --offload | True | Offloads model components to CPU to reduce VRAM usage (recommended) || --use_usp | True | Enables multi-GPU acceleration with xDiT USP || --outdir | ./video_out | Directory where generated videos will be saved |-| --prompt_enhancer | True | expand the prompt into a more detailed description |+| --prompt_enhancer | True | Expand the prompt into a more detailed description |+| --teacache | False | Enables teacache for faster inference |+| --teacache_thresh | 0.2 | Higher speedup will cause to worse quality |+| --use_ret_steps | False | Retention Steps for teacache |**Diffusion Forcing Additional Parameters**| Parameter | Recommended Value | Description |diff --git a/generate_video.py b/generate_video.pyindex 2e09350..72b1174 100644--- a/generate_video.py+++ b/generate_video.py@@ -48,6 +48,16 @@ if __name__ == "__main__":default="A serene lake surrounded by towering mountains, with a few swans gracefully gliding across the water and sunlight dancing on the surface.",)parser.add_argument("--prompt_enhancer", action="store_true")+ parser.add_argument("--teacache", action="store_true")+ parser.add_argument(+ "--teacache_thresh",+ type=float,+ default=0.2,+ help="Higher speedup will cause to worse quality -- 0.1 for 2.0x speedup -- 0.2 for 3.0x speedup")+ parser.add_argument(+ "--use_ret_steps",+ action="store_true",+ help="Using Retention Steps will result in faster generation speed and better generation quality.")args = parser.parse_args()args.model_id = download_model(args.model_id)@@ -116,6 +126,11 @@ if __name__ == "__main__":height, width = width, heightargs.image = resizecrop(args.image, height, width)+ if args.teacache:+ pipe.transformer.initialize_teacache(enable_teacache=True, num_steps=args.inference_steps,+ teacache_thresh=args.teacache_thresh, use_ret_steps=args.use_ret_steps,+ ckpt_dir=args.model_id)+prompt_input = args.promptif args.prompt_enhancer and image is not None:prompt_input = prompt_enhancer(prompt_input)diff --git a/generate_video_df.py b/generate_video_df.pyindex 4bc3d55..7b73cd2 100644--- a/generate_video_df.py+++ b/generate_video_df.py@@ -39,6 +39,16 @@ if __name__ == "__main__":default="A woman in a leather jacket and sunglasses riding a vintage motorcycle through a desert highway at sunset, her hair blowing wildly in the wind as the motorcycle kicks up dust, with the golden sun casting long shadows across the barren landscape.",)parser.add_argument("--prompt_enhancer", action="store_true")+ parser.add_argument("--teacache", action="store_true")+ parser.add_argument(+ "--teacache_thresh",+ type=float,+ default=0.2,+ help="Higher speedup will cause to worse quality -- 0.1 for 2.0x speedup -- 0.2 for 3.0x speedup")+ parser.add_argument(+ "--use_ret_steps",+ action="store_true",+ help="Using Retention Steps will result in faster generation speed and better generation quality.")args = parser.parse_args()args.model_id = download_model(args.model_id)@@ -117,6 +127,16 @@ if __name__ == "__main__":if args.causal_attention:pipe.transformer.set_ar_attention(args.causal_block_size)++ if args.teacache:+ if args.ar_step > 0:+ num_steps = args.inference_steps + (((args.base_num_frames - 1)//4 + 1) // args.causal_block_size - 1) * args.ar_step+ print('num_steps:', num_steps)+ else:+ num_steps = args.inference_steps+ pipe.transformer.initialize_teacache(enable_teacache=True, num_steps=num_steps,+ teacache_thresh=args.teacache_thresh, use_ret_steps=args.use_ret_steps,+ ckpt_dir=args.model_id)print(f"prompt:{prompt_input}")print(f"guidance_scale:{guidance_scale}")diff --git a/skyreels_v2_infer/modules/transformer.py b/skyreels_v2_infer/modules/transformer.pyindex 6f41558..3bef5ce 100644--- a/skyreels_v2_infer/modules/transformer.py+++ b/skyreels_v2_infer/modules/transformer.py@@ -1,6 +1,6 @@# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved.import math-+import numpy as npimport torchimport torch.amp as ampimport torch.nn as nn@@ -484,6 +484,7 @@ class WanModel(ModelMixin, ConfigMixin, PeftAdapterMixin):self.num_frame_per_block = 1self.flag_causal_attention = Falseself.block_mask = None+ self.enable_teacache = False# embeddingsself.patch_embedding = nn.Conv3d(in_dim, dim, kernel_size=patch_size, stride=patch_size)@@ -574,6 +575,50 @@ class WanModel(ModelMixin, ConfigMixin, PeftAdapterMixin):return block_mask+ def initialize_teacache(self, enable_teacache=True, num_steps=25, teacache_thresh=0.15, use_ret_steps=False, ckpt_dir=''):+ self.enable_teacache = enable_teacache+ print('using teacache')+ self.cnt = 0+ self.num_steps = num_steps+ self.teacache_thresh = teacache_thresh+ self.accumulated_rel_l1_distance_even = 0+ self.accumulated_rel_l1_distance_odd = 0+ self.previous_e0_even = None+ self.previous_e0_odd = None+ self.previous_residual_even = None+ self.previous_residual_odd = None+ self.use_ref_steps = use_ret_steps+ if "I2V" in ckpt_dir:+ if use_ret_steps:+ if '540P' in ckpt_dir:+ self.coefficients = [ 2.57151496e+05, -3.54229917e+04, 1.40286849e+03, -1.35890334e+01, 1.32517977e-01]+ if '720P' in ckpt_dir:+ self.coefficients = [ 8.10705460e+03, 2.13393892e+03, -3.72934672e+02, 1.66203073e+01, -4.17769401e-02]+ self.ret_steps = 5*2+ self.cutoff_steps = num_steps*2+ else:+ if '540P' in ckpt_dir:+ self.coefficients = [-3.02331670e+02, 2.23948934e+02, -5.25463970e+01, 5.87348440e+00, -2.01973289e-01]+ if '720P' in ckpt_dir:+ self.coefficients = [-114.36346466, 65.26524496, -18.82220707, 4.91518089, -0.23412683]+ self.ret_steps = 1*2+ self.cutoff_steps = num_steps*2 - 2+ else:+ if use_ret_steps:+ if '1.3B' in ckpt_dir:+ self.coefficients = [-5.21862437e+04, 9.23041404e+03, -5.28275948e+02, 1.36987616e+01, -4.99875664e-02]+ if '14B' in ckpt_dir:+ self.coefficients = [-3.03318725e+05, 4.90537029e+04, -2.65530556e+03, 5.87365115e+01, -3.15583525e-01]+ self.ret_steps = 5*2+ self.cutoff_steps = num_steps*2+ else:+ if '1.3B' in ckpt_dir:+ self.coefficients = [2.39676752e+03, -1.31110545e+03, 2.01331979e+02, -8.29855975e+00, 1.37887774e-01]+ if '14B' in ckpt_dir:+ self.coefficients = [-5784.54975374, 5449.50911966, -1811.16591783, 256.27178429, -13.02252404]+ self.ret_steps = 1*2+ self.cutoff_steps = num_steps*2 - 2+def forward(self, x, t, context, clip_fea=None, y=None, fps=None):r"""Forward pass through the diffusion model@@ -664,13 +709,68 @@ class WanModel(ModelMixin, ConfigMixin, PeftAdapterMixin):# argumentskwargs = dict(e=e0, grid_sizes=grid_sizes, freqs=self.freqs, context=context, block_mask=self.block_mask)- for block in self.blocks:- x = block(x, **kwargs)+ if self.enable_teacache:+ modulated_inp = e0 if self.use_ref_steps else e+ # teacache+ if self.cnt%2==0: # even -> conditon+ self.is_even = True+ if self.cnt < self.ret_steps or self.cnt >= self.cutoff_steps:+ should_calc_even = True+ self.accumulated_rel_l1_distance_even = 0+ else:+ rescale_func = np.poly1d(self.coefficients)+ self.accumulated_rel_l1_distance_even += rescale_func(((modulated_inp-self.previous_e0_even).abs().mean() / self.previous_e0_even.abs().mean()).cpu().item())+ if self.accumulated_rel_l1_distance_even < self.teacache_thresh:+ should_calc_even = False+ else:+ should_calc_even = True+ self.accumulated_rel_l1_distance_even = 0+ self.previous_e0_even = modulated_inp.clone()++ else: # odd -> unconditon+ self.is_even = False+ if self.cnt < self.ret_steps or self.cnt >= self.cutoff_steps:+ should_calc_odd = True+ self.accumulated_rel_l1_distance_odd = 0+ else:+ rescale_func = np.poly1d(self.coefficients)+ self.accumulated_rel_l1_distance_odd += rescale_func(((modulated_inp-self.previous_e0_odd).abs().mean() / self.previous_e0_odd.abs().mean()).cpu().item())+ if self.accumulated_rel_l1_distance_odd < self.teacache_thresh:+ should_calc_odd = False+ else:+ should_calc_odd = True+ self.accumulated_rel_l1_distance_odd = 0+ self.previous_e0_odd = modulated_inp.clone()++ if self.enable_teacache:+ if self.is_even:+ if not should_calc_even:+ x += self.previous_residual_even+ else:+ ori_x = x.clone()+ for block in self.blocks:+ x = block(x, **kwargs)+ self.previous_residual_even = x - ori_x+ else:+ if not should_calc_odd:+ x += self.previous_residual_odd+ else:+ ori_x = x.clone()+ for block in self.blocks:+ x = block(x, **kwargs)+ self.previous_residual_odd = x - ori_x++ else:+ for block in self.blocks:+ x = block(x, **kwargs)x = self.head(x, e)# unpatchifyx = self.unpatchify(x, grid_sizes)+ self.cnt += 1+ if self.cnt >= self.num_steps:+ self.cnt = 0return x.float()def unpatchify(self, x, grid_sizes):diff --git a/skyreels_v2_infer/pipelines/diffusion_forcing_pipeline.py b/skyreels_v2_infer/pipelines/diffusion_forcing_pipeline.pyindex c6f39ec..b27f1d8 100644--- a/skyreels_v2_infer/pipelines/diffusion_forcing_pipeline.py+++ b/skyreels_v2_infer/pipelines/diffusion_forcing_pipeline.py@@ -328,6 +328,9 @@ class DiffusionForcingPipeline:finished_frame_num = i * (base_num_frames - overlap_history_frames) + overlap_history_framesleft_frame_num = latent_length - finished_frame_numbase_num_frames_iter = min(left_frame_num + overlap_history_frames, base_num_frames)+ if ar_step > 0 and self.transformer.enable_teacache:+ num_steps = num_inference_steps + ((base_num_frames_iter - overlap_history_frames) // causal_block_size - 1) * ar_step+ self.transformer.num_steps = num_stepselse: # i == 0base_num_frames_iter = base_num_frameslatent_shape = [16, base_num_frames_iter, latent_height, latent_width]