Skip to content
Python

support spatial_refine_only

zhangtong65fdf183ecde8c
diff --git a/run_demo_image_to_video.py b/run_demo_image_to_video.py
index fb85287..aa3668f 100644
--- a/run_demo_image_to_video.py
+++ b/run_demo_image_to_video.py
@@ -30,6 +30,7 @@ def generate(args):
image = load_image(image_path)
prompt = "A woman sits at a wooden table by the window in a cozy café. She reaches out with her right hand, picks up the white coffee cup from the saucer, and gently brings it to her lips to take a sip. After drinking, she places the cup back on the table and looks out the window, enjoying the peaceful atmosphere."
negative_prompt = "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards"
+ spatial_refine_only = False
# load parsed args
checkpoint_dir = args.checkpoint_dir
@@ -147,6 +148,7 @@ def generate(args):
num_cond_frames=1,
num_inference_steps=50,
generator=generator,
+ spatial_refine_only=spatial_refine_only
)[0]
pipe.dit.disable_all_loras()
@@ -158,7 +160,8 @@ def generate(args):
output_refine = [frame.resize(target_size, PIL.Image.BICUBIC) for frame in output_refine]
output_tensor = torch.from_numpy(np.array(output_refine))
- write_video("output_i2v_refine.mp4", output_tensor, fps=30, video_codec="libx264", options={"crf": f"{10}"})
+ fps = 15 if spatial_refine_only else 30
+ write_video("output_i2v_refine.mp4", output_tensor, fps=fps, video_codec="libx264", options={"crf": f"{10}"})
def _parse_args():
diff --git a/run_demo_interactive_video.py b/run_demo_interactive_video.py
index c68fefc..dd13f36 100644
--- a/run_demo_interactive_video.py
+++ b/run_demo_interactive_video.py
@@ -36,6 +36,7 @@ def generate(args):
num_segments = len(prompt_list) - 1 # 1 minute video
num_frames = 93
num_cond_frames = 13
+ spatial_refine_only = False
# load parsed args
checkpoint_dir = args.checkpoint_dir
@@ -157,7 +158,7 @@ def generate(args):
for segment_idx in range(num_segments+1):
if local_rank == 0:
- print(f"Refine segment {segment_idx + 1}/{num_segments}...")
+ print(f"Refine segment {segment_idx + 1}/{num_segments+1}...")
output_refine = pipe.generate_refine(
video=cur_condition_video,
@@ -166,6 +167,7 @@ def generate(args):
num_cond_frames=cur_num_cond_frames,
num_inference_steps=50,
generator=generator,
+ spatial_refine_only=spatial_refine_only
)[0]
new_video = [(output_refine[i] * 255).astype(np.uint8) for i in range(output_refine.shape[0])]
@@ -174,12 +176,13 @@ def generate(args):
all_refine_frames.extend(new_video[cur_num_cond_frames:])
cur_condition_video = new_video
- cur_num_cond_frames = num_cond_frames * 2
+ cur_num_cond_frames = num_cond_frames if spatial_refine_only else num_cond_frames * 2
start_id = start_id + num_frames - num_cond_frames
if local_rank == 0:
output_tensor = torch.from_numpy(np.array(all_refine_frames))
- write_video(f"output_interactive_refine_{segment_idx}.mp4", output_tensor, fps=30, video_codec="libx264", options={"crf": f"{10}"})
+ fps = 15 if spatial_refine_only else 30
+ write_video(f"output_interactive_refine_{segment_idx}.mp4", output_tensor, fps=fps, video_codec="libx264", options={"crf": f"{10}"})
def _parse_args():
parser = argparse.ArgumentParser()
diff --git a/run_demo_long_video.py b/run_demo_long_video.py
index 54018aa..6b2aee8 100644
--- a/run_demo_long_video.py
+++ b/run_demo_long_video.py
@@ -29,6 +29,7 @@ def generate(args):
num_segments = 11 # 1 minute video
num_frames = 93
num_cond_frames = 13
+ spatial_refine_only = False
# load parsed args
checkpoint_dir = args.checkpoint_dir
@@ -150,7 +151,7 @@ def generate(args):
for segment_idx in range(num_segments+1):
if local_rank == 0:
- print(f"Refine segment {segment_idx + 1}/{num_segments}...")
+ print(f"Refine segment {segment_idx + 1}/{num_segments+1}...")
output_refine = pipe.generate_refine(
video=cur_condition_video,
@@ -159,6 +160,7 @@ def generate(args):
num_cond_frames=cur_num_cond_frames,
num_inference_steps=50,
generator=generator,
+ spatial_refine_only=spatial_refine_only
)[0]
new_video = [(output_refine[i] * 255).astype(np.uint8) for i in range(output_refine.shape[0])]
@@ -167,12 +169,13 @@ def generate(args):
all_refine_frames.extend(new_video[cur_num_cond_frames:])
cur_condition_video = new_video
- cur_num_cond_frames = num_cond_frames * 2
+ cur_num_cond_frames = num_cond_frames if spatial_refine_only else num_cond_frames * 2
start_id = start_id + num_frames - num_cond_frames
if local_rank == 0:
output_tensor = torch.from_numpy(np.array(all_refine_frames))
- write_video(f"output_longvideo_refine_{segment_idx}.mp4", output_tensor, fps=30, video_codec="libx264", options={"crf": f"{10}"})
+ fps = 15 if spatial_refine_only else 30
+ write_video(f"output_longvideo_refine_{segment_idx}.mp4", output_tensor, fps=fps, video_codec="libx264", options={"crf": f"{10}"})
def _parse_args():
parser = argparse.ArgumentParser()
diff --git a/run_demo_text_to_video.py b/run_demo_text_to_video.py
index 51d67f1..4fe8a28 100644
--- a/run_demo_text_to_video.py
+++ b/run_demo_text_to_video.py
@@ -27,6 +27,7 @@ def generate(args):
# case setup
prompt = "In a realistic photography style, a white boy around seven or eight years old sits on a park bench, wearing a light blue T-shirt, denim shorts, and white sneakers. He holds an ice cream cone with vanilla and chocolate flavors, and beside him is a medium-sized golden Labrador. Smiling, the boy offers the ice cream to the dog, who eagerly licks it with its tongue. The sun is shining brightly, and the background features a green lawn and several tall trees, creating a warm and loving scene."
negative_prompt = "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards"
+ spatial_refine_only = False
# load parsed args
checkpoint_dir = args.checkpoint_dir
@@ -134,6 +135,7 @@ def generate(args):
stage1_video=stage1_video,
num_inference_steps=50,
generator=generator,
+ spatial_refine_only=spatial_refine_only
)[0]
pipe.dit.disable_all_loras()
@@ -142,7 +144,8 @@ def generate(args):
if local_rank == 0:
output_tensor = torch.from_numpy(output_refine)
output_tensor = (output_tensor * 255).clamp(0, 255).to(torch.uint8)
- write_video("output_t2v_refine.mp4", output_tensor, fps=30, video_codec="libx264", options={"crf": f"{10}"})
+ fps = 15 if spatial_refine_only else 30
+ write_video("output_t2v_refine.mp4", output_tensor, fps=fps, video_codec="libx264", options={"crf": f"{10}"})
def _parse_args():
diff --git a/run_demo_video_continuation.py b/run_demo_video_continuation.py
index 2c45c5f..937e1c4 100644
--- a/run_demo_video_continuation.py
+++ b/run_demo_video_continuation.py
@@ -41,6 +41,7 @@ def generate(args):
prompt = "A person rides a motorcycle along a long, straight road that stretches between a body of water and a forested hillside. The rider steadily accelerates, keeping the motorcycle centered between the guardrails, while the scenery passes by on both sides. The video captures the journey from the rider’s perspective, emphasizing the sense of motion and adventure."
negative_prompt = "Bright tones, overexposed, static, blurred details, subtitles, style, works, paintings, images, static, overall gray, worst quality, low quality, JPEG compression residue, ugly, incomplete, extra fingers, poorly drawn hands, poorly drawn faces, deformed, disfigured, misshapen limbs, fused fingers, still picture, messy background, three legs, many people in the background, walking backwards"
num_cond_frames = 13
+ spatial_refine_only = False
# load parsed args
checkpoint_dir = args.checkpoint_dir
@@ -165,14 +166,16 @@ def generate(args):
target_fps = 30
stride = max(1, round(current_fps / target_fps))
+ cur_num_cond_frames = num_cond_frames if spatial_refine_only else num_cond_frames * 2
output_refine = pipe.generate_refine(
video=video[::stride],
prompt=prompt,
stage1_video=stage1_video,
- num_cond_frames=num_cond_frames*2,
+ num_cond_frames=cur_num_cond_frames,
num_inference_steps=50,
generator=generator,
+ spatial_refine_only=spatial_refine_only
)[0]
pipe.dit.disable_all_loras()
@@ -182,10 +185,11 @@ def generate(args):
output_refine = [(output_refine[i] * 255).astype(np.uint8) for i in range(output_refine.shape[0])]
output_refine = [PIL.Image.fromarray(img) for img in output_refine]
output_refine = [frame.resize(target_size, PIL.Image.BICUBIC) for frame in output_refine]
- output_refine = video[::stride] + output_refine[num_cond_frames*2:]
+ output_refine = video[::stride] + output_refine[cur_num_cond_frames:]
output_tensor = torch.from_numpy(np.array(output_refine))
- write_video("output_vc_refine.mp4", output_tensor, fps=30, video_codec="libx264", options={"crf": f"{10}"})
+ fps = 15 if spatial_refine_only else 30
+ write_video("output_vc_refine.mp4", output_tensor, fps=fps, video_codec="libx264", options={"crf": f"{10}"})
def _parse_args():