Open-Sora: Democratizing Efficient Video Production for All
Python
merge video edit
zhengzangw11f1822cd2a3
diff --git a/.gitignore b/.gitignoreindex 2b9e525..b9f8121 100644--- a/.gitignore+++ b/.gitignore@@ -174,6 +174,7 @@ sampleslogs/pretrained_models/evaluation_results/+cache/*.swp# Secret filesdiff --git a/assets/texts/t2v_ref.txt b/assets/texts/t2v_ref.txtindex 49252b6..c0debe5 100644--- a/assets/texts/t2v_ref.txt+++ b/assets/texts/t2v_ref.txt@@ -3,3 +3,4 @@ In an ornate, historical hall, a massive tidal wave peaks and begins to crash. TPirate ship in a cosmic maelstrom nebula.Drone view of waves crashing against the rugged cliffs along Big Sur’s garay point beach. The crashing blue waters create white-tipped waves, while the golden light of the setting sun illuminates the rocky shore. A small island with a lighthouse sits in the distance, and green shrubbery covers the cliff’s edge. The steep drop from the road down to the beach is a dramatic feat, with the cliff's edges jutting out over the sea. This is a view that captures the raw beauty of the coast and the rugged landscape of the Pacific Coast Highway.A sad small cactus with in the Sahara desert becomes happy.+A car driving on a road in the middle of a desert.diff --git a/configs/opensora-v1-1/inference/sample-ref.py b/configs/opensora-v1-1/inference/sample-ref.pyindex 5099164..6e8e805 100644--- a/configs/opensora-v1-1/inference/sample-ref.py+++ b/configs/opensora-v1-1/inference/sample-ref.py@@ -6,19 +6,28 @@ multi_resolution = "STDiT2"# Conditionprompt_path = None-prompt = None+prompt = [+ "A car driving on a road in the middle of a desert.",+ # "A man smiling",+ # "Drone view of waves crashing against the rugged cliffs along Big Sur's garay point beach. The crashing blue waters create white-tipped waves, while the golden light of the setting sun illuminates the rocky shore. A small island with a lighthouse sits in the distance, and green shrubbery covers the cliff’s edge. The steep drop from the road down to the beach is a dramatic feat, with the cliff's edges jutting out over the sea. This is a view that captures the raw beauty of the coast and the rugged landscape of the Pacific Coast Highway.",+ # "In an ornate, historical hall, a massive tidal wave peaks and begins to crash. Two surfers, seizing the moment, skillfully navigate the face of the wave.",+]-loop = 10+loop = 1condition_frame_length = 4reference_path = [- "assets/images/condition/cliff.png",- "assets/images/condition/wave.png",+ "https://cdn.openai.com/tmp/s/interp/d0.mp4",+ # "https://www.comp.nus.edu.sg/~youy/index_files/yangyou3.png",+ # "assets/images/condition/cliff.png",+ # "assets/images/condition/wave.png",]# valid when reference_path is not None# (loop id, ref id, ref start, length, target start)mask_strategy = [- "0,0,0,1,0",- "0,0,0,1,0",+ "0,0,0,8,0,0.5",+ # "0,0,0,1,0",+ # "0,0,0,1,0",+ # "0,0,0,1,0",]# Define modeldiff --git a/configs/opensora/inference-long/16x256x256-sdedit.py b/configs/opensora/inference-long/16x256x256-sdedit.pyindex 4447455..4ade991 100644--- a/configs/opensora/inference-long/16x256x256-sdedit.py+++ b/configs/opensora/inference-long/16x256x256-sdedit.py@@ -24,7 +24,6 @@ text_encoder = dict()scheduler = dict(type="iddpm",- # type="dpm-solver",num_sampling_steps=100,cfg_scale=7.0,)@@ -33,31 +32,16 @@ dtype = "fp16"# Conditionprompt_path = Noneprompt = [- # "Drone view of waves crashing against the rugged cliffs along Big Sur’s garay point beach. The crashing blue waters create white-tipped waves, while the golden light of the setting sun illuminates the rocky shore. A small island with a lighthouse sits in the distance, and green shrubbery covers the cliff’s edge. The steep drop from the road down to the beach is a dramatic feat, with the cliff's edges jutting out over the sea. This is a view that captures the raw beauty of the coast and the rugged landscape of the Pacific Coast Highway.",- "In an ornate, historical hall, a massive tidal wave peaks and begins to crash. Two surfers, seizing the moment, skillfully navigate the face of the wave.",- # "put the video in space with a rainbow road",- # "make it have dinosaurs",- # "make it in claymation animation style",- # "make it go underwater"+ "A car driving on a road in the middle of a desert.",]loop = 1condition_frame_length = 4reference_path = [- # "assets/images/condition/cliff.png",- "assets/images/condition/wave.png",- # "/home/zhouyukun/Open-Sora-dev/assets/videos/base.mp4",- # "/home/zhouyukun/Open-Sora-dev/assets/videos/base.mp4",- # "/home/zhouyukun/Open-Sora-dev/assets/videos/base.mp4",- # "/home/zhouyukun/Open-Sora-dev/assets/videos/base.mp4",+ "https://cdn.openai.com/tmp/s/interp/d0.mp4",]mask_strategy = [- # "0,0,0,1,0,0","0,0,0,1,0,0",- # "0,0,0,12,0,0", # 噪声率- # "0,0,0,12,0,0", # 噪声率- # "0,0,0,12,0,0", # 噪声率- # "0,0,0,12,0,0", # 噪声率] # valid when reference_path is not None# (loop id, ref id, ref start, length, target start)diff --git a/eval/sample.sh b/eval/sample.shindex 297205e..37320b0 100644--- a/eval/sample.sh+++ b/eval/sample.sh@@ -120,11 +120,11 @@ function run_video_edit() { # 23min# 3.2eval $CMD_REF --ckpt-path $CKPT --save-dir $OUTPUT --sample-name ref_L1_128x240x426 \- --prompt-path assets/texts/t2v_ref.txt --start-index 3 --end-index 5 \+ --prompt-path assets/texts/t2v_ref.txt --start-index 3 --end-index 6 \--num-frames 128 --image-size 240 426 \--loop 1 \- --reference-path assets/images/condition/cliff.png "assets/images/condition/cactus-sad.png\;assets/images/condition/cactus-happy.png" \- --mask-strategy "0,0,0,1,0\;0,0,0,1,-1" "0,0,0,1,0\;0,1,0,1,-1"+ --reference-path assets/images/condition/cliff.png "assets/images/condition/cactus-sad.png\;assets/images/condition/cactus-happy.png" https://cdn.openai.com/tmp/s/interp/d0.mp4 \+ --mask-strategy "0,0,0,1,0\;0,0,0,1,-1" "0,0,0,1,0\;0,1,0,1,-1" "0,0,0,64,0,0.5"}# vbench has 950 samplesdiff --git a/opensora/datasets/utils.py b/opensora/datasets/utils.pyindex 907ba6a..0e0ba6a 100644--- a/opensora/datasets/utils.py+++ b/opensora/datasets/utils.py@@ -1,7 +1,9 @@import os+import reimport numpy as npimport pandas as pd+import requestsimport torchimport torchvisionimport torchvision.transforms as transforms@@ -14,6 +16,20 @@ from . import video_transformsVID_EXTENSIONS = (".mp4", ".avi", ".mov", ".mkv")+regex = re.compile(+ r"^(?:http|ftp)s?://" # http:// or https://+ r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|" # domain...+ r"localhost|" # localhost...+ r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})" # ...or ip+ r"(?::\d+)?" # optional port+ r"(?:/?|[/?]\S+)$",+ re.IGNORECASE,+)+++def is_url(url):+ return re.match(regex, url) is not None+def read_file(input_path):if input_path.endswith(".csv"):@@ -24,6 +40,19 @@ def read_file(input_path):raise NotImplementedError(f"Unsupported file format: {input_path}")+def download_url(input_path):+ output_dir = "cache"+ if not os.path.exists(output_dir):+ os.makedirs(output_dir)+ base_name = os.path.basename(input_path)+ output_path = os.path.join(output_dir, base_name)+ img_data = requests.get(input_path).content+ with open(output_path, "wb") as handler:+ handler.write(img_data)+ print(f"URL {input_path} downloaded to {output_path}")+ return output_path++def temporal_random_crop(vframes, num_frames, frame_interval):temporal_sample = video_transforms.TemporalRandomCrop(num_frames * frame_interval)total_frames = len(vframes)@@ -106,6 +135,8 @@ def read_video_from_path(path, transform=None, transform_name="center", image_sidef read_from_path(path, image_size, transform_name="center"):+ if is_url(path):+ path = download_url(path)ext = os.path.splitext(path)[-1].lower()if ext.lower() in VID_EXTENSIONS:return read_video_from_path(path, image_size=image_size, transform_name=transform_name)diff --git a/opensora/schedulers/iddpm/gaussian_diffusion.py b/opensora/schedulers/iddpm/gaussian_diffusion.pyindex fa734f4..55292b2 100644--- a/opensora/schedulers/iddpm/gaussian_diffusion.py+++ b/opensora/schedulers/iddpm/gaussian_diffusion.py@@ -408,15 +408,21 @@ class GaussianDiffusion:if mask is not None:if mask.shape[0] != x.shape[0]:mask = mask.repeat(2, 1) # HACK- # copy unchanged x values to x0+ mask_t = (mask * len(self.betas) - 1).to(torch.int)++ # x0: copy unchanged x values+ # x_noise: add noise to x valuesx0 = x.clone()- mask_t = (mask * len(self.betas)).to(torch.int)+ x_noise = x0 * _extract_into_tensor(self.sqrt_alphas_cumprod, t, x.shape) + torch.randn_like(+ x+ ) * _extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, x.shape)++ # active noise additionmask_t_equall = (mask_t == t.unsqueeze(1))[:, None, :, None, None]+ x = torch.where(mask_t_equall, x_noise, x0)++ # create x_maskmask_t_upper = (mask_t > t.unsqueeze(1))[:, None, :, None, None]- x_noise = x0 * _extract_into_tensor(self.sqrt_alphas_cumprod, t, x.shape) + torch.randn_like(x) * _extract_into_tensor(- self.sqrt_one_minus_alphas_cumprod, t, x.shape)- # active noise addition- x = torch.where(mask_t_equall, x_noise, x)batch_size = x.shape[0]model_kwargs["x_mask"] = mask_t_upper.reshape(batch_size, -1).to(torch.bool)diff --git a/scripts/inference-long.py b/scripts/inference-long.pyindex 1ab74d3..69060dc 100644--- a/scripts/inference-long.py+++ b/scripts/inference-long.py@@ -34,9 +34,11 @@ def apply_mask_strategy(z, refs_x, mask_strategys, loop_i):masks = []for i, mask_strategy in enumerate(mask_strategys):mask_strategy = mask_strategy.split(";")- mask = torch.ones(z.shape[2], dtype=torch.bool, device=z.device)+ mask = torch.ones(z.shape[2], dtype=torch.float, device=z.device)for mst in mask_strategy:- loop_id, m_id, m_ref_start, m_length, m_target_start = mst.split(",")+ mask_batch = mst.split(",")+ loop_id, m_id, m_ref_start, m_length, m_target_start = mask_batch[:5]+ edit_ratio = mask_batch[5] if len(mask_batch) == 6 else 0.0loop_id = int(loop_id)if loop_id != loop_i:continue@@ -44,6 +46,7 @@ def apply_mask_strategy(z, refs_x, mask_strategys, loop_i):m_ref_start = int(m_ref_start)m_length = int(m_length)m_target_start = int(m_target_start)+ edit_ratio = float(edit_ratio)ref = refs_x[i][m_id] # [C, T, H, W]if m_ref_start < 0:m_ref_start = ref.shape[1] + m_ref_start@@ -51,7 +54,7 @@ def apply_mask_strategy(z, refs_x, mask_strategys, loop_i):# z: [B, C, T, H, W]m_target_start = z.shape[2] + m_target_startz[i, :, m_target_start : m_target_start + m_length] = ref[:, m_ref_start : m_ref_start + m_length]- mask[m_target_start : m_target_start + m_length] = 0+ mask[m_target_start : m_target_start + m_length] = edit_ratiomasks.append(mask)masks = torch.stack(masks)return masks@@ -69,7 +72,7 @@ def process_prompts(prompts, num_loop):text = text_preprocessing(text)end_loop = int(prompt_list[i + 2]) if i + 2 < len(prompt_list) else num_looptext_list.extend([text] * (end_loop - start_loop))- assert len(text_list) == num_loop+ assert len(text_list) == num_loop, f"Prompt loop mismatch: {len(text_list)} != {num_loop}"ret_prompts.append(text_list)else:prompt = text_preprocessing(prompt)@@ -161,8 +164,12 @@ def main():# 3.5 referenceif cfg.reference_path is not None:- assert len(cfg.reference_path) == len(prompts)- assert len(cfg.reference_path) == len(cfg.mask_strategy)+ assert len(cfg.reference_path) == len(+ prompts+ ), f"Reference path mismatch: {len(cfg.reference_path)} != {len(prompts)}"+ assert len(cfg.reference_path) == len(+ cfg.mask_strategy+ ), f"Mask strategy mismatch: {len(cfg.mask_strategy)} != {len(prompts)}"# ======================================================# 4. inference@@ -204,7 +211,6 @@ def main():j] += f";{loop_i},{len(refs)-1},-{cfg.condition_frame_length},{cfg.condition_frame_length},0"masks = apply_mask_strategy(z, refs_x, mask_strategy, loop_i)- model_args["x_mask"] = masks# 4.6. diffusion samplingold_sample_idx = sample_idxdiff --git a/scripts/inference-sdedit.py b/scripts/inference-sdedit.pydeleted file mode 100644index 9adafcf..0000000--- a/scripts/inference-sdedit.py+++ /dev/null@@ -1,220 +0,0 @@-import os--import colossalai-import torch-import torch.distributed as dist-from colossalai.cluster import DistCoordinator-from mmengine.runner import set_random_seed--from opensora.acceleration.parallel_states import set_sequence_parallel_group-from opensora.datasets import save_sample-from opensora.datasets.utils import read_from_path-from opensora.registry import MODELS, SCHEDULERS, build_module-from opensora.utils.config_utils import parse_configs-from opensora.utils.misc import to_torch_dtype---def collect_references_batch(reference_paths, vae, image_size):- refs_x = []- for reference_path in reference_paths:- ref_path = reference_path.split(";")- ref = []- for r_path in ref_path:- r = read_from_path(r_path, image_size)- r_x = vae.encode(r.unsqueeze(0).to(vae.device, vae.dtype))- r_x = r_x.squeeze(0)- ref.append(r_x)- refs_x.append(ref)- # refs_x: [batch, ref_num, C, T, H, W]- return refs_x---def apply_mask_strategy(z, refs_x, mask_strategys, loop_i):- masks = []- for i, mask_strategy in enumerate(mask_strategys):- mask_strategy = mask_strategy.split(";")- mask = torch.ones(z.shape[2], dtype=torch.float, device=z.device)- for mst in mask_strategy:- mask_batch = mst.split(",")- loop_id, m_id, m_ref_start, m_length, m_target_start = mask_batch[:5]- edit_ratio = mask_batch[5] if len(mask_batch) == 6 else 0.0- loop_id = int(loop_id)- if loop_id != loop_i:- continue- m_id = int(m_id)- m_ref_start = int(m_ref_start)- m_length = int(m_length)- m_target_start = int(m_target_start)- edit_ratio = float(edit_ratio)- ref = refs_x[i][m_id] # [C, T, H, W]- if m_ref_start < 0:- m_ref_start = ref.shape[1] + m_ref_start- if m_target_start < 0:- # z: [B, C, T, H, W]- m_target_start = z.shape[2] + m_target_start- z[i, :, m_target_start : m_target_start + m_length] = ref[:, m_ref_start : m_ref_start + m_length]- mask[m_target_start : m_target_start + m_length] = edit_ratio- masks.append(mask)- masks = torch.stack(masks)- return masks, z---def process_prompts(prompts, num_loop):- ret_prompts = []- for prompt in prompts:- if prompt.startswith("|0|"):- prompt_list = prompt.split("|")[1:]- text_list = []- for i in range(0, len(prompt_list), 2):- start_loop = int(prompt_list[i])- text = prompt_list[i + 1]- end_loop = int(prompt_list[i + 2]) if i + 2 < len(prompt_list) else num_loop- text_list.extend([text] * (end_loop - start_loop))- assert len(text_list) == num_loop- ret_prompts.append(text_list)- else:- ret_prompts.append([prompt] * num_loop)- return ret_prompts---def main():- # ======================================================- # 1. cfg and init distributed env- # ======================================================- cfg = parse_configs(training=False)- print(cfg)-- # init distributed- colossalai.launch_from_torch({})- coordinator = DistCoordinator()-- if coordinator.world_size > 1:- set_sequence_parallel_group(dist.group.WORLD)- enable_sequence_parallelism = True- else:- enable_sequence_parallelism = False-- # ======================================================- # 2. runtime variables- # ======================================================- torch.set_grad_enabled(False)- torch.backends.cuda.matmul.allow_tf32 = True- torch.backends.cudnn.allow_tf32 = True- device = "cuda" if torch.cuda.is_available() else "cpu"- dtype = to_torch_dtype(cfg.dtype)- set_random_seed(seed=cfg.seed)- prompts = cfg.prompt-- # ======================================================- # 3. build model & load weights- # ======================================================- # 3.1. build model- input_size = (cfg.num_frames, *cfg.image_size)- vae = build_module(cfg.vae, MODELS)- latent_size = vae.get_latent_size(input_size)- text_encoder = build_module(cfg.text_encoder, MODELS, device=device) # T5 must be fp32- model = build_module(- cfg.model,- MODELS,- input_size=latent_size,- in_channels=vae.out_channels,- caption_channels=text_encoder.output_dim,- model_max_length=text_encoder.model_max_length,- dtype=dtype,- enable_sequence_parallelism=enable_sequence_parallelism,- )- text_encoder.y_embedder = model.y_embedder # hack for classifier-free guidance-- # 3.2. move to device & eval- vae = vae.to(device, dtype).eval()- model = model.to(device, dtype).eval()-- # 3.3. build scheduler- scheduler = build_module(cfg.scheduler, SCHEDULERS)-- # 3.4. support for multi-resolution- model_args = dict()- if cfg.multi_resolution == "PixArtMS":- image_size = cfg.image_size- hw = torch.tensor([image_size], device=device, dtype=dtype).repeat(cfg.batch_size, 1)- ar = torch.tensor([[image_size[0] / image_size[1]]], device=device, dtype=dtype).repeat(cfg.batch_size, 1)- model_args["data_info"] = dict(ar=ar, hw=hw)- elif cfg.multi_resolution == "STDiT2":- image_size = cfg.image_size- height = torch.tensor([image_size[0]], device=device, dtype=dtype).repeat(cfg.batch_size)- width = torch.tensor([image_size[1]], device=device, dtype=dtype).repeat(cfg.batch_size)- num_frames = torch.tensor([cfg.num_frames], device=device, dtype=dtype).repeat(cfg.batch_size)- ar = torch.tensor([image_size[0] / image_size[1]], device=device, dtype=dtype).repeat(cfg.batch_size)- model_args["height"] = height- model_args["width"] = width- model_args["num_frames"] = num_frames- model_args["ar"] = ar-- # 3.5 reference- if cfg.reference_path is not None:- assert len(cfg.reference_path) == len(prompts)- assert len(cfg.reference_path) == len(cfg.mask_strategy)-- # ======================================================- # 4. inference- # ======================================================- sample_idx = 0- save_dir = cfg.save_dir- os.makedirs(save_dir, exist_ok=True)-- # 4.1. batch generation- for i in range(0, len(prompts), cfg.batch_size):- batch_prompts_loops = process_prompts(prompts[i : i + cfg.batch_size], cfg.loop)- video_clips = []-- # 4.2. load reference videos & images- if cfg.reference_path is not None:- refs_x = collect_references_batch(cfg.reference_path[i : i + cfg.batch_size], vae, cfg.image_size)- mask_strategy = cfg.mask_strategy[i : i + cfg.batch_size]-- # 4.3. long video generation- for loop_i in range(cfg.loop):- # 4.4 sample in hidden space- batch_prompts = [prompt[loop_i] for prompt in batch_prompts_loops]- z = torch.randn(len(batch_prompts), vae.out_channels, *latent_size, device=device, dtype=dtype)-- # 4.5. apply mask strategy- masks = None- if cfg.reference_path is not None:- if loop_i > 0:- ref_x = vae.encode(video_clips[-1])- for j, refs in enumerate(refs_x):- refs.append(ref_x[j])- mask_strategy[- j- ] += f";{loop_i},{len(refs)-1},-{cfg.condition_frame_length},{cfg.condition_frame_length},0,0"- masks, z = apply_mask_strategy(z, refs_x, mask_strategy, loop_i)- # 4.6. diffusion sampling- samples = scheduler.sample(- model,- text_encoder,- z=z,- prompts=batch_prompts,- device=device,- additional_args=model_args,- mask=masks, # scheduler must support mask- )- samples = vae.decode(samples.to(dtype))- video_clips.append(samples)-- # 4.7. save video- if loop_i == cfg.loop - 1:- if coordinator.is_master():- for idx in range(len(video_clips[0])):- video_clips_i = [video_clips[0][idx]] + [- video_clips[i][idx][:, cfg.condition_frame_length :] for i in range(1, cfg.loop)- ]- video = torch.cat(video_clips_i, dim=1)- print(f"Prompt: {prompts[i + idx]}")- save_path = os.path.join(save_dir, f"sample_{prompts[i + idx]}")- save_sample(video, fps=cfg.fps, save_path=save_path)- sample_idx += 1---if __name__ == "__main__":- main()