Open-Sora: Democratizing Efficient Video Production for All
Python
[feat] image generation
zhengzangw4543bc217cf0
diff --git a/opensora/datasets/utils.py b/opensora/datasets/utils.pyindex 9b0be4c..18ec6e3 100644--- a/opensora/datasets/utils.py+++ b/opensora/datasets/utils.py@@ -145,7 +145,7 @@ def read_from_path(path, image_size, transform_name="center"):return read_image_from_path(path, image_size=image_size, transform_name=transform_name)-def save_sample(x, fps=8, save_path=None, normalize=True, value_range=(-1, 1), force_video=False):+def save_sample(x, fps=8, save_path=None, normalize=True, value_range=(-1, 1), force_video=False, verbose=True):"""Args:x (Tensor): shape [C, T, H, W]@@ -165,7 +165,8 @@ def save_sample(x, fps=8, save_path=None, normalize=True, value_range=(-1, 1), fx = x.mul(255).add_(0.5).clamp_(0, 255).permute(1, 2, 3, 0).to("cpu", torch.uint8)write_video(save_path, x, fps=fps, video_codec="h264")- print(f"Saved to {save_path}")+ if verbose:+ print(f"Saved to {save_path}")return save_pathdiff --git a/opensora/schedulers/dpms/__init__.py b/opensora/schedulers/dpms/__init__.pyindex ed74427..df10477 100644--- a/opensora/schedulers/dpms/__init__.py+++ b/opensora/schedulers/dpms/__init__.py@@ -22,6 +22,7 @@ class DPM_SOLVER:device,additional_args=None,mask=None,+ progress=True,):assert mask is None, "mask is not supported in dpm-solver"n = len(prompts)@@ -38,7 +39,14 @@ class DPM_SOLVER:cfg_scale=self.cfg_scale,model_kwargs=model_args,)- samples = dpms.sample(z, steps=self.num_sampling_steps, order=2, skip_type="time_uniform", method="multistep")+ samples = dpms.sample(+ z,+ steps=self.num_sampling_steps,+ order=2,+ skip_type="time_uniform",+ method="multistep",+ progress=progress,+ )return samplesdiff --git a/opensora/schedulers/dpms/dpm_solver.py b/opensora/schedulers/dpms/dpm_solver.pyindex 106e59e..d422a0a 100644--- a/opensora/schedulers/dpms/dpm_solver.py+++ b/opensora/schedulers/dpms/dpm_solver.py@@ -1255,6 +1255,7 @@ class DPM_Solver:atol=0.0078,rtol=0.05,return_intermediate=False,+ progress=True,):"""Compute the sample at time `t_end` by DPM-Solver, given the initial `x` at time `t_start`.@@ -1414,7 +1415,8 @@ class DPM_Solver:t_prev_list.append(t)model_prev_list.append(self.model_fn(x, t))# Compute the remaining values by `order`-th order multistep DPM-Solver.- for step in tqdm(range(order, steps + 1)):+ progress_fn = tqdm if progress else lambda x: x+ for step in progress_fn(range(order, steps + 1)):t = timesteps[step]# We only use lower order for steps < 10if lower_order_final and steps < 10:diff --git a/opensora/schedulers/iddpm/__init__.py b/opensora/schedulers/iddpm/__init__.pyindex 26a856d..7b65cad 100644--- a/opensora/schedulers/iddpm/__init__.py+++ b/opensora/schedulers/iddpm/__init__.py@@ -61,6 +61,7 @@ class IDDPM(SpacedDiffusion):device,additional_args=None,mask=None,+ progress=True,):n = len(prompts)z = torch.cat([z, z], 0)@@ -76,7 +77,7 @@ class IDDPM(SpacedDiffusion):z,clip_denoised=False,model_kwargs=model_args,- progress=True,+ progress=progress,device=device,mask=mask,)diff --git a/opensora/schedulers/rf/__init__.py b/opensora/schedulers/rf/__init__.pyindex 3e71d15..e1c1e82 100644--- a/opensora/schedulers/rf/__init__.py+++ b/opensora/schedulers/rf/__init__.py@@ -43,6 +43,7 @@ class RFLOW:additional_args=None,mask=None,guidance_scale=None,+ progress=True,):assert mask is None, "mask is not supported in rectified flow inference yet"# if no specific guidance scale is provided, use the default scale when initializing the scheduler@@ -68,7 +69,8 @@ class RFLOW:for t in timesteps]- for i, t in tqdm(enumerate(timesteps)):+ progress_wrap = tqdm if progress else (lambda x: x)+ for i, t in progress_wrap(enumerate(timesteps)):z_in = torch.cat([z, z], 0)pred = model(z_in, torch.tensor([t] * z_in.shape[0], device=device), **model_args).chunk(2, dim=1)[0]pred_cond, pred_uncond = pred.chunk(2, dim=0)diff --git a/opensora/utils/config_utils.py b/opensora/utils/config_utils.pyindex 9e60a69..2ec2b71 100644--- a/opensora/utils/config_utils.py+++ b/opensora/utils/config_utils.py@@ -37,6 +37,7 @@ def parse_args(training=False):parser.add_argument("--end-index", default=None, type=int, help="end index for sample name")parser.add_argument("--num-sample", default=None, type=int, help="number of samples to generate for one prompt")parser.add_argument("--prompt-as-path", action="store_true", help="use prompt as path to save samples")+ parser.add_argument("--verbose", default=None, type=int, help="verbose level")# promptparser.add_argument("--prompt-path", default=None, type=str, help="path to prompt txt file")diff --git a/scripts/inference-long.py b/scripts/inference-long.pyindex eec19cf..81b996d 100644--- a/scripts/inference-long.py+++ b/scripts/inference-long.py@@ -217,7 +217,7 @@ def main():# ======================================================# 4. inference# ======================================================- sample_idx = 0+ sample_idx = cfg.get("start_index", 0)if cfg.sample_name is not None:sample_name = cfg.sample_nameelif cfg.prompt_as_path:diff --git a/scripts/inference.py b/scripts/inference.pyindex 7a83ead..5eb114c 100644--- a/scripts/inference.py+++ b/scripts/inference.py@@ -5,6 +5,7 @@ import torchimport torch.distributed as distfrom colossalai.cluster import DistCoordinatorfrom mmengine.runner import set_random_seed+from tqdm import tqdmfrom opensora.acceleration.parallel_states import set_sequence_parallel_groupfrom opensora.datasets import IMG_FPS, save_sample@@ -19,6 +20,7 @@ def main():# 1. cfg and init distributed env# ======================================================cfg = parse_configs(training=False)+ verbose = cfg.get("verbose", 2)print(cfg)# init distributed@@ -99,7 +101,7 @@ def main():# ======================================================# 4. inference# ======================================================- sample_idx = 0+ sample_idx = cfg.get("start_index", 0)if cfg.sample_name is not None:sample_name = cfg.sample_nameelif cfg.prompt_as_path:@@ -110,7 +112,8 @@ def main():os.makedirs(save_dir, exist_ok=True)# 4.1. batch generation- for i in range(0, len(prompts), cfg.batch_size):+ progress_wrap = tqdm if verbose == 1 else (lambda x: x)+ for i in progress_wrap(range(0, len(prompts), cfg.batch_size)):# 4.2 sample in hidden spacebatch_prompts_raw = prompts[i : i + cfg.batch_size]batch_prompts = [text_preprocessing(prompt) for prompt in batch_prompts_raw]@@ -152,13 +155,15 @@ def main():prompts=batch_prompts,device=device,additional_args=model_args,+ progress=verbose >= 2,)samples = vae.decode(samples.to(dtype))# 4.4. save samplesif not use_dist or coordinator.is_master():for idx, sample in enumerate(samples):- print(f"Prompt: {batch_prompts_raw[idx]}")+ if verbose >= 2:+ print(f"Prompt: {batch_prompts_raw[idx]}")if cfg.prompt_as_path:sample_name_suffix = batch_prompts_raw[idx]else:@@ -166,7 +171,12 @@ def main():save_path = os.path.join(save_dir, f"{sample_name}{sample_name_suffix}")if cfg.num_sample != 1:save_path = f"{save_path}-{k}"- save_sample(sample, fps=cfg.fps // cfg.frame_interval, save_path=save_path)+ save_sample(+ sample,+ fps=cfg.fps // cfg.frame_interval,+ save_path=save_path,+ verbose=verbose >= 2,+ )sample_idx += 1diff --git a/scripts/misc/generate.sh b/scripts/misc/generate.shnew file mode 100644index 0000000..6056d20--- /dev/null+++ b/scripts/misc/generate.sh@@ -0,0 +1,21 @@+#!/bin/bash++set -x+set -e++TEXT_PATH=/home/data/sora_data/pixart-sigma-generated/text.txt+OUTPUT_PATH=/home/data/sora_data/pixart-sigma-generated/raw+CMD="python scripts/inference.py configs/pixart/inference/1x2048MS.py"+LOG_BASE=logs/sample/generate+NUM_PER_GPU=10000+N_LAUNCH=6+NUM_START=$(($N_LAUNCH * $NUM_PER_GPU * 8))++CUDA_VISIBLE_DEVICES=0 $CMD --prompt-path $TEXT_PATH --save-dir $OUTPUT_PATH --start-index $(($NUM_START + $NUM_PER_GPU * 0)) --end-index $(($NUM_START + $NUM_PER_GPU * 1)) --image-size 2048 2048 --verbose 1 --batch-size 2 >${LOG_BASE}_${N_LAUNCH}_1.log 2>&1 &+CUDA_VISIBLE_DEVICES=1 $CMD --prompt-path $TEXT_PATH --save-dir $OUTPUT_PATH --start-index $(($NUM_START + $NUM_PER_GPU * 1)) --end-index $(($NUM_START + $NUM_PER_GPU * 2)) --image-size 1408 2816 --verbose 1 --batch-size 2 >${LOG_BASE}_${N_LAUNCH}_2.log 2>&1 &+CUDA_VISIBLE_DEVICES=2 $CMD --prompt-path $TEXT_PATH --save-dir $OUTPUT_PATH --start-index $(($NUM_START + $NUM_PER_GPU * 2)) --end-index $(($NUM_START + $NUM_PER_GPU * 3)) --image-size 2816 1408 --verbose 1 --batch-size 2 >${LOG_BASE}_${N_LAUNCH}_3.log 2>&1 &+CUDA_VISIBLE_DEVICES=3 $CMD --prompt-path $TEXT_PATH --save-dir $OUTPUT_PATH --start-index $(($NUM_START + $NUM_PER_GPU * 3)) --end-index $(($NUM_START + $NUM_PER_GPU * 4)) --image-size 1664 2304 --verbose 1 --batch-size 2 >${LOG_BASE}_${N_LAUNCH}_4.log 2>&1 &+CUDA_VISIBLE_DEVICES=4 $CMD --prompt-path $TEXT_PATH --save-dir $OUTPUT_PATH --start-index $(($NUM_START + $NUM_PER_GPU * 4)) --end-index $(($NUM_START + $NUM_PER_GPU * 5)) --image-size 2304 1664 --verbose 1 --batch-size 2 >${LOG_BASE}_${N_LAUNCH}_5.log 2>&1 &+CUDA_VISIBLE_DEVICES=5 $CMD --prompt-path $TEXT_PATH --save-dir $OUTPUT_PATH --start-index $(($NUM_START + $NUM_PER_GPU * 5)) --end-index $(($NUM_START + $NUM_PER_GPU * 6)) --image-size 1536 2560 --verbose 1 --batch-size 2 >${LOG_BASE}_${N_LAUNCH}_6.log 2>&1 &+CUDA_VISIBLE_DEVICES=6 $CMD --prompt-path $TEXT_PATH --save-dir $OUTPUT_PATH --start-index $(($NUM_START + $NUM_PER_GPU * 6)) --end-index $(($NUM_START + $NUM_PER_GPU * 7)) --image-size 2560 1536 --verbose 1 --batch-size 2 >${LOG_BASE}_${N_LAUNCH}_7.log 2>&1 &+CUDA_VISIBLE_DEVICES=7 $CMD --prompt-path $TEXT_PATH --save-dir $OUTPUT_PATH --start-index $(($NUM_START + $NUM_PER_GPU * 7)) --end-index $(($NUM_START + $NUM_PER_GPU * 8)) --image-size 2048 2048 --verbose 1 --batch-size 2 >${LOG_BASE}_${N_LAUNCH}_8.log 2>&1 &diff --git a/tools/architecture/net2net.py b/tools/architecture/net2net.pyindex cb23b77..80d346a 100644--- a/tools/architecture/net2net.py+++ b/tools/architecture/net2net.py@@ -7,7 +7,7 @@ Numpy modules for Net2NetWritten by Kyunghyun Paeng"""-import numpy as np+def net2net(teach_param, stu_param):# teach param with shape (a, b)@@ -24,26 +24,29 @@ def net2net(teach_param, stu_param):assert len(teach_param_shape) == len(stu_param_shape), "teach_param and stu_param must have same dimension"if len(teach_param_shape) == 1:- stu_param[:teach_param_shape[0]] = teach_param-+ stu_param[: teach_param_shape[0]] = teach_paramelif len(teach_param_shape) == 2:- stu_param[:teach_param_shape[0], :teach_param_shape[1]] = teach_param+ stu_param[: teach_param_shape[0], : teach_param_shape[1]] = teach_param+ else:+ breakpoint()if stu_param.shape != stu_param_shape:stu_param = stu_param.reshape(stu_param_shape)return stu_param-if __name__ == '__main__':- """ Net2Net Class Test """- from opensora.models.pixart import PixArt_Sigma_XL_2, PixArt_1B_2+if __name__ == "__main__":+ """Net2Net Class Test"""+import torch+ from opensora.models.pixart import PixArt_1B_2+model = PixArt_1B_2(no_temporal_pos_emb=True, space_scale=4, enable_flashattn=True, enable_layernorm_kernel=True)print("load model done")- ckpt = torch.load('/home/zhouyukun/projs/opensora/pretrained_models/PixArt-Sigma-XL-2-2K-MS.pth')+ ckpt = torch.load("/home/zhouyukun/projs/opensora/pretrained_models/PixArt-Sigma-XL-2-2K-MS.pth")print("load ckpt done")ckpt = ckpt["state_dict"]diff --git a/train_pixart_1B.sh b/train_pixart_1B.shdeleted file mode 100644index d3d12ce..0000000--- a/train_pixart_1B.sh+++ /dev/null@@ -1 +0,0 @@-colossalai run --nproc_per_node 8 scripts/train.py configs/pixart/train/1x2048x2048.py --data-path /home/zhaowangbo/data/csv/image-v1_1_ext_noempty_rcp_clean_info.csv