Open-Sora: Democratizing Efficient Video Production for All
Python
restore train vae script
Shen-Chenhuiae6bffb8c525
diff --git a/configs/vae_3d/inference/16x256x256.py b/configs/vae_3d/inference/16x256x256.pyindex 0c34d4e..5cda9bd 100644--- a/configs/vae_3d/inference/16x256x256.py+++ b/configs/vae_3d/inference/16x256x256.py@@ -46,7 +46,7 @@ save_dir = "outputs/samples"wandb = False-epochs = 5+epochs = 10log_every = 1ckpt_every = 1000load = Nonediff --git a/opensora/models/vae/README.md b/opensora/models/vae/README.mdindex dfa186e..f0a83af 100644--- a/opensora/models/vae/README.md+++ b/opensora/models/vae/README.md@@ -21,12 +21,18 @@ CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scrip# debug train and inference on the same few samplesWANDB_API_KEY=7bc1ce71b2dc0b8cd40c500eb256747583f6c07e CUDA_VISIBLE_DEVICES=5 torchrun --master_port=29530 --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/train/16x256x256.py --data-path /home/shenchenhui/data/pexels/debug.csv --wandb True-CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-debug.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/006-F16S3-VAE_3D_B/epoch49-global_step50 --data-path /home/shenchenhui/data/pexels/debug.csv+CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/006-F16S3-VAE_3D_B/epoch49-global_step50 --data-path /home/shenchenhui/data/pexels/debug.csv# resume training debugWANDB_API_KEY=7bc1ce71b2dc0b8cd40c500eb256747583f6c07e CUDA_VISIBLE_DEVICES=5 torchrun --master_port=29530 --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/train/16x256x256.py --data-path /home/shenchenhui/data/pexels/debug.csv --load /home/shenchenhui/Open-Sora-dev/outputs/006-F16S3-VAE_3D_B/epoch49-global_step50 --wandb True```+# inference++CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/train_pexel_028/epoch9-global_step52000/ --data-path /home/shenchenhui/data/pexels/test.csv+++```yamlscp -P 31081 [email protected]:/home/shenchenhui/Open-Sora-dev/outputs/samples/sample_0.mp4 /Users/shenchenhui/Desktopdiff --git a/scripts/train-vae.py b/scripts/train-vae.pyindex b9ca67e..cb6a213 100644--- a/scripts/train-vae.py+++ b/scripts/train-vae.py@@ -175,162 +175,97 @@ def main():if cfg.load is not None:logger.info("Loading checkpoint")booster.load_model(vae, os.path.join(cfg.load, "model"))- # booster.load_optimizer(optimizer, os.path.join(cfg.load, "optimizer"))- # if lr_scheduler is not None:- # booster.load_lr_scheduler(lr_scheduler, os.path.join(cfg.load, "lr_scheduler"))- # running_states = load_json(os.path.join(cfg.load, "running_states.json"))+ booster.load_optimizer(optimizer, os.path.join(cfg.load, "optimizer"))+ if lr_scheduler is not None:+ booster.load_lr_scheduler(lr_scheduler, os.path.join(cfg.load, "lr_scheduler"))+ running_states = load_json(os.path.join(cfg.load, "running_states.json"))dist.barrier()- # start_epoch, start_step, sampler_start_idx = running_states["epoch"], running_states["step"], running_states["sample_start_index"]- # logger.info(f"Loaded checkpoint {cfg.load} at epoch {start_epoch} step {start_step}")- # logger.info(f"Training for {cfg.epochs} epochs with {num_steps_per_epoch} steps per epoch")-- # dataloader.sampler.set_start_index(sampler_start_idx)-- # # define loss function- # loss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)--- # # 6.2. training loop- # for epoch in range(start_epoch, cfg.epochs):- # dataloader.sampler.set_epoch(epoch)- # dataloader_iter = iter(dataloader)- # logger.info(f"Beginning epoch {epoch}...")-- # with tqdm(- # range(start_step, num_steps_per_epoch),- # desc=f"Epoch {epoch}",- # disable=not coordinator.is_master(),- # total=num_steps_per_epoch,- # initial=start_step,- # ) as pbar:- # for step in pbar:- # batch = next(dataloader_iter)- # x = batch["video"].to(device, dtype) # [B, C, T, H, W]-- # # loss = vae.get_loss(x)- # reconstructions, posterior = vae(x)- # loss = loss_function(x, reconstructions, posterior)-- # # Backward & update- # booster.backward(loss=loss, optimizer=optimizer)- # optimizer.step()- # optimizer.zero_grad()-- # # Log loss values:- # all_reduce_mean(loss)- # running_loss += loss.item()- # global_step = epoch * num_steps_per_epoch + step- # log_step += 1-- # # Log to tensorboard- # if coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:- # avg_loss = running_loss / log_step- # pbar.set_postfix({"loss": avg_loss, "step": step, "global_step": global_step})- # running_loss = 0- # log_step = 0- # writer.add_scalar("loss", loss.item(), global_step)- # if cfg.wandb:- # wandb.log(- # {- # "iter": global_step,- # "num_samples": global_step * total_batch_size,- # "epoch": epoch,- # "loss": loss.item(),- # "avg_loss": avg_loss,- # },- # step=global_step,- # )-- # # Save checkpoint- # if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:- # save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")- # os.makedirs(os.path.join(save_dir, "model"), exist_ok=True)- # # TODO: save in model?- # booster.save_model(vae, os.path.join(save_dir, "model"), shard=True)- # booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)- # if lr_scheduler is not None:- # booster.save_lr_scheduler(lr_scheduler, os.path.join(save_dir, "lr_scheduler"))- # running_states = {- # "epoch": epoch,- # "step": step+1,- # "global_step": global_step+1,- # "sample_start_index": (step+1) * cfg.batch_size,- # }- # if coordinator.is_master():- # save_json(running_states, os.path.join(save_dir, "running_states.json"))- # dist.barrier()- # logger.info(- # f"Saved checkpoint at epoch {epoch} step {step + 1} global_step {global_step + 1} to {exp_dir}"- # )-- # # the continue epochs are not resumed, so we need to reset the sampler start index and start step- # dataloader.sampler.set_start_index(0)- # start_step = 0-- # DEBUG inference-- # 4.1. batch generation-+ start_epoch, start_step, sampler_start_idx = running_states["epoch"], running_states["step"], running_states["sample_start_index"]+ logger.info(f"Loaded checkpoint {cfg.load} at epoch {start_epoch} step {start_step}")+ logger.info(f"Training for {cfg.epochs} epochs with {num_steps_per_epoch} steps per epoch")++ dataloader.sampler.set_start_index(sampler_start_idx)+# define loss functionloss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)- running_loss = 0.0- loss_steps = 0- from opensora.datasets import save_sample-- # get data again- print("loading test data...")- dataset = DatasetFromCSV(- cfg.data_path,- # TODO: change transforms- transform=(- get_transforms_video(cfg.image_size[0])- if not cfg.use_image_transform- else get_transforms_image(cfg.image_size[0])- ),- num_frames=cfg.num_frames,- frame_interval=cfg.frame_interval,- root=cfg.root,- )- dataloader = prepare_dataloader(- dataset,- batch_size=cfg.batch_size,- num_workers=cfg.num_workers,- shuffle=False,- drop_last=True,- pin_memory=True,- process_group=get_data_parallel_group(),- )- print(f"Dataset contains {len(dataset):,} videos ({cfg.data_path})")-- total_steps = len(dataloader)- dataloader_iter = iter(dataloader)-- print("total steps:", total_steps)-- with tqdm(- range(total_steps),- # desc=f"Avg Loss: {running_loss}",- disable=not coordinator.is_master(),- total=total_steps,- initial=0,- ) as pbar:- for step in pbar:- batch = next(dataloader_iter)- x = batch["video"].to(device, dtype) # [B, C, T, H, W]- reconstructions, posterior = vae(x)- loss = loss_function(x, reconstructions, posterior)- loss_steps += 1- running_loss = loss.item()/ loss_steps + running_loss * ((loss_steps - 1) / loss_steps)-- # if coordinator.is_master():- # for idx, sample in enumerate(reconstructions):- # pos = step * cfg.batch_size + idx- # save_path = os.path.join("outputs/debug", f"sample_{pos}")- # save_sample(sample, fps=8, save_path=save_path)-- print("test loss:", running_loss)+ # 6.2. training loop+ for epoch in range(start_epoch, cfg.epochs):+ dataloader.sampler.set_epoch(epoch)+ dataloader_iter = iter(dataloader)+ logger.info(f"Beginning epoch {epoch}...")++ with tqdm(+ range(start_step, num_steps_per_epoch),+ desc=f"Epoch {epoch}",+ disable=not coordinator.is_master(),+ total=num_steps_per_epoch,+ initial=start_step,+ ) as pbar:+ for step in pbar:+ batch = next(dataloader_iter)+ x = batch["video"].to(device, dtype) # [B, C, T, H, W]++ # loss = vae.get_loss(x)+ reconstructions, posterior = vae(x)+ loss = loss_function(x, reconstructions, posterior)++ # Backward & update+ booster.backward(loss=loss, optimizer=optimizer)+ optimizer.step()+ optimizer.zero_grad()++ # Log loss values:+ all_reduce_mean(loss)+ running_loss += loss.item()+ global_step = epoch * num_steps_per_epoch + step+ log_step += 1++ # Log to tensorboard+ if coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:+ avg_loss = running_loss / log_step+ pbar.set_postfix({"loss": avg_loss, "step": step, "global_step": global_step})+ running_loss = 0+ log_step = 0+ writer.add_scalar("loss", loss.item(), global_step)+ if cfg.wandb:+ wandb.log(+ {+ "iter": global_step,+ "num_samples": global_step * total_batch_size,+ "epoch": epoch,+ "loss": loss.item(),+ "avg_loss": avg_loss,+ },+ step=global_step,+ )++ # Save checkpoint+ if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:+ save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")+ os.makedirs(os.path.join(save_dir, "model"), exist_ok=True)+ # TODO: save in model?+ booster.save_model(vae, os.path.join(save_dir, "model"), shard=True)+ booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)+ if lr_scheduler is not None:+ booster.save_lr_scheduler(lr_scheduler, os.path.join(save_dir, "lr_scheduler"))+ running_states = {+ "epoch": epoch,+ "step": step+1,+ "global_step": global_step+1,+ "sample_start_index": (step+1) * cfg.batch_size,+ }+ if coordinator.is_master():+ save_json(running_states, os.path.join(save_dir, "running_states.json"))+ dist.barrier()+ logger.info(+ f"Saved checkpoint at epoch {epoch} step {step + 1} global_step {global_step + 1} to {exp_dir}"+ )++ # the continue epochs are not resumed, so we need to reset the sampler start index and start step+ dataloader.sampler.set_start_index(0)+ start_step = 0if __name__ == "__main__":main()