Skip to content

Open-Sora: Democratizing Efficient Video Production for All

Python

restore train vae script

Shen-Chenhuiae6bffb8c525
diff --git a/configs/vae_3d/inference/16x256x256.py b/configs/vae_3d/inference/16x256x256.py
index 0c34d4e..5cda9bd 100644
--- a/configs/vae_3d/inference/16x256x256.py
+++ b/configs/vae_3d/inference/16x256x256.py
@@ -46,7 +46,7 @@ save_dir = "outputs/samples"
wandb = False
-epochs = 5
+epochs = 10
log_every = 1
ckpt_every = 1000
load = None
diff --git a/opensora/models/vae/README.md b/opensora/models/vae/README.md
index dfa186e..f0a83af 100644
--- a/opensora/models/vae/README.md
+++ b/opensora/models/vae/README.md
@@ -21,12 +21,18 @@ CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scrip
# debug train and inference on the same few samples
WANDB_API_KEY=7bc1ce71b2dc0b8cd40c500eb256747583f6c07e CUDA_VISIBLE_DEVICES=5 torchrun --master_port=29530 --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/train/16x256x256.py --data-path /home/shenchenhui/data/pexels/debug.csv --wandb True
-CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-debug.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/006-F16S3-VAE_3D_B/epoch49-global_step50 --data-path /home/shenchenhui/data/pexels/debug.csv
+CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/006-F16S3-VAE_3D_B/epoch49-global_step50 --data-path /home/shenchenhui/data/pexels/debug.csv
# resume training debug
WANDB_API_KEY=7bc1ce71b2dc0b8cd40c500eb256747583f6c07e CUDA_VISIBLE_DEVICES=5 torchrun --master_port=29530 --nnodes=1 --nproc_per_node=1 scripts/train-vae.py configs/vae_3d/train/16x256x256.py --data-path /home/shenchenhui/data/pexels/debug.csv --load /home/shenchenhui/Open-Sora-dev/outputs/006-F16S3-VAE_3D_B/epoch49-global_step50 --wandb True
```
+# inference
+
+CUDA_VISIBLE_DEVICES=6 torchrun --standalone --nnodes=1 --nproc_per_node=1 scripts/inference-vae.py configs/vae_3d/inference/16x256x256.py --ckpt-path /home/shenchenhui/Open-Sora-dev/outputs/train_pexel_028/epoch9-global_step52000/ --data-path /home/shenchenhui/data/pexels/test.csv
+
+
+
```yaml
scp -P 31081 [email protected]:/home/shenchenhui/Open-Sora-dev/outputs/samples/sample_0.mp4 /Users/shenchenhui/Desktop
diff --git a/scripts/train-vae.py b/scripts/train-vae.py
index b9ca67e..cb6a213 100644
--- a/scripts/train-vae.py
+++ b/scripts/train-vae.py
@@ -175,162 +175,97 @@ def main():
if cfg.load is not None:
logger.info("Loading checkpoint")
booster.load_model(vae, os.path.join(cfg.load, "model"))
- # booster.load_optimizer(optimizer, os.path.join(cfg.load, "optimizer"))
- # if lr_scheduler is not None:
- # booster.load_lr_scheduler(lr_scheduler, os.path.join(cfg.load, "lr_scheduler"))
- # running_states = load_json(os.path.join(cfg.load, "running_states.json"))
+ booster.load_optimizer(optimizer, os.path.join(cfg.load, "optimizer"))
+ if lr_scheduler is not None:
+ booster.load_lr_scheduler(lr_scheduler, os.path.join(cfg.load, "lr_scheduler"))
+ running_states = load_json(os.path.join(cfg.load, "running_states.json"))
dist.barrier()
- # start_epoch, start_step, sampler_start_idx = running_states["epoch"], running_states["step"], running_states["sample_start_index"]
- # logger.info(f"Loaded checkpoint {cfg.load} at epoch {start_epoch} step {start_step}")
- # logger.info(f"Training for {cfg.epochs} epochs with {num_steps_per_epoch} steps per epoch")
-
- # dataloader.sampler.set_start_index(sampler_start_idx)
-
- # # define loss function
- # loss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)
-
-
- # # 6.2. training loop
- # for epoch in range(start_epoch, cfg.epochs):
- # dataloader.sampler.set_epoch(epoch)
- # dataloader_iter = iter(dataloader)
- # logger.info(f"Beginning epoch {epoch}...")
-
- # with tqdm(
- # range(start_step, num_steps_per_epoch),
- # desc=f"Epoch {epoch}",
- # disable=not coordinator.is_master(),
- # total=num_steps_per_epoch,
- # initial=start_step,
- # ) as pbar:
- # for step in pbar:
- # batch = next(dataloader_iter)
- # x = batch["video"].to(device, dtype) # [B, C, T, H, W]
-
- # # loss = vae.get_loss(x)
- # reconstructions, posterior = vae(x)
- # loss = loss_function(x, reconstructions, posterior)
-
- # # Backward & update
- # booster.backward(loss=loss, optimizer=optimizer)
- # optimizer.step()
- # optimizer.zero_grad()
-
- # # Log loss values:
- # all_reduce_mean(loss)
- # running_loss += loss.item()
- # global_step = epoch * num_steps_per_epoch + step
- # log_step += 1
-
- # # Log to tensorboard
- # if coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:
- # avg_loss = running_loss / log_step
- # pbar.set_postfix({"loss": avg_loss, "step": step, "global_step": global_step})
- # running_loss = 0
- # log_step = 0
- # writer.add_scalar("loss", loss.item(), global_step)
- # if cfg.wandb:
- # wandb.log(
- # {
- # "iter": global_step,
- # "num_samples": global_step * total_batch_size,
- # "epoch": epoch,
- # "loss": loss.item(),
- # "avg_loss": avg_loss,
- # },
- # step=global_step,
- # )
-
- # # Save checkpoint
- # if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:
- # save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")
- # os.makedirs(os.path.join(save_dir, "model"), exist_ok=True)
- # # TODO: save in model?
- # booster.save_model(vae, os.path.join(save_dir, "model"), shard=True)
- # booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)
- # if lr_scheduler is not None:
- # booster.save_lr_scheduler(lr_scheduler, os.path.join(save_dir, "lr_scheduler"))
- # running_states = {
- # "epoch": epoch,
- # "step": step+1,
- # "global_step": global_step+1,
- # "sample_start_index": (step+1) * cfg.batch_size,
- # }
- # if coordinator.is_master():
- # save_json(running_states, os.path.join(save_dir, "running_states.json"))
- # dist.barrier()
- # logger.info(
- # f"Saved checkpoint at epoch {epoch} step {step + 1} global_step {global_step + 1} to {exp_dir}"
- # )
-
- # # the continue epochs are not resumed, so we need to reset the sampler start index and start step
- # dataloader.sampler.set_start_index(0)
- # start_step = 0
-
- # DEBUG inference
-
- # 4.1. batch generation
-
+ start_epoch, start_step, sampler_start_idx = running_states["epoch"], running_states["step"], running_states["sample_start_index"]
+ logger.info(f"Loaded checkpoint {cfg.load} at epoch {start_epoch} step {start_step}")
+ logger.info(f"Training for {cfg.epochs} epochs with {num_steps_per_epoch} steps per epoch")
+
+ dataloader.sampler.set_start_index(sampler_start_idx)
+
# define loss function
loss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)
- running_loss = 0.0
- loss_steps = 0
- from opensora.datasets import save_sample
-
- # get data again
- print("loading test data...")
- dataset = DatasetFromCSV(
- cfg.data_path,
- # TODO: change transforms
- transform=(
- get_transforms_video(cfg.image_size[0])
- if not cfg.use_image_transform
- else get_transforms_image(cfg.image_size[0])
- ),
- num_frames=cfg.num_frames,
- frame_interval=cfg.frame_interval,
- root=cfg.root,
- )
- dataloader = prepare_dataloader(
- dataset,
- batch_size=cfg.batch_size,
- num_workers=cfg.num_workers,
- shuffle=False,
- drop_last=True,
- pin_memory=True,
- process_group=get_data_parallel_group(),
- )
- print(f"Dataset contains {len(dataset):,} videos ({cfg.data_path})")
-
- total_steps = len(dataloader)
- dataloader_iter = iter(dataloader)
-
- print("total steps:", total_steps)
-
- with tqdm(
- range(total_steps),
- # desc=f"Avg Loss: {running_loss}",
- disable=not coordinator.is_master(),
- total=total_steps,
- initial=0,
- ) as pbar:
- for step in pbar:
- batch = next(dataloader_iter)
- x = batch["video"].to(device, dtype) # [B, C, T, H, W]
- reconstructions, posterior = vae(x)
- loss = loss_function(x, reconstructions, posterior)
- loss_steps += 1
- running_loss = loss.item()/ loss_steps + running_loss * ((loss_steps - 1) / loss_steps)
-
- # if coordinator.is_master():
- # for idx, sample in enumerate(reconstructions):
- # pos = step * cfg.batch_size + idx
- # save_path = os.path.join("outputs/debug", f"sample_{pos}")
- # save_sample(sample, fps=8, save_path=save_path)
-
- print("test loss:", running_loss)
+ # 6.2. training loop
+ for epoch in range(start_epoch, cfg.epochs):
+ dataloader.sampler.set_epoch(epoch)
+ dataloader_iter = iter(dataloader)
+ logger.info(f"Beginning epoch {epoch}...")
+
+ with tqdm(
+ range(start_step, num_steps_per_epoch),
+ desc=f"Epoch {epoch}",
+ disable=not coordinator.is_master(),
+ total=num_steps_per_epoch,
+ initial=start_step,
+ ) as pbar:
+ for step in pbar:
+ batch = next(dataloader_iter)
+ x = batch["video"].to(device, dtype) # [B, C, T, H, W]
+
+ # loss = vae.get_loss(x)
+ reconstructions, posterior = vae(x)
+ loss = loss_function(x, reconstructions, posterior)
+
+ # Backward & update
+ booster.backward(loss=loss, optimizer=optimizer)
+ optimizer.step()
+ optimizer.zero_grad()
+
+ # Log loss values:
+ all_reduce_mean(loss)
+ running_loss += loss.item()
+ global_step = epoch * num_steps_per_epoch + step
+ log_step += 1
+
+ # Log to tensorboard
+ if coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:
+ avg_loss = running_loss / log_step
+ pbar.set_postfix({"loss": avg_loss, "step": step, "global_step": global_step})
+ running_loss = 0
+ log_step = 0
+ writer.add_scalar("loss", loss.item(), global_step)
+ if cfg.wandb:
+ wandb.log(
+ {
+ "iter": global_step,
+ "num_samples": global_step * total_batch_size,
+ "epoch": epoch,
+ "loss": loss.item(),
+ "avg_loss": avg_loss,
+ },
+ step=global_step,
+ )
+
+ # Save checkpoint
+ if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:
+ save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")
+ os.makedirs(os.path.join(save_dir, "model"), exist_ok=True)
+ # TODO: save in model?
+ booster.save_model(vae, os.path.join(save_dir, "model"), shard=True)
+ booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)
+ if lr_scheduler is not None:
+ booster.save_lr_scheduler(lr_scheduler, os.path.join(save_dir, "lr_scheduler"))
+ running_states = {
+ "epoch": epoch,
+ "step": step+1,
+ "global_step": global_step+1,
+ "sample_start_index": (step+1) * cfg.batch_size,
+ }
+ if coordinator.is_master():
+ save_json(running_states, os.path.join(save_dir, "running_states.json"))
+ dist.barrier()
+ logger.info(
+ f"Saved checkpoint at epoch {epoch} step {step + 1} global_step {global_step + 1} to {exp_dir}"
+ )
+
+ # the continue epochs are not resumed, so we need to reset the sampler start index and start step
+ dataloader.sampler.set_start_index(0)
+ start_step = 0
if __name__ == "__main__":
main()