Open-Sora: Democratizing Efficient Video Production for All
Python
debug
Shen-Chenhuieae30f9f896c
diff --git a/scripts/train-vae.py b/scripts/train-vae.pyindex 0d3ac2f..4481ddd 100644--- a/scripts/train-vae.py+++ b/scripts/train-vae.py@@ -164,173 +164,173 @@ def main():num_steps_per_epoch = len(dataloader)logger.info("Boost vae for distributed training")- # =======================================================- # 6. training loop- # =======================================================- start_epoch = start_step = log_step = sampler_start_idx = 0- running_loss = 0.0+ # # =======================================================+ # # 6. training loop+ # # =======================================================+ # start_epoch = start_step = log_step = sampler_start_idx = 0+ # running_loss = 0.0- # 6.1. resume training- if cfg.load is not None:- logger.info("Loading checkpoint")- booster.load_model(vae, os.path.join(cfg.load, "model"))- booster.load_optimizer(optimizer, os.path.join(cfg.load, "optimizer"))- if lr_scheduler is not None:- booster.load_lr_scheduler(lr_scheduler, os.path.join(cfg.load, "lr_scheduler"))- running_states = load_json(os.path.join(cfg.load, "running_states.json"))- dist.barrier()- start_epoch, start_step, sampler_start_idx = running_states["epoch"], running_states["step"], running_states["sample_start_index"]- logger.info(f"Loaded checkpoint {cfg.load} at epoch {start_epoch} step {start_step}")- logger.info(f"Training for {cfg.epochs} epochs with {num_steps_per_epoch} steps per epoch")+ # # 6.1. resume training+ # if cfg.load is not None:+ # logger.info("Loading checkpoint")+ # booster.load_model(vae, os.path.join(cfg.load, "model"))+ # booster.load_optimizer(optimizer, os.path.join(cfg.load, "optimizer"))+ # if lr_scheduler is not None:+ # booster.load_lr_scheduler(lr_scheduler, os.path.join(cfg.load, "lr_scheduler"))+ # running_states = load_json(os.path.join(cfg.load, "running_states.json"))+ # dist.barrier()+ # start_epoch, start_step, sampler_start_idx = running_states["epoch"], running_states["step"], running_states["sample_start_index"]+ # logger.info(f"Loaded checkpoint {cfg.load} at epoch {start_epoch} step {start_step}")+ # logger.info(f"Training for {cfg.epochs} epochs with {num_steps_per_epoch} steps per epoch")- dataloader.sampler.set_start_index(sampler_start_idx)+ # dataloader.sampler.set_start_index(sampler_start_idx)+ # # define loss function+ # loss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)+++ # # 6.2. training loop+ # for epoch in range(start_epoch, cfg.epochs):+ # dataloader.sampler.set_epoch(epoch)+ # dataloader_iter = iter(dataloader)+ # logger.info(f"Beginning epoch {epoch}...")++ # with tqdm(+ # range(start_step, num_steps_per_epoch),+ # desc=f"Epoch {epoch}",+ # disable=not coordinator.is_master(),+ # total=num_steps_per_epoch,+ # initial=start_step,+ # ) as pbar:+ # for step in pbar:+ # batch = next(dataloader_iter)+ # x = batch["video"].to(device, dtype) # [B, C, T, H, W]++ # # loss = vae.get_loss(x)+ # reconstructions, posterior = vae(x)+ # loss = loss_function(x, reconstructions, posterior)++ # # Backward & update+ # booster.backward(loss=loss, optimizer=optimizer)+ # optimizer.step()+ # optimizer.zero_grad()++ # # Log loss values:+ # all_reduce_mean(loss)+ # running_loss += loss.item()+ # global_step = epoch * num_steps_per_epoch + step+ # log_step += 1++ # # Log to tensorboard+ # if coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:+ # avg_loss = running_loss / log_step+ # pbar.set_postfix({"loss": avg_loss, "step": step, "global_step": global_step})+ # running_loss = 0+ # log_step = 0+ # writer.add_scalar("loss", loss.item(), global_step)+ # if cfg.wandb:+ # wandb.log(+ # {+ # "iter": global_step,+ # "num_samples": global_step * total_batch_size,+ # "epoch": epoch,+ # "loss": loss.item(),+ # "avg_loss": avg_loss,+ # },+ # step=global_step,+ # )++ # # Save checkpoint+ # if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:+ # save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")+ # os.makedirs(os.path.join(save_dir, "model"), exist_ok=True)+ # # TODO: save in model?+ # booster.save_model(vae, os.path.join(save_dir, "model"), shard=True)+ # booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)+ # if lr_scheduler is not None:+ # booster.save_lr_scheduler(lr_scheduler, os.path.join(save_dir, "lr_scheduler"))+ # running_states = {+ # "epoch": epoch,+ # "step": step+1,+ # "global_step": global_step+1,+ # "sample_start_index": (step+1) * cfg.batch_size,+ # }+ # if coordinator.is_master():+ # save_json(running_states, os.path.join(save_dir, "running_states.json"))+ # dist.barrier()+ # logger.info(+ # f"Saved checkpoint at epoch {epoch} step {step + 1} global_step {global_step + 1} to {exp_dir}"+ # )++ # # the continue epochs are not resumed, so we need to reset the sampler start index and start step+ # dataloader.sampler.set_start_index(0)+ # start_step = 0++ # DEBUG inference++ # 4.1. batch generation+# define loss functionloss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)+ running_loss = 0.0+ loss_steps = 0+ from opensora.datasets import save_sample- # 6.2. training loop- for epoch in range(start_epoch, cfg.epochs):- dataloader.sampler.set_epoch(epoch)- dataloader_iter = iter(dataloader)- logger.info(f"Beginning epoch {epoch}...")-- with tqdm(- range(start_step, num_steps_per_epoch),- desc=f"Epoch {epoch}",- disable=not coordinator.is_master(),- total=num_steps_per_epoch,- initial=start_step,- ) as pbar:- for step in pbar:- batch = next(dataloader_iter)- x = batch["video"].to(device, dtype) # [B, C, T, H, W]-- # loss = vae.get_loss(x)- reconstructions, posterior = vae(x)- loss = loss_function(x, reconstructions, posterior)-- # Backward & update- booster.backward(loss=loss, optimizer=optimizer)- optimizer.step()- optimizer.zero_grad()-- # Log loss values:- all_reduce_mean(loss)- running_loss += loss.item()- global_step = epoch * num_steps_per_epoch + step- log_step += 1-- # Log to tensorboard- if coordinator.is_master() and (global_step + 1) % cfg.log_every == 0:- avg_loss = running_loss / log_step- pbar.set_postfix({"loss": avg_loss, "step": step, "global_step": global_step})- running_loss = 0- log_step = 0- writer.add_scalar("loss", loss.item(), global_step)- if cfg.wandb:- wandb.log(- {- "iter": global_step,- "num_samples": global_step * total_batch_size,- "epoch": epoch,- "loss": loss.item(),- "avg_loss": avg_loss,- },- step=global_step,- )-- # Save checkpoint- if cfg.ckpt_every > 0 and (global_step + 1) % cfg.ckpt_every == 0:- save_dir = os.path.join(exp_dir, f"epoch{epoch}-global_step{global_step+1}")- os.makedirs(os.path.join(save_dir, "model"), exist_ok=True)- # TODO: save in model?- booster.save_model(vae, os.path.join(save_dir, "model"), shard=True)- booster.save_optimizer(optimizer, os.path.join(save_dir, "optimizer"), shard=True, size_per_shard=4096)- if lr_scheduler is not None:- booster.save_lr_scheduler(lr_scheduler, os.path.join(save_dir, "lr_scheduler"))- running_states = {- "epoch": epoch,- "step": step+1,- "global_step": global_step+1,- "sample_start_index": (step+1) * cfg.batch_size,- }- if coordinator.is_master():- save_json(running_states, os.path.join(save_dir, "running_states.json"))- dist.barrier()- logger.info(- f"Saved checkpoint at epoch {epoch} step {step + 1} global_step {global_step + 1} to {exp_dir}"- )-- # the continue epochs are not resumed, so we need to reset the sampler start index and start step- dataloader.sampler.set_start_index(0)- start_step = 0-- # # DEBUG inference-- # # 4.1. batch generation-- # # define loss function- # loss_function = VEA3DLoss(kl_weight=cfg.kl_weight, perceptual_weight=cfg.perceptual_weight).to(device, dtype)- # running_loss = 0.0- # loss_steps = 0-- # from opensora.datasets import save_sample-- # # get data again- # print("loading test data...")- # dataset = DatasetFromCSV(- # cfg.data_path,- # # TODO: change transforms- # transform=(- # get_transforms_video(cfg.image_size[0])- # if not cfg.use_image_transform- # else get_transforms_image(cfg.image_size[0])- # ),- # num_frames=cfg.num_frames,- # frame_interval=cfg.frame_interval,- # root=cfg.root,- # )-- # dataloader = prepare_dataloader(- # dataset,- # batch_size=cfg.batch_size,- # num_workers=cfg.num_workers,- # shuffle=False,- # drop_last=True,- # pin_memory=True,- # process_group=get_data_parallel_group(),- # )- # print(f"Dataset contains {len(dataset):,} videos ({cfg.data_path})")-- # total_steps = len(dataloader)- # dataloader_iter = iter(dataloader)-- # print("total steps:", total_steps)-- # with tqdm(- # range(total_steps),- # # desc=f"Avg Loss: {running_loss}",- # disable=not coordinator.is_master(),- # total=total_steps,- # initial=0,- # ) as pbar:- # for step in pbar:- # batch = next(dataloader_iter)- # x = batch["video"].to(device, dtype) # [B, C, T, H, W]- # reconstructions, posterior = vae(x)- # loss = loss_function(x, reconstructions, posterior)- # loss_steps += 1- # running_loss = loss.item()/ loss_steps + running_loss * ((loss_steps - 1) / loss_steps)-- # # if coordinator.is_master():- # # for idx, sample in enumerate(reconstructions):- # # pos = step * cfg.batch_size + idx- # # save_path = os.path.join("outputs/debug", f"sample_{pos}")- # # save_sample(sample, fps=8, save_path=save_path)-- # print("test loss:", running_loss)+ # get data again+ print("loading test data...")+ dataset = DatasetFromCSV(+ cfg.data_path,+ # TODO: change transforms+ transform=(+ get_transforms_video(cfg.image_size[0])+ if not cfg.use_image_transform+ else get_transforms_image(cfg.image_size[0])+ ),+ num_frames=cfg.num_frames,+ frame_interval=cfg.frame_interval,+ root=cfg.root,+ )++ dataloader = prepare_dataloader(+ dataset,+ batch_size=cfg.batch_size,+ num_workers=cfg.num_workers,+ shuffle=False,+ drop_last=True,+ pin_memory=True,+ process_group=get_data_parallel_group(),+ )+ print(f"Dataset contains {len(dataset):,} videos ({cfg.data_path})")++ total_steps = len(dataloader)+ dataloader_iter = iter(dataloader)++ print("total steps:", total_steps)++ with tqdm(+ range(total_steps),+ # desc=f"Avg Loss: {running_loss}",+ disable=not coordinator.is_master(),+ total=total_steps,+ initial=0,+ ) as pbar:+ for step in pbar:+ batch = next(dataloader_iter)+ x = batch["video"].to(device, dtype) # [B, C, T, H, W]+ reconstructions, posterior = vae(x)+ loss = loss_function(x, reconstructions, posterior)+ loss_steps += 1+ running_loss = loss.item()/ loss_steps + running_loss * ((loss_steps - 1) / loss_steps)++ # if coordinator.is_master():+ # for idx, sample in enumerate(reconstructions):+ # pos = step * cfg.batch_size + idx+ # save_path = os.path.join("outputs/debug", f"sample_{pos}")+ # save_sample(sample, fps=8, save_path=save_path)++ print("test loss:", running_loss)if __name__ == "__main__":main()