Skip to content

[CVPR 2026] PersonaLive! : Expressive Portrait Image Animation for Live Streaming

update inference

Zhiyuan Lia2d84307e9d6
diff --git a/inference_offline.py b/inference_offline.py
new file mode 100644
index 0000000..924ab18
--- /dev/null
+++ b/inference_offline.py
@@ -0,0 +1,254 @@
+import argparse
+import os
+import sys
+from datetime import datetime
+import mediapipe as mp
+import numpy as np
+import cv2
+import torch
+from skimage.transform import resize
+from diffusers import AutoencoderKLTemporalDecoder, AutoencoderKL, AutoencoderTiny
+from src.scheduler.scheduler_ddim import DDIMScheduler
+import random
+from omegaconf import OmegaConf
+from PIL import Image
+from torchvision import transforms
+from transformers import CLIPVisionModelWithProjection
+from src.models.unet_2d_condition import UNet2DConditionModel
+from src.models.unet_3d import UNet3DConditionModel
+from src.pipelines.pipeline_pose2vid import Pose2VideoPipeline
+from src.utils.util import save_videos_grid, crop_face
+from decord import VideoReader
+from diffusers.utils.import_utils import is_xformers_available
+
+from src.models.motion_encoder.encoder import MotEncoder
+from src.liveportrait.motion_extractor import MotionExtractor
+from src.models.pose_guider import PoseGuider
+from tqdm import tqdm
+
+def parse_args():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--config", type=str, default='configs/prompts/personalive_offline.yaml')
+ parser.add_argument("--name", type=str, default='personalive_offline')
+ parser.add_argument("-W", type=int, default=512)
+ parser.add_argument("-H", type=int, default=512)
+ parser.add_argument("-L", type=int, default=100)
+ parser.add_argument("--seed", type=int, default=42)
+ parser.add_argument("--device", type=str, default="cuda")
+ args = parser.parse_args()
+
+ return args
+
+def main(args):
+ device = args.device
+ print('device', device)
+ config = OmegaConf.load(args.config)
+
+ if config.weight_dtype == "fp16":
+ weight_dtype = torch.float16
+ else:
+ weight_dtype = torch.float32
+
+ vae = AutoencoderKL.from_pretrained(config.vae_path).to(device, dtype=weight_dtype)
+ # if use tiny VAE
+ # vae_tiny = AutoencoderTiny.from_pretrained(config.vae_tiny_path).to(device, dtype=weight_dtype)
+
+ infer_config = OmegaConf.load(config.inference_config)
+ reference_unet = UNet2DConditionModel.from_pretrained(
+ config.pretrained_base_model_path,
+ subfolder="unet",
+ ).to(device=device, dtype=weight_dtype)
+ denoising_unet = UNet3DConditionModel.from_pretrained_2d(
+ config.pretrained_base_model_path,
+ "",
+ subfolder="unet",
+ unet_additional_kwargs=infer_config.unet_additional_kwargs,
+ ).to(dtype=weight_dtype, device=device)
+
+ motion_encoder = MotEncoder().to(dtype=weight_dtype, device=device).eval()
+ pose_guider = PoseGuider().to(device=device, dtype=weight_dtype)
+ pose_encoder = MotionExtractor(num_kp=21).to(device=device, dtype=weight_dtype).eval()
+
+ image_enc = CLIPVisionModelWithProjection.from_pretrained(
+ config.image_encoder_path
+ ).to(dtype=weight_dtype, device=device)
+
+ sched_kwargs = OmegaConf.to_container(
+ OmegaConf.load(config.inference_config).noise_scheduler_kwargs
+ )
+ scheduler = DDIMScheduler(**sched_kwargs)
+
+ generator = torch.manual_seed(args.seed)
+ width, height = args.W, args.H
+
+ # load pretrained weights
+ denoising_unet.load_state_dict(
+ torch.load(config.denoising_unet_path, map_location="cpu"), strict=False
+ )
+ reference_unet.load_state_dict(
+ torch.load(
+ config.denoising_unet_path.replace('denoising_unet', 'reference_unet'),
+ map_location="cpu",
+ ),
+ strict=True,
+ )
+ motion_encoder.load_state_dict(
+ torch.load(
+ config.denoising_unet_path.replace('denoising_unet', 'motion_encoder'),
+ map_location="cpu",
+ ),
+ strict=True,
+ )
+ pose_guider.load_state_dict(
+ torch.load(
+ config.denoising_unet_path.replace('denoising_unet', 'pose_guider'),
+ map_location="cpu",
+ ),
+ strict=True,
+ )
+ denoising_unet.load_state_dict(
+ torch.load(
+ config.denoising_unet_path.replace('denoising_unet', 'temporal_module'),
+ map_location="cpu",
+ ),
+ strict=False,
+ )
+ pose_encoder.load_state_dict(
+ torch.load(
+ config.denoising_unet_path.replace('denoising_unet', 'motion_extractor'),
+ map_location="cpu",
+ ),
+ strict=False,
+ )
+
+ if is_xformers_available():
+ reference_unet.enable_xformers_memory_efficient_attention()
+ denoising_unet.enable_xformers_memory_efficient_attention()
+ else:
+ raise ValueError(
+ "xformers is not available. Make sure it is installed correctly"
+ )
+
+ mp_face_mesh = mp.solutions.face_mesh
+ face_mesh = mp_face_mesh.FaceMesh(static_image_mode=True, max_num_faces=1)
+
+ pipe = Pose2VideoPipeline(
+ vae=vae,
+ # vae_tiny=vae_tiny,
+ image_encoder=image_enc,
+ reference_unet=reference_unet,
+ denoising_unet=denoising_unet,
+ motion_encoder=motion_encoder,
+ pose_encoder=pose_encoder,
+ pose_guider=pose_guider,
+ scheduler=scheduler,
+ )
+ pipe = pipe.to(device)
+
+ date_str = datetime.now().strftime("%Y%m%d")
+ if args.name is None:
+ time_str = datetime.now().strftime("%H%M")
+ save_dir_name = f"{date_str}--{time_str}"
+ else:
+ save_dir_name = f"{date_str}--{args.name}"
+ save_vid_dir = os.path.join('results', save_dir_name, 'concat_vid')
+ os.makedirs(save_vid_dir, exist_ok=True)
+ save_split_vid_dir = os.path.join('results', save_dir_name, 'split_vid')
+ os.makedirs(save_split_vid_dir, exist_ok=True)
+
+ pose_transform = transforms.Compose(
+ [transforms.Resize((height, width)), transforms.ToTensor()]
+ )
+
+ args.test_cases = OmegaConf.load(args.config)["test_cases"]
+
+ for ref_image_path in list(args.test_cases.keys()):
+ for pose_video_path in args.test_cases[ref_image_path]:
+ video_name = os.path.basename(pose_video_path).split(".")[0]
+ source_name = os.path.basename(ref_image_path).split(".")[0]
+
+ vid_name = f"{source_name}_{video_name}.mp4"
+ save_vid_path = os.path.join(save_vid_dir, vid_name)
+ print(save_vid_path)
+ if os.path.exists(save_vid_path):
+ continue
+
+ if ref_image_path.endswith('.mp4'):
+ src_vid = VideoReader(ref_image_path)
+ ref_img = src_vid[0].asnumpy()
+ ref_img = Image.fromarray(ref_img).convert("RGB")
+ else:
+ ref_img = Image.open(ref_image_path).convert("RGB")
+
+ control = VideoReader(pose_video_path)
+ video_length = min(len(control) // 4 * 4, args.L)
+ sel_idx = range(len(control))[:video_length]
+ control = control.get_batch([sel_idx]).asnumpy() # N, H, W, C
+
+ ref_image_pil = ref_img.copy()
+ ref_patch = crop_face(ref_image_pil, face_mesh)
+ ref_face_pil = Image.fromarray(ref_patch).convert("RGB")
+
+ size = args.H
+ generator = torch.Generator(device=device)
+ generator.manual_seed(42)
+
+ dri_faces = []
+ ori_pose_images = []
+ for idx_control, pose_image_pil in tqdm(enumerate(control[:video_length]), total=video_length, desc='cropping faces'):
+ pose_image_pil = Image.fromarray(pose_image_pil).convert("RGB")
+ ori_pose_images.append(pose_image_pil)
+ dri_face = crop_face(pose_image_pil, face_mesh)
+ dri_face_pil = Image.fromarray(dri_face).convert("RGB")
+ dri_faces.append(dri_face_pil)
+
+ face_tensor_list = []
+ ori_pose_tensor_list = []
+ ref_tensor_list = []
+
+ for idx, pose_image_pil in enumerate(ori_pose_images):
+ face_tensor_list.append(pose_transform(dri_faces[idx]))
+ ori_pose_tensor_list.append(pose_transform(pose_image_pil))
+ ref_tensor_list.append(pose_transform(ref_image_pil))
+
+ ref_tensor = torch.stack(ref_tensor_list, dim=0) # (f, c, h, w)
+ ref_tensor = ref_tensor.transpose(0, 1).unsqueeze(0) # (c, f, h, w)
+
+ face_tensor = torch.stack(face_tensor_list, dim=0) # (f, c, h, w)
+ face_tensor = face_tensor.transpose(0, 1).unsqueeze(0)
+
+ ori_pose_tensor = torch.stack(ori_pose_tensor_list, dim=0) # (f, c, h, w)
+ ori_pose_tensor = ori_pose_tensor.transpose(0, 1).unsqueeze(0)
+
+ gen_video = pipe(
+ ori_pose_images,
+ ref_image_pil,
+ dri_faces,
+ ref_face_pil,
+ width,
+ height,
+ len(dri_faces),
+ num_inference_steps=4,
+ guidance_scale=1.0,
+ generator=generator,
+ temporal_window_size = 4,
+ temporal_adaptive_step = 4,
+ ).videos
+
+ #Concat it with pose tensor
+ video = torch.cat([ref_tensor, face_tensor, ori_pose_tensor, gen_video], dim=0)
+
+ save_videos_grid(
+ video,
+ save_vid_path,
+ n_rows=4,
+ fps=25,
+ )
+
+ if True:
+ save_vid_path = save_vid_path.replace(save_vid_dir, save_split_vid_dir)
+ save_videos_grid(gen_video, save_vid_path, n_rows=1, fps=25, crf=18)
+
+if __name__ == "__main__":
+ args = parse_args()
+ main(args)
diff --git a/inference_online.py b/inference_online.py
new file mode 100644
index 0000000..a426ed4
--- /dev/null
+++ b/inference_online.py
@@ -0,0 +1,323 @@
+import os
+import signal
+import sys
+import json
+
+from fastapi import FastAPI, WebSocket, HTTPException, WebSocketDisconnect, UploadFile, File
+from fastapi.responses import JSONResponse
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+from fastapi import Request
+
+import markdown2
+import threading
+import logging
+import uuid
+import time
+from types import SimpleNamespace
+import asyncio
+import mimetypes
+import torch
+
+from webcam.config import config, Args
+from webcam.util import pil_to_frame, bytes_to_pil, is_firefox, bytes_to_tensor
+from webcam.connection_manager import ConnectionManager, ServerFullException
+import multiprocessing as mp
+
+use_trt = True
+
+if use_trt:
+ from webcam.vid2vid_trt import Pipeline
+else:
+ from webcam.vid2vid import Pipeline
+
+mimetypes.add_type("application/javascript", ".js")
+
+THROTTLE = 0.001
+
+
+class App:
+ def __init__(self, config: Args, pipeline: Pipeline):
+ self.args = config
+ self.pipeline = pipeline
+ self.app = FastAPI()
+ self.conn_manager = ConnectionManager()
+
+ self.produce_predictions_stop_event = None
+ self.produce_predictions_task = None
+ self.shutdown_event = asyncio.Event()
+
+ self.init_app()
+
+ def init_app(self):
+ self.app.add_middleware(
+ CORSMiddleware,
+ allow_origins=["*"],
+ allow_credentials=True,
+ allow_methods=["*"],
+ allow_headers=["*"],
+ )
+
+ @self.app.websocket("/api/ws/{user_id}")
+ async def websocket_endpoint(user_id: uuid.UUID, websocket: WebSocket):
+ try:
+ await self.conn_manager.connect(
+ user_id, websocket, self.args.max_queue_size
+ )
+
+ sender_task = asyncio.create_task(push_results_to_client(user_id, websocket))
+
+ if self.produce_predictions_task is None or self.produce_predictions_task.done():
+ start_prediction_thread(user_id)
+
+ await handle_websocket_input(user_id, websocket)
+
+ except ServerFullException as e:
+ logging.error(f"Server Full: {e}")
+ except WebSocketDisconnect:
+ logging.info(f"User disconnected: {user_id}")
+ except Exception as e:
+ logging.error(f"WS Error: {e}")
+ finally:
+ if 'sender_task' in locals():
+ sender_task.cancel()
+
+ await self.conn_manager.disconnect(user_id, self.pipeline)
+
+ if self.produce_predictions_stop_event is not None:
+ self.produce_predictions_stop_event.set()
+ logging.info(f"Cleaned up user: {user_id}")
+
+ async def handle_websocket_input(user_id: uuid.UUID, websocket: WebSocket):
+ if not self.conn_manager.check_user(user_id):
+ raise HTTPException(status_code=404, detail="User not found")
+
+ try:
+ while True:
+ message = await websocket.receive()
+
+ if "text" in message:
+ try:
+ text_data = message["text"]
+ data = json.loads(text_data)
+ status = data.get("status")
+
+ if status == "pause":
+ params = SimpleNamespace(**{"restart": True})
+ await self.conn_manager.update_data(user_id, params)
+ elif status == "resume":
+ await self.conn_manager.send_json(user_id, {"status": "send_frame"})
+ except Exception as e:
+ logging.error(f"JSON Parse Error: {e}")
+
+ elif "bytes" in message:
+ image_data = message["bytes"]
+ if len(image_data) > 0:
+ input_tensor = bytes_to_tensor(image_data)
+ params = SimpleNamespace()
+ params.image = input_tensor
+ self.pipeline.accept_new_params(params)
+
+ except WebSocketDisconnect:
+ raise
+ except Exception as e:
+ logging.error(f"Input Loop Error: {e}")
+ raise
+
+ async def push_results_to_client(user_id: uuid.UUID, websocket: WebSocket):
+ MIN_FPS = 10
+ MAX_FPS = 30
+ SMOOTHING = 0.8 # EMA smoothing factor
+
+ last_burst_time = time.time()
+ last_queue_size = 0
+ sleep_time = 1 / 40 # Initial guess
+
+ last_frame_time = None
+ frame_time_list = []
+
+ ema_frame_interval = sleep_time
+
+ try:
+ while True:
+ queue_size = await self.conn_manager.get_output_queue_size(user_id)
+ if queue_size > last_queue_size:
+ current_burst_time = time.time()
+ elapsed = current_burst_time - last_burst_time
+
+ if queue_size > 0 and elapsed > 0:
+ raw_interval = elapsed / queue_size
+ ema_frame_interval = SMOOTHING * ema_frame_interval + (1 - SMOOTHING) * raw_interval
+ sleep_time = min(max(ema_frame_interval, 1 / MAX_FPS), 1 / MIN_FPS)
+
+ last_burst_time = current_burst_time
+
+ last_queue_size = queue_size
+
+ frame = await self.conn_manager.get_frame(user_id)
+ if frame is None:
+ await asyncio.sleep(0.001)
+ continue
+
+ await websocket.send_bytes(frame)
+
+ if last_frame_time is None:
+ last_frame_time = time.time()
+ else:
+ frame_time_list.append(time.time() - last_frame_time)
+ if len(frame_time_list) > 100:
+ frame_time_list.pop(0)
+ last_frame_time = time.time()
+
+ await asyncio.sleep(sleep_time)
+
+ except asyncio.CancelledError:
+ pass
+ except Exception as e:
+ logging.error(f"Push Result Error: {e}")
+
+ def start_prediction_thread(user_id):
+ self.produce_predictions_stop_event = threading.Event()
+
+ def prediction_loop(uid, loop, stop_event):
+ while not stop_event.is_set():
+ images = self.pipeline.produce_outputs()
+ if len(images) == 0:
+ time.sleep(THROTTLE)
+ continue
+
+ frames = list(map(pil_to_frame, images))
+ asyncio.run_coroutine_threadsafe(
+ self.conn_manager.put_frames_to_output_queue(uid, frames),
+ loop
+ )
+
+ self.produce_predictions_task = asyncio.create_task(asyncio.to_thread(
+ prediction_loop, user_id, asyncio.get_running_loop(), self.produce_predictions_stop_event
+ ))
+
+ @self.app.get("/api/queue")
+ async def get_queue_size():
+ queue_size = self.conn_manager.get_user_count()
+ return JSONResponse({"queue_size": queue_size})
+
+ @self.app.get("/api/settings")
+ async def settings():
+ info_schema = pipeline.Info.schema()
+ info = pipeline.Info()
+ if info.page_content:
+ page_content = markdown2.markdown(info.page_content)
+
+ input_params = pipeline.InputParams.schema()
+ return JSONResponse(
+ {
+ "info": info_schema,
+ "input_params": input_params,
+ "max_queue_size": self.args.max_queue_size,
+ "page_content": page_content if info.page_content else "",
+ }
+ )
+
+ @self.app.post("/api/upload_reference_image")
+ async def upload_reference_image(ref_image: UploadFile = File(...)):
+ try:
+ data = await ref_image.read()
+ img = bytes_to_pil(data)
+ self.pipeline.fuse_reference(img)
+ return {"status": "ok"}
+ except Exception as e:
+ logging.error(f"Reference image error: {e}")
+ raise HTTPException(status_code=500, detail="Failed to process reference image")
+
+ if not os.path.exists("./demo_w_camera/frontend/public"):
+ os.makedirs("./demo_w_camera/frontend/public")
+
+ self.app.mount(
+ "/", StaticFiles(directory="./demo_w_camera/frontend/public", html=True), name="public"
+ )
+
+ @self.app.on_event("shutdown")
+ async def shutdown_event():
+ await self.cleanup()
+
+ async def cleanup(self):
+ print("[App] Starting cleanup process...")
+ self.shutdown_event.set()
+
+ if self.produce_predictions_stop_event is not None:
+ self.produce_predictions_stop_event.set()
+
+ if self.produce_predictions_task is not None:
+ self.produce_predictions_task.cancel()
+ try:
+ await self.produce_predictions_task
+ except asyncio.CancelledError:
+ pass
+
+ try:
+ await self.conn_manager.disconnect_all(self.pipeline)
+ except Exception as e:
+ print(f"[App] Error during disconnect_all: {e}")
+
+ print("[App] Cleanup completed")
+
+app_instance = None
+
+def signal_handler(signum, frame):
+ print(f"\n[Main] Received signal {signum}, shutting down gracefully...")
+ if app_instance:
+ import threading
+ def trigger_cleanup():
+ try:
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+ loop.run_until_complete(app_instance.cleanup())
+ loop.close()
+ except Exception as e:
+ print(f"[Main] Error during cleanup: {e}")
+
+ cleanup_thread = threading.Thread(target=trigger_cleanup)
+ cleanup_thread.daemon = True
+ cleanup_thread.start()
+ cleanup_thread.join(timeout=5)
+
+ sys.exit(0)
+
+
+if __name__ == "__main__":
+ import uvicorn
+ signal.signal(signal.SIGINT, signal_handler)
+ signal.signal(signal.SIGTERM, signal_handler)
+ mp.set_start_method("spawn", force=True)
+
+ device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+ pipeline = Pipeline(config, device)
+
+ app_obj = App(config, pipeline)
+ app = app_obj.app
+ app_instance = app_obj
+
+ print('init done')
+
+ try:
+ uvicorn.run(
+ app,
+ host=config.host,
+ port=config.port,
+ reload=config.reload,
+ ssl_certfile=config.ssl_certfile,
+ ssl_keyfile=config.ssl_keyfile,
+ )
+ except KeyboardInterrupt:
+ try:
+ import asyncio
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+ loop.run_until_complete(app_obj.cleanup())
+ loop.close()
+ except Exception as e:
+ print(f"[Main] Error during cleanup: {e}")
+ sys.exit(0)
+ except Exception as e:
+ print(f"[Main] Error: {e}")
+ sys.exit(1)
\ No newline at end of file
diff --git a/torch2trt.py b/torch2trt.py
new file mode 100644
index 0000000..6d8f89c
--- /dev/null
+++ b/torch2trt.py
@@ -0,0 +1,131 @@
+import os
+import torch
+from omegaconf import OmegaConf
+import gc
+from src.modeling.framed_models import unet_work
+from diffusers import AutoencoderKL
+from diffusers.models.attention_processor import AttnProcessor
+from src.scheduler.scheduler_ddim import DDIMScheduler
+from src.models.unet_3d_explicit_reference import UNet3DConditionModel
+from src.models.motion_encoder.encoder import MotEncoder
+from src.models.pose_guider import PoseGuider
+
+from src.modeling.onnx_export import export_onnx, handle_onnx_batch_norm, optimize_onnx
+from polygraphy.backend.trt import engine_from_network, network_from_onnx_path, save_engine, CreateConfig
+from polygraphy.logger import G_LOGGER
+import tensorrt as trt
+G_LOGGER.severity = G_LOGGER.VERBOSE
+
+def map_device(device_or_str):
+ return device_or_str if isinstance(device_or_str, torch.device) else torch.device(device_or_str)
+
+# parameters
+batch_size = 1
+height = 256
+width = 256
+onnx_opset = 17
+device = torch.device("cuda:0")
+device = map_device(device)
+dtype = torch.float16
+
+config_path = './configs/prompts/personalive_trt.yaml'
+cfg = OmegaConf.load(config_path)
+onnx_path = cfg.onnx_path
+onnx_opt_path = cfg.onnx_opt_path
+tensorrt_target_model = cfg.tensorrt_target_model
+
+infer_config = OmegaConf.load(cfg.inference_config)
+sched_kwargs = OmegaConf.to_container(
+ infer_config.noise_scheduler_kwargs
+)
+
+pose_guider = PoseGuider().to(device=device, dtype=dtype)
+pose_guider_state_dict = torch.load(cfg.pose_guider_path, map_location="cpu")
+pose_guider.load_state_dict(pose_guider_state_dict)
+del pose_guider_state_dict
+
+motion_encoder:MotEncoder = MotEncoder().to(dtype=dtype, device=device).eval()
+motion_encoder.set_attn_processor(AttnProcessor())
+motion_encoder_state_dict = torch.load(cfg.motion_encoder_path, map_location="cpu")
+motion_encoder.load_state_dict(motion_encoder_state_dict)
+del motion_encoder_state_dict
+
+denoising_unet:UNet3DConditionModel = UNet3DConditionModel.from_pretrained_2d(
+ cfg.pretrained_base_model_path,
+ "",
+ subfolder="unet",
+ unet_additional_kwargs=infer_config.unet_additional_kwargs,
+).to(dtype=dtype, device=device)
+
+denoising_unet.load_state_dict(
+ torch.load(cfg.denoising_unet_path, map_location="cpu"), strict=False
+)
+denoising_unet.load_state_dict(
+ torch.load(
+ cfg.temporal_module_path,
+ map_location="cpu",
+ ),
+ strict=False,
+)
+denoising_unet.set_attn_processor(AttnProcessor())
+
+vae:AutoencoderKL = AutoencoderKL.from_pretrained(cfg.vae_model_path).to(device=device, dtype=dtype)
+vae.set_default_attn_processor()
+
+scheduler = DDIMScheduler(**sched_kwargs)
+scheduler.to(device)
+timesteps = torch.tensor([0,0,0,0,333,333,333,333,666,666,666,666,999,999,999,999], device=device).long()
+scheduler.set_step_length(333)
+
+model = unet_work(
+ pose_guider,
+ motion_encoder,
+ denoising_unet,
+ vae,
+ scheduler,
+ timesteps,
+ )
+
+if(not os.path.exists(os.path.dirname(onnx_path))):
+ os.mkdir(os.path.dirname(onnx_path))
+
+if not os.path.exists(onnx_path):
+ export_onnx(
+ model,
+ onnx_path=onnx_path,
+ opt_image_height=height,
+ opt_image_width=width,
+ opt_batch_size=batch_size,
+ onnx_opset=onnx_opset,
+ auto_cast=True,
+ dtype=dtype,
+ device=device,
+ )
+
+batch_size = 1
+height = 512
+width = 512
+profile = model.get_dynamic_map(batch_size, height, width)
+del model
+gc.collect()
+torch.cuda.empty_cache()
+
+print('finished')
+
+print("Optimizing Onnx Model...")
+if(not os.path.exists(os.path.dirname(onnx_opt_path))):
+ os.mkdir(os.path.dirname(onnx_opt_path))
+optimize_onnx(
+ onnx_path=onnx_path,
+ onnx_opt_path=onnx_opt_path,
+)
+
+engine = engine_from_network(
+ network_from_onnx_path(onnx_opt_path, flags=[trt.OnnxParserFlag.NATIVE_INSTANCENORM]),
+ config=CreateConfig(
+ fp16=True, refittable=False, profiles=[profile]
+ ),
+)
+save_engine(engine, path=tensorrt_target_model)
+gc.collect()
+torch.cuda.empty_cache()
\ No newline at end of file