[CVPR 2026] PersonaLive! : Expressive Portrait Image Animation for Live Streaming
update inference
Zhiyuan Lia2d84307e9d6
diff --git a/inference_offline.py b/inference_offline.pynew file mode 100644index 0000000..924ab18--- /dev/null+++ b/inference_offline.py@@ -0,0 +1,254 @@+import argparse+import os+import sys+from datetime import datetime+import mediapipe as mp+import numpy as np+import cv2+import torch+from skimage.transform import resize+from diffusers import AutoencoderKLTemporalDecoder, AutoencoderKL, AutoencoderTiny+from src.scheduler.scheduler_ddim import DDIMScheduler+import random+from omegaconf import OmegaConf+from PIL import Image+from torchvision import transforms+from transformers import CLIPVisionModelWithProjection+from src.models.unet_2d_condition import UNet2DConditionModel+from src.models.unet_3d import UNet3DConditionModel+from src.pipelines.pipeline_pose2vid import Pose2VideoPipeline+from src.utils.util import save_videos_grid, crop_face+from decord import VideoReader+from diffusers.utils.import_utils import is_xformers_available++from src.models.motion_encoder.encoder import MotEncoder+from src.liveportrait.motion_extractor import MotionExtractor+from src.models.pose_guider import PoseGuider+from tqdm import tqdm++def parse_args():+ parser = argparse.ArgumentParser()+ parser.add_argument("--config", type=str, default='configs/prompts/personalive_offline.yaml')+ parser.add_argument("--name", type=str, default='personalive_offline')+ parser.add_argument("-W", type=int, default=512)+ parser.add_argument("-H", type=int, default=512)+ parser.add_argument("-L", type=int, default=100)+ parser.add_argument("--seed", type=int, default=42)+ parser.add_argument("--device", type=str, default="cuda")+ args = parser.parse_args()++ return args++def main(args):+ device = args.device+ print('device', device)+ config = OmegaConf.load(args.config)++ if config.weight_dtype == "fp16":+ weight_dtype = torch.float16+ else:+ weight_dtype = torch.float32++ vae = AutoencoderKL.from_pretrained(config.vae_path).to(device, dtype=weight_dtype)+ # if use tiny VAE+ # vae_tiny = AutoencoderTiny.from_pretrained(config.vae_tiny_path).to(device, dtype=weight_dtype)++ infer_config = OmegaConf.load(config.inference_config)+ reference_unet = UNet2DConditionModel.from_pretrained(+ config.pretrained_base_model_path,+ subfolder="unet",+ ).to(device=device, dtype=weight_dtype)+ denoising_unet = UNet3DConditionModel.from_pretrained_2d(+ config.pretrained_base_model_path,+ "",+ subfolder="unet",+ unet_additional_kwargs=infer_config.unet_additional_kwargs,+ ).to(dtype=weight_dtype, device=device)++ motion_encoder = MotEncoder().to(dtype=weight_dtype, device=device).eval()+ pose_guider = PoseGuider().to(device=device, dtype=weight_dtype)+ pose_encoder = MotionExtractor(num_kp=21).to(device=device, dtype=weight_dtype).eval()++ image_enc = CLIPVisionModelWithProjection.from_pretrained(+ config.image_encoder_path+ ).to(dtype=weight_dtype, device=device)++ sched_kwargs = OmegaConf.to_container(+ OmegaConf.load(config.inference_config).noise_scheduler_kwargs+ )+ scheduler = DDIMScheduler(**sched_kwargs)++ generator = torch.manual_seed(args.seed)+ width, height = args.W, args.H++ # load pretrained weights+ denoising_unet.load_state_dict(+ torch.load(config.denoising_unet_path, map_location="cpu"), strict=False+ )+ reference_unet.load_state_dict(+ torch.load(+ config.denoising_unet_path.replace('denoising_unet', 'reference_unet'),+ map_location="cpu",+ ),+ strict=True,+ )+ motion_encoder.load_state_dict(+ torch.load(+ config.denoising_unet_path.replace('denoising_unet', 'motion_encoder'),+ map_location="cpu",+ ),+ strict=True,+ )+ pose_guider.load_state_dict(+ torch.load(+ config.denoising_unet_path.replace('denoising_unet', 'pose_guider'),+ map_location="cpu",+ ),+ strict=True,+ )+ denoising_unet.load_state_dict(+ torch.load(+ config.denoising_unet_path.replace('denoising_unet', 'temporal_module'),+ map_location="cpu",+ ),+ strict=False,+ )+ pose_encoder.load_state_dict(+ torch.load(+ config.denoising_unet_path.replace('denoising_unet', 'motion_extractor'),+ map_location="cpu",+ ),+ strict=False,+ )++ if is_xformers_available():+ reference_unet.enable_xformers_memory_efficient_attention()+ denoising_unet.enable_xformers_memory_efficient_attention()+ else:+ raise ValueError(+ "xformers is not available. Make sure it is installed correctly"+ )++ mp_face_mesh = mp.solutions.face_mesh+ face_mesh = mp_face_mesh.FaceMesh(static_image_mode=True, max_num_faces=1)++ pipe = Pose2VideoPipeline(+ vae=vae,+ # vae_tiny=vae_tiny,+ image_encoder=image_enc,+ reference_unet=reference_unet,+ denoising_unet=denoising_unet,+ motion_encoder=motion_encoder,+ pose_encoder=pose_encoder,+ pose_guider=pose_guider,+ scheduler=scheduler,+ )+ pipe = pipe.to(device)++ date_str = datetime.now().strftime("%Y%m%d")+ if args.name is None:+ time_str = datetime.now().strftime("%H%M")+ save_dir_name = f"{date_str}--{time_str}"+ else:+ save_dir_name = f"{date_str}--{args.name}"+ save_vid_dir = os.path.join('results', save_dir_name, 'concat_vid')+ os.makedirs(save_vid_dir, exist_ok=True)+ save_split_vid_dir = os.path.join('results', save_dir_name, 'split_vid')+ os.makedirs(save_split_vid_dir, exist_ok=True)++ pose_transform = transforms.Compose(+ [transforms.Resize((height, width)), transforms.ToTensor()]+ )++ args.test_cases = OmegaConf.load(args.config)["test_cases"]++ for ref_image_path in list(args.test_cases.keys()):+ for pose_video_path in args.test_cases[ref_image_path]:+ video_name = os.path.basename(pose_video_path).split(".")[0]+ source_name = os.path.basename(ref_image_path).split(".")[0]++ vid_name = f"{source_name}_{video_name}.mp4"+ save_vid_path = os.path.join(save_vid_dir, vid_name)+ print(save_vid_path)+ if os.path.exists(save_vid_path):+ continue++ if ref_image_path.endswith('.mp4'):+ src_vid = VideoReader(ref_image_path)+ ref_img = src_vid[0].asnumpy()+ ref_img = Image.fromarray(ref_img).convert("RGB")+ else:+ ref_img = Image.open(ref_image_path).convert("RGB")++ control = VideoReader(pose_video_path)+ video_length = min(len(control) // 4 * 4, args.L)+ sel_idx = range(len(control))[:video_length]+ control = control.get_batch([sel_idx]).asnumpy() # N, H, W, C++ ref_image_pil = ref_img.copy()+ ref_patch = crop_face(ref_image_pil, face_mesh)+ ref_face_pil = Image.fromarray(ref_patch).convert("RGB")++ size = args.H+ generator = torch.Generator(device=device)+ generator.manual_seed(42)++ dri_faces = []+ ori_pose_images = []+ for idx_control, pose_image_pil in tqdm(enumerate(control[:video_length]), total=video_length, desc='cropping faces'):+ pose_image_pil = Image.fromarray(pose_image_pil).convert("RGB")+ ori_pose_images.append(pose_image_pil)+ dri_face = crop_face(pose_image_pil, face_mesh)+ dri_face_pil = Image.fromarray(dri_face).convert("RGB")+ dri_faces.append(dri_face_pil)++ face_tensor_list = []+ ori_pose_tensor_list = []+ ref_tensor_list = []++ for idx, pose_image_pil in enumerate(ori_pose_images):+ face_tensor_list.append(pose_transform(dri_faces[idx]))+ ori_pose_tensor_list.append(pose_transform(pose_image_pil))+ ref_tensor_list.append(pose_transform(ref_image_pil))++ ref_tensor = torch.stack(ref_tensor_list, dim=0) # (f, c, h, w)+ ref_tensor = ref_tensor.transpose(0, 1).unsqueeze(0) # (c, f, h, w)++ face_tensor = torch.stack(face_tensor_list, dim=0) # (f, c, h, w)+ face_tensor = face_tensor.transpose(0, 1).unsqueeze(0)++ ori_pose_tensor = torch.stack(ori_pose_tensor_list, dim=0) # (f, c, h, w)+ ori_pose_tensor = ori_pose_tensor.transpose(0, 1).unsqueeze(0)++ gen_video = pipe(+ ori_pose_images,+ ref_image_pil,+ dri_faces,+ ref_face_pil,+ width,+ height,+ len(dri_faces),+ num_inference_steps=4,+ guidance_scale=1.0,+ generator=generator,+ temporal_window_size = 4,+ temporal_adaptive_step = 4,+ ).videos++ #Concat it with pose tensor+ video = torch.cat([ref_tensor, face_tensor, ori_pose_tensor, gen_video], dim=0)++ save_videos_grid(+ video,+ save_vid_path,+ n_rows=4,+ fps=25,+ )++ if True:+ save_vid_path = save_vid_path.replace(save_vid_dir, save_split_vid_dir)+ save_videos_grid(gen_video, save_vid_path, n_rows=1, fps=25, crf=18)++if __name__ == "__main__":+ args = parse_args()+ main(args)diff --git a/inference_online.py b/inference_online.pynew file mode 100644index 0000000..a426ed4--- /dev/null+++ b/inference_online.py@@ -0,0 +1,323 @@+import os+import signal+import sys+import json++from fastapi import FastAPI, WebSocket, HTTPException, WebSocketDisconnect, UploadFile, File+from fastapi.responses import JSONResponse+from fastapi.middleware.cors import CORSMiddleware+from fastapi.staticfiles import StaticFiles+from fastapi import Request++import markdown2+import threading+import logging+import uuid+import time+from types import SimpleNamespace+import asyncio+import mimetypes+import torch++from webcam.config import config, Args+from webcam.util import pil_to_frame, bytes_to_pil, is_firefox, bytes_to_tensor+from webcam.connection_manager import ConnectionManager, ServerFullException+import multiprocessing as mp++use_trt = True++if use_trt:+ from webcam.vid2vid_trt import Pipeline+else:+ from webcam.vid2vid import Pipeline++mimetypes.add_type("application/javascript", ".js")++THROTTLE = 0.001+++class App:+ def __init__(self, config: Args, pipeline: Pipeline):+ self.args = config+ self.pipeline = pipeline+ self.app = FastAPI()+ self.conn_manager = ConnectionManager()++ self.produce_predictions_stop_event = None+ self.produce_predictions_task = None+ self.shutdown_event = asyncio.Event()++ self.init_app()++ def init_app(self):+ self.app.add_middleware(+ CORSMiddleware,+ allow_origins=["*"],+ allow_credentials=True,+ allow_methods=["*"],+ allow_headers=["*"],+ )++ @self.app.websocket("/api/ws/{user_id}")+ async def websocket_endpoint(user_id: uuid.UUID, websocket: WebSocket):+ try:+ await self.conn_manager.connect(+ user_id, websocket, self.args.max_queue_size+ )++ sender_task = asyncio.create_task(push_results_to_client(user_id, websocket))++ if self.produce_predictions_task is None or self.produce_predictions_task.done():+ start_prediction_thread(user_id)++ await handle_websocket_input(user_id, websocket)++ except ServerFullException as e:+ logging.error(f"Server Full: {e}")+ except WebSocketDisconnect:+ logging.info(f"User disconnected: {user_id}")+ except Exception as e:+ logging.error(f"WS Error: {e}")+ finally:+ if 'sender_task' in locals():+ sender_task.cancel()++ await self.conn_manager.disconnect(user_id, self.pipeline)++ if self.produce_predictions_stop_event is not None:+ self.produce_predictions_stop_event.set()+ logging.info(f"Cleaned up user: {user_id}")++ async def handle_websocket_input(user_id: uuid.UUID, websocket: WebSocket):+ if not self.conn_manager.check_user(user_id):+ raise HTTPException(status_code=404, detail="User not found")++ try:+ while True:+ message = await websocket.receive()++ if "text" in message:+ try:+ text_data = message["text"]+ data = json.loads(text_data)+ status = data.get("status")++ if status == "pause":+ params = SimpleNamespace(**{"restart": True})+ await self.conn_manager.update_data(user_id, params)+ elif status == "resume":+ await self.conn_manager.send_json(user_id, {"status": "send_frame"})+ except Exception as e:+ logging.error(f"JSON Parse Error: {e}")++ elif "bytes" in message:+ image_data = message["bytes"]+ if len(image_data) > 0:+ input_tensor = bytes_to_tensor(image_data)+ params = SimpleNamespace()+ params.image = input_tensor+ self.pipeline.accept_new_params(params)++ except WebSocketDisconnect:+ raise+ except Exception as e:+ logging.error(f"Input Loop Error: {e}")+ raise++ async def push_results_to_client(user_id: uuid.UUID, websocket: WebSocket):+ MIN_FPS = 10+ MAX_FPS = 30+ SMOOTHING = 0.8 # EMA smoothing factor++ last_burst_time = time.time()+ last_queue_size = 0+ sleep_time = 1 / 40 # Initial guess++ last_frame_time = None+ frame_time_list = []++ ema_frame_interval = sleep_time++ try:+ while True:+ queue_size = await self.conn_manager.get_output_queue_size(user_id)+ if queue_size > last_queue_size:+ current_burst_time = time.time()+ elapsed = current_burst_time - last_burst_time++ if queue_size > 0 and elapsed > 0:+ raw_interval = elapsed / queue_size+ ema_frame_interval = SMOOTHING * ema_frame_interval + (1 - SMOOTHING) * raw_interval+ sleep_time = min(max(ema_frame_interval, 1 / MAX_FPS), 1 / MIN_FPS)++ last_burst_time = current_burst_time++ last_queue_size = queue_size++ frame = await self.conn_manager.get_frame(user_id)+ if frame is None:+ await asyncio.sleep(0.001)+ continue++ await websocket.send_bytes(frame)++ if last_frame_time is None:+ last_frame_time = time.time()+ else:+ frame_time_list.append(time.time() - last_frame_time)+ if len(frame_time_list) > 100:+ frame_time_list.pop(0)+ last_frame_time = time.time()++ await asyncio.sleep(sleep_time)++ except asyncio.CancelledError:+ pass+ except Exception as e:+ logging.error(f"Push Result Error: {e}")++ def start_prediction_thread(user_id):+ self.produce_predictions_stop_event = threading.Event()++ def prediction_loop(uid, loop, stop_event):+ while not stop_event.is_set():+ images = self.pipeline.produce_outputs()+ if len(images) == 0:+ time.sleep(THROTTLE)+ continue++ frames = list(map(pil_to_frame, images))+ asyncio.run_coroutine_threadsafe(+ self.conn_manager.put_frames_to_output_queue(uid, frames),+ loop+ )++ self.produce_predictions_task = asyncio.create_task(asyncio.to_thread(+ prediction_loop, user_id, asyncio.get_running_loop(), self.produce_predictions_stop_event+ ))++ @self.app.get("/api/queue")+ async def get_queue_size():+ queue_size = self.conn_manager.get_user_count()+ return JSONResponse({"queue_size": queue_size})++ @self.app.get("/api/settings")+ async def settings():+ info_schema = pipeline.Info.schema()+ info = pipeline.Info()+ if info.page_content:+ page_content = markdown2.markdown(info.page_content)++ input_params = pipeline.InputParams.schema()+ return JSONResponse(+ {+ "info": info_schema,+ "input_params": input_params,+ "max_queue_size": self.args.max_queue_size,+ "page_content": page_content if info.page_content else "",+ }+ )++ @self.app.post("/api/upload_reference_image")+ async def upload_reference_image(ref_image: UploadFile = File(...)):+ try:+ data = await ref_image.read()+ img = bytes_to_pil(data)+ self.pipeline.fuse_reference(img)+ return {"status": "ok"}+ except Exception as e:+ logging.error(f"Reference image error: {e}")+ raise HTTPException(status_code=500, detail="Failed to process reference image")++ if not os.path.exists("./demo_w_camera/frontend/public"):+ os.makedirs("./demo_w_camera/frontend/public")++ self.app.mount(+ "/", StaticFiles(directory="./demo_w_camera/frontend/public", html=True), name="public"+ )++ @self.app.on_event("shutdown")+ async def shutdown_event():+ await self.cleanup()++ async def cleanup(self):+ print("[App] Starting cleanup process...")+ self.shutdown_event.set()++ if self.produce_predictions_stop_event is not None:+ self.produce_predictions_stop_event.set()++ if self.produce_predictions_task is not None:+ self.produce_predictions_task.cancel()+ try:+ await self.produce_predictions_task+ except asyncio.CancelledError:+ pass++ try:+ await self.conn_manager.disconnect_all(self.pipeline)+ except Exception as e:+ print(f"[App] Error during disconnect_all: {e}")++ print("[App] Cleanup completed")++app_instance = None++def signal_handler(signum, frame):+ print(f"\n[Main] Received signal {signum}, shutting down gracefully...")+ if app_instance:+ import threading+ def trigger_cleanup():+ try:+ loop = asyncio.new_event_loop()+ asyncio.set_event_loop(loop)+ loop.run_until_complete(app_instance.cleanup())+ loop.close()+ except Exception as e:+ print(f"[Main] Error during cleanup: {e}")++ cleanup_thread = threading.Thread(target=trigger_cleanup)+ cleanup_thread.daemon = True+ cleanup_thread.start()+ cleanup_thread.join(timeout=5)++ sys.exit(0)+++if __name__ == "__main__":+ import uvicorn+ signal.signal(signal.SIGINT, signal_handler)+ signal.signal(signal.SIGTERM, signal_handler)+ mp.set_start_method("spawn", force=True)++ device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")+ pipeline = Pipeline(config, device)++ app_obj = App(config, pipeline)+ app = app_obj.app+ app_instance = app_obj++ print('init done')++ try:+ uvicorn.run(+ app,+ host=config.host,+ port=config.port,+ reload=config.reload,+ ssl_certfile=config.ssl_certfile,+ ssl_keyfile=config.ssl_keyfile,+ )+ except KeyboardInterrupt:+ try:+ import asyncio+ loop = asyncio.new_event_loop()+ asyncio.set_event_loop(loop)+ loop.run_until_complete(app_obj.cleanup())+ loop.close()+ except Exception as e:+ print(f"[Main] Error during cleanup: {e}")+ sys.exit(0)+ except Exception as e:+ print(f"[Main] Error: {e}")+ sys.exit(1)\ No newline at end of filediff --git a/torch2trt.py b/torch2trt.pynew file mode 100644index 0000000..6d8f89c--- /dev/null+++ b/torch2trt.py@@ -0,0 +1,131 @@+import os+import torch+from omegaconf import OmegaConf+import gc+from src.modeling.framed_models import unet_work+from diffusers import AutoencoderKL+from diffusers.models.attention_processor import AttnProcessor+from src.scheduler.scheduler_ddim import DDIMScheduler+from src.models.unet_3d_explicit_reference import UNet3DConditionModel+from src.models.motion_encoder.encoder import MotEncoder+from src.models.pose_guider import PoseGuider++from src.modeling.onnx_export import export_onnx, handle_onnx_batch_norm, optimize_onnx+from polygraphy.backend.trt import engine_from_network, network_from_onnx_path, save_engine, CreateConfig+from polygraphy.logger import G_LOGGER+import tensorrt as trt+G_LOGGER.severity = G_LOGGER.VERBOSE++def map_device(device_or_str):+ return device_or_str if isinstance(device_or_str, torch.device) else torch.device(device_or_str)++# parameters+batch_size = 1+height = 256+width = 256+onnx_opset = 17+device = torch.device("cuda:0")+device = map_device(device)+dtype = torch.float16++config_path = './configs/prompts/personalive_trt.yaml'+cfg = OmegaConf.load(config_path)+onnx_path = cfg.onnx_path+onnx_opt_path = cfg.onnx_opt_path+tensorrt_target_model = cfg.tensorrt_target_model++infer_config = OmegaConf.load(cfg.inference_config)+sched_kwargs = OmegaConf.to_container(+ infer_config.noise_scheduler_kwargs+)++pose_guider = PoseGuider().to(device=device, dtype=dtype)+pose_guider_state_dict = torch.load(cfg.pose_guider_path, map_location="cpu")+pose_guider.load_state_dict(pose_guider_state_dict)+del pose_guider_state_dict++motion_encoder:MotEncoder = MotEncoder().to(dtype=dtype, device=device).eval()+motion_encoder.set_attn_processor(AttnProcessor())+motion_encoder_state_dict = torch.load(cfg.motion_encoder_path, map_location="cpu")+motion_encoder.load_state_dict(motion_encoder_state_dict)+del motion_encoder_state_dict++denoising_unet:UNet3DConditionModel = UNet3DConditionModel.from_pretrained_2d(+ cfg.pretrained_base_model_path,+ "",+ subfolder="unet",+ unet_additional_kwargs=infer_config.unet_additional_kwargs,+).to(dtype=dtype, device=device)++denoising_unet.load_state_dict(+ torch.load(cfg.denoising_unet_path, map_location="cpu"), strict=False+)+denoising_unet.load_state_dict(+ torch.load(+ cfg.temporal_module_path,+ map_location="cpu",+ ),+ strict=False,+)+denoising_unet.set_attn_processor(AttnProcessor())++vae:AutoencoderKL = AutoencoderKL.from_pretrained(cfg.vae_model_path).to(device=device, dtype=dtype)+vae.set_default_attn_processor()++scheduler = DDIMScheduler(**sched_kwargs)+scheduler.to(device)+timesteps = torch.tensor([0,0,0,0,333,333,333,333,666,666,666,666,999,999,999,999], device=device).long()+scheduler.set_step_length(333)++model = unet_work(+ pose_guider,+ motion_encoder,+ denoising_unet,+ vae,+ scheduler,+ timesteps,+ )++if(not os.path.exists(os.path.dirname(onnx_path))):+ os.mkdir(os.path.dirname(onnx_path))++if not os.path.exists(onnx_path):+ export_onnx(+ model,+ onnx_path=onnx_path,+ opt_image_height=height,+ opt_image_width=width,+ opt_batch_size=batch_size,+ onnx_opset=onnx_opset,+ auto_cast=True,+ dtype=dtype,+ device=device,+ )++batch_size = 1+height = 512+width = 512+profile = model.get_dynamic_map(batch_size, height, width)+del model+gc.collect()+torch.cuda.empty_cache()++print('finished')++print("Optimizing Onnx Model...")+if(not os.path.exists(os.path.dirname(onnx_opt_path))):+ os.mkdir(os.path.dirname(onnx_opt_path))+optimize_onnx(+ onnx_path=onnx_path,+ onnx_opt_path=onnx_opt_path,+)++engine = engine_from_network(+ network_from_onnx_path(onnx_opt_path, flags=[trt.OnnxParserFlag.NATIVE_INSTANCENORM]),+ config=CreateConfig(+ fp16=True, refittable=False, profiles=[profile]+ ),+)+save_engine(engine, path=tensorrt_target_model)+gc.collect()+torch.cuda.empty_cache()\ No newline at end of file