Skip to content

[CVPR 2026] PersonaLive! : Expressive Portrait Image Animation for Live Streaming

3.2 KB
Raw Blame
import mediapipe as mp
import cv2
import os
import torch
import numpy as np
from concurrent.futures import ProcessPoolExecutor
from src.utils.util import read_frames
import logging
logging.getLogger('mediapipe').setLevel(logging.ERROR)
import argparse
from functools import partial

mp_face_mesh = mp.solutions.face_mesh

face_indices = list(range(468))
left_eye_indices = [226, 230, 223, 245]
right_eye_indices = [446, 450, 465, 443]
mouth_indices = [61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291] + \
                [78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308] + \
                [146, 91, 181, 84, 17, 314, 405, 321, 375] + \
                [191, 80, 81, 82, 13, 312, 311, 310, 415]

def get_region_box(landmarks, indices):
    xs = [int(landmarks[i][0]) for i in indices]
    ys = [int(landmarks[i][1]) for i in indices]
    x_min, x_max = min(xs), max(xs)
    y_min, y_max = min(ys), max(ys)
    return [x_min, y_min, x_max, y_max]

def process_video(name, video_dir, save_dir):
    video_path = os.path.join(video_dir, name)
    save_path = os.path.join(save_dir, name.replace('.mp4', '.pt'))
    if os.path.exists(save_path):
        return name
        
    video = read_frames(video_path)
    boxes = []

    with mp_face_mesh.FaceMesh(static_image_mode=True, max_num_faces=1) as face_mesh:
        for image_pil in video:
            image = np.array(image_pil)
            h, w, _ = image.shape
            results = face_mesh.process(image)

            if results.multi_face_landmarks is not None:
                face_landmarks = results.multi_face_landmarks[0]
                landmarks = [(int(l.x * w), int(l.y * h)) for l in face_landmarks.landmark]
                face_box = get_region_box(landmarks, face_indices)
                left_eye_box = get_region_box(landmarks, left_eye_indices)
                right_eye_box = get_region_box(landmarks, right_eye_indices)
                mouth_box = get_region_box(landmarks, mouth_indices)
                boxes.append({
                    'face': face_box,
                    'left_eye': left_eye_box,
                    'right_eye': right_eye_box,
                    'mouth': mouth_box
                })
            else:
                boxes.append(boxes[-1] if boxes else {
                    'face': [],
                    'left_eye': [],
                    'right_eye': [],
                    'mouth': []
                })
    torch.save(boxes, save_path)

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Extract facial boxes from videos and save as .pt files.")
    parser.add_argument('--video_dir', type=str, default='/home/zyli/Repositories/x-nemo-inference/lv100/videos')
    parser.add_argument('--save_dir', type=str, default='/home/zyli/Repositories/x-nemo-inference/lv100/boxes_zyli')
    parser.add_argument('--workers', type=int, default=8)

    args = parser.parse_args()
    os.makedirs(args.save_dir, exist_ok=True)
    video_files = [f for f in os.listdir(args.video_dir) if f.endswith('.mp4')]

    process_func = partial(process_video, video_dir=args.video_dir, save_dir=args.save_dir)

    with ProcessPoolExecutor(max_workers=8) as executor:
        list(executor.map(process_func, video_files))