[CVPR 2026] PersonaLive! : Expressive Portrait Image Animation for Live Streaming
add audio track from driving_video
andrewb45f959f797a
diff --git a/inference_offline.py b/inference_offline.pyindex 0e06e34..a1d72fe 100644--- a/inference_offline.py+++ b/inference_offline.py@@ -261,7 +261,7 @@ def main(args):if True:save_vid_path = save_vid_path.replace(save_vid_dir, save_split_vid_dir)- save_videos_grid(gen_video, save_vid_path, n_rows=1, fps=25, crf=18)+ save_videos_grid(gen_video, save_vid_path, n_rows=1, fps=25, crf=18, audio_source=pose_video_path)if __name__ == "__main__":args = parse_args()diff --git a/src/utils/util.py b/src/utils/util.pyindex 7f9ccbf..1d9d589 100644--- a/src/utils/util.py+++ b/src/utils/util.py@@ -2,7 +2,9 @@ import importlibimport osimport os.path as ospimport shutil+import subprocessimport sys+import tempfilefrom pathlib import Pathimport av@@ -98,7 +100,121 @@ def delete_additional_ckpt(base_path, num_keep):shutil.rmtree(path_to_dir)-def save_videos_from_pil(pil_images, path, fps=8, crf=None):+def has_audio_stream(video_path):+ """Check if a video file has an audio stream."""+ try:+ container = av.open(video_path)+ for stream in container.streams:+ if stream.type == "audio":+ container.close()+ return True+ container.close()+ return False+ except Exception:+ return False+++def add_audio_to_video(video_path, audio_source_path, output_path=None, verbose=False):+ """+ Add audio from audio_source_path to video_path.++ The audio will be trimmed to match the video duration if it's longer.+ If the video is longer than the audio, the audio will end when it ends.++ Args:+ video_path: Path to the video file (without audio or with audio to replace)+ audio_source_path: Path to the source file to extract audio from+ output_path: Path for the output file. If None, replaces the original video.+ verbose: If True, print debug information++ Returns:+ True if audio was successfully added, False otherwise+ """+ if not has_audio_stream(audio_source_path):+ if verbose:+ print(f"No audio stream found in {audio_source_path}")+ return False++ if output_path is None:+ output_path = video_path++ # Create a temporary file for the output+ temp_output = None+ try:+ # Get video duration+ video_container = av.open(video_path)+ video_stream = next(s for s in video_container.streams if s.type == "video")+ video_duration = float(video_stream.duration * video_stream.time_base)+ video_container.close()++ if verbose:+ print(f"Video duration: {video_duration:.2f}s")++ # Create temp file in the same directory as output to ensure same filesystem+ output_dir = os.path.dirname(output_path) or "."+ temp_fd, temp_output = tempfile.mkstemp(suffix=".mp4", dir=output_dir)+ os.close(temp_fd)++ # Use ffmpeg to combine video and audio with proper duration handling+ # -t limits the output duration to the video duration+ # -shortest would stop when the shortest stream ends, but we use -t for more control+ cmd = [+ "ffmpeg", "-y",+ "-i", video_path,+ "-i", audio_source_path,+ "-c:v", "copy",+ "-c:a", "aac",+ "-map", "0:v:0",+ "-map", "1:a:0",+ "-t", str(video_duration),+ "-shortest",+ temp_output+ ]++ if verbose:+ print(f"Running: {' '.join(cmd)}")++ result = subprocess.run(+ cmd,+ capture_output=True,+ text=True+ )++ if result.returncode != 0:+ if verbose:+ print(f"ffmpeg error: {result.stderr}")+ return False++ # Replace the original file with the new one+ shutil.move(temp_output, output_path)+ temp_output = None # Mark as moved++ if verbose:+ print(f"Successfully added audio to {output_path}")+ return True++ except Exception as e:+ if verbose:+ print(f"Error adding audio: {e}")+ return False+ finally:+ # Clean up temp file if it wasn't moved+ if temp_output and os.path.exists(temp_output):+ os.remove(temp_output)+++def save_videos_from_pil(pil_images, path, fps=8, crf=None, audio_source=None):+ """+ Save a list of PIL images as a video file.++ Args:+ pil_images: List of PIL Image objects+ path: Output path for the video+ fps: Frames per second+ crf: Constant Rate Factor for video quality (lower = better quality)+ audio_source: Optional path to a video file to extract audio from.+ The audio will be trimmed to match the output video duration.+ """import avsave_fmt = Path(path).suffix@@ -144,8 +260,12 @@ def save_videos_from_pil(pil_images, path, fps=8, crf=None):else:raise ValueError("Unsupported file type. Use .mp4 or .gif.")+ # Add audio from source video if provided (only for mp4)+ if audio_source is not None and save_fmt == ".mp4":+ add_audio_to_video(path, audio_source, verbose=False)+-def save_videos_grid(videos_, path: str, rescale=False, n_rows=6, fps=8, crf=None):+def save_videos_grid(videos_, path: str, rescale=False, n_rows=6, fps=8, crf=None, audio_source=None):if not isinstance(videos_, list): videos_ = [videos_]outputs = []@@ -167,7 +287,7 @@ def save_videos_grid(videos_, path: str, rescale=False, n_rows=6, fps=8, crf=Nonoutputs.append(output)os.makedirs(os.path.dirname(path), exist_ok=True)- save_videos_from_pil(outputs, path, fps, crf)+ save_videos_from_pil(outputs, path, fps, crf, audio_source=audio_source)def save_videos_grid_ori(videos: torch.Tensor, path: str, rescale=False, n_rows=6, fps=8):