Industry leading face manipulation platform
Introduce video manager to handle broken videos
henryruhs62483fdfad5b
diff --git a/facefusion/face_store.py b/facefusion/face_store.pyindex 518b9e3c..59e6d5d1 100644--- a/facefusion/face_store.py+++ b/facefusion/face_store.py@@ -30,7 +30,7 @@ def set_static_faces(vision_frame : VisionFrame, faces : List[Face]) -> None:def clear_static_faces() -> None:- FACE_STORE['static_faces'] = {}+ FACE_STORE['static_faces'].clear()def create_frame_hash(vision_frame : VisionFrame) -> Optional[str]:@@ -53,4 +53,4 @@ def append_reference_face(name : str, face : Face) -> None:def clear_reference_faces() -> None:- FACE_STORE['reference_faces'] = {}+ FACE_STORE['reference_faces'].clear()diff --git a/facefusion/processors/modules/age_modifier.py b/facefusion/processors/modules/age_modifier.pyindex 925808b6..47197442 100755--- a/facefusion/processors/modules/age_modifier.py+++ b/facefusion/processors/modules/age_modifier.py@@ -9,7 +9,7 @@ import facefusion.choicesimport facefusion.jobs.job_managerimport facefusion.jobs.job_storeimport facefusion.processors.core as processors-from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, wording+from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, video_manager, wordingfrom facefusion.common_helper import create_int_metavarfrom facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_urlfrom facefusion.execution import has_execution_provider@@ -115,6 +115,7 @@ def pre_process(mode : ProcessMode) -> bool:def post_process() -> None:read_static_image.cache_clear()+ video_manager.clear_video_pool()if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]:clear_inference_pool()if state_manager.get_item('video_memory_strategy') == 'strict':diff --git a/facefusion/processors/modules/deep_swapper.py b/facefusion/processors/modules/deep_swapper.pyindex 14e23708..54438c71 100755--- a/facefusion/processors/modules/deep_swapper.py+++ b/facefusion/processors/modules/deep_swapper.py@@ -9,7 +9,7 @@ from cv2.typing import Sizeimport facefusion.jobs.job_managerimport facefusion.jobs.job_storeimport facefusion.processors.core as processors-from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, wording+from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, video_manager, wordingfrom facefusion.common_helper import create_int_metavarfrom facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url_by_providerfrom facefusion.face_analyser import get_many_faces, get_one_face@@ -311,6 +311,7 @@ def pre_process(mode : ProcessMode) -> bool:def post_process() -> None:read_static_image.cache_clear()+ video_manager.clear_video_pool()if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]:clear_inference_pool()if state_manager.get_item('video_memory_strategy') == 'strict':diff --git a/facefusion/processors/modules/expression_restorer.py b/facefusion/processors/modules/expression_restorer.pyindex 4d301a40..0e307ddc 100755--- a/facefusion/processors/modules/expression_restorer.py+++ b/facefusion/processors/modules/expression_restorer.py@@ -8,7 +8,7 @@ import numpyimport facefusion.jobs.job_managerimport facefusion.jobs.job_storeimport facefusion.processors.core as processors-from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, wording+from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, video_manager, wordingfrom facefusion.common_helper import create_int_metavarfrom facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_urlfrom facefusion.face_analyser import get_many_faces, get_one_face@@ -129,6 +129,7 @@ def pre_process(mode : ProcessMode) -> bool:def post_process() -> None:read_static_image.cache_clear()+ video_manager.clear_video_pool()if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]:clear_inference_pool()if state_manager.get_item('video_memory_strategy') == 'strict':diff --git a/facefusion/processors/modules/face_debugger.py b/facefusion/processors/modules/face_debugger.pyindex e4397f23..ea53b7d5 100755--- a/facefusion/processors/modules/face_debugger.py+++ b/facefusion/processors/modules/face_debugger.py@@ -7,7 +7,7 @@ import numpyimport facefusion.jobs.job_managerimport facefusion.jobs.job_storeimport facefusion.processors.core as processors-from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, logger, process_manager, state_manager, wording+from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, logger, process_manager, state_manager, video_manager, wordingfrom facefusion.face_analyser import get_many_faces, get_one_facefrom facefusion.face_helper import warp_face_by_face_landmark_5from facefusion.face_masker import create_occlusion_mask, create_region_mask, create_static_box_mask@@ -56,6 +56,7 @@ def pre_process(mode : ProcessMode) -> bool:def post_process() -> None:read_static_image.cache_clear()+ video_manager.clear_video_pool()if state_manager.get_item('video_memory_strategy') == 'strict':content_analyser.clear_inference_pool()face_classifier.clear_inference_pool()diff --git a/facefusion/processors/modules/face_editor.py b/facefusion/processors/modules/face_editor.pyindex 3f494d8d..2414d146 100755--- a/facefusion/processors/modules/face_editor.py+++ b/facefusion/processors/modules/face_editor.py@@ -8,7 +8,7 @@ import numpyimport facefusion.jobs.job_managerimport facefusion.jobs.job_storeimport facefusion.processors.core as processors-from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, wording+from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, video_manager, wordingfrom facefusion.common_helper import create_float_metavarfrom facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_urlfrom facefusion.face_analyser import get_many_faces, get_one_face@@ -182,6 +182,7 @@ def pre_process(mode : ProcessMode) -> bool:def post_process() -> None:read_static_image.cache_clear()+ video_manager.clear_video_pool()if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]:clear_inference_pool()if state_manager.get_item('video_memory_strategy') == 'strict':diff --git a/facefusion/processors/modules/face_enhancer.py b/facefusion/processors/modules/face_enhancer.pyindex 1787d692..ad1b0a52 100755--- a/facefusion/processors/modules/face_enhancer.py+++ b/facefusion/processors/modules/face_enhancer.py@@ -8,7 +8,7 @@ import numpyimport facefusion.jobs.job_managerimport facefusion.jobs.job_storeimport facefusion.processors.core as processors-from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, wording+from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, video_manager, wordingfrom facefusion.common_helper import create_float_metavar, create_int_metavarfrom facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_urlfrom facefusion.face_analyser import get_many_faces, get_one_face@@ -275,6 +275,7 @@ def pre_process(mode : ProcessMode) -> bool:def post_process() -> None:read_static_image.cache_clear()+ video_manager.clear_video_pool()if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]:clear_inference_pool()if state_manager.get_item('video_memory_strategy') == 'strict':diff --git a/facefusion/processors/modules/face_swapper.py b/facefusion/processors/modules/face_swapper.pyindex 7ce0ed42..1df1fa02 100755--- a/facefusion/processors/modules/face_swapper.py+++ b/facefusion/processors/modules/face_swapper.py@@ -8,7 +8,7 @@ import facefusion.choicesimport facefusion.jobs.job_managerimport facefusion.jobs.job_storeimport facefusion.processors.core as processors-from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, wording+from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, video_manager, wordingfrom facefusion.common_helper import get_firstfrom facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_urlfrom facefusion.execution import has_execution_provider@@ -406,9 +406,10 @@ def pre_process(mode : ProcessMode) -> bool:def post_process() -> None:read_static_image.cache_clear()+ video_manager.clear_video_pool()if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]:- clear_inference_pool()get_static_model_initializer.cache_clear()+ clear_inference_pool()if state_manager.get_item('video_memory_strategy') == 'strict':content_analyser.clear_inference_pool()face_classifier.clear_inference_pool()diff --git a/facefusion/processors/modules/frame_colorizer.py b/facefusion/processors/modules/frame_colorizer.pyindex 12a1942c..abf3fc91 100644--- a/facefusion/processors/modules/frame_colorizer.py+++ b/facefusion/processors/modules/frame_colorizer.py@@ -8,7 +8,7 @@ import numpyimport facefusion.jobs.job_managerimport facefusion.jobs.job_storeimport facefusion.processors.core as processors-from facefusion import config, content_analyser, inference_manager, logger, process_manager, state_manager, wording+from facefusion import config, content_analyser, inference_manager, logger, process_manager, state_manager, video_manager, wordingfrom facefusion.common_helper import create_int_metavarfrom facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_urlfrom facefusion.execution import has_execution_provider@@ -188,6 +188,7 @@ def pre_process(mode : ProcessMode) -> bool:def post_process() -> None:read_static_image.cache_clear()+ video_manager.clear_video_pool()if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]:clear_inference_pool()if state_manager.get_item('video_memory_strategy') == 'strict':diff --git a/facefusion/processors/modules/frame_enhancer.py b/facefusion/processors/modules/frame_enhancer.pyindex 5b653a75..27a9b6c1 100644--- a/facefusion/processors/modules/frame_enhancer.py+++ b/facefusion/processors/modules/frame_enhancer.py@@ -8,7 +8,7 @@ import numpyimport facefusion.jobs.job_managerimport facefusion.jobs.job_storeimport facefusion.processors.core as processors-from facefusion import config, content_analyser, inference_manager, logger, process_manager, state_manager, wording+from facefusion import config, content_analyser, inference_manager, logger, process_manager, state_manager, video_manager, wordingfrom facefusion.common_helper import create_int_metavarfrom facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_urlfrom facefusion.execution import has_execution_provider@@ -450,6 +450,7 @@ def pre_process(mode : ProcessMode) -> bool:def post_process() -> None:read_static_image.cache_clear()+ video_manager.clear_video_pool()if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]:clear_inference_pool()if state_manager.get_item('video_memory_strategy') == 'strict':diff --git a/facefusion/processors/modules/lip_syncer.py b/facefusion/processors/modules/lip_syncer.pyindex 1b6faa2e..ac38cbb9 100755--- a/facefusion/processors/modules/lip_syncer.py+++ b/facefusion/processors/modules/lip_syncer.py@@ -8,7 +8,7 @@ import numpyimport facefusion.jobs.job_managerimport facefusion.jobs.job_storeimport facefusion.processors.core as processors-from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, voice_extractor, wording+from facefusion import config, content_analyser, face_classifier, face_detector, face_landmarker, face_masker, face_recognizer, inference_manager, logger, process_manager, state_manager, video_manager, voice_extractor, wordingfrom facefusion.audio import create_empty_audio_frame, get_voice_frame, read_static_voicefrom facefusion.common_helper import get_firstfrom facefusion.download import conditional_download_hashes, conditional_download_sources, resolve_download_url@@ -127,6 +127,7 @@ def pre_process(mode : ProcessMode) -> bool:def post_process() -> None:read_static_image.cache_clear()read_static_voice.cache_clear()+ video_manager.clear_video_pool()if state_manager.get_item('video_memory_strategy') in [ 'strict', 'moderate' ]:clear_inference_pool()if state_manager.get_item('video_memory_strategy') == 'strict':diff --git a/facefusion/types.py b/facefusion/types.pyindex def3aa09..57210b01 100755--- a/facefusion/types.py+++ b/facefusion/types.py@@ -1,6 +1,7 @@from collections import namedtuplefrom typing import Any, Callable, Dict, List, Literal, Optional, Tuple, TypeAlias, TypedDict+import cv2import numpyfrom numpy.typing import NDArrayfrom onnxruntime import InferenceSession@@ -49,6 +50,7 @@ FaceStore = TypedDict('FaceStore','static_faces' : FaceSet,'reference_faces' : FaceSet})+VideoPoolSet : TypeAlias = Dict[str, cv2.VideoCapture]VisionFrame : TypeAlias = NDArray[Any]Mask : TypeAlias = NDArray[Any]diff --git a/facefusion/video_manager.py b/facefusion/video_manager.pynew file mode 100644index 00000000..f38bc19a--- /dev/null+++ b/facefusion/video_manager.py@@ -0,0 +1,21 @@+import cv2++from facefusion.types import VideoPoolSet++VIDEO_POOL_SET : VideoPoolSet = {}+++def get_video_capture(video_path : str) -> cv2.VideoCapture:+ if video_path not in VIDEO_POOL_SET:+ VIDEO_POOL_SET[video_path] = cv2.VideoCapture(video_path)++ return VIDEO_POOL_SET.get(video_path)+++def clear_video_pool() -> None:+ global VIDEO_POOL_SET++ for video_capture in VIDEO_POOL_SET.values():+ video_capture.release()++ VIDEO_POOL_SET.clear()diff --git a/facefusion/vision.py b/facefusion/vision.pyindex d03e3834..b146170c 100644--- a/facefusion/vision.py+++ b/facefusion/vision.py@@ -9,7 +9,9 @@ from cv2.typing import Sizeimport facefusion.choicesfrom facefusion.common_helper import is_windowsfrom facefusion.filesystem import get_file_extension, is_image, is_video+from facefusion.thread_helper import thread_semaphorefrom facefusion.types import Duration, Fps, Orientation, Resolution, VisionFrame+from facefusion.video_manager import get_video_capture@lru_cache()@@ -81,24 +83,30 @@ def create_image_resolutions(resolution : Resolution) -> List[str]:def read_video_frame(video_path : str, frame_number : int = 0) -> Optional[VisionFrame]:if is_video(video_path):- video_capture = cv2.VideoCapture(video_path)+ video_capture = get_video_capture(video_path)+if video_capture.isOpened():frame_total = video_capture.get(cv2.CAP_PROP_FRAME_COUNT)- video_capture.set(cv2.CAP_PROP_POS_FRAMES, min(frame_total, frame_number - 1))- has_vision_frame, vision_frame = video_capture.read()- video_capture.release()++ with thread_semaphore():+ video_capture.set(cv2.CAP_PROP_POS_FRAMES, min(frame_total, frame_number - 1))+ has_vision_frame, vision_frame = video_capture.read()+if has_vision_frame:return vision_frame+return Nonedef count_video_frame_total(video_path : str) -> int:if is_video(video_path):- video_capture = cv2.VideoCapture(video_path)+ video_capture = get_video_capture(video_path)+if video_capture.isOpened():- video_frame_total = int(video_capture.get(cv2.CAP_PROP_FRAME_COUNT))- video_capture.release()- return video_frame_total+ with thread_semaphore():+ video_frame_total = int(video_capture.get(cv2.CAP_PROP_FRAME_COUNT))+ return video_frame_total+return 0@@ -112,11 +120,13 @@ def predict_video_frame_total(video_path : str, fps : Fps, trim_frame_start : indef detect_video_fps(video_path : str) -> Optional[float]:if is_video(video_path):- video_capture = cv2.VideoCapture(video_path)+ video_capture = get_video_capture(video_path)+if video_capture.isOpened():- video_fps = video_capture.get(cv2.CAP_PROP_FPS)- video_capture.release()- return video_fps+ with thread_semaphore():+ video_fps = video_capture.get(cv2.CAP_PROP_FPS)+ return video_fps+return None@@ -163,12 +173,14 @@ def restrict_trim_frame(video_path : str, trim_frame_start : Optional[int], trimdef detect_video_resolution(video_path : str) -> Optional[Resolution]:if is_video(video_path):- video_capture = cv2.VideoCapture(video_path)+ video_capture = get_video_capture(video_path)+if video_capture.isOpened():- width = video_capture.get(cv2.CAP_PROP_FRAME_WIDTH)- height = video_capture.get(cv2.CAP_PROP_FRAME_HEIGHT)- video_capture.release()- return int(width), int(height)+ with thread_semaphore():+ width = video_capture.get(cv2.CAP_PROP_FRAME_WIDTH)+ height = video_capture.get(cv2.CAP_PROP_FRAME_HEIGHT)+ return int(width), int(height)+return None