Free Motion Capture for Everyone πβ¨
TypeScript
mediapipe is in there but its not doing anything and apparently the frames are still out of sync lol
Jond4cc1a04b8ec
diff --git a/src/api/routes/session/session_router.py b/src/api/routes/session/session_router.pyindex 2736e769..677967df 100644--- a/src/api/routes/session/session_router.py+++ b/src/api/routes/session/session_router.py@@ -65,7 +65,8 @@ def calibrate_session(session_calibrate_model: SessionCalibrateModel = SessionCashow_camera_views_in_windows=True,)calibration_orchestrator.run_anipose_camera_calibration(- charuco_square_size=session_calibrate_model.charuco_square_size)+ charuco_square_size=session_calibrate_model.charuco_square_size,+ pin_camera_0_to_origin = True)@session_router.post("/session/record")diff --git a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.pyindex eae4f6b4..d16a0cc9 100644--- a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py+++ b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py@@ -1,7 +1,7 @@import loggingfrom dataclasses import dataclassfrom pathlib import Path-from typing import List, Dict, Union+from typing import List, Dict, Union, Anyimport cv2import numpy as np@@ -20,7 +20,7 @@ logger = logging.getLogger(__name__)@dataclass-class Mediapipe2dSingleCameraNpyArrays:+class Mediapipe2dNumpyArrays:body2d_frameNumber_trackedPointNumber_XY: np.ndarray = NonerightHand2d_frameNumber_trackedPointNumber_XY: np.ndarray = NoneleftHand2d_frameNumber_trackedPointNumber_XY: np.ndarray = None@@ -37,6 +37,14 @@ class Mediapipe2dSingleCameraNpyArrays:self.face2d_frameNumber_trackedPointNumber_XY])+@dataclass+class Mediapipe2dDataPayload:+ raw_frame_payload: FramePayload = None+ mediapipe_results: Any = None+ annotated_image: np.ndarray = None+ pixel_data_numpy_arrays: Mediapipe2dNumpyArrays = None++class MediaPipeSkeletonDetector:def __init__(self, session_id: str = None):self._session_id = session_id@@ -59,9 +67,39 @@ class MediaPipeSkeletonDetector:min_tracking_confidence=self.min_tracking_confidence)self._mediapipe_tracked_point_names_dict = mediapipe_tracked_point_names_dict- def detect_skeleton_in_image(self, raw_image, annotate_image=True):- mediapipe_results = self._holistic_tracker.process(raw_image) # <-this is where the magic happens- return mediapipe_results+ self.body_names_list = self._mediapipe_tracked_point_names_dict["body"]+ self.right_hand_names_list = self._mediapipe_tracked_point_names_dict["right_hand"]+ self.left_hand_names_list = self._mediapipe_tracked_point_names_dict["left_hand"]+ self.face_names_list = self._mediapipe_tracked_point_names_dict["face"]++ # TODO - build a better iterator and list of `face_marker_names` that will only pull out the face_counters & iris edges (mp.python.solutions.face_mesh_connections.FACEMESH_CONTOURS, FACE_MESH_IRISES)++ self.number_of_body_tracked_points = len(self.body_names_list)+ self.number_of_right_hand_tracked_points = len(self.right_hand_names_list)+ self.number_of_left_hand_tracked_points = len(self.left_hand_names_list)+ self.number_of_face_tracked_points = mp.python.solutions.face_mesh.FACEMESH_NUM_LANDMARKS_WITH_IRISES++ self.number_of_tracked_points_total = self.number_of_body_tracked_points + \+ self.number_of_left_hand_tracked_points + \+ self.number_of_right_hand_tracked_points + \+ self.number_of_face_tracked_points++ def detect_skeleton_in_image(self, raw_frame_payload: FramePayload,+ annotated_image: np.ndarray = None) -> Mediapipe2dDataPayload:++ mediapipe_results = self._holistic_tracker.process(+ raw_frame_payload.image) # <-this is where the magic happens, i.e. where the raw image is processed by a convolutional neural network to provide an estimate of joint position in pixel coordinates. Please don't forget that this is insane and should not be possible lol++ if annotated_image is None:+ annotated_image = raw_frame_payload.image.copy()++ annotated_image = self._annotate_image(annotated_image, mediapipe_results)++ mediapipe_single_frame_npy_data = self._list_of_mediapipe_results_to_npy_arrays([mediapipe_results])+ return Mediapipe2dDataPayload(raw_frame_payload=raw_frame_payload,+ mediapipe_results=mediapipe_results,+ annotated_image=annotated_image,+ pixel_data_numpy_arrays=mediapipe_single_frame_npy_data)def process_session_folder(self,save_annotated_videos: bool = True):@@ -194,45 +232,35 @@ class MediaPipeSkeletonDetector:mediapipe_results_list: List,image_width: int = 1,image_height: int = 1- ) -> Mediapipe2dSingleCameraNpyArrays:-- body_names_list = self._mediapipe_tracked_point_names_dict["body"]- right_hand_names_list = self._mediapipe_tracked_point_names_dict["right_hand"]- left_hand_names_list = self._mediapipe_tracked_point_names_dict["left_hand"]- face_names_list = self._mediapipe_tracked_point_names_dict["face"]+ ) -> Mediapipe2dNumpyArrays:# apparently `mediapipe_results.pose_landmarks.landmark` returns something iterable Β―\_(γ)_/Β―mediapipe_pose_landmark_iterator = mp.python.solutions.pose.PoseLandmarkmediapipe_hand_landmark_iterator = mp.python.solutions.hands.HandLandmark- # TODO - build a better iterator and list of `face_marker_names` that will only pull out the face_counters & iris edges (mp.python.solutions.face_mesh_connections.FACEMESH_CONTOURS, FACE_MESH_IRISES)number_of_frames = len(mediapipe_results_list)-- number_of_body_trackedPoints = len(body_names_list)- number_of_right_hand_trackedPoints = len(right_hand_names_list)- number_of_left_hand_trackedPoints = len(left_hand_names_list)- number_of_face_trackedPoints = mp.python.solutions.face_mesh.FACEMESH_NUM_LANDMARKS_WITH_IRISESnumber_of_spatial_dimensions = 2 # this will be 2d XY pixel databody2d_frameNumber_trackedPointNumber_XY = np.zeros(- (number_of_frames, number_of_body_trackedPoints, number_of_spatial_dimensions))+ (number_of_frames, self.number_of_body_tracked_points, number_of_spatial_dimensions))body2d_frameNumber_trackedPointNumber_XY[:] = np.nan- body2d_frameNumber_trackedPointNumber_confidence = np.zeros((number_of_frames, number_of_body_trackedPoints))+ body2d_frameNumber_trackedPointNumber_confidence = np.zeros(+ (number_of_frames, self.number_of_body_tracked_points))body2d_frameNumber_trackedPointNumber_confidence[:] = np.nan # only body markers get a 'confidence' valuerightHand2d_frameNumber_trackedPointNumber_XY = np.zeros((number_of_frames,- number_of_right_hand_trackedPoints,+ self.number_of_right_hand_tracked_points,number_of_spatial_dimensions))rightHand2d_frameNumber_trackedPointNumber_XY[:] = np.nanleftHand2d_frameNumber_trackedPointNumber_XY = np.zeros((number_of_frames,- number_of_left_hand_trackedPoints,+ self.number_of_left_hand_tracked_points,number_of_spatial_dimensions))leftHand2d_frameNumber_trackedPointNumber_XY[:] = np.nanface2d_frameNumber_trackedPointNumber_XY = np.zeros((number_of_frames,- number_of_face_trackedPoints,+ self.number_of_face_tracked_points,number_of_spatial_dimensions))face2d_frameNumber_trackedPointNumber_XY[:] = np.nan@@ -312,7 +340,7 @@ class MediaPipeSkeletonDetector:all_tracked_points_visible_on_this_frame_list.append(all_points_visible)- return Mediapipe2dSingleCameraNpyArrays(+ return Mediapipe2dNumpyArrays(body2d_frameNumber_trackedPointNumber_XY=body2d_frameNumber_trackedPointNumber_XY,rightHand2d_frameNumber_trackedPointNumber_XY=rightHand2d_frameNumber_trackedPointNumber_XY,leftHand2d_frameNumber_trackedPointNumber_XY=leftHand2d_frameNumber_trackedPointNumber_XY,diff --git a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py b/src/pipelines/session_pipeline/session_pipeline_orchestrator.pyindex 446ab473..cdf765bc 100644--- a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py+++ b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py@@ -157,6 +157,7 @@ class SessionPipelineOrchestrator:self.anipose_camera_calibration_object = CalibrationPipelineOrchestrator().load_most_recent_calibration()dictionary_of_charuco_frame_payloads_on_this_multi_frame = {}+ dictionary_of_mediapipe_payloads_on_this_multi_frame = {}with self._open_cv_camera_manager.start_capture_session_all_cams() as connected_cameras_dict:@@ -173,11 +174,19 @@ class SessionPipelineOrchestrator:if detect_charuco:if len(dictionary_of_charuco_frame_payloads_on_this_multi_frame) > 0:- charuco3d_multi_frame_payload = self._reconstruct_3d_charuco(dictionary_of_charuco_frame_payloads_on_this_multi_frame)+ charuco3d_multi_frame_payload = self._reconstruct_3d_charuco(+ dictionary_of_charuco_frame_payloads_on_this_multi_frame)if show_visualizer_gui and charuco3d_multi_frame_payload is not None:self._visualizer_gui.update_charuco_3d_dottos(charuco3d_multi_frame_payload)dictionary_of_charuco_frame_payloads_on_this_multi_frame = {}+ if detect_mediapipe:+ if len(dictionary_of_mediapipe_payloads_on_this_multi_frame) > 0:+ mediapipe3d_multi_frame_payload = self._reconstruct_3d_mediapipe(+ dictionary_of_mediapipe_payloads_on_this_multi_frame)+ if show_visualizer_gui and mediapipe3d_multi_frame_payload is not None:+ self._visualizer_gui.update_medaiapipe3d_skel(mediapipe3d_multi_frame_payload)+ dictionary_of_mediapipe_payloads_on_this_multi_frame = {}timestamp_manager.log_new_timestamp_for_main_loop_ns(time.perf_counter_ns())@@ -226,6 +235,17 @@ class SessionPipelineOrchestrator:dictionary_of_charuco_frame_payloads_on_this_multi_frame[this_webcam_id] = this_charuco_frame_payload+ if detect_mediapipe:+ # detect mediapipe skeleton+ this_mediapipe_frame_payload = self._mediapipe_skeleton_detector.detect_skeleton_in_image(+ this_cam_latest_frame,+ annotated_image=image_to_display)+ image_to_display = this_mediapipe_frame_payload.annotated_image++ if reconstruct_3d:+ dictionary_of_mediapipe_payloads_on_this_multi_frame[+ this_webcam_id] = this_mediapipe_frame_payload+if show_camera_views_in_windows:should_continue = show_cam_window(this_webcam_id, image_to_display, timestamp_manager@@ -290,18 +310,17 @@ class SessionPipelineOrchestrator:min_cameras_to_reconstruct = 2if sum(is_there_any_charuco_data) >= min_cameras_to_reconstruct: # if at least 2 cameras have any data, then we can try to make some 3d dottos- charuco2d_data_per_cam_dict = self.format_charuco2d_data(dictionary_of_charuco_payloads_on_this_multiframe)+ charuco2d_data_per_cam_dict = self._format_charuco2d_data(dictionary_of_charuco_payloads_on_this_multiframe)charuco_3d_data_payload = self._triangulate_2d_data(charuco2d_data_per_cam_dict,self._charuco_board_detector.number_of_charuco_corners)x = charuco_3d_data_payload.data3d_trackedPointNum_xyz[:, 0]return charuco_3d_data_payload- def format_charuco2d_data(self, dictionary_of_charuco_payloads_on_this_multiframe) -> Dict:+ def _format_charuco2d_data(self, dictionary_of_charuco_payloads_on_this_multiframe) -> Dict:number_of_tracked_points = self._charuco_board_detector.number_of_charuco_corners-base_charuco_data_npy_with_nans_for_missing_data_xy = np.zeros((number_of_tracked_points, 2))base_charuco_data_npy_with_nans_for_missing_data_xy[:] = np.nan@@ -320,6 +339,25 @@ class SessionPipelineOrchestrator:return charuco2d_data_per_cam_dict+ def _reconstruct_3d_mediapipe(self, dictionary_of_mediapipe_payloads_on_this_multi_frame) -> \+ Union[None, Data3dMultiFramePayload]:++ return++ is_there_any_mediapipe_data = []+ for this_cam_medipipe_data in dictionary_of_mediapipe_payloads_on_this_multi_frame.values():+ is_there_any_mediapipe_data.append(+ this_cam_medipipe_data.pixel_data_numpy_arrays.all_data2d_nFrames_nTrackedPts_XY is not None)++ min_cameras_to_reconstruct = 2+ if sum(is_there_any_mediapipe_data) >= min_cameras_to_reconstruct: # if at least 2 cameras have any data, then we can try to make some 3d dottos+ mediapipe2d_data_per_cam_dict = self._format_charuco2d_data(+ dictionary_of_mediapipe_payloads_on_this_multi_frame)+ mediapipe_3d_data_payload = self._triangulate_2d_data(mediapipe2d_data_per_cam_dict,+ self._mediapipe_skeleton_detector.number_of_tracked_points_total)++ return mediapipe_3d_data_payload+def _triangulate_2d_data(self, data2d_per_cam_dict: Dict,number_of_tracked_points: int) -> Data3dMultiFramePayload:diff --git a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.pyindex d7fee482..62fb0381 100644--- a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py+++ b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py@@ -201,6 +201,15 @@ class QTVisualizerAndGui:self.opengl_3d_plot_dock.addWidget(self.opengl_3d_plot_widget)self._main_dock_area.addDock(self.opengl_3d_plot_dock, 'bottom', self._camera_views_dock)+ def initialize_charuco_dottos(self, number_of_charuco_corners: int):+ dummy_charuco_points = np.zeros((number_of_charuco_corners, 3))+ self._charuco_scatter_item = gl.GLScatterPlotItem(+ pos=dummy_charuco_points,+ color=(1, 1, 0, 1),+ size=20+ )+ self.opengl_3d_plot_widget.addItem(self._charuco_scatter_item)+def update_charuco_3d_dottos(self, charuco_frame_payload: Data3dMultiFramePayload):# if not self._camera_set:# mean_xyz = np.nanmean(charuco_frame_payload.data3d_trackedPointNum_xyz,axis=0)@@ -211,14 +220,9 @@ class QTVisualizerAndGui:pos=charuco_frame_payload.data3d_trackedPointNum_xyz)- def initialize_charuco_dottos(self, number_of_charuco_corners: int):- dummy_charuco_points = np.zeros((number_of_charuco_corners, 3))- self._charuco_scatter_item = gl.GLScatterPlotItem(- pos=dummy_charuco_points,- color=(1, 1, 0, 1),- size=20- )- self.opengl_3d_plot_widget.addItem(self._charuco_scatter_item)++ def update_medaiapipe3d_skel(self, mediapipe3d_multi_frame_payload):+ passif __name__ == "__main__":