Free Motion Capture for Everyone πβ¨
TypeScript
say hello to mr skreleton (still pretty borked tho lol)
Jon2b7d9721de02
diff --git a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.pyindex 7283dcda..5267d594 100644--- a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py+++ b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py@@ -40,10 +40,10 @@ class Mediapipe2dNumpyArrays:# if there's no body data, there's no hand or face data eitherreturn- return np.squeeze(np.hstack([self.body2d_frameNumber_trackedPointNumber_XY,- self.rightHand2d_frameNumber_trackedPointNumber_XY,- self.leftHand2d_frameNumber_trackedPointNumber_XY,- self.face2d_frameNumber_trackedPointNumber_XY]))+ return np.vstack([self.body2d_frameNumber_trackedPointNumber_XY,+ self.rightHand2d_frameNumber_trackedPointNumber_XY,+ self.leftHand2d_frameNumber_trackedPointNumber_XY,+ self.face2d_frameNumber_trackedPointNumber_XY])@dataclass@@ -104,7 +104,16 @@ class MediaPipeSkeletonDetector:annotated_image = self._annotate_image(annotated_image, mediapipe_results)- mediapipe_single_frame_npy_data = self._list_of_mediapipe_results_to_npy_arrays([mediapipe_results])+ mediapipe_single_frame_npy_data = self._list_of_mediapipe_results_to_npy_arrays([mediapipe_results],+ image_width=+ annotated_image.shape[0],+ image_height=+ annotated_image.shape[1])+ mediapipe_single_frame_npy_data.body2d_frameNumber_trackedPointNumber_XY = self._threshold_by_confidence(+ mediapipe_single_frame_npy_data.body2d_frameNumber_trackedPointNumber_XY,+ mediapipe_single_frame_npy_data.body2d_frameNumber_trackedPointNumber_confidence,+ confidence_threshold=.5)+return Mediapipe2dDataPayload(raw_frame_payload=raw_frame_payload,mediapipe_results=mediapipe_results,annotated_image=annotated_image,@@ -239,8 +248,8 @@ class MediaPipeSkeletonDetector:def _list_of_mediapipe_results_to_npy_arrays(self,mediapipe_results_list: List,- image_width: int = 1,- image_height: int = 1+ image_width: Union[int, float],+ image_height: Union[int, float],) -> Mediapipe2dNumpyArrays:# apparently `mediapipe_results.pose_landmarks.landmark` returns something iterable Β―\_(γ)_/Β―@@ -350,8 +359,18 @@ class MediaPipeSkeletonDetector:all_tracked_points_visible_on_this_frame_list.append(all_points_visible)return Mediapipe2dNumpyArrays(- body2d_frameNumber_trackedPointNumber_XY=body2d_frameNumber_trackedPointNumber_XY,- rightHand2d_frameNumber_trackedPointNumber_XY=rightHand2d_frameNumber_trackedPointNumber_XY,- leftHand2d_frameNumber_trackedPointNumber_XY=leftHand2d_frameNumber_trackedPointNumber_XY,- face2d_frameNumber_trackedPointNumber_XY=face2d_frameNumber_trackedPointNumber_XY,- body2d_frameNumber_trackedPointNumber_confidence=body2d_frameNumber_trackedPointNumber_confidence)+ body2d_frameNumber_trackedPointNumber_XY=np.squeeze(body2d_frameNumber_trackedPointNumber_XY),+ rightHand2d_frameNumber_trackedPointNumber_XY=np.squeeze(rightHand2d_frameNumber_trackedPointNumber_XY),+ leftHand2d_frameNumber_trackedPointNumber_XY=np.squeeze(leftHand2d_frameNumber_trackedPointNumber_XY),+ face2d_frameNumber_trackedPointNumber_XY=np.squeeze(face2d_frameNumber_trackedPointNumber_XY),+ body2d_frameNumber_trackedPointNumber_confidence=np.squeeze(+ body2d_frameNumber_trackedPointNumber_confidence))++ def _threshold_by_confidence(self,+ data2d_trackedPoint_dim: np.ndarray,+ data2d_trackedPoint_confidence: np.ndarray,+ confidence_threshold: float):++ threshold_mask = data2d_trackedPoint_confidence < confidence_threshold+ data2d_trackedPoint_dim[threshold_mask, :] = np.nan+ return data2d_trackedPoint_dimdiff --git a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py b/src/pipelines/session_pipeline/session_pipeline_orchestrator.pyindex d20a0226..0fd53966 100644--- a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py+++ b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py@@ -15,7 +15,8 @@ from src.config.home_dir import create_session_id, get_session_folder_path, get_get_most_recent_session_id, create_session_folderfrom src.core_processor.camera_calibration.camera_calibrator import CameraCalibratorfrom src.core_processor.timestamp_manager.timestamp_manager import TimestampManager-from src.core_processor.mediapipe_skeleton_detector.mediapipe_skeleton_detector import MediaPipeSkeletonDetector+from src.core_processor.mediapipe_skeleton_detector.mediapipe_skeleton_detector import MediaPipeSkeletonDetector, \+ Mediapipe2dDataPayloadfrom src.core_processor.show_cam_window import show_cam_windowfrom src.core_processor.utils.image_fps_writer import write_fps_to_imagefrom src.debug_plot_makers.simple_plot_3d_points import simple_plot_3d@@ -172,21 +173,22 @@ class SessionPipelineOrchestrator:should_continue = Truewhile should_continue: # BIG FRAME LOOP STARTS HERE- if detect_charuco:- if len(dictionary_of_charuco_frame_payloads_on_this_multi_frame) > 0:- charuco3d_multi_frame_payload = self._reconstruct_3d_charuco(- dictionary_of_charuco_frame_payloads_on_this_multi_frame)- if show_visualizer_gui and charuco3d_multi_frame_payload is not None:- self._visualizer_gui.update_charuco_3d_dottos(charuco3d_multi_frame_payload)- dictionary_of_charuco_frame_payloads_on_this_multi_frame = {}-- if detect_mediapipe:- if len(dictionary_of_mediapipe_payloads_on_this_multi_frame) > 0:- mediapipe3d_multi_frame_payload = self._reconstruct_3d_mediapipe(- dictionary_of_mediapipe_payloads_on_this_multi_frame)- if show_visualizer_gui and mediapipe3d_multi_frame_payload is not None:- self._visualizer_gui.update_medaiapipe3d_skeleton(mediapipe3d_multi_frame_payload)- dictionary_of_mediapipe_payloads_on_this_multi_frame = {}+ if reconstruct_3d:+ if detect_charuco:+ if len(dictionary_of_charuco_frame_payloads_on_this_multi_frame) > 0:+ charuco3d_multi_frame_payload = self._reconstruct_3d_charuco(+ dictionary_of_charuco_frame_payloads_on_this_multi_frame)+ if show_visualizer_gui and charuco3d_multi_frame_payload is not None:+ self._visualizer_gui.update_charuco_3d_dottos(charuco3d_multi_frame_payload)+ dictionary_of_charuco_frame_payloads_on_this_multi_frame = {}++ if detect_mediapipe:+ if len(dictionary_of_mediapipe_payloads_on_this_multi_frame) > 0:+ mediapipe3d_multi_frame_payload = self._reconstruct_3d_mediapipe(+ dictionary_of_mediapipe_payloads_on_this_multi_frame)+ if show_visualizer_gui and mediapipe3d_multi_frame_payload is not None:+ self._visualizer_gui.update_medaiapipe3d_skeleton(mediapipe3d_multi_frame_payload)+ dictionary_of_mediapipe_payloads_on_this_multi_frame = {}timestamp_manager.log_new_timestamp_for_main_loop_ns(time.perf_counter_ns())@@ -350,6 +352,9 @@ class SessionPipelineOrchestrator:if sum(is_there_any_mediapipe_data) >= min_cameras_to_reconstruct: # if at least 2 cameras have any data, then we can try to make some 3d dottosmediapipe2d_data_per_cam_dict = {}for this_webcam_id, this_mediapipe_payload in dictionary_of_mediapipe_payloads_on_this_multi_frame.items():++ self._threshold_2d_data_by_confidence(this_mediapipe_payload.pixel_data_numpy_arrays, .5)+mediapipe2d_data_per_cam_dict[this_webcam_id] = this_mediapipe_payload.pixel_data_numpy_arrays.all_data2d_nFrames_nTrackedPts_XYmediapipe_3d_data_payload = self._triangulate_2d_data(mediapipe2d_data_per_cam_dict,@@ -441,6 +446,9 @@ class SessionPipelineOrchestrator:logger.info(f"saving: {self._mediapipe_reprojection_error_save_path}")np.save(str(self._mediapipe_reprojection_error_save_path), data3d_numFrames_numTrackedPoints_reprojectionError)+ def _threshold_2d_data_by_confidence(self, mediapipe2d_data_payload:Mediapipe2dDataPayload, confidence_threshold:float):+ pass+def load_mediapipe3d_skeleton_data(session_id: str = None):if session_id is None: