Free Motion Capture for Everyone πβ¨
TypeScript
Trent/single camera 2 (#342)
trentwirth1bc2a2c635b5
diff --git a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/mediapipe_skeleton_detector.py b/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/mediapipe_skeleton_detector.pyindex 92818239..0f65cb70 100644--- a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/mediapipe_skeleton_detector.py+++ b/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/mediapipe_skeleton_detector.py@@ -1,6 +1,6 @@import logging-import multiprocessingfrom dataclasses import dataclass+import multiprocessingfrom pathlib import Pathfrom typing import Any, List, Union@@ -11,52 +11,54 @@ from skellycam.detection.models.frame_payload import FramePayloadfrom skellycam.opencv.video_recorder.video_recorder import VideoRecorderfrom tqdm import tqdm+from freemocap.system.paths_and_files_names import MEDIAPIPE_2D_NPY_FILE_NAME, ANNOTATED_VIDEOS_FOLDER_NAME, \+ MEDIAPIPE_BODY_WORLD_FILE_NAMEfrom freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.mediapipe_skeleton_names_and_connections import (mediapipe_tracked_point_names_dict,)from freemocap.parameter_info_models.recording_processing_parameter_models import MediapipeParametersModel-from freemocap.system.paths_and_files_names import MEDIAPIPE_2D_NPY_FILE_NAME, ANNOTATED_VIDEOS_FOLDER_NAMElogger = logging.getLogger(__name__)@dataclassclass Mediapipe2dNumpyArrays:- body2d_frameNumber_trackedPointNumber_XY: np.ndarray = None- rightHand2d_frameNumber_trackedPointNumber_XY: np.ndarray = None- leftHand2d_frameNumber_trackedPointNumber_XY: np.ndarray = None- face2d_frameNumber_trackedPointNumber_XY: np.ndarray = None+ body_frameNumber_trackedPointNumber_XYZ: np.ndarray = None+ body_world_frameNumber_trackedPointNumber_XYZ: np.ndarray = None+ rightHand_frameNumber_trackedPointNumber_XYZ: np.ndarray = None+ leftHand_frameNumber_trackedPointNumber_XYZ: np.ndarray = None+ face_frameNumber_trackedPointNumber_XYZ: np.ndarray = None- body2d_frameNumber_trackedPointNumber_confidence: np.ndarray = None+ body_frameNumber_trackedPointNumber_confidence: np.ndarray = None@propertydef has_data(self):- return not np.isnan(self.body2d_frameNumber_trackedPointNumber_XY).all()+ return not np.isnan(self.body_frameNumber_trackedPointNumber_XYZ).all()@propertydef all_data2d_nFrames_nTrackedPts_XY(self):"""dimensions will be [number_of_frames , number_of_markers, XY]"""- if self.body2d_frameNumber_trackedPointNumber_XY is None:+ if self.body_frameNumber_trackedPointNumber_XYZ is None:# if there's no body data, there's no hand or face data eitherreturn- if len(self.body2d_frameNumber_trackedPointNumber_XY.shape) == 3: # multiple frames+ if len(self.body_frameNumber_trackedPointNumber_XYZ.shape) == 3: # multiple framesreturn np.hstack([- self.body2d_frameNumber_trackedPointNumber_XY,- self.rightHand2d_frameNumber_trackedPointNumber_XY,- self.leftHand2d_frameNumber_trackedPointNumber_XY,- self.face2d_frameNumber_trackedPointNumber_XY,+ self.body_frameNumber_trackedPointNumber_XYZ,+ self.rightHand_frameNumber_trackedPointNumber_XYZ,+ self.leftHand_frameNumber_trackedPointNumber_XYZ,+ self.face_frameNumber_trackedPointNumber_XYZ,])- elif len(self.body2d_frameNumber_trackedPointNumber_XY.shape) == 2: # single frame+ elif len(self.body_frameNumber_trackedPointNumber_XYZ.shape) == 2: # single framereturn np.vstack([- self.body2d_frameNumber_trackedPointNumber_XY,- self.rightHand2d_frameNumber_trackedPointNumber_XY,- self.leftHand2d_frameNumber_trackedPointNumber_XY,- self.face2d_frameNumber_trackedPointNumber_XY,+ self.body_frameNumber_trackedPointNumber_XYZ,+ self.rightHand_frameNumber_trackedPointNumber_XYZ,+ self.leftHand_frameNumber_trackedPointNumber_XYZ,+ self.face_frameNumber_trackedPointNumber_XYZ,])else:@@ -133,9 +135,9 @@ class MediaPipeSkeletonDetector:image_width=annotated_image.shape[0],image_height=annotated_image.shape[1],)- mediapipe_single_frame_npy_data.body2d_frameNumber_trackedPointNumber_XY = self._threshold_by_confidence(- mediapipe_single_frame_npy_data.body2d_frameNumber_trackedPointNumber_XY,- mediapipe_single_frame_npy_data.body2d_frameNumber_trackedPointNumber_confidence,+ mediapipe_single_frame_npy_data.body_frameNumber_trackedPointNumber_XYZ = self._threshold_by_confidence(+ mediapipe_single_frame_npy_data.body_frameNumber_trackedPointNumber_XYZ,+ mediapipe_single_frame_npy_data.body_frameNumber_trackedPointNumber_confidence,confidence_threshold=0.5,)@@ -223,14 +225,28 @@ class MediaPipeSkeletonDetector:m2d.all_data2d_nFrames_nTrackedPts_XY for m2d in mediapipe2d_single_camera_npy_arrays_list]+ all_cameras_pose_world_data_list = [+ m2d.body_world_frameNumber_trackedPointNumber_XYZ for m2d in mediapipe2d_single_camera_npy_arrays_list+ ]++ all_cameras_right_hand_world_data_list = [+ m2d.rightHand_frameNumber_trackedPointNumber_XYZ for m2d in mediapipe2d_single_camera_npy_arrays_list+ ]++ all_cameras_left_hand_world_data_list = [+ m2d.leftHand_frameNumber_trackedPointNumber_XYZ for m2d in mediapipe2d_single_camera_npy_arrays_list+ ]++ all_cameras_face_world_data_list = [+ m2d.face_frameNumber_trackedPointNumber_XYZ for m2d in mediapipe2d_single_camera_npy_arrays_list+ ]+number_of_cameras = len(all_cameras_data2d_list)number_of_frames = all_cameras_data2d_list[0].shape[0]number_of_tracked_points = all_cameras_data2d_list[0].shape[1]- number_of_spatial_dimensions = all_cameras_data2d_list[0].shape[2] # XY, 2d data+ number_of_spatial_dimensions = all_cameras_data2d_list[0].shape[2]- if not number_of_spatial_dimensions == 2:- logger.error(f"this should be 2D data (XY pixel coordinates), but we founds {number_of_spatial_dimensions}")- raise Exception+ number_of_body_points = all_cameras_pose_world_data_list[0].shape[1]data2d_numCams_numFrames_numTrackedPts_XY = np.empty((@@ -241,11 +257,37 @@ class MediaPipeSkeletonDetector:))+ body_world_numCams_numFrames_numTrackedPts_XYZ = np.empty(+ (+ number_of_cameras,+ number_of_frames,+ number_of_tracked_points,+ number_of_spatial_dimensions,+ )+ )+for cam_num in range(number_of_cameras):data2d_numCams_numFrames_numTrackedPts_XY[cam_num, :, :, :] = all_cameras_data2d_list[cam_num]- mediapipe_data_2d_npy_path = self._save_mediapipe2d_data_to_npy(+ pose_3d = all_cameras_pose_world_data_list[cam_num]+ right_hand_3d = all_cameras_right_hand_world_data_list[cam_num]+ left_hand_3d = all_cameras_left_hand_world_data_list[cam_num]+ face_3d = all_cameras_face_world_data_list[cam_num]++ body_world_numCams_numFrames_numTrackedPts_XYZ[cam_num, :, :, :] = np.concatenate((+ pose_3d,+ right_hand_3d,+ left_hand_3d,+ face_3d),+ axis=1)++ logger.info(+ f"The shape of body_world_numCams_numFrames_numTrackedPts_XYZ is "+ f"{body_world_numCams_numFrames_numTrackedPts_XYZ.shape}")++ self._save_mediapipe2d_data_to_npy(data2d_numCams_numFrames_numTrackedPts_XY=data2d_numCams_numFrames_numTrackedPts_XY,+ body_world_numCams_numFrames_numTrackedPts_XYZ=body_world_numCams_numFrames_numTrackedPts_XYZ,output_data_folder_path=Path(output_data_folder_path),)return data2d_numCams_numFrames_numTrackedPts_XY@@ -253,14 +295,18 @@ class MediaPipeSkeletonDetector:def _save_mediapipe2d_data_to_npy(self,data2d_numCams_numFrames_numTrackedPts_XY: np.ndarray,+ body_world_numCams_numFrames_numTrackedPts_XYZ: np.ndarray,output_data_folder_path: Union[str, Path],):mediapipe_2dData_save_path = Path(output_data_folder_path) / MEDIAPIPE_2D_NPY_FILE_NAMEmediapipe_2dData_save_path.parent.mkdir(exist_ok=True, parents=True)- logger.info(f"saving: {mediapipe_2dData_save_path}")+ logger.info(f"saving mediapipe image npy file: {mediapipe_2dData_save_path}")np.save(str(mediapipe_2dData_save_path), data2d_numCams_numFrames_numTrackedPts_XY)- return mediapipe_2dData_save_path+ mediapipe_body_world_save_path = Path(output_data_folder_path) / MEDIAPIPE_BODY_WORLD_FILE_NAME+ mediapipe_body_world_save_path.parent.mkdir(exist_ok=True, parents=True)+ logger.info(f"saving mediapipe body world npy xyz: {mediapipe_body_world_save_path}")+ np.save(str(mediapipe_body_world_save_path), body_world_numCams_numFrames_numTrackedPts_XYZ)def _annotate_image(self, image, mediapipe_results):self._mp_drawing.draw_landmarks(@@ -308,48 +354,57 @@ class MediaPipeSkeletonDetector:) -> Mediapipe2dNumpyArrays:number_of_frames = len(mediapipe_results_list)- number_of_spatial_dimensions = 2 # this will be 2d XY pixel data+ number_of_spatial_dimensions = 3 # this will be 2d XY pixel data, with mediapipe's estimate of Z++ body_frameNumber_trackedPointNumber_XYZ = np.zeros(+ (+ number_of_frames,+ self.number_of_body_tracked_points,+ number_of_spatial_dimensions,+ )+ )+ body_frameNumber_trackedPointNumber_XYZ[:] = np.nan- body2d_frameNumber_trackedPointNumber_XY = np.zeros(+ body_world_frameNumber_trackedPointNumber_XYZ = np.zeros((number_of_frames,self.number_of_body_tracked_points,number_of_spatial_dimensions,))- body2d_frameNumber_trackedPointNumber_XY[:] = np.nan+ body_world_frameNumber_trackedPointNumber_XYZ[:] = np.nan- body2d_frameNumber_trackedPointNumber_confidence = np.zeros(+ body_frameNumber_trackedPointNumber_confidence = np.zeros((number_of_frames, self.number_of_body_tracked_points))- body2d_frameNumber_trackedPointNumber_confidence[:] = np.nan # only body markers get a 'confidence' value+ body_frameNumber_trackedPointNumber_confidence[:] = np.nan # only body markers get a 'confidence' value- rightHand2d_frameNumber_trackedPointNumber_XY = np.zeros(+ rightHand_frameNumber_trackedPointNumber_XYZ = np.zeros((number_of_frames,self.number_of_right_hand_tracked_points,number_of_spatial_dimensions,))- rightHand2d_frameNumber_trackedPointNumber_XY[:] = np.nan+ rightHand_frameNumber_trackedPointNumber_XYZ[:] = np.nan- leftHand2d_frameNumber_trackedPointNumber_XY = np.zeros(+ leftHand_frameNumber_trackedPointNumber_XYZ = np.zeros((number_of_frames,self.number_of_left_hand_tracked_points,number_of_spatial_dimensions,))- leftHand2d_frameNumber_trackedPointNumber_XY[:] = np.nan+ leftHand_frameNumber_trackedPointNumber_XYZ[:] = np.nan- face2d_frameNumber_trackedPointNumber_XY = np.zeros(+ face_frameNumber_trackedPointNumber_XYZ = np.zeros((number_of_frames,self.number_of_face_tracked_points,number_of_spatial_dimensions,))- face2d_frameNumber_trackedPointNumber_XY[:] = np.nan+ face_frameNumber_trackedPointNumber_XYZ[:] = np.nanall_body_tracked_points_visible_on_frame_bool_list = []all_right_hand_points_visible_on_frame_bool_list = []@@ -363,61 +418,84 @@ class MediaPipeSkeletonDetector:if frame_results.pose_landmarks is not None:for landmark_number, landmark_data in enumerate(frame_results.pose_landmarks.landmark):- body2d_frameNumber_trackedPointNumber_XY[frame_number, landmark_number, 0] = (+ body_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (landmark_data.x * image_width)- body2d_frameNumber_trackedPointNumber_XY[frame_number, landmark_number, 1] = (+ body_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (landmark_data.y * image_height)- body2d_frameNumber_trackedPointNumber_confidence[+ body_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (+ landmark_data.z * image_width # z is on roughly the same scale as x, according to mediapipe docs+ )+ body_frameNumber_trackedPointNumber_confidence[frame_number, landmark_number] = landmark_data.visibility # mediapipe calls their 'confidence' score 'visibility'+ for landmark_number, landmark_data in enumerate(frame_results.pose_world_landmarks.landmark):+ body_world_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (+ landmark_data.x * image_width+ )+ body_world_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (+ landmark_data.y * image_height+ )+ body_world_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (+ landmark_data.z * image_width+ )+# get Right Hand dataif frame_results.right_hand_landmarks is not None:for landmark_number, landmark_data in enumerate(frame_results.right_hand_landmarks.landmark):- rightHand2d_frameNumber_trackedPointNumber_XY[frame_number, landmark_number, 0] = (+ rightHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (landmark_data.x * image_width)- rightHand2d_frameNumber_trackedPointNumber_XY[frame_number, landmark_number, 1] = (+ rightHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (landmark_data.y * image_height)+ rightHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (+ landmark_data.z * image_width+ )# get Left Hand dataif frame_results.left_hand_landmarks is not None:for landmark_number, landmark_data in enumerate(frame_results.left_hand_landmarks.landmark):- leftHand2d_frameNumber_trackedPointNumber_XY[frame_number, landmark_number, 0] = (+ leftHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (landmark_data.x * image_width)- leftHand2d_frameNumber_trackedPointNumber_XY[frame_number, landmark_number, 1] = (+ leftHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (landmark_data.y * image_height)+ leftHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (+ landmark_data.z * image_width+ )# get Face dataif frame_results.face_landmarks is not None:for landmark_number, landmark_data in enumerate(frame_results.face_landmarks.landmark):- face2d_frameNumber_trackedPointNumber_XY[frame_number, landmark_number, 0] = (+ face_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (landmark_data.x * image_width)- face2d_frameNumber_trackedPointNumber_XY[frame_number, landmark_number, 1] = (+ face_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (landmark_data.y * image_height)+ face_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (+ landmark_data.z * image_width+ )# check if all tracked points are visible on this frame- all_body_visible = all(sum(np.isnan(body2d_frameNumber_trackedPointNumber_XY[frame_number, :, :])) == 0)+ all_body_visible = all(sum(np.isnan(body_frameNumber_trackedPointNumber_XYZ[frame_number, :, :])) == 0)all_body_tracked_points_visible_on_frame_bool_list.append(all_body_visible)all_right_hand_visible = all(- sum(np.isnan(rightHand2d_frameNumber_trackedPointNumber_XY[frame_number, :, :])) == 0+ sum(np.isnan(rightHand_frameNumber_trackedPointNumber_XYZ[frame_number, :, :])) == 0)all_right_hand_points_visible_on_frame_bool_list.append(all_right_hand_visible)all_left_hand_visible = all(- sum(np.isnan(leftHand2d_frameNumber_trackedPointNumber_XY[frame_number, :, :])) == 0+ sum(np.isnan(leftHand_frameNumber_trackedPointNumber_XYZ[frame_number, :, :])) == 0)all_left_hand_points_visible_on_frame_bool_list.append(all_left_hand_visible)- all_face_visible = all(sum(np.isnan(face2d_frameNumber_trackedPointNumber_XY[frame_number, :, :])) == 0)+ all_face_visible = all(sum(np.isnan(face_frameNumber_trackedPointNumber_XYZ[frame_number, :, :])) == 0)all_face_points_visible_on_frame_bool_list.append(all_face_visible)all_points_visible = all(@@ -432,12 +510,13 @@ class MediaPipeSkeletonDetector:all_tracked_points_visible_on_frame_list.append(all_points_visible)return Mediapipe2dNumpyArrays(- body2d_frameNumber_trackedPointNumber_XY=np.squeeze(body2d_frameNumber_trackedPointNumber_XY),- rightHand2d_frameNumber_trackedPointNumber_XY=np.squeeze(rightHand2d_frameNumber_trackedPointNumber_XY),- leftHand2d_frameNumber_trackedPointNumber_XY=np.squeeze(leftHand2d_frameNumber_trackedPointNumber_XY),- face2d_frameNumber_trackedPointNumber_XY=np.squeeze(face2d_frameNumber_trackedPointNumber_XY),- body2d_frameNumber_trackedPointNumber_confidence=np.squeeze(- body2d_frameNumber_trackedPointNumber_confidence+ body_frameNumber_trackedPointNumber_XYZ=np.squeeze(body_frameNumber_trackedPointNumber_XYZ),+ body_world_frameNumber_trackedPointNumber_XYZ=np.squeeze(body_world_frameNumber_trackedPointNumber_XYZ),+ rightHand_frameNumber_trackedPointNumber_XYZ=np.squeeze(rightHand_frameNumber_trackedPointNumber_XYZ),+ leftHand_frameNumber_trackedPointNumber_XYZ=np.squeeze(leftHand_frameNumber_trackedPointNumber_XYZ),+ face_frameNumber_trackedPointNumber_XYZ=np.squeeze(face_frameNumber_trackedPointNumber_XYZ),+ body_frameNumber_trackedPointNumber_confidence=np.squeeze(+ body_frameNumber_trackedPointNumber_confidence),)diff --git a/freemocap/core_processes/post_process_skeleton_data/process_single_camera_skeleton_data.py b/freemocap/core_processes/post_process_skeleton_data/process_single_camera_skeleton_data.pynew file mode 100644index 00000000..265ce2fa--- /dev/null+++ b/freemocap/core_processes/post_process_skeleton_data/process_single_camera_skeleton_data.py@@ -0,0 +1,33 @@+from pathlib import Path+import logging+from typing import Union++import numpy as np++from freemocap.core_processes.capture_volume_calibration.triangulate_3d_data import save_mediapipe_3d_data_to_npy+from freemocap.utilities.project_3d_data_to_z_plane import project_3d_data_to_z_plane++logger = logging.getLogger(__name__)+++def process_single_camera_skeleton_data(input_image_data_frame_marker_xyz: np.ndarray,+ raw_data_folder_path: Union[str, Path],+ project_to_z_plane: bool = True) -> [np.ndarray, np.ndarray]:++ if project_to_z_plane:++ logger.info("Single camera detected - "+ "Altering image-derived 3d data to resemble multi-camera reconstructed data.")++ skeleton_reprojection_error_fr_mar = np.zeros(input_image_data_frame_marker_xyz.shape[0:2])++ raw_skel3d_frame_marker_xyz = project_3d_data_to_z_plane(+ skel3d_frame_marker_xyz=input_image_data_frame_marker_xyz)++ save_mediapipe_3d_data_to_npy(+ data3d_numFrames_numTrackedPoints_XYZ=raw_skel3d_frame_marker_xyz,+ data3d_numFrames_numTrackedPoints_reprojectionError=skeleton_reprojection_error_fr_mar,+ path_to_folder_where_data_will_be_saved=raw_data_folder_path,+ )++ return raw_skel3d_frame_marker_xyz, skeleton_reprojection_error_fr_mardiff --git a/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py b/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.pyindex dae8634a..03ada531 100644--- a/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py+++ b/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py@@ -4,6 +4,15 @@ from pathlib import Pathimport pandas as pd+from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.mediapipe_skeleton_names_and_connections import (+ mediapipe_names_and_connections_dict,+)+from freemocap.core_processes.post_process_skeleton_data.process_single_camera_skeleton_data import \+ process_single_camera_skeleton_data+from freemocap.system.paths_and_files_names import (+ MEDIAPIPE_BODY_3D_DATAFRAME_CSV_FILE_NAME,+ RAW_DATA_FOLDER_NAME,+)from freemocap.core_processes.capture_volume_calibration.anipose_camera_calibration.get_anipose_calibration_object import (load_anipose_calibration_toml_from_path,)@@ -16,9 +25,6 @@ from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.convfrom freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.mediapipe_skeleton_detector import (MediaPipeSkeletonDetector,)-from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.mediapipe_skeleton_names_and_connections import (- mediapipe_names_and_connections_dict,-)from freemocap.core_processes.post_process_skeleton_data.estimate_skeleton_segment_lengths import (estimate_skeleton_segment_lengths,mediapipe_skeleton_segment_definitions,@@ -27,14 +33,12 @@ from freemocap.core_processes.post_process_skeleton_data.gap_fill_filter_and_origap_fill_filter_origin_align_3d_data_and_then_calculate_center_of_mass,)from freemocap.parameter_info_models.recording_processing_parameter_models import RecordingProcessingParameterModel-from freemocap.system.paths_and_files_names import (- MEDIAPIPE_BODY_3D_DATAFRAME_CSV_FILE_NAME,- RAW_DATA_FOLDER_NAME,-)+from freemocap.tests.test_mediapipe_2d_data_shape import (test_mediapipe_2d_data_shape,)from freemocap.tests.test_mediapipe_3d_data_shape import test_mediapipe_3d_data_shape+from freemocap.utilities.rotate_by_90_degrees_around_x_axis import rotate_by_90_degrees_around_x_axisfrom freemocap.utilities.save_dictionary_to_json import save_dictionary_to_jsonlogger = logging.getLogger(__name__)@@ -66,7 +70,7 @@ def process_recording_folder(parameter_model=rec.mediapipe_parameters_model,)- mediapipe_2d_data = mediapipe_skeleton_detector.process_folder_full_of_videos(+ mediapipe_image_data_numCams_numFrames_numTrackedPts_XYZ = mediapipe_skeleton_detector.process_folder_full_of_videos(rec.recording_info_model.synchronized_videos_folder_path,Path(rec.recording_info_model.output_data_folder_path) / RAW_DATA_FOLDER_NAME,kill_event=kill_event,@@ -80,37 +84,49 @@ def process_recording_folder(mediapipe_2d_data_file_path=rec.recording_info_model.mediapipe_2d_data_npy_file_path,)- logger.info("Triangulating 3d skeletons...")+ # spoof 3D data if single camera+ if mediapipe_image_data_numCams_numFrames_numTrackedPts_XYZ.shape[0] == 1:- assert rec.recording_info_model.calibration_toml_check, f"No calibration file found at: {rec.recording_info_model.calibration_toml_path}"- assert rec.recording_info_model.data2d_status_check, f"No mediapipe 2d data found at: {rec.recording_info_model.mediapipe_2d_data_npy_file_path}"+ (raw_skel3d_frame_marker_xyz, skeleton_reprojection_error_fr_mar) = process_single_camera_skeleton_data(+ input_image_data_frame_marker_xyz=mediapipe_image_data_numCams_numFrames_numTrackedPts_XYZ[0],+ raw_data_folder_path=Path(rec.recording_info_model.raw_data_folder_path))- anipose_calibration_object = load_anipose_calibration_toml_from_path(- camera_calibration_data_toml_path=rec.recording_info_model.calibration_toml_path,- save_copy_of_calibration_to_this_path=rec.recording_info_model.path,- )- (raw_skel3d_frame_marker_xyz, skeleton_reprojection_error_fr_mar,) = triangulate_3d_data(- anipose_calibration_object=anipose_calibration_object,- mediapipe_2d_data=mediapipe_2d_data,- output_data_folder_path=rec.recording_info_model.raw_data_folder_path,- mediapipe_confidence_cutoff_threshold=rec.anipose_triangulate_3d_parameters_model.confidence_threshold_cutoff,- use_triangulate_ransac=rec.anipose_triangulate_3d_parameters_model.use_triangulate_ransac_method,- kill_event=kill_event,- )+ else:- if kill_event is not None and kill_event.is_set():- return+ logger.info("Triangulating 3d skeletons...")- assert test_mediapipe_3d_data_shape(- synchronized_videos_folder=rec.recording_info_model.synchronized_videos_folder_path,- mediapipe_3d_data_npy_path=rec.recording_info_model.raw_mediapipe_3d_data_npy_file_path,- medipipe_reprojection_error_data_npy_path=rec.recording_info_model.mediapipe_reprojection_error_data_npy_file_path,- )+ assert rec.recording_info_model.calibration_toml_check, f"No calibration file found at: {rec.recording_info_model.calibration_toml_path}"+ assert rec.recording_info_model.data2d_status_check, f"No mediapipe 2d data found at: {rec.recording_info_model.mediapipe_2d_data_npy_file_path}"++ anipose_calibration_object = load_anipose_calibration_toml_from_path(+ camera_calibration_data_toml_path=rec.recording_info_model.calibration_toml_path,+ save_copy_of_calibration_to_this_path=rec.recording_info_model.path,+ )+ (raw_skel3d_frame_marker_xyz, skeleton_reprojection_error_fr_mar,) = triangulate_3d_data(+ anipose_calibration_object=anipose_calibration_object,+ mediapipe_2d_data=mediapipe_image_data_numCams_numFrames_numTrackedPts_XYZ,+ output_data_folder_path=rec.recording_info_model.raw_data_folder_path,+ mediapipe_confidence_cutoff_threshold=rec.anipose_triangulate_3d_parameters_model.confidence_threshold_cutoff,+ use_triangulate_ransac=rec.anipose_triangulate_3d_parameters_model.use_triangulate_ransac_method,+ kill_event=kill_event,+ )++ if kill_event is not None and kill_event.is_set():+ return++ assert test_mediapipe_3d_data_shape(+ synchronized_videos_folder=rec.recording_info_model.synchronized_videos_folder_path,+ mediapipe_3d_data_npy_path=rec.recording_info_model.raw_mediapipe_3d_data_npy_file_path,+ medipipe_reprojection_error_data_npy_path=rec.recording_info_model.mediapipe_reprojection_error_data_npy_file_path,+ )++ #rotate so skeleton is closer to 'vertical' in a z-up reference frame+ rotated_raw_skel3d_frame_marker_xyz = rotate_by_90_degrees_around_x_axis(raw_skel3d_frame_marker_xyz)logger.info("Gap-filling, butterworth filtering, origin aligning 3d skeletons, then calculating center of mass ...")skel3d_frame_marker_xyz = gap_fill_filter_origin_align_3d_data_and_then_calculate_center_of_mass(- raw_skel3d_frame_marker_xyz=raw_skel3d_frame_marker_xyz,+ raw_skel3d_frame_marker_xyz=rotated_raw_skel3d_frame_marker_xyz,skeleton_reprojection_error_fr_mar=skeleton_reprojection_error_fr_mar,path_to_folder_where_we_will_save_this_data=rec.recording_info_model.output_data_folder_path,sampling_rate=rec.post_processing_parameters_model.framerate,diff --git a/freemocap/system/paths_and_files_names.py b/freemocap/system/paths_and_files_names.pyindex c8b9a7ca..bf1d10a4 100644--- a/freemocap/system/paths_and_files_names.py+++ b/freemocap/system/paths_and_files_names.py@@ -26,6 +26,7 @@ CENTER_OF_MASS_FOLDER_NAME = "center_of_mass"MOST_RECENT_RECORDING_TOML_FILENAME = "most_recent_recording.toml"LAST_SUCCESSFUL_CALIBRATION_TOML_FILENAME = "last_successful_calibration.toml"MEDIAPIPE_2D_NPY_FILE_NAME = "mediapipe2dData_numCams_numFrames_numTrackedPoints_pixelXY.npy"+MEDIAPIPE_BODY_WORLD_FILE_NAME = "mediapipeBodyWorld_numCams_numFrames_numTrackedPoints_XYZ.npy"MEDIAPIPE_3D_NPY_FILE_NAME = "mediaPipeSkel_3d_body_hands_face.npy"RAW_MEDIAPIPE_3D_NPY_FILE_NAME = "mediapipe3dData_numFrames_numTrackedPoints_spatialXYZ.npy"MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME = "mediapipe3dData_numFrames_numTrackedPoints_reprojectionError.npy"diff --git a/freemocap/tests/test_mediapipe_2d_data_shape.py b/freemocap/tests/test_mediapipe_2d_data_shape.pyindex aff8e3b5..6fd42488 100644--- a/freemocap/tests/test_mediapipe_2d_data_shape.py+++ b/freemocap/tests/test_mediapipe_2d_data_shape.py@@ -43,6 +43,6 @@ def test_mediapipe_2d_data_shape(# TODO - check number of tracked points vs 'expected' number of tracked points- assert mediapipe_2d_data.shape[3] == 2+ assert mediapipe_2d_data.shape[3] == 3 # x y zreturn Truediff --git a/freemocap/utilities/project_3d_data_to_z_plane.py b/freemocap/utilities/project_3d_data_to_z_plane.pynew file mode 100644index 00000000..273f34e8--- /dev/null+++ b/freemocap/utilities/project_3d_data_to_z_plane.py@@ -0,0 +1,12 @@+import numpy as np+++def project_3d_data_to_z_plane(skel3d_frame_marker_xyz: np.ndarray) -> np.ndarray:++ projected_skel3d_frame_marker_xyz = np.zeros(skel3d_frame_marker_xyz.shape)++ projected_skel3d_frame_marker_xyz[:, :, 0] = skel3d_frame_marker_xyz[:, :, 0]+ projected_skel3d_frame_marker_xyz[:, :, 1] = skel3d_frame_marker_xyz[:, :, 1]+ projected_skel3d_frame_marker_xyz[:, :, 2] = 0++ return projected_skel3d_frame_marker_xyzdiff --git a/freemocap/utilities/rotate_by_90_degrees_around_x_axis.py b/freemocap/utilities/rotate_by_90_degrees_around_x_axis.pynew file mode 100644index 00000000..93cac3ea--- /dev/null+++ b/freemocap/utilities/rotate_by_90_degrees_around_x_axis.py@@ -0,0 +1,12 @@+import numpy as np+++def rotate_by_90_degrees_around_x_axis(raw_skel3d_frame_marker_xyz: np.ndarray) -> np.ndarray:++ swapped_skel3d_frame_marker_xyz = np.zeros(raw_skel3d_frame_marker_xyz.shape)++ swapped_skel3d_frame_marker_xyz[:, :, 0] = raw_skel3d_frame_marker_xyz[:, :, 0]+ swapped_skel3d_frame_marker_xyz[:, :, 1] = raw_skel3d_frame_marker_xyz[:, :, 2]+ swapped_skel3d_frame_marker_xyz[:, :, 2] = -raw_skel3d_frame_marker_xyz[:, :, 1]++ return swapped_skel3d_frame_marker_xyz