Free Motion Capture for Everyone πβ¨
TypeScript
Generalize mediapipe code to other trackers (#536)
philipqueenc83c8bd99c7d
diff --git a/.flake8 b/.flake8index b3d0a521..a1263f6d 100644--- a/.flake8+++ b/.flake8@@ -12,12 +12,10 @@ exclude =per-file-ignores =./freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py:C901- ./freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/mediapipe_skeleton_detector.py:C901- ./freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/convert_mediapipe_npy_to_csv.py:E203+ ./freemocap/core_processes/export_data/blender_stuff/export_to_blender/methods/ajc_addon/run_ajc_addon_main.py:C901./freemocap/gui/qt/widgets/control_panel/calibration_control_panel.py:C901./experimental/batch_process/batch_process.py:E226# C901: function is too complex-# E203: whitespace before ':' (PEP8 says this doesn't apply to slicing)# E226: missing whitespace around arithmetic operator (file slashes misread as arithmetic)max-line-length = 88diff --git a/experimental/openpose/run_openpose.py b/experimental/openpose/run_openpose.pynew file mode 100644index 00000000..2d0a5531--- /dev/null+++ b/experimental/openpose/run_openpose.py@@ -0,0 +1,43 @@+from pathlib import Path++def main():+ try:+ from freemocap.data_layer.recording_models.post_processing_parameter_models import ProcessingParameterModel, AniposeTriangulate3DParametersModel, PostProcessingParametersModel+ from freemocap.data_layer.recording_models.recording_info_model import RecordingInfoModel+ from freemocap.core_processes.process_motion_capture_videos.process_recording_folder import process_recording_folder+ from skellytracker.trackers.openpose_tracker.openpose_model_info import OpenPoseModelInfo, OpenPoseTrackingParams+ except ImportError as e:+ if 'ProcessingParameterModel' not in dir() or 'process_folder_of_videos' not in dir() or 'OpenPoseModelInfo' not in dir():+ raise e++++ recording_folder = Path(r'D:\mdn_treadmill_for_testing')+ openpose_root_folder_path = r'C:\openpose'+++ recording_model = RecordingInfoModel(+ recording_folder_path=recording_folder,+ active_tracker="openpose"+ )++ openpose_processing_parameters = ProcessingParameterModel(+ recording_info_model= recording_model,+ tracking_parameters_model=OpenPoseTrackingParams(+ # net_resolution="-1x656",+ openpose_root_folder_path=str(openpose_root_folder_path),+ run_image_tracking=True+ ),+ anipose_triangulate_3d_parameters_model=AniposeTriangulate3DParametersModel(),+ post_processing_parameters_model=PostProcessingParametersModel(),+ tracking_model_info=OpenPoseModelInfo()+ )++ process_recording_folder(+ recording_processing_parameter_model=openpose_processing_parameters,+ kill_event=None,+ logging_queue=None,+ )++if __name__ == "__main__":+ main()diff --git a/experimental/react_fastapi/api/routes/session/session_router.py b/experimental/react_fastapi/api/routes/session/session_router.pyindex fa70caaa..04fb0a98 100644--- a/experimental/react_fastapi/api/routes/session/session_router.py+++ b/experimental/react_fastapi/api/routes/session/session_router.py@@ -6,7 +6,7 @@ from fastapi import APIRouterfrom jon_scratch.pupil_calibration_pipeline.qt_gl_laser_skeleton_visualizer import (QtGlLaserSkeletonVisualizer,)-from pydantic import BaseModel+from pydantic import BaseModel, ConfigDictfrom src.cameras.launch_camera_frame_loop import launch_camera_frame_loopfrom src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManagerfrom src.config.home_dir import (@@ -33,8 +33,7 @@ session_router = APIRouter()class TweakedPydanticBaseModel(BaseModel):- class Config:- arbitrary_types_allowed = True+ model_config = ConfigDict(arbitrary_types_allowed = True)class SessionCreateModel(BaseModel):diff --git a/freemocap/core_processes/capture_volume_calibration/by_camera_reprojection_filtering.py b/freemocap/core_processes/capture_volume_calibration/by_camera_reprojection_filtering.pyindex 72822de5..94b375b1 100644--- a/freemocap/core_processes/capture_volume_calibration/by_camera_reprojection_filtering.py+++ b/freemocap/core_processes/capture_volume_calibration/by_camera_reprojection_filtering.py@@ -5,14 +5,10 @@ from matplotlib import pyplot as pltimport numpy as np-from freemocap.core_processes.capture_volume_calibration.save_mediapipe_3d_data_to_npy import (- save_mediapipe_3d_data_to_npy,+from freemocap.core_processes.capture_volume_calibration.save_3d_data_to_npy import (+ save_3d_data_to_npy,)from freemocap.core_processes.capture_volume_calibration.triangulate_3d_data import triangulate_3d_data--from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.data_models.mediapipe_skeleton_names_and_connections import (- NUMBER_OF_MEDIAPIPE_BODY_MARKERS,-)from freemocap.data_layer.recording_models.post_processing_parameter_models import ProcessingParameterModellogger = logging.getLogger(__name__)@@ -38,6 +34,13 @@ def run_reprojection_error_filtering(:return: filtered 3d data"""+ if hasattr(+ processing_parameters.tracking_model_info, "num_tracked_points_body"+ ): # we don't want to reproject hand and face data+ num_tracked_points = processing_parameters.tracking_model_info.num_tracked_points_body+ else:+ num_tracked_points = processing_parameters.tracking_model_info.num_tracked_points+(reprojection_filtered_skel3d_frame_marker_xyz,reprojection_filtered_skeleton_reprojection_error_fr_mar,@@ -46,19 +49,20 @@ def run_reprojection_error_filtering(reprojection_error_camera_frame_marker=skeleton_reprojection_error_cam_fr_mar,reprojection_error_frame_marker=skeleton_reprojection_error_fr_mar,reprojection_error_confidence_threshold=processing_parameters.anipose_triangulate_3d_parameters_model.reprojection_error_confidence_cutoff,- mediapipe_2d_data=image_data_numCams_numFrames_numTrackedPts_XYZ[:, :, :, :2],+ image_2d_data=image_data_numCams_numFrames_numTrackedPts_XYZ[:, :, :, :2],raw_skel3d_frame_marker_xyz=raw_skel3d_frame_marker_xyz,anipose_calibration_object=anipose_calibration_object,- output_data_folder_path=processing_parameters.recording_info_model.raw_data_folder_path,+ num_tracked_points=num_tracked_points,use_triangulate_ransac=processing_parameters.anipose_triangulate_3d_parameters_model.use_triangulate_ransac_method,minimum_cameras_to_reproject=processing_parameters.anipose_triangulate_3d_parameters_model.minimum_cameras_to_reproject,)- save_mediapipe_3d_data_to_npy(+ save_3d_data_to_npy(data3d_numFrames_numTrackedPoints_XYZ=reprojection_filtered_skel3d_frame_marker_xyz,data3d_numFrames_numTrackedPoints_reprojectionError=reprojection_filtered_skeleton_reprojection_error_fr_mar,data3d_numCams_numFrames_numTrackedPoints_reprojectionError=reprojection_filtered_skeleton_reprojection_error_cam_fr_mar,path_to_folder_where_data_will_be_saved=processing_parameters.recording_info_model.raw_data_folder_path,processing_level="reprojection_filtered",+ file_prefix=processing_parameters.tracking_model_info.name,)plot_reprojection_error(raw_reprojection_error_frame_marker=skeleton_reprojection_error_fr_mar,@@ -76,14 +80,14 @@ def filter_by_reprojection_error(reprojection_error_camera_frame_marker: np.ndarray,reprojection_error_frame_marker: np.ndarray,reprojection_error_confidence_threshold: float,- mediapipe_2d_data: np.ndarray,+ image_2d_data: np.ndarray,raw_skel3d_frame_marker_xyz: np.ndarray,anipose_calibration_object,- output_data_folder_path: Union[str, Path],+ num_tracked_points: int,use_triangulate_ransac: bool = False,minimum_cameras_to_reproject: int = 3,) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:- total_cameras = mediapipe_2d_data.shape[0]+ total_cameras = image_2d_data.shape[0]num_cameras_to_remove = 1if total_cameras <= minimum_cameras_to_reproject:@@ -96,8 +100,8 @@ def filter_by_reprojection_error(reprojection_error_camera_frame_marker,)- body2d_camera_frame_marker_xy = mediapipe_2d_data[:, :, :NUMBER_OF_MEDIAPIPE_BODY_MARKERS, :2]- bodyReprojErr_camera_frame_marker = reprojection_error_camera_frame_marker[:, :, :NUMBER_OF_MEDIAPIPE_BODY_MARKERS]+ body2d_camera_frame_marker_xy = image_2d_data[:, :, :num_tracked_points, :2]+ bodyReprojErr_camera_frame_marker = reprojection_error_camera_frame_marker[:, :, :num_tracked_points]reprojection_error_confidence_threshold = min(max(reprojection_error_confidence_threshold, 0), 100)@@ -121,7 +125,7 @@ def filter_by_reprojection_error(new_reprojError_cam_frame_marker,) = triangulate_3d_data(anipose_calibration_object=anipose_calibration_object,- mediapipe_2d_data=data_to_reproject_camera_frame_marker_xy,+ image_2d_data=data_to_reproject_camera_frame_marker_xy,use_triangulate_ransac=use_triangulate_ransac,)@@ -139,15 +143,13 @@ def filter_by_reprojection_error(# put retriangulated data back in placefiltered_skel3d_frame_marker_xyz = raw_skel3d_frame_marker_xyz.copy()- filtered_skel3d_frame_marker_xyz[:, :NUMBER_OF_MEDIAPIPE_BODY_MARKERS, :] = retriangulated_data_frame_marker_xyz+ filtered_skel3d_frame_marker_xyz[:, :num_tracked_points, :] = retriangulated_data_frame_marker_xyzfiltered_reprojection_error_frame_marker = reprojection_error_frame_marker.copy()- filtered_reprojection_error_frame_marker[:, :NUMBER_OF_MEDIAPIPE_BODY_MARKERS] = new_reprojection_error_flat+ filtered_reprojection_error_frame_marker[:, :num_tracked_points] = new_reprojection_error_flatfiltered_reprojection_error_camera_frame_marker = reprojection_error_camera_frame_marker.copy()- filtered_reprojection_error_camera_frame_marker[- :, :, :NUMBER_OF_MEDIAPIPE_BODY_MARKERS- ] = new_reprojError_cam_frame_marker+ filtered_reprojection_error_camera_frame_marker[:, :, :num_tracked_points] = new_reprojError_cam_frame_markerreturn (filtered_skel3d_frame_marker_xyz,@@ -172,7 +174,7 @@ def _get_data_to_reproject(input_2d_data_camera_frame_marker_xy: np.ndarray,) -> tuple[np.ndarray, list]:indices_above_threshold = np.nonzero(reprojError_cam_frame_marker > reprojection_error_threshold)- logger.info(f"SHAPE OF INDICES ABOVE THRESHOLD: {indices_above_threshold[0].shape}")+ logger.debug(f"SHAPE OF INDICES ABOVE THRESHOLD: {indices_above_threshold[0].shape}")total_frame_marker_combos = (input_2d_data_camera_frame_marker_xy.shape[1] * input_2d_data_camera_frame_marker_xy.shape[2]diff --git a/freemocap/core_processes/capture_volume_calibration/save_3d_data_to_npy.py b/freemocap/core_processes/capture_volume_calibration/save_3d_data_to_npy.pynew file mode 100644index 00000000..c82ffc2f--- /dev/null+++ b/freemocap/core_processes/capture_volume_calibration/save_3d_data_to_npy.py@@ -0,0 +1,67 @@+import logging+from pathlib import Path+from typing import Union++import numpy as np++from freemocap.system.paths_and_filenames.file_and_folder_names import (+ RAW_3D_NPY_FILE_NAME,+ FULL_REPROJECTION_ERROR_NPY_FILE_NAME,+ REPROJECTION_ERROR_NPY_FILE_NAME,+ REPROJECTION_FILTERED_PREFIX,+)++logger = logging.getLogger(__name__)+++def save_3d_data_to_npy(+ data3d_numFrames_numTrackedPoints_XYZ: np.ndarray,+ data3d_numFrames_numTrackedPoints_reprojectionError: np.ndarray,+ data3d_numCams_numFrames_numTrackedPoints_reprojectionError: np.ndarray,+ path_to_folder_where_data_will_be_saved: Union[str, Path],+ processing_level: str,+ file_prefix: str = "",+):+ path_to_folder_where_data_will_be_saved = Path(path_to_folder_where_data_will_be_saved)+ Path(path_to_folder_where_data_will_be_saved).mkdir(parents=True, exist_ok=True) # save spatial XYZ data+ if processing_level == "raw":+ data_3d_save_path = path_to_folder_where_data_will_be_saved / f"{file_prefix}_{RAW_3D_NPY_FILE_NAME}"+ reprojection_error_save_path = (+ path_to_folder_where_data_will_be_saved / f"{file_prefix}_{REPROJECTION_ERROR_NPY_FILE_NAME}"+ )+ full_reprojection_error_save_path = (+ path_to_folder_where_data_will_be_saved / f"{file_prefix}_{FULL_REPROJECTION_ERROR_NPY_FILE_NAME}"+ )+ elif processing_level == "reprojection_filtered":+ data_3d_save_path = (+ path_to_folder_where_data_will_be_saved+ / f"{REPROJECTION_FILTERED_PREFIX}{file_prefix}_{RAW_3D_NPY_FILE_NAME}"+ )+ reprojection_error_save_path = (+ path_to_folder_where_data_will_be_saved+ / f"{REPROJECTION_FILTERED_PREFIX}{file_prefix}_{REPROJECTION_ERROR_NPY_FILE_NAME}"+ )+ full_reprojection_error_save_path = (+ path_to_folder_where_data_will_be_saved+ / f"{REPROJECTION_FILTERED_PREFIX}{file_prefix}_{FULL_REPROJECTION_ERROR_NPY_FILE_NAME}"+ )+ else:+ logger.exception(f"processing_level: {processing_level} not recognized")+ raise Exception++ logger.info(f"saving: {data_3d_save_path}")+ np.save(str(data_3d_save_path), data3d_numFrames_numTrackedPoints_XYZ)++ # save reprojection error++ logger.info(f"saving: {reprojection_error_save_path}")+ np.save(+ str(reprojection_error_save_path),+ data3d_numFrames_numTrackedPoints_reprojectionError,+ )++ logger.info(f"saving: {full_reprojection_error_save_path}")+ np.save(+ str(full_reprojection_error_save_path),+ data3d_numCams_numFrames_numTrackedPoints_reprojectionError,+ )diff --git a/freemocap/core_processes/capture_volume_calibration/save_mediapipe_3d_data_to_npy.py b/freemocap/core_processes/capture_volume_calibration/save_mediapipe_3d_data_to_npy.pydeleted file mode 100644index 710b7dae..00000000--- a/freemocap/core_processes/capture_volume_calibration/save_mediapipe_3d_data_to_npy.py+++ /dev/null@@ -1,65 +0,0 @@-import logging-from pathlib import Path-from typing import Union--import numpy as np--from freemocap.system.paths_and_filenames.file_and_folder_names import (- RAW_MEDIAPIPE_3D_NPY_FILE_NAME,- RAW_MEDIAPIPE_FULL_REPROJECTION_ERROR_NPY_FILE_NAME,- RAW_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME,- REPROJECTION_FILTERED_MEDIAPIPE_3D_NPY_FILE_NAME,- REPROJECTION_FILTERED_MEDIAPIPE_FULL_REPROJECTION_ERROR_NPY_FILE_NAME,- REPROJECTION_FILTERED_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME,-)--logger = logging.getLogger(__name__)---def save_mediapipe_3d_data_to_npy(- data3d_numFrames_numTrackedPoints_XYZ: np.ndarray,- data3d_numFrames_numTrackedPoints_reprojectionError: np.ndarray,- data3d_numCams_numFrames_numTrackedPoints_reprojectionError: np.ndarray,- path_to_folder_where_data_will_be_saved: Union[str, Path],- processing_level: str,-):- path_to_folder_where_data_will_be_saved = Path(path_to_folder_where_data_will_be_saved)- Path(path_to_folder_where_data_will_be_saved).mkdir(parents=True, exist_ok=True) # save spatial XYZ data- if processing_level == "raw":- mediapipe_3dData_save_path = path_to_folder_where_data_will_be_saved / RAW_MEDIAPIPE_3D_NPY_FILE_NAME- mediapipe_reprojection_error_save_path = (- path_to_folder_where_data_will_be_saved / RAW_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME- )- mediapipe_full_reprojection_error_save_path = (- path_to_folder_where_data_will_be_saved / RAW_MEDIAPIPE_FULL_REPROJECTION_ERROR_NPY_FILE_NAME- )- elif processing_level == "reprojection_filtered":- mediapipe_3dData_save_path = (- path_to_folder_where_data_will_be_saved / REPROJECTION_FILTERED_MEDIAPIPE_3D_NPY_FILE_NAME- )- mediapipe_reprojection_error_save_path = (- path_to_folder_where_data_will_be_saved / REPROJECTION_FILTERED_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME- )- mediapipe_full_reprojection_error_save_path = (- path_to_folder_where_data_will_be_saved / REPROJECTION_FILTERED_MEDIAPIPE_FULL_REPROJECTION_ERROR_NPY_FILE_NAME- )- else:- logger.exception(f"processing_level: {processing_level} not recognized")- raise Exception-- logger.info(f"saving: {mediapipe_3dData_save_path}")- np.save(str(mediapipe_3dData_save_path), data3d_numFrames_numTrackedPoints_XYZ)-- # save reprojection error-- logger.info(f"saving: {mediapipe_reprojection_error_save_path}")- np.save(- str(mediapipe_reprojection_error_save_path),- data3d_numFrames_numTrackedPoints_reprojectionError,- )-- logger.info(f"saving: {mediapipe_full_reprojection_error_save_path}")- np.save(- str(mediapipe_full_reprojection_error_save_path),- data3d_numCams_numFrames_numTrackedPoints_reprojectionError,- )diff --git a/freemocap/core_processes/capture_volume_calibration/triangulate_3d_data.py b/freemocap/core_processes/capture_volume_calibration/triangulate_3d_data.pyindex b6089da3..a411bec0 100644--- a/freemocap/core_processes/capture_volume_calibration/triangulate_3d_data.py+++ b/freemocap/core_processes/capture_volume_calibration/triangulate_3d_data.py@@ -4,8 +4,8 @@ from pathlib import Pathimport numpy as np-from freemocap.core_processes.capture_volume_calibration.save_mediapipe_3d_data_to_npy import (- save_mediapipe_3d_data_to_npy,+from freemocap.core_processes.capture_volume_calibration.save_3d_data_to_npy import (+ save_3d_data_to_npy,)logger = logging.getLogger(__name__)@@ -13,14 +13,14 @@ logger = logging.getLogger(__name__)def triangulate_3d_data(anipose_calibration_object,- mediapipe_2d_data: np.ndarray,+ image_2d_data: np.ndarray,use_triangulate_ransac: bool = False,kill_event: multiprocessing.Event = None,) -> tuple[np.ndarray, np.ndarray, np.ndarray]:- number_of_cameras = mediapipe_2d_data.shape[0]- number_of_frames = mediapipe_2d_data.shape[1]- number_of_tracked_points = mediapipe_2d_data.shape[2]- number_of_spatial_dimensions = mediapipe_2d_data.shape[3]+ number_of_cameras = image_2d_data.shape[0]+ number_of_frames = image_2d_data.shape[1]+ number_of_tracked_points = image_2d_data.shape[2]+ number_of_spatial_dimensions = image_2d_data.shape[3]if not number_of_spatial_dimensions == 2:logger.error(@@ -30,7 +30,7 @@ def triangulate_3d_data(# reshape data to collapse across 'frames' so it becomes [number_of_cameras,# number_of_2d_points(numFrames*numPoints), XY]- data2d_flat = mediapipe_2d_data.reshape(number_of_cameras, -1, 2)+ data2d_flat = image_2d_data.reshape(number_of_cameras, -1, 2)logger.info(f"Reconstructing 3d points from 2d points with shape: \n"@@ -74,9 +74,9 @@ if __name__ == "__main__":parser = argparse.ArgumentParser()parser.add_argument(- "--mediapipe_2d_data_path",+ "--data_2d_path",type=str,- help="path to mediapipe 2d data",+ help="path to 2d data",required=True,)parser.add_argument(@@ -103,12 +103,18 @@ if __name__ == "__main__":help="use triangulate ransac anipose method ",required=False,)+ parser.add_argument(+ "--file_prefix",+ type=str,+ help="file prefix",+ required=False,+ )args = parser.parse_args()- mediapipe_2d_data = np.load(args.mediapipe_2d_data_path)+ data_2d = np.load(args.data_2d_path)if args.output_data_folder_path is None:- args.output_data_folder_path = Path(args.mediapipe_2d_data_path).parent+ args.output_data_folder_path = Path(args.data_2d_path).parentif args.save_data_as_csv is None:args.save_data_as_csv = True # default@@ -116,11 +122,14 @@ if __name__ == "__main__":if args.use_triangulate_ransac is None:args.use_triangulate_ransac = True+ if args.file_prefix is None:+ args.file_prefix = ""+if args.calibration_file_path:anipose_calibration_object = load_anipose_calibration_toml_from_path(args.calibration_file_path)else:anipose_calibration_object = load_anipose_calibration_toml_from_path(- Path(args.mediapipe_2d_data_path).parent.parent / "camera_calibration_data.toml"+ Path(args.data_2d_path).parent.parent / "camera_calibration_data.toml")(@@ -129,13 +138,14 @@ if __name__ == "__main__":skeleton_reprojection_error_cam_fr_mar,) = triangulate_3d_data(anipose_calibration_object=anipose_calibration_object,- mediapipe_2d_data=mediapipe_2d_data,+ image_2d_data=data_2d,use_triangulate_ransac=args.use_triangulate_ransac,)- save_mediapipe_3d_data_to_npy(+ save_3d_data_to_npy(data3d_numFrames_numTrackedPoints_XYZ=skel3d_frame_marker_xyz,data3d_numFrames_numTrackedPoints_reprojectionError=skeleton_reprojection_error_fr_mar,data3d_numCams_numFrames_numTrackedPoints_reprojectionError=skeleton_reprojection_error_cam_fr_mar,path_to_folder_where_data_will_be_saved=args.output_data_folder_path,processing_level="raw",+ file_prefix=args.file_prefix,)diff --git a/freemocap/core_processes/detecting_things_in_2d_images/__init__.py b/freemocap/core_processes/detecting_things_in_2d_images/__init__.pydeleted file mode 100644index e69de29b..00000000diff --git a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/__init__.py b/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/__init__.pydeleted file mode 100644index e69de29b..00000000diff --git a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/convert_mediapipe_npy_to_csv.py b/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/convert_mediapipe_npy_to_csv.pydeleted file mode 100644index ee7dcf76..00000000--- a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/convert_mediapipe_npy_to_csv.py+++ /dev/null@@ -1,331 +0,0 @@-# %%-import logging-from pathlib import Path-from typing import Union--import numpy as np-import pandas as pd-from mediapipe.python.solutions import holistic as mp_holistic--from freemocap.system.paths_and_filenames.file_and_folder_names import (- MEDIAPIPE_RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME,- MEDIAPIPE_BODY_3D_DATAFRAME_CSV_FILE_NAME,- MEDIAPIPE_LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME,-)--logger = logging.getLogger(__name__)---def convert_mediapipe_npy_to_csv(- mediapipe_3d_frame_trackedPoint_xyz: np.ndarray,- output_data_folder_path: Union[str, Path],-):- logger.info(- f"Converting npy data with shape: {mediapipe_3d_frame_trackedPoint_xyz.shape} into `csv` and smaller `npy` files"- )-- # %%- mediapipe_pose_landmark_names = [landmark.name.lower() for landmark in mp_holistic.PoseLandmark]- mediapipe_hand_landmark_names = [landmark.name.lower() for landmark in mp_holistic.HandLandmark]- # face_landmark_names = [landmark.name.lower() for landmark in mp_holistic.PoseLandmark] #gonna have the clever for the face- # logger.info(f"Body tracked point names: {mediapipe_pose_landmark_names}")- # logger.info(mediapipe_hand_landmark_names)-- # %%- # get number of points in body, hands, face-- number_of_body_points = len(mediapipe_pose_landmark_names)- number_of_hand_points = len(mediapipe_hand_landmark_names)-- first_body_marker_index = 0- last_body_marker_index = number_of_body_points - 1-- first_right_hand_marker_index = last_body_marker_index + 1- last_right_hand_marker_index = number_of_body_points + number_of_hand_points - 1-- first_left_hand_marker_index = last_right_hand_marker_index + 1- last_left_hand_marker_index = last_right_hand_marker_index + 1 + number_of_hand_points - 1-- first_face_marker_index = last_left_hand_marker_index + 1- last_face_marker_index = mediapipe_3d_frame_trackedPoint_xyz.shape[1]-- number_of_face_points = last_face_marker_index - first_face_marker_index-- # logger.info(- # f"body tracked point indices: {first_body_marker_index}:{last_body_marker_index}"- # )- # logger.info(- # f"right hand tracked point indices: {first_right_hand_marker_index}:{last_right_hand_marker_index}"- # )- # logger.info(- # f"left hand tracked point indices: {first_left_hand_marker_index}:{last_left_hand_marker_index}"- # )- # logger.info(- # f"face tracked point indices: {first_face_marker_index}:{last_face_marker_index}"- # )- #- # logger.info(- # f"number of body points: {last_body_marker_index - first_body_marker_index + 1}"- # )- # logger.info(- # f"number of right hand points: {last_right_hand_marker_index - first_right_hand_marker_index + 1}"- # )- # logger.info(- # f"number of left hand points: {last_left_hand_marker_index - first_left_hand_marker_index + 1}"- # )- # logger.info(- # f"number of face points: {last_face_marker_index - first_face_marker_index + 1}"- # )-- # %%- body_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[:, first_body_marker_index : last_body_marker_index + 1, :]- right_hand_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[- :, first_right_hand_marker_index : last_right_hand_marker_index + 1, :- ]- left_hand_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[- :, first_left_hand_marker_index : last_left_hand_marker_index + 1, :- ]- face_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[:, first_face_marker_index : last_face_marker_index + 1, :]-- # logger.info(f"body 3d xyz shape: {body_3d_xyz.shape}")- # logger.info(f"right hand 3d xyz shape: {right_hand_3d_xyz.shape}")- # logger.info(f"left hand 3d xyz shape: {left_hand_3d_xyz.shape}")- # logger.info(f"face 3d xyz shape: {face_3d_xyz.shape}")-- # %%- # save broken up npy files- np.save(str(Path(output_data_folder_path) / "mediapipe_body_3d_xyz.npy"), body_3d_xyz)- np.save(- str(Path(output_data_folder_path) / "mediapipe_right_hand_3d_xyz.npy"),- right_hand_3d_xyz,- )- np.save(- str(Path(output_data_folder_path) / "mediapipe_left_hand_3d_xyz.npy"),- left_hand_3d_xyz,- )- np.save(str(Path(output_data_folder_path) / "mediapipe_face_3d_xyz.npy"), face_3d_xyz)-- # %%- # create pandas data frame headers-- body_3d_xyz_header = []- for landmark_name in mediapipe_pose_landmark_names:- body_3d_xyz_header.append(f"{landmark_name}_x")- body_3d_xyz_header.append(f"{landmark_name}_y")- body_3d_xyz_header.append(f"{landmark_name}_z")-- right_hand_3d_xyz_header = []- for landmark_name in mediapipe_hand_landmark_names:- right_hand_3d_xyz_header.append(f"right_hand_{landmark_name}_x")- right_hand_3d_xyz_header.append(f"right_hand_{landmark_name}_y")- right_hand_3d_xyz_header.append(f"right_hand_{landmark_name}_z")-- left_hand_3d_xyz_header = []- for landmark_name in mediapipe_hand_landmark_names:- left_hand_3d_xyz_header.append(f"left_hand_{landmark_name}_x")- left_hand_3d_xyz_header.append(f"left_hand_{landmark_name}_y")- left_hand_3d_xyz_header.append(f"left_hand_{landmark_name}_z")-- face_3d_xyz_header = []- for landmark_number in range(last_face_marker_index - first_face_marker_index):- face_3d_xyz_header.append(f"face_{str(landmark_number).zfill(4)}_x")- face_3d_xyz_header.append(f"face_{str(landmark_number).zfill(4)}_y")- face_3d_xyz_header.append(f"face_{str(landmark_number).zfill(4)}_z")- #- # logger.debug(- # f"length of body 3d xyz header: {len(body_3d_xyz_header)}, should be: {number_of_body_points * 3}"- # )- # logger.debug(- # f"length of right hand 3d xyz header: {len(right_hand_3d_xyz_header)}, should be: {number_of_hand_points * 3}"- # )- # logger.debug(- # f"length of left hand 3d xyz header: {len(left_hand_3d_xyz_header)}, should be: {number_of_hand_points * 3}"- # )- # logger.debug(- # f"length of face 3d xyz header: {len(face_3d_xyz_header)}, should be: {number_of_face_points * 3}"- # )-- # %%- number_of_frames = mediapipe_3d_frame_trackedPoint_xyz.shape[0]- body_flat = body_3d_xyz.reshape(number_of_frames, number_of_body_points * 3)-- body_dataframe = pd.DataFrame(body_flat, columns=body_3d_xyz_header)- body_dataframe.to_csv(str(Path(output_data_folder_path) / MEDIAPIPE_BODY_3D_DATAFRAME_CSV_FILE_NAME), index=False)-- right_hand_flat = right_hand_3d_xyz.reshape(number_of_frames, number_of_hand_points * 3)- right_hand_dataframe = pd.DataFrame(right_hand_flat, columns=right_hand_3d_xyz_header)- right_hand_dataframe.to_csv(- str(Path(output_data_folder_path) / MEDIAPIPE_RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME),- index=False,- )-- left_hand_flat = left_hand_3d_xyz.reshape(number_of_frames, number_of_hand_points * 3)- left_hand_dataframe = pd.DataFrame(left_hand_flat, columns=left_hand_3d_xyz_header)- left_hand_dataframe.to_csv(- str(Path(output_data_folder_path) / MEDIAPIPE_LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME),- index=False,- )-- face_flat = face_3d_xyz.reshape(number_of_frames, number_of_face_points * 3)- face_dataframe = pd.DataFrame(face_flat, columns=face_3d_xyz_header)- face_dataframe.to_csv(str(Path(output_data_folder_path) / "mediapipe_face_3d_xyz.csv"), index=False)-- # %%-- # %%- mediapipe_pose_landmark_names = [landmark.name.lower() for landmark in mp_holistic.PoseLandmark]- mediapipe_hand_landmark_names = [landmark.name.lower() for landmark in mp_holistic.HandLandmark]- # face_landmark_names = [landmark.name.lower() for landmark in mp_holistic.PoseLandmark] #gonna have the clever for the face-- # logger.debug(f"Body tracked point names: {mediapipe_pose_landmark_names}")- # logger.debug(mediapipe_hand_landmark_names)-- # %%- # get number of points in body, hands, face-- number_of_body_points = len(mediapipe_pose_landmark_names)- number_of_hand_points = len(mediapipe_hand_landmark_names)-- first_body_marker_index = 0- last_body_marker_index = number_of_body_points - 1-- first_right_hand_marker_index = last_body_marker_index + 1- last_right_hand_marker_index = number_of_body_points + number_of_hand_points - 1-- first_left_hand_marker_index = last_right_hand_marker_index + 1- last_left_hand_marker_index = last_right_hand_marker_index + 1 + number_of_hand_points - 1-- first_face_marker_index = last_left_hand_marker_index + 1- last_face_marker_index = mediapipe_3d_frame_trackedPoint_xyz.shape[1]-- number_of_face_points = last_face_marker_index - first_face_marker_index-- # logger.debug(- # f"body tracked point indices: {first_body_marker_index}:{last_body_marker_index}"- # )- # logger.debug(- # f"right hand tracked point indices: {first_right_hand_marker_index}:{last_right_hand_marker_index}"- # )- # logger.debug(- # f"left hand tracked point indices: {first_left_hand_marker_index}:{last_left_hand_marker_index}"- # )- # logger.debug(- # f"face tracked point indices: {first_face_marker_index}:{last_face_marker_index}"- # )- #- # logger.debug(- # f"number of body points: {last_body_marker_index - first_body_marker_index + 1}"- # )- # logger.debug(- # f"number of right hand points: {last_right_hand_marker_index - first_right_hand_marker_index + 1}"- # )- # logger.debug(- # f"number of left hand points: {last_left_hand_marker_index - first_left_hand_marker_index + 1}"- # )- # logger.debug(- # f"number of face points: {last_face_marker_index - first_face_marker_index + 1}"- # )-- # %%- body_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[:, first_body_marker_index : last_body_marker_index + 1, :]- right_hand_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[- :, first_right_hand_marker_index : last_right_hand_marker_index + 1, :- ]- left_hand_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[- :, first_left_hand_marker_index : last_left_hand_marker_index + 1, :- ]- face_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[:, first_face_marker_index : last_face_marker_index + 1, :]-- logger.debug(f"body 3d xyz shape: {body_3d_xyz.shape}")- logger.debug(f"right hand 3d xyz shape: {right_hand_3d_xyz.shape}")- logger.debug(f"left hand 3d xyz shape: {left_hand_3d_xyz.shape}")- logger.debug(f"face 3d xyz shape: {face_3d_xyz.shape}")-- # %%- # save broken up npy files- np.save(str(Path(output_data_folder_path) / "mediapipe_body_3d_xyz.npy"), body_3d_xyz)- np.save(- str(Path(output_data_folder_path) / "mediapipe_right_hand_3d_xyz.npy"),- right_hand_3d_xyz,- )- np.save(- str(Path(output_data_folder_path) / "mediapipe_left_hand_3d_xyz.npy"),- left_hand_3d_xyz,- )- np.save(str(Path(output_data_folder_path) / "mediapipe_face_3d_xyz.npy"), face_3d_xyz)-- # %%- # create pandas data frame headers-- body_3d_xyz_header = []- for landmark_name in mediapipe_pose_landmark_names:- body_3d_xyz_header.append(f"{landmark_name}_x")- body_3d_xyz_header.append(f"{landmark_name}_y")- body_3d_xyz_header.append(f"{landmark_name}_z")-- right_hand_3d_xyz_header = []- for landmark_name in mediapipe_hand_landmark_names:- right_hand_3d_xyz_header.append(f"right_hand_{landmark_name}_x")- right_hand_3d_xyz_header.append(f"right_hand_{landmark_name}_y")- right_hand_3d_xyz_header.append(f"right_hand_{landmark_name}_z")-- left_hand_3d_xyz_header = []- for landmark_name in mediapipe_hand_landmark_names:- left_hand_3d_xyz_header.append(f"left_hand_{landmark_name}_x")- left_hand_3d_xyz_header.append(f"left_hand_{landmark_name}_y")- left_hand_3d_xyz_header.append(f"left_hand_{landmark_name}_z")-- face_3d_xyz_header = []- for landmark_number in range(last_face_marker_index - first_face_marker_index):- face_3d_xyz_header.append(f"face_{str(landmark_number).zfill(4)}_x")- face_3d_xyz_header.append(f"face_{str(landmark_number).zfill(4)}_y")- face_3d_xyz_header.append(f"face_{str(landmark_number).zfill(4)}_z")-- logger.debug(- f"length of body 3d xyz `.csv` header: {len(body_3d_xyz_header)}, should be: {number_of_body_points * 3}"- )- logger.debug(- f"length of right hand 3d xyz `.csv` header: {len(right_hand_3d_xyz_header)}, should be: {number_of_hand_points * 3}"- )- logger.debug(- f"length of left hand 3d xyz `.csv` header: {len(left_hand_3d_xyz_header)}, should be: {number_of_hand_points * 3}"- )- logger.debug(- f"length of face 3d xyz `.csv` header: {len(face_3d_xyz_header)}, should be: {number_of_face_points * 3}"- )-- # %%- number_of_frames = mediapipe_3d_frame_trackedPoint_xyz.shape[0]- body_flat = body_3d_xyz.reshape(number_of_frames, number_of_body_points * 3)-- body_dataframe = pd.DataFrame(body_flat, columns=body_3d_xyz_header)- body_dataframe.to_csv(str(Path(output_data_folder_path) / MEDIAPIPE_BODY_3D_DATAFRAME_CSV_FILE_NAME), index=False)-- right_hand_flat = right_hand_3d_xyz.reshape(number_of_frames, number_of_hand_points * 3)- right_hand_dataframe = pd.DataFrame(right_hand_flat, columns=right_hand_3d_xyz_header)- right_hand_dataframe.to_csv(- str(Path(output_data_folder_path) / MEDIAPIPE_RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME),- index=False,- )-- left_hand_flat = left_hand_3d_xyz.reshape(number_of_frames, number_of_hand_points * 3)- left_hand_dataframe = pd.DataFrame(left_hand_flat, columns=left_hand_3d_xyz_header)- left_hand_dataframe.to_csv(- str(Path(output_data_folder_path) / MEDIAPIPE_LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME),- index=False,- )-- face_flat = face_3d_xyz.reshape(number_of_frames, number_of_face_points * 3)- face_dataframe = pd.DataFrame(face_flat, columns=face_3d_xyz_header)- face_dataframe.to_csv(str(Path(output_data_folder_path) / "mediapipe_face_3d_xyz.csv"), index=False)-- logger.info("Done saving out `csv` and broken up `npy` files")---if __name__ == "__main__":- mediapipe_3d_frame_trackedPoint_xyz = np.load(- r"C:\Users\jonma\freemocap_data\session_10-15-2022-09_50_10\output_data\post_processed_data\mediaPipeSkel_3d_origin_aligned.npy"- )- output_data_folder_path = r"C:\Users\jonma\freemocap_data\session_10-15-2022-09_50_10\output_data"-- convert_mediapipe_npy_to_csv(mediapipe_3d_frame_trackedPoint_xyz, output_data_folder_path)diff --git a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/data_models/__init__.py b/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/data_models/__init__.pydeleted file mode 100644index e69de29b..00000000diff --git a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/data_models/mediapipe_dataclasses.py b/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/data_models/mediapipe_dataclasses.pydeleted file mode 100644index 14b88588..00000000--- a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/data_models/mediapipe_dataclasses.py+++ /dev/null@@ -1,60 +0,0 @@-import logging-from dataclasses import dataclass-from typing import Any--import numpy as np-from skellycam.detection.models.frame_payload import FramePayload--logger = logging.getLogger(__name__)---@dataclass-class Mediapipe2dNumpyArrays:- body_frameNumber_trackedPointNumber_XYZ: np.ndarray = None- body_world_frameNumber_trackedPointNumber_XYZ: np.ndarray = None- rightHand_frameNumber_trackedPointNumber_XYZ: np.ndarray = None- leftHand_frameNumber_trackedPointNumber_XYZ: np.ndarray = None- face_frameNumber_trackedPointNumber_XYZ: np.ndarray = None-- body_frameNumber_trackedPointNumber_confidence: np.ndarray = None-- @property- def has_data(self):- return not np.isnan(self.body_frameNumber_trackedPointNumber_XYZ).all()-- @property- def all_data2d_nFrames_nTrackedPts_XY(self):- """dimensions will be [number_of_frames , number_of_markers, XY]"""-- if self.body_frameNumber_trackedPointNumber_XYZ is None:- # if there's no body data, there's no hand or face data either- return-- if len(self.body_frameNumber_trackedPointNumber_XYZ.shape) == 3: # multiple frames- return np.hstack(- [- self.body_frameNumber_trackedPointNumber_XYZ,- self.rightHand_frameNumber_trackedPointNumber_XYZ,- self.leftHand_frameNumber_trackedPointNumber_XYZ,- self.face_frameNumber_trackedPointNumber_XYZ,- ]- )- elif len(self.body_frameNumber_trackedPointNumber_XYZ.shape) == 2: # single frame- return np.vstack(- [- self.body_frameNumber_trackedPointNumber_XYZ,- self.rightHand_frameNumber_trackedPointNumber_XYZ,- self.leftHand_frameNumber_trackedPointNumber_XYZ,- self.face_frameNumber_trackedPointNumber_XYZ,- ]- )- else:- logger.error("data should have either 2 or 3 dimensions")---@dataclass-class Mediapipe2dDataPayload:- raw_frame_payload: FramePayload = None- mediapipe_results: Any = None- annotated_image: np.ndarray = None- pixel_data_numpy_arrays: Mediapipe2dNumpyArrays = Nonediff --git a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/data_models/mediapipe_skeleton_names_and_connections.py b/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/data_models/mediapipe_skeleton_names_and_connections.pydeleted file mode 100644index 572ff38d..00000000--- a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/data_models/mediapipe_skeleton_names_and_connections.py+++ /dev/null@@ -1,101 +0,0 @@-from mediapipe.python.solutions import holistic as mp_holistic--mediapipe_body_connections = [connection for connection in mp_holistic.POSE_CONNECTIONS]--mediapipe_hand_connections = [connection for connection in mp_holistic.HAND_CONNECTIONS]--mediapipe_face_connections = [connection for connection in mp_holistic.FACEMESH_CONTOURS]--mediapipe_body_landmark_names = [landmark.name.lower() for landmark in mp_holistic.PoseLandmark]-NUMBER_OF_MEDIAPIPE_BODY_MARKERS = len(mediapipe_body_landmark_names)--mediapipe_hand_landmark_names = [landmark.name.lower() for landmark in mp_holistic.HandLandmark]--mediapipe_face_landmark_names = [- "right_eye",- "left_eye",- "nose_tip",- "mouth_center",- "right_ear_tragion",- "left_ear_tragion",-]--mediapipe_tracked_point_names_dict = {- "body": mediapipe_body_landmark_names,- "right_hand": [f"right_hand_{name}" for name in mediapipe_hand_landmark_names],- "left_hand": [f"left_hand_{name}" for name in mediapipe_hand_landmark_names],- "face": mediapipe_face_landmark_names,-}--mediapipe_names_and_connections_dict = {- "body": {- "names": mediapipe_body_landmark_names,- "connections": mediapipe_body_connections,- },- "right_hand": {- "names": [f"right_hand_{name}" for name in mediapipe_hand_landmark_names],- "connections": mediapipe_hand_connections,- },- "left_hand": {- "names": [f"left_hand_{name}" for name in mediapipe_hand_landmark_names],- "connections": mediapipe_hand_connections,- },- "face": {- "names": mediapipe_face_landmark_names,- "connections": mediapipe_face_connections,- },-}-mediapipe_virtual_marker_definitions_dict = {- "head_center": {- "marker_names": ["left_ear", "right_ear"],- "marker_weights": [0.5, 0.5],- },- "neck_center": {- "marker_names": ["left_shoulder", "right_shoulder"],- "marker_weights": [0.5, 0.5],- },- "trunk_center": {- "marker_names": ["left_shoulder", "right_shoulder", "left_hip", "right_hip"],- "marker_weights": [0.25, 0.25, 0.25, 0.25],- },- "hips_center": {- "marker_names": ["left_hip", "right_hip"],- "marker_weights": [0.5, 0.5],- },-}-mediapipe_skeleton_schema = {- "body": {- "point_names": mediapipe_body_landmark_names,- "connections": mediapipe_body_connections,- "virtual_marker_definitions": mediapipe_virtual_marker_definitions_dict,- "parent": "hips_center",- },- "hands": {- "right": {- "point_names": [name for name in mediapipe_hand_landmark_names],- "connections": mediapipe_hand_connections,- "parent": "right_wrist",- },- "left": {- "point_names": [name for name in mediapipe_hand_landmark_names],- "connections": mediapipe_hand_connections,- "parent": "left_wrist",- },- },- "face": {- "point_names": mediapipe_face_landmark_names,- "connections": mediapipe_face_connections,- "parent": "nose",- },-}--if __name__ == "__main__":- import pprint-- #- # print("mediapipe_body_connections:")- # pprint.pp(mediapipe_tracked_point_names_dict)-- print("=====================================================")- print("mediapipe_skeleton_schema:")- pprint.pp(mediapipe_skeleton_schema)diff --git a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/mediapipe_skeleton_detector.py b/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/mediapipe_skeleton_detector.pydeleted file mode 100644index 70c7920d..00000000--- a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/mediapipe_skeleton_detector.py+++ /dev/null@@ -1,507 +0,0 @@-import logging-import multiprocessing-from pathlib import Path-from typing import List, Optional, Union, Callable, Tuple--import cv2-import mediapipe as mp-import numpy as np-from skellycam.opencv.video_recorder.video_recorder import VideoRecorder-from skellytracker.trackers.mediapipe_tracker.mediapipe_model_info import (- MediapipeTrackingParams-)-from tqdm import tqdm--from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.data_models.mediapipe_dataclasses import (- Mediapipe2dNumpyArrays,-)-from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.data_models.mediapipe_skeleton_names_and_connections import (- mediapipe_tracked_point_names_dict,-)-from freemocap.system.paths_and_filenames.file_and_folder_names import (- MEDIAPIPE_2D_NPY_FILE_NAME,- ANNOTATED_VIDEOS_FOLDER_NAME,- MEDIAPIPE_BODY_WORLD_FILE_NAME,-)-from freemocap.utilities.get_video_paths import get_video_paths--logger = logging.getLogger(__name__)--mp_drawing = mp.solutions.drawing_utils-mp_drawing_styles = mp.solutions.drawing_styles-mp_holistic = mp.solutions.holistic--body_drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)-hand_drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)-face_drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)---class MediaPipeSkeletonDetector:- def __init__(- self,- parameter_model: Optional[MediapipeTrackingParams] = None,- use_tqdm: bool = True,- ):- if parameter_model is None:- parameter_model = MediapipeTrackingParams()-- self._parameter_model = parameter_model- self._use_tqdm = use_tqdm- self._mediapipe_payload_list = []-- self._mediapipe_tracked_point_names_dict = mediapipe_tracked_point_names_dict-- self.body_names_list = self._mediapipe_tracked_point_names_dict["body"]- self.right_hand_names_list = self._mediapipe_tracked_point_names_dict["right_hand"]- self.left_hand_names_list = self._mediapipe_tracked_point_names_dict["left_hand"]- self.face_names_list = self._mediapipe_tracked_point_names_dict["face"]-- # TODO - build a better iterator and list of `face_marker_names` that will only pull out the face_counters & iris edges (mp.python.solutions.face_mesh_connections.FACEMESH_CONTOURS, FACE_MESH_IRISES)-- self.number_of_body_tracked_points = len(self.body_names_list)- self.number_of_right_hand_tracked_points = len(self.right_hand_names_list)- self.number_of_left_hand_tracked_points = len(self.left_hand_names_list)- self.number_of_face_tracked_points = mp.solutions.face_mesh.FACEMESH_NUM_LANDMARKS_WITH_IRISES-- self.number_of_tracked_points_total = (- self.number_of_body_tracked_points- + self.number_of_left_hand_tracked_points- + self.number_of_right_hand_tracked_points- + self.number_of_face_tracked_points- )-- def process_folder_full_of_videos(- self,- path_to_folder_of_videos_to_process: Union[Path, str],- output_data_folder_path: Union[str, Path],- kill_event: multiprocessing.Event = None,- use_multiprocessing: bool = True,- ) -> Union[np.ndarray, None]:- path_to_folder_of_videos_to_process = Path(path_to_folder_of_videos_to_process)- logger.info(f"processing videos from: {path_to_folder_of_videos_to_process}")-- tasks = self._create_video_processing_tasks(- output_data_folder_path=output_data_folder_path,- path_to_folder_of_videos_to_process=path_to_folder_of_videos_to_process,- )-- if use_multiprocessing:- with multiprocessing.Pool() as pool:- mediapipe2d_single_camera_npy_arrays_list = pool.starmap(self.process_single_video, tasks)- else:- mediapipe2d_single_camera_npy_arrays_list = []- for task in tasks:- if kill_event is not None:- if kill_event.is_set():- break- mediapipe2d_single_camera_npy_arrays_list.append(self.process_single_video(*task))-- (- body_world_numCams_numFrames_numTrackedPts_XYZ,- data2d_numCams_numFrames_numTrackedPts_XY,- ) = self._build_output_numpy_array(mediapipe2d_single_camera_npy_arrays_list)-- self._save_mediapipe2d_data_to_npy(- data2d_numCams_numFrames_numTrackedPts_XY=data2d_numCams_numFrames_numTrackedPts_XY,- body_world_numCams_numFrames_numTrackedPts_XYZ=body_world_numCams_numFrames_numTrackedPts_XYZ,- output_data_folder_path=Path(output_data_folder_path),- )- return data2d_numCams_numFrames_numTrackedPts_XY-- @staticmethod- def process_single_video(- synchronized_video_file_path: Path,- output_data_folder_path: Path,- mediapipe_parameters_model: MediapipeTrackingParams,- annotate_image: Callable,- list_of_mediapipe_results_to_npy_arrays: Callable,- use_tqdm: bool = True,- ):- logger.info(f"Running `mediapipe` skeleton detection on video: {str(synchronized_video_file_path)}")-- holistic_tracker = mp_holistic.Holistic(- model_complexity=mediapipe_parameters_model.mediapipe_model_complexity,- min_detection_confidence=mediapipe_parameters_model.min_detection_confidence,- min_tracking_confidence=mediapipe_parameters_model.min_tracking_confidence,- )-- video_capture_object = cv2.VideoCapture(str(synchronized_video_file_path))-- video_width = video_capture_object.get(cv2.CAP_PROP_FRAME_WIDTH)- video_height = video_capture_object.get(cv2.CAP_PROP_FRAME_HEIGHT)- video_framerate = video_capture_object.get(cv2.CAP_PROP_FPS)-- video_mediapipe_results_list = []- video_annotated_images_list = []-- success, image = video_capture_object.read()-- number_of_frames = int(video_capture_object.get(cv2.CAP_PROP_FRAME_COUNT))-- if use_tqdm:- iterator = tqdm(- range(number_of_frames),- desc=f"mediapiping video: {synchronized_video_file_path.name}",- total=number_of_frames,- colour="magenta",- unit="frames",- dynamic_ncols=True,- )- else:- iterator = range(number_of_frames)-- for _frame_number in iterator:- if not success or image is None:- logger.error(f"Failed to load an image from: {str(synchronized_video_file_path)}")- raise Exception-- mediapipe_results = holistic_tracker.process(- image- ) # <-this is where the magic happens, i.e. where the raw image is processed by a convolutional neural network to provide an estimate of joint position in pixel coordinates. Please don't forget that this is insane and should not be possible lol-- mediapipe_single_frame_npy_data = list_of_mediapipe_results_to_npy_arrays(- [mediapipe_results],- image_width=image.shape[0],- image_height=image.shape[1],- )-- confidence_threshold = 0.5-- threshold_mask = (- mediapipe_single_frame_npy_data.body_frameNumber_trackedPointNumber_confidence < confidence_threshold- )- mediapipe_single_frame_npy_data.body_frameNumber_trackedPointNumber_XYZ[threshold_mask, :] = np.nan-- video_mediapipe_results_list.append(mediapipe_results)- annotated_image = annotate_image(image, mediapipe_results)- video_annotated_images_list.append(annotated_image)-- success, image = video_capture_object.read()-- try:- annotated_video_path = output_data_folder_path.parent.parent / ANNOTATED_VIDEOS_FOLDER_NAME- annotated_video_path.mkdir(exist_ok=True, parents=True)- annotated_video_name = synchronized_video_file_path.stem + "_mediapipe.mp4"- annotated_video_save_path = annotated_video_path / annotated_video_name-- logger.info(f"Saving mediapipe annotated video to : {annotated_video_save_path}")-- video_recorder = VideoRecorder()-- video_recorder.save_image_list_to_disk(- image_list=video_annotated_images_list,- path_to_save_video_file=annotated_video_save_path,- frames_per_second=video_framerate,- )- except Exception as e:- logger.error(f"Failed to save annotated video to disk: {e}")- raise e-- camera_mediapipe_2d_single_camera_npy_arrays = list_of_mediapipe_results_to_npy_arrays(- video_mediapipe_results_list,- image_width=video_width,- image_height=video_height,- )-- # return the numpy array for this video- return camera_mediapipe_2d_single_camera_npy_arrays-- def _build_output_numpy_array(self, mediapipe2d_single_camera_npy_arrays_list):- all_cameras_data2d_list = [- m2d.all_data2d_nFrames_nTrackedPts_XY for m2d in mediapipe2d_single_camera_npy_arrays_list- ]- all_cameras_pose_world_data_list = [- m2d.body_world_frameNumber_trackedPointNumber_XYZ for m2d in mediapipe2d_single_camera_npy_arrays_list- ]- all_cameras_right_hand_world_data_list = [- m2d.rightHand_frameNumber_trackedPointNumber_XYZ for m2d in mediapipe2d_single_camera_npy_arrays_list- ]- all_cameras_left_hand_world_data_list = [- m2d.leftHand_frameNumber_trackedPointNumber_XYZ for m2d in mediapipe2d_single_camera_npy_arrays_list- ]- all_cameras_face_world_data_list = [- m2d.face_frameNumber_trackedPointNumber_XYZ for m2d in mediapipe2d_single_camera_npy_arrays_list- ]- number_of_cameras = len(all_cameras_data2d_list)- number_of_frames = all_cameras_data2d_list[0].shape[0]- number_of_tracked_points = all_cameras_data2d_list[0].shape[1]- number_of_spatial_dimensions = all_cameras_data2d_list[0].shape[2]- number_of_body_points = all_cameras_pose_world_data_list[0].shape[1] # noqa- data2d_numCams_numFrames_numTrackedPts_XY = np.empty(- (- number_of_cameras,- number_of_frames,- number_of_tracked_points,- number_of_spatial_dimensions,- )- )- body_world_numCams_numFrames_numTrackedPts_XYZ = np.empty(- (- number_of_cameras,- number_of_frames,- number_of_tracked_points,- number_of_spatial_dimensions,- )- )- for cam_num in range(number_of_cameras):- data2d_numCams_numFrames_numTrackedPts_XY[cam_num, :, :, :] = all_cameras_data2d_list[cam_num]-- pose_3d = all_cameras_pose_world_data_list[cam_num]- right_hand_3d = all_cameras_right_hand_world_data_list[cam_num]- left_hand_3d = all_cameras_left_hand_world_data_list[cam_num]- face_3d = all_cameras_face_world_data_list[cam_num]-- body_world_numCams_numFrames_numTrackedPts_XYZ[cam_num, :, :, :] = np.concatenate(- (pose_3d, right_hand_3d, left_hand_3d, face_3d), axis=1- )-- logger.info(- f"The shape of body_world_numCams_numFrames_numTrackedPts_XYZ is "- f"{body_world_numCams_numFrames_numTrackedPts_XYZ.shape}"- )- return body_world_numCams_numFrames_numTrackedPts_XYZ, data2d_numCams_numFrames_numTrackedPts_XY-- def _create_video_processing_tasks(- self,- output_data_folder_path: Union[str, Path],- path_to_folder_of_videos_to_process: Union[Path, str],- ) -> List[Tuple]:- video_paths = get_video_paths(path_to_folder_of_videos_to_process)- tasks = [- (- video_path,- output_data_folder_path,- self._parameter_model,- self._annotate_image,- self._list_of_mediapipe_results_to_npy_arrays,- self._use_tqdm,- )- for video_path in video_paths- ]- return tasks-- def _save_mediapipe2d_data_to_npy(- self,- data2d_numCams_numFrames_numTrackedPts_XY: np.ndarray,- body_world_numCams_numFrames_numTrackedPts_XYZ: np.ndarray,- output_data_folder_path: Union[str, Path],- ):- mediapipe_2dData_save_path = Path(output_data_folder_path) / MEDIAPIPE_2D_NPY_FILE_NAME- mediapipe_2dData_save_path.parent.mkdir(exist_ok=True, parents=True)- logger.info(f"saving mediapipe image npy file: {mediapipe_2dData_save_path}")- np.save(str(mediapipe_2dData_save_path), data2d_numCams_numFrames_numTrackedPts_XY)-- mediapipe_body_world_save_path = Path(output_data_folder_path) / MEDIAPIPE_BODY_WORLD_FILE_NAME- mediapipe_body_world_save_path.parent.mkdir(exist_ok=True, parents=True)- logger.info(f"saving mediapipe body world npy xyz: {mediapipe_body_world_save_path}")- np.save(str(mediapipe_body_world_save_path), body_world_numCams_numFrames_numTrackedPts_XYZ)-- @staticmethod- def _annotate_image(image, mediapipe_results):- mp_drawing.draw_landmarks(- image=image,- landmark_list=mediapipe_results.face_landmarks,- connections=mp_holistic.FACEMESH_CONTOURS,- landmark_drawing_spec=None,- connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_contours_style(),- )- mp_drawing.draw_landmarks(- image=image,- landmark_list=mediapipe_results.face_landmarks,- connections=mp_holistic.FACEMESH_TESSELATION,- landmark_drawing_spec=None,- connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_tesselation_style(),- )- mp_drawing.draw_landmarks(- image=image,- landmark_list=mediapipe_results.pose_landmarks,- connections=mp_holistic.POSE_CONNECTIONS,- landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style(),- )- mp_drawing.draw_landmarks(- image=image,- landmark_list=mediapipe_results.left_hand_landmarks,- connections=mp_holistic.HAND_CONNECTIONS,- landmark_drawing_spec=None,- connection_drawing_spec=mp_drawing_styles.get_default_hand_connections_style(),- )-- mp_drawing.draw_landmarks(- image=image,- landmark_list=mediapipe_results.right_hand_landmarks,- connections=mp_holistic.HAND_CONNECTIONS,- landmark_drawing_spec=None,- connection_drawing_spec=mp_drawing_styles.get_default_hand_connections_style(),- )- return image-- def _list_of_mediapipe_results_to_npy_arrays(- self,- mediapipe_results_list: List,- image_width: Union[int, float],- image_height: Union[int, float],- ) -> Mediapipe2dNumpyArrays:- number_of_frames = len(mediapipe_results_list)- number_of_spatial_dimensions = 3 # this will be 2d XY pixel data, with mediapipe's estimate of Z-- body_frameNumber_trackedPointNumber_XYZ = np.zeros(- (- number_of_frames,- self.number_of_body_tracked_points,- number_of_spatial_dimensions,- )- )- body_frameNumber_trackedPointNumber_XYZ[:] = np.nan-- body_world_frameNumber_trackedPointNumber_XYZ = np.zeros(- (- number_of_frames,- self.number_of_body_tracked_points,- number_of_spatial_dimensions,- )- )- body_world_frameNumber_trackedPointNumber_XYZ[:] = np.nan-- body_frameNumber_trackedPointNumber_confidence = np.zeros(- (number_of_frames, self.number_of_body_tracked_points)- )- body_frameNumber_trackedPointNumber_confidence[:] = np.nan # only body markers get a 'confidence' value-- rightHand_frameNumber_trackedPointNumber_XYZ = np.zeros(- (- number_of_frames,- self.number_of_right_hand_tracked_points,- number_of_spatial_dimensions,- )- )- rightHand_frameNumber_trackedPointNumber_XYZ[:] = np.nan-- leftHand_frameNumber_trackedPointNumber_XYZ = np.zeros(- (- number_of_frames,- self.number_of_left_hand_tracked_points,- number_of_spatial_dimensions,- )- )- leftHand_frameNumber_trackedPointNumber_XYZ[:] = np.nan-- face_frameNumber_trackedPointNumber_XYZ = np.zeros(- (- number_of_frames,- self.number_of_face_tracked_points,- number_of_spatial_dimensions,- )- )- face_frameNumber_trackedPointNumber_XYZ[:] = np.nan-- all_body_tracked_points_visible_on_frame_bool_list = []- all_right_hand_points_visible_on_frame_bool_list = []- all_left_hand_points_visible_on_frame_bool_list = []- all_face_points_visible_on_frame_bool_list = []- all_tracked_points_visible_on_frame_list = []-- for frame_number, frame_results in enumerate(mediapipe_results_list):- # get the Body data (aka 'pose')- if frame_results.pose_landmarks is not None:- for landmark_number, landmark_data in enumerate(frame_results.pose_landmarks.landmark):- body_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (- landmark_data.x * image_width- )- body_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (- landmark_data.y * image_height- )- body_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (- landmark_data.z- * image_width- # z is on roughly the same scale as x, according to mediapipe docs- )- body_frameNumber_trackedPointNumber_confidence[- frame_number, landmark_number- ] = landmark_data.visibility # mediapipe calls their 'confidence' score 'visibility'-- for landmark_number, landmark_data in enumerate(frame_results.pose_world_landmarks.landmark):- body_world_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (- landmark_data.x * image_width- )- body_world_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (- landmark_data.y * image_height- )- body_world_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (- landmark_data.z * image_width- )-- # get Right Hand data- if frame_results.right_hand_landmarks is not None:- for landmark_number, landmark_data in enumerate(frame_results.right_hand_landmarks.landmark):- rightHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (- landmark_data.x * image_width- )- rightHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (- landmark_data.y * image_height- )- rightHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (- landmark_data.z * image_width- )-- # get Left Hand data- if frame_results.left_hand_landmarks is not None:- for landmark_number, landmark_data in enumerate(frame_results.left_hand_landmarks.landmark):- leftHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (- landmark_data.x * image_width- )- leftHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (- landmark_data.y * image_height- )- leftHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (- landmark_data.z * image_width- )-- # get Face data- if frame_results.face_landmarks is not None:- for landmark_number, landmark_data in enumerate(frame_results.face_landmarks.landmark):- face_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (- landmark_data.x * image_width- )- face_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (- landmark_data.y * image_height- )- face_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (- landmark_data.z * image_width- )-- # check if all tracked points are visible on this frame- all_body_visible = all(sum(np.isnan(body_frameNumber_trackedPointNumber_XYZ[frame_number, :, :])) == 0)- all_body_tracked_points_visible_on_frame_bool_list.append(all_body_visible)-- all_right_hand_visible = all(- sum(np.isnan(rightHand_frameNumber_trackedPointNumber_XYZ[frame_number, :, :])) == 0- )- all_right_hand_points_visible_on_frame_bool_list.append(all_right_hand_visible)-- all_left_hand_visible = all(- sum(np.isnan(leftHand_frameNumber_trackedPointNumber_XYZ[frame_number, :, :])) == 0- )- all_left_hand_points_visible_on_frame_bool_list.append(all_left_hand_visible)-- all_face_visible = all(sum(np.isnan(face_frameNumber_trackedPointNumber_XYZ[frame_number, :, :])) == 0)- all_face_points_visible_on_frame_bool_list.append(all_face_visible)-- all_points_visible = all(- [- all_body_visible,- all_right_hand_visible,- all_left_hand_visible,- all_face_visible,- ],- )-- all_tracked_points_visible_on_frame_list.append(all_points_visible)-- return Mediapipe2dNumpyArrays(- body_frameNumber_trackedPointNumber_XYZ=np.squeeze(body_frameNumber_trackedPointNumber_XYZ),- body_world_frameNumber_trackedPointNumber_XYZ=np.squeeze(body_world_frameNumber_trackedPointNumber_XYZ),- rightHand_frameNumber_trackedPointNumber_XYZ=np.squeeze(rightHand_frameNumber_trackedPointNumber_XYZ),- leftHand_frameNumber_trackedPointNumber_XYZ=np.squeeze(leftHand_frameNumber_trackedPointNumber_XYZ),- face_frameNumber_trackedPointNumber_XYZ=np.squeeze(face_frameNumber_trackedPointNumber_XYZ),- body_frameNumber_trackedPointNumber_confidence=np.squeeze(body_frameNumber_trackedPointNumber_confidence),- )diff --git a/freemocap/core_processes/export_data/blender_stuff/export_to_blender/methods/ajc_addon/run_ajc_addon_main.py b/freemocap/core_processes/export_data/blender_stuff/export_to_blender/methods/ajc_addon/run_ajc_addon_main.pyindex d5789cd7..cacb92e9 100644--- a/freemocap/core_processes/export_data/blender_stuff/export_to_blender/methods/ajc_addon/run_ajc_addon_main.py+++ b/freemocap/core_processes/export_data/blender_stuff/export_to_blender/methods/ajc_addon/run_ajc_addon_main.py@@ -13,6 +13,14 @@ from freemocap.core_processes.export_data.blender_stuff.export_to_blender.methodinstall_ajc_addon,)from freemocap.core_processes.export_data.blender_stuff.export_to_blender.methods.ajc_addon.run_simple import run_simple+from freemocap.system.paths_and_filenames.file_and_folder_names import (+ OLD_TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME,+ OLD_SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME,+ OLD_REPROJECTION_ERROR_NPY_FILE_NAME,+ REPROJECTION_ERROR_NPY_FILE_NAME,+ SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME,+ TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME,+)logger = logging.getLogger(__name__)@@ -37,6 +45,7 @@ def run_ajc_blender_addon_subprocess(blender_exe_path: Union[str, Path],):try:+ # TODO: pass active tracker into hereajc_blender_addon_validator(recording_folder_path=recording_folder_path)except FileNotFoundError as e:logger.error("Missing required files to run AJC addon, did something go wrong during processing?")@@ -114,42 +123,63 @@ def get_blenders_numpy(blender_exe_path: Union[str, Path]) -> str:return numpy_path-def ajc_blender_addon_validator(recording_folder_path: Union[str, Path]):+def ajc_blender_addon_validator(recording_folder_path: Union[str, Path], active_tracker: str = "mediapipe"):"""Check if the required files exist in the recording folder"""recording_path = Path(recording_folder_path)output_data_path = recording_path / "output_data"- if not (output_data_path / "mediapipe_body_3d_xyz.npy").exists():- raise FileNotFoundError(f"Could not find required file: {output_data_path / 'mediapipe_body_3d_xyz.npy'}")-- if not (output_data_path / "mediapipe_right_hand_3d_xyz.npy").exists():- raise FileNotFoundError(f"Could not find required file: {output_data_path / 'mediapipe_right_hand_3d_xyz.npy'}")-- if not (output_data_path / "mediapipe_left_hand_3d_xyz.npy").exists():- raise FileNotFoundError(f"Could not find required file: {output_data_path / 'mediapipe_left_hand_3d_xyz.npy'}")+ if active_tracker[-1] != "_":+ active_tracker = active_tracker + "_"- if not (output_data_path / "mediapipe_face_3d_xyz.npy").exists():- raise FileNotFoundError(f"Could not find required file: {output_data_path / 'mediapipe_face_3d_xyz.npy'}")+ if not (output_data_path / (active_tracker + "body_3d_xyz.npy")).exists():+ raise FileNotFoundError(+ f"Could not find required file: {output_data_path / (active_tracker + 'body_3d_xyz.npy')}"+ )- if not (output_data_path / "center_of_mass" / "total_body_center_of_mass_xyz.npy").exists():+ if not (output_data_path / (active_tracker + "right_hand_3d_xyz.npy")).exists():raise FileNotFoundError(- f"Could not find required file: {output_data_path / 'center_of_mass' / 'total_body_center_of_mass_xyz.npy'}"+ f"Could not find required file: {output_data_path / (active_tracker + 'right_hand_3d_xyz.npy')}")- if not (output_data_path / "center_of_mass" / "segmentCOM_frame_joint_xyz.npy").exists():+ if not (output_data_path / (active_tracker + "left_hand_3d_xyz.npy")).exists():raise FileNotFoundError(- f"Could not find required file: {output_data_path / 'center_of_mass' / 'segmentCOM_frame_joint_xyz.npy'}"+ f"Could not find required file: {output_data_path / (active_tracker + 'left_hand_3d_xyz.npy')}")- if not (- output_data_path / "raw_data" / "mediapipe3dData_numFrames_numTrackedPoints_reprojectionError.npy"- ).exists():+ if not (output_data_path / (active_tracker + "face_3d_xyz.npy")).exists():raise FileNotFoundError(- f"Could not find required file: {output_data_path / 'raw_data' / 'mediapipe3dData_numFrames_numTrackedPoints_reprojectionError.npy'}"+ f"Could not find required file: {output_data_path / (active_tracker + 'face_3d_xyz.npy')}")+ if not (output_data_path / "center_of_mass" / (active_tracker + TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME)).exists():+ if (+ active_tracker != "mediapipe_"+ or not (output_data_path / "center_of_mass" / OLD_TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME).exists()+ ):+ raise FileNotFoundError(+ f"Could not find required file: {output_data_path / 'center_of_mass' / (active_tracker + TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME)}"+ )++ if not (output_data_path / "center_of_mass" / (active_tracker + SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME)).exists():+ if (+ active_tracker != "mediapipe_"+ or not (output_data_path / "center_of_mass" / OLD_SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME).exists()+ ):+ raise FileNotFoundError(+ f"Could not find required file: {output_data_path / 'center_of_mass' / (active_tracker + SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME)}"+ )++ if not (output_data_path / "raw_data" / (active_tracker + REPROJECTION_ERROR_NPY_FILE_NAME)).exists():+ if (+ active_tracker != "mediapipe_"+ or not (output_data_path / "raw_data" / OLD_REPROJECTION_ERROR_NPY_FILE_NAME).exists()+ ):+ raise FileNotFoundError(+ f"Could not find required file: {output_data_path / 'raw_data' / (active_tracker + REPROJECTION_ERROR_NPY_FILE_NAME)}"+ )+if __name__ == "__main__":from freemocap.core_processes.export_data.blender_stuff.get_best_guess_of_blender_path import (diff --git a/freemocap/core_processes/post_process_skeleton_data/calculate_center_of_mass.py b/freemocap/core_processes/post_process_skeleton_data/calculate_center_of_mass.pyindex 187627eb..bdf8e02a 100644--- a/freemocap/core_processes/post_process_skeleton_data/calculate_center_of_mass.py+++ b/freemocap/core_processes/post_process_skeleton_data/calculate_center_of_mass.py@@ -1,405 +1,140 @@import logging-from typing import List-+from typing import Dict, Tupleimport numpy as np-import pandas as pd-from rich.progress import track-from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.data_models.mediapipe_skeleton_names_and_connections import (- mediapipe_body_landmark_names,-)+from freemocap.data_layer.skeleton_models.segments import SegmentAnthropometry+from freemocap.data_layer.skeleton_models.skeleton import Skeletonlogger = logging.getLogger(__name__)-def mediapipe_body_names_match_expected(mediapipe_body_landmark_names: List[str]) -> bool:+def calculate_all_segments_com(+ segment_positions: Dict[str, Dict[str, np.ndarray]], center_of_mass_definitions: Dict[str, SegmentAnthropometry]+) -> Dict[str, np.ndarray]:+ """+ Calculates the center of mass (COM) for each segment based on provided segment positions+ and anthropometric data.++ Parameters:+ - segment_positions: A dictionary where each key is a segment name, and the value is another+ dictionary with 'proximal' and 'distal' keys containing the respective marker positions+ as numpy arrays.+ - center_of_mass_definitions: A dictionary containing anthropometric information for each segment.+ Each key is a segment name with a value that is a dictionary, which includes the+ 'segment_com_length' key representing the percentage distance from the proximal marker+ to the segment's COM.++ Returns:+ - A dictionary where each key is a segment name, and the value is the calculated COM+ position as a numpy array."""- Check if the mediapipe folks have changed their landmark names. If they have, then this function may need to be updated.- Args:- mediapipe_body_landmark_names: List of strings, each string is the name of a mediapipe landmark.+ segment_com_data = {}++ for segment_name, segment_info in center_of_mass_definitions.items():+ proximal = segment_positions[segment_name]["proximal"]+ distal = segment_positions[segment_name]["distal"]++ com_length = segment_info.segment_com_length++ segment_com = proximal + (distal - proximal) * com_length++ segment_com_data[segment_name] = segment_com++ return segment_com_data+++def calculate_total_body_center_of_mass(+ segment_center_of_mass_data: Dict[str, np.ndarray],+ center_of_mass_definitions: Dict[str, SegmentAnthropometry],+ num_frames: int,+) -> np.ndarray:+ """+ Calculates the total body center of mass for each frame based on segment COM positions and anthropometric data.++ Parameters:+ - segment_com_data: A dictionary with segment names as keys and COM positions as values for each frame.+ - center_of_mass_definitions: A dictionary containing segment mass percentages.+ - num_frames: The number of frames in the data.Returns:- bool: True if the mediapipe landmark names are as expected, False otherwise.+ - A numpy array containing the position of the total body center of mass for each frame."""- expected_mediapipe_body_landmark_names = [- "nose",- "left_eye_inner",- "left_eye",- "left_eye_outer",- "right_eye_inner",- "right_eye",- "right_eye_outer",- "left_ear",- "right_ear",- "mouth_left",- "mouth_right",- "left_shoulder",- "right_shoulder",- "left_elbow",- "right_elbow",- "left_wrist",- "right_wrist",- "left_pinky",- "right_pinky",- "left_index",- "right_index",- "left_thumb",- "right_thumb",- "left_hip",- "right_hip",- "left_knee",- "right_knee",- "left_ankle",- "right_ankle",- "left_heel",- "right_heel",- "left_foot_index",- "right_foot_index",- ]- return mediapipe_body_landmark_names == expected_mediapipe_body_landmark_names---def return_indices_of_joints(list_of_indices, list_of_joint_names):- indices = []- for name in list_of_joint_names:- this_name_index = list_of_indices.index(name)- indices.append(this_name_index)-- return indices---# %%---def return_XYZ_coordinates_of_markers(freemocap_data, indices_list, frame):- XYZ_coordinates = []- for index in indices_list:- this_joint_coordinate = freemocap_data[frame, index, :]- XYZ_coordinates.append(this_joint_coordinate)-- return XYZ_coordinates---# %%---def build_virtual_trunk_marker(freemocap_data, list_of_indices, trunk_joint_connection, frame):- trunk_marker_indices = return_indices_of_joints(list_of_indices, trunk_joint_connection)-- trunk_XYZ_coordinates = return_XYZ_coordinates_of_markers(freemocap_data, trunk_marker_indices, frame)-- trunk_proximal = (trunk_XYZ_coordinates[0] + trunk_XYZ_coordinates[1]) / 2- trunk_distal = (trunk_XYZ_coordinates[2] + trunk_XYZ_coordinates[3]) / 2-- return trunk_proximal, trunk_distal---# %%---def build_mediapipe_skeleton(mediapipe_pose_data, segment_dataframe, mediapipe_indices) -> list:- """This function takes in the mediapipe pose data array and the segment_conn_len_perc_dataframe.- For each frame of data, it loops through each segment we want to find and identifies the names- of the proximal and distal joints of that segment. Then it searches the mediapipe_indices list- to find the index that corresponds to the name of that segment. We plug the index into the- mediapipe_pose_data array to find the proximal/distal joints' XYZ coordinates at that frame.- The segment, its proximal joint and its distal joint gets thrown into a dictionary.- And then that dictionary gets saved to a list for each frame. By the end of the function, you- have a list that contains the skeleton segment XYZ coordinates for each frame."""-- num_frames = mediapipe_pose_data.shape[0]- num_frame_range = range(num_frames)-- mediapipe_frame_segment_joint_XYZ = [] # empty list to hold all the skeleton XYZ coordinates/frame-- for frame in track(num_frame_range, description="Building a MediaPipe Skeleton"):- trunk_joint_connection = [- "left_shoulder",- "right_shoulder",- "left_hip",- "right_hip",- ]- trunk_virtual_markers = build_virtual_trunk_marker(- mediapipe_pose_data, mediapipe_indices, trunk_joint_connection, frame- )-- mediapipe_pose_skeleton_coordinates = {}- for (- segment,- segment_info,- ) in (- segment_dataframe.iterrows()- ): # iterate through the data frame by the segment name and all the info for that segment- if segment == "trunk":- # based on index, extract coordinate data from fmc mediapipe data- mediapipe_pose_skeleton_coordinates[segment] = [- trunk_virtual_markers[0],- trunk_virtual_markers[1],- ]- elif segment == "left_hand" or segment == "right_hand":- proximal_joint_hand = segment_info["Joint_Connection"][0]- if segment == "left_hand":- distal_joint_hand = "left_index"- else:- distal_joint_hand = "right_index"-- proximal_joint_hand_index = mediapipe_indices.index(proximal_joint_hand)- distal_joint_hand_index = mediapipe_indices.index(distal_joint_hand)-- mediapipe_pose_skeleton_coordinates[segment] = [- mediapipe_pose_data[frame, proximal_joint_hand_index, :],- mediapipe_pose_data[frame, distal_joint_hand_index, :],- ]-- elif segment == "left_foot" or segment == "right_foot":- if segment == "left_foot":- proximal_joint_foot_name = "left_ankle"- else:- proximal_joint_foot_name = "right_ankle"-- proximal_joint_foot_index = mediapipe_indices.index(proximal_joint_foot_name)-- distal_joint_foot = segment_info["Joint_Connection"][1]- distal_joint_foot_index = mediapipe_indices.index(distal_joint_foot)- mediapipe_pose_skeleton_coordinates[segment] = [- mediapipe_pose_data[frame, proximal_joint_foot_index, :],- mediapipe_pose_data[frame, distal_joint_foot_index, :],- ]-- else:- proximal_joint_name = segment_info["Joint_Connection"][0]- distal_joint_name = segment_info["Joint_Connection"][1]-- # get the mediapipe index for the proximal and distal joint for this segment- proximal_joint_index = mediapipe_indices.index(proximal_joint_name)- distal_joint_index = mediapipe_indices.index(distal_joint_name)-- # use the mediapipe indices to get the XYZ coordinates for the prox/distal joints and throw it in a dictionary- # mediapipe_pose_skeleton_coordinates[segment] = {'proximal':mediapipe_pose_data[frame,proximal_joint_index,:],'distal':mediapipe_pose_data[frame,distal_joint_index,:]}- mediapipe_pose_skeleton_coordinates[segment] = [- mediapipe_pose_data[frame, proximal_joint_index, :],- mediapipe_pose_data[frame, distal_joint_index, :],- ]-- mediapipe_frame_segment_joint_XYZ.append(mediapipe_pose_skeleton_coordinates)-- return mediapipe_frame_segment_joint_XYZ---# %%-# values for segment weight and segment mass percentages taken from Winter anthropometry tables-# https://imgur.com/a/aD74j-# Winter, D.A. (2005) Biomechanics and Motor Control of Human Movement. 3rd Edition, John Wiley & Sons, Inc., Hoboken.---BODY_SEGMENT_NAMES = [- "head",- "trunk",- "right_upper_arm",- "left_upper_arm",- "right_forearm",- "left_forearm",- "right_hand",- "left_hand",- "right_thigh",- "left_thigh",- "right_shin",- "left_shin",- "right_foot",- "left_foot",-]--joint_connections = [- ["left_ear", "right_ear"],- ["mid_chest_marker", "mid_hip_marker"],- ["right_shoulder", "right_elbow"],- ["left_shoulder", "left_elbow"],- ["right_elbow", "right_wrist"],- ["left_elbow", "left_wrist"],- ["right_wrist", "right_hand_marker"],- ["left_wrist", "left_hand_marker"],- ["right_hip", "right_knee"],- ["left_hip", "left_knee"],- ["right_knee", "right_ankle"],- ["left_knee", "left_ankle"],- ["right_back_of_foot_marker", "right_foot_index"],- ["left_back_of_foot_marker", "left_foot_index"],-]--segment_COM_lengths = [- 0.5,- 0.5,- 0.436,- 0.436,- 0.430,- 0.430,- 0.506,- 0.506,- 0.433,- 0.433,- 0.433,- 0.433,- 0.5,- 0.5,-]--segment_COM_percentages = [- 0.081,- 0.497,- 0.028,- 0.028,- 0.016,- 0.016,- 0.006,- 0.006,- 0.1,- 0.1,- 0.0465,- 0.0465,- 0.0145,- 0.0145,-]---# %%-def calculate_segment_COM(- segment_conn_len_perc_dataframe,- skelcoordinates_frame_segment_joint_XYZ,- num_frame_range,-):- segment_COM_frame_dict = []- for frame in track(num_frame_range, description="Calculating Segment Center of Mass"):- segment_COM_dict = {}- for segment, segment_info in segment_conn_len_perc_dataframe.iterrows():- this_segment_XYZ = skelcoordinates_frame_segment_joint_XYZ[frame][segment]-- # for mediapipe- this_segment_proximal = this_segment_XYZ[0]- this_segment_distal = this_segment_XYZ[1]- this_segment_COM_length = segment_info["Segment_COM_Length"]-- this_segment_COM = this_segment_proximal + this_segment_COM_length * (- this_segment_distal - this_segment_proximal- )- segment_COM_dict[segment] = this_segment_COM- segment_COM_frame_dict.append(segment_COM_dict)- return segment_COM_frame_dict---# %%---def reformat_segment_COM(segment_COM_frame_dict, num_frame_range, num_segments):- segment_COM_frame_imgPoint_XYZ = np.empty([int(len(num_frame_range)), int(num_segments), 3])- for frame in num_frame_range:- this_frame_skeleton = segment_COM_frame_dict[frame]- for joint_count, segment in enumerate(this_frame_skeleton.keys()):- segment_COM_frame_imgPoint_XYZ[frame, joint_count, :] = this_frame_skeleton[segment]- return segment_COM_frame_imgPoint_XYZ---# %%---def calculate_total_body_COM(segment_conn_len_perc_dataframe, segment_COM_frame_dict, num_frame_range):- logger.info("Calculating Total Body Center of Mass")- totalBodyCOM_frame_XYZ = np.empty([int(len(num_frame_range)), 3])-- for frame in track(num_frame_range, description="Calculating Total Body Center of Mass..."):- this_frame_total_body_percentages = []- this_frame_skeleton = segment_COM_frame_dict[frame]-- for segment, segment_info in segment_conn_len_perc_dataframe.iterrows():- this_segment_COM = this_frame_skeleton[segment]- this_segment_COM_percentage = segment_info["Segment_COM_Percentage"]-- this_segment_total_body_percentage = this_segment_COM * this_segment_COM_percentage- this_frame_total_body_percentages.append(this_segment_total_body_percentage)-- this_frame_total_body_COM = np.nansum(this_frame_total_body_percentages, axis=0)-- totalBodyCOM_frame_XYZ[frame, :] = this_frame_total_body_COM-- return totalBodyCOM_frame_XYZ---# %%---def build_anthropometric_dataframe(- segments: list,- joint_connections: list,- segment_COM_lengths: list,- segment_COM_percentages: list,-) -> pd.DataFrame:- # load anthropometric data into a pandas dataframe- df = pd.DataFrame(- list(- zip(- segments,- joint_connections,- segment_COM_lengths,- segment_COM_percentages,- )- ),- columns=[- "Segment_Name",- "Joint_Connection",- "Segment_COM_Length",- "Segment_COM_Percentage",- ],- )- segment_conn_len_perc_dataframe = df.set_index("Segment_Name")- return segment_conn_len_perc_dataframe+ total_body_com = np.zeros((num_frames, 3))+ for segment_name, segment_info in center_of_mass_definitions.items():-def calculate_center_of_mass(- freemocap_marker_data_array: np.ndarray,- pose_estimation_skeleton: list,- anthropometric_info_dataframe: pd.DataFrame,-):- num_frames = freemocap_marker_data_array.shape[0]- num_frame_range = range(num_frames)- num_segments = len(anthropometric_info_dataframe)+ segment_com = segment_center_of_mass_data.get(segment_name)+ if segment_com is None:+ raise ValueError(f"Segment {segment_name} not found in segment center of mass data.")- segment_COM_frame_dict = calculate_segment_COM(- anthropometric_info_dataframe, pose_estimation_skeleton, num_frame_range- )- segment_COM_frame_imgPoint_XYZ = reformat_segment_COM(segment_COM_frame_dict, num_frame_range, num_segments)- totalBodyCOM_frame_XYZ = calculate_total_body_COM(- anthropometric_info_dataframe, segment_COM_frame_dict, num_frame_range- )+ segment_mass_percentage = segment_info.segment_com_percentage- return (- segment_COM_frame_dict,- segment_COM_frame_imgPoint_XYZ,- totalBodyCOM_frame_XYZ,- )+ total_body_com += segment_com * segment_mass_percentage+ return total_body_com-def run_center_of_mass_calculations(processed_skel3d_frame_marker_xyz: np.ndarray):- anthropometric_info_dataframe = build_anthropometric_dataframe(- BODY_SEGMENT_NAMES, joint_connections, segment_COM_lengths, segment_COM_percentages- )- if not mediapipe_body_names_match_expected(mediapipe_body_landmark_names):- raise ValueError(- "Mediapipe body landmark names do not match expected names - Perhaps they altered the names in a new version? This code will need to be updated"- )-- skelcoordinates_frame_segment_joint_XYZ = build_mediapipe_skeleton(- processed_skel3d_frame_marker_xyz,- anthropometric_info_dataframe,- mediapipe_body_landmark_names,- )- (- segment_COM_frame_dict,- segment_COM_frame_imgPoint_XYZ,- totalBodyCOM_frame_XYZ,- ) = calculate_center_of_mass(- processed_skel3d_frame_marker_xyz,- skelcoordinates_frame_segment_joint_XYZ,- anthropometric_info_dataframe,++def get_all_segment_markers(skeleton: Skeleton) -> Dict[str, Dict[str, np.ndarray]]:+ """+ Retrieves the proximal and distal marker positions for all segments within a skeleton.++ Parameters:+ - skeleton: The Skeleton instance containing segment information and marker data.++ Returns:+ - A dictionary where each key is a segment name, and the value is another dictionary+ with 'proximal' and 'distal' keys containing the respective marker positions as numpy arrays.+ """+ segment_positions = {}++ if skeleton.segments is None:+ raise ValueError("Segments must be defined before getting segment markers.")++ for segment_name in skeleton.segments.keys():+ segment_positions[segment_name] = skeleton.get_segment_markers(segment_name)++ return segment_positions+++def merge_segment_com_data(segment_com_data: Dict[str, np.ndarray]) -> np.ndarray:+ """+ Merges the center of mass data from multiple segments into a single array.++ Parameters:+ - segment_com_data: A dictionary where each key is a segment name and the value is the center of mass data for that segment.++ Returns:+ - A numpy array containing the merged center of mass data.+ """+ com_data_list = list(segment_com_data.values())++ return np.stack(com_data_list, axis=1)+++def calculate_center_of_mass_from_skeleton(skeleton: Skeleton) -> Tuple[np.ndarray, np.ndarray]:+ """+ Calculates the segment and total body center of mass for a skeleton based on anthropometric data.++ Parameters:+ - skeleton: The Skeleton instance containing marker data and segment information.+ - anthropometric_data: A dictionary containing segment mass percentages++ Returns:+ - A tuple containing the segment center of mass data and the total body center of mass.+ """+ if skeleton.center_of_mass_definitions is None:+ raise ValueError("Segment center of mass definitions must be defined before calculating center of mass.")++ if skeleton.segments is None:+ raise ValueError("Segments must be defined before calculating center of mass.")++ segment_3d_positions = get_all_segment_markers(skeleton) # TODO: Should this be a method of the skeleton model?+ segment_com_data = calculate_all_segments_com(segment_3d_positions, skeleton.center_of_mass_definitions)+ total_body_com = calculate_total_body_center_of_mass(+ segment_com_data, skeleton.center_of_mass_definitions, skeleton.num_frames)- return segment_COM_frame_imgPoint_XYZ, totalBodyCOM_frame_XYZ+ merged_segment_com_data = merge_segment_com_data(segment_com_data)++ return merged_segment_com_data, total_body_comdiff --git a/freemocap/core_processes/post_process_skeleton_data/create_skeleton.py b/freemocap/core_processes/post_process_skeleton_data/create_skeleton.pynew file mode 100644index 00000000..5e329c9c--- /dev/null+++ b/freemocap/core_processes/post_process_skeleton_data/create_skeleton.py@@ -0,0 +1,67 @@+from typing import List, Dict, Optional+++from freemocap.data_layer.skeleton_models.marker_info import MarkerInfo+from freemocap.data_layer.skeleton_models.segments import Segment, SegmentAnthropometry+from freemocap.data_layer.skeleton_models.skeleton import Skeleton+++def create_marker_info(+ marker_list: List[str], virtual_markers: Optional[Dict[str, Dict[str, List]]] = None+) -> MarkerInfo:+ """+ Creates a MarkerInfo instance from a list of actual marker names and optional virtual markers.++ Parameters:+ - marker_list: A list of strings representing the names of actual markers.+ - virtual_markers: A dictionary defining virtual markers and their related data. Each key is a virtual marker name,+ and its value is another dictionary with 'marker_names' and 'marker_weights' as keys.++ Returns:+ - An instance of MarkerInfo populated with actual and, if provided, virtual markers.+ """+ marker_hub = MarkerInfo.create(marker_list=marker_list)++ if virtual_markers:+ marker_hub.add_virtual_markers(virtual_markers)+ return marker_hub+++def create_skeleton_model(+ actual_markers: List[str],+ num_tracked_points: int,+ segment_connections: Optional[Dict[str, Segment]] = None,+ virtual_markers: Optional[Dict[str, Dict[str, List]]] = None,+ joint_hierarchy: Optional[Dict[str, List[str]]] = None,+ center_of_mass_info: Optional[Dict[str, SegmentAnthropometry]] = None,+) -> Skeleton:+ """+ Creates a Skeleton model that includes both actual and optionally virtual markers+ Optionally integrates segment connections, joint hierarchy, and anthro data if needed+ Parameters:+ - actual_markers: A list of strings representing the names of actual markers.+ - num_tracked_points: The number of tracked points expected in the 3D data.+ - segment_connections: Optional; A dictionary where each key is a segment name and its value is a dictionary+ with information about that segment (e.g., 'proximal', 'distal' marker names).+ - virtual_markers: Optional; a dictionary with information necessary to compute virtual markers.+ - joint_hierarchy: Optional; a dictionary with joint names as keys and lists of connected marker names as values.+ - center_of_mass_info: Optional; a dictionary containing segment mass percentages++ Returns:+ - An instance of the Skeleton class that represents the complete skeletal model including markers, segments,+ and optionally, joint hierarchy data.+ """+ marker_hub = create_marker_info(marker_list=actual_markers, virtual_markers=virtual_markers)++ skeleton_model = Skeleton(markers=marker_hub, num_tracked_points=num_tracked_points)++ if segment_connections:+ skeleton_model.add_segments(segment_connections)++ if joint_hierarchy:+ skeleton_model.add_joint_hierarchy(joint_hierarchy)++ if center_of_mass_info:+ skeleton_model.add_center_of_mass_definitions(center_of_mass_info)++ return skeleton_modeldiff --git a/freemocap/core_processes/post_process_skeleton_data/enforce_rigid_bones.py b/freemocap/core_processes/post_process_skeleton_data/enforce_rigid_bones.pynew file mode 100644index 00000000..3dd7970b--- /dev/null+++ b/freemocap/core_processes/post_process_skeleton_data/enforce_rigid_bones.py@@ -0,0 +1,149 @@+from copy import deepcopy+import numpy as np+from typing import Dict, List, Union++from freemocap.data_layer.skeleton_models.segments import Segment+from freemocap.data_layer.skeleton_models.skeleton import Skeleton+++def calculate_bone_lengths_and_statistics(+ marker_data: Dict[str, np.ndarray], segment_connections: Dict[str, Segment]+) -> Dict[str, Dict[str, Union[np.ndarray, float]]]:+ """+ Calculates bone lengths for each frame and their statistics (median and standard deviation)+ based on marker positions and segment connections.++ Parameters:+ - marker_data: A dictionary containing marker trajectories with marker names as keys and+ 3D positions as values (numpy arrays).+ - segment_connections: A dictionary defining the segments (bones) with segment names as keys+ and dictionaries with 'proximal' and 'distal' markers as values.++ Returns:+ - A dictionary with segment names as keys and dictionaries with lengths, median lengths,+ and standard deviations as values.+ """+ bone_statistics = {}++ for segment_name, segment in segment_connections.items():+ proximal_positions = marker_data[segment.proximal]+ distal_positions = marker_data[segment.distal]++ lengths = np.linalg.norm(distal_positions - proximal_positions, axis=1)+ valid_lengths = lengths[~np.isnan(lengths)]++ median_length = np.median(valid_lengths)+ stdev_length = np.std(valid_lengths)++ bone_statistics[segment_name] = {"lengths": lengths, "median": median_length, "stdev": stdev_length}++ return bone_statistics+++def enforce_rigid_bones(+ marker_data: Dict[str, np.ndarray],+ segment_connections: Dict[str, Segment],+ bone_lengths_and_statistics: Dict[str, Dict[str, Union[np.ndarray, float]]],+ joint_hierarchy: Dict[str, List[str]],+) -> Dict[str, np.ndarray]:+ """+ Enforces rigid bones by adjusting the distal joints of each segment to match the median length.++ Parameters:+ - marker_data: The original marker positions.+ - segment_connections: Information about how segments (bones) are connected.+ - bone_lengths_and_statistics: The desired bone lengths and statistics for each segment.+ - joint_hierarchy: The hierarchy of joints, indicating parent-child relationships.++ Returns:+ - A dictionary of adjusted marker positions.+ """+ rigid_marker_data = deepcopy(marker_data)++ for segment_name, stats in bone_lengths_and_statistics.items():+ desired_length = stats["median"]+ lengths = stats["lengths"]++ segment = segment_connections[segment_name]+ proximal_marker, distal_marker = segment.proximal, segment.distal++ for frame_index, current_length in enumerate(lengths):+ if current_length != desired_length:+ proximal_position = marker_data[proximal_marker][frame_index]+ distal_position = marker_data[distal_marker][frame_index]+ direction = distal_position - proximal_position+ try:+ direction /= np.linalg.norm(direction) # Normalize to unit vector+ except ZeroDivisionError:+ direction /= 1e-5 # Set to a small value if the direction is zero+ adjustment = (desired_length - current_length) * direction++ rigid_marker_data[distal_marker][frame_index] += adjustment++ adjust_children(distal_marker, frame_index, adjustment, rigid_marker_data, joint_hierarchy)++ return rigid_marker_data+++def adjust_children(+ parent_marker: str,+ frame_index: int,+ adjustment: np.ndarray,+ marker_data: Dict[str, np.ndarray],+ joint_hierarchy: Dict[str, List[str]],+):+ """+ Recursively adjusts the positions of child markers based on the adjustment of the parent marker.+ """+ if parent_marker in joint_hierarchy:+ for child_marker in joint_hierarchy[parent_marker]:+ marker_data[child_marker][frame_index] += adjustment+ adjust_children(child_marker, frame_index, adjustment, marker_data, joint_hierarchy)+++def merge_rigid_marker_data(rigid_marker_data: Dict[str, np.ndarray]) -> np.ndarray:+ """+ Merges the center of mass data from multiple segments into a single array.++ Parameters:+ - segment_com_data: A dictionary where each key is a segment name and the value is the center of mass data for that segment.++ Returns:+ - A numpy array containing the merged center of mass data.+ """+ # TODO: We could use this more broadly as a skeleton method++ rigid_marker_data_list = list(rigid_marker_data.values())++ return np.stack(rigid_marker_data_list, axis=1)+++def enforce_rigid_bones_from_skeleton(skeleton: Skeleton) -> np.ndarray:+ """+ Calculates bone lengths and statistics from skeleton data and enforces rigid bones.++ Parameters:+ - skeleton: The Skeleton instance containing segment information, joint hierarchy, and marker data.++ Returns:+ - A numpy array of adjusted marker positions.+ """+ # TODO: Should this be a method of Skeleton?+ if not skeleton.segments:+ raise ValueError("Segments must be defined before rigid bones can be enforced.")++ if not skeleton.joint_hierarchy:+ raise ValueError("Joint hierarchy must be defined before rigid bones can be enforced.")++ bone_lengths_and_statistcs = calculate_bone_lengths_and_statistics(+ marker_data=skeleton.marker_data, segment_connections=skeleton.segments+ )++ rigid_marker_data = enforce_rigid_bones(+ marker_data=skeleton.marker_data,+ segment_connections=skeleton.segments,+ bone_lengths_and_statistics=bone_lengths_and_statistcs,+ joint_hierarchy=skeleton.joint_hierarchy,+ )++ return merge_rigid_marker_data(rigid_marker_data=rigid_marker_data)diff --git a/freemocap/core_processes/post_process_skeleton_data/post_process_skeleton.py b/freemocap/core_processes/post_process_skeleton_data/post_process_skeleton.pyindex 535f0c0f..f238d121 100644--- a/freemocap/core_processes/post_process_skeleton_data/post_process_skeleton.py+++ b/freemocap/core_processes/post_process_skeleton_data/post_process_skeleton.py@@ -16,6 +16,7 @@ from skellyforge.freemocap_utils.constants import (TASK_FINDING_GOOD_FRAME,)from skellyforge.freemocap_utils.postprocessing_widgets.task_worker_thread import TaskWorkerThread+from skellytracker.trackers.base_tracker.model_info import ModelInfofrom freemocap.system.logging.configure_logging import log_view_logging_format_stringfrom freemocap.system.logging.queue_logger import DirectQueueHandler@@ -32,11 +33,7 @@ class PostProcessedDataHandler:self.processed_skeleton = processed_skeleton-def save_numpy_array_to_disk(- array_to_save: np.ndarray,- file_name: str,- save_directory: Union[str, Path]-):+def save_numpy_array_to_disk(array_to_save: np.ndarray, file_name: str, save_directory: Union[str, Path]):if not file_name.endswith(".npy"):file_name += ".npy"Path(save_directory).mkdir(parents=True, exist_ok=True)@@ -67,7 +64,9 @@ def adjust_default_settings(filter_sampling_rate, filter_cutoff_frequency, filtereturn adjusted_settings-def run_post_processing_worker(raw_skel3d_frame_marker_xyz: np.ndarray, settings_dictionary: dict):+def run_post_processing_worker(+ raw_skel3d_frame_marker_xyz: np.ndarray, settings_dictionary: dict, landmark_names: list+) -> np.ndarray:def handle_thread_finished(results, post_processed_data_handler: PostProcessedDataHandler):processed_skeleton = results[TASK_FILTERING]["result"]post_processed_data_handler.data_callback(processed_skeleton=processed_skeleton)@@ -84,6 +83,7 @@ def run_post_processing_worker(raw_skel3d_frame_marker_xyz: np.ndarray, settingstask_list=task_list,settings=settings_dictionary,all_tasks_finished_callback=lambda results: handle_thread_finished(results, post_processed_data_handler),+ landmark_names=landmark_names,)worker_thread.start()worker_thread.join()@@ -93,7 +93,7 @@ def run_post_processing_worker(raw_skel3d_frame_marker_xyz: np.ndarray, settingsdef post_process_data(- recording_processing_parameter_model, raw_skel3d_frame_marker_xyz: np.ndarray, queue: multiprocessing.Queue+ recording_processing_parameter_model, raw_skel3d_frame_marker_xyz: np.ndarray, queue: multiprocessing.Queue) -> np.ndarray:if queue:handler = DirectQueueHandler(queue)@@ -104,7 +104,18 @@ def post_process_data()adjusted_settings = adjust_default_settings(filter_sampling_rate, filter_cutoff_frequency, filter_order)processed_skeleton_array = run_post_processing_worker(- raw_skel3d_frame_marker_xyz=raw_skel3d_frame_marker_xyz, settings_dictionary=adjusted_settings+ raw_skel3d_frame_marker_xyz=raw_skel3d_frame_marker_xyz,+ settings_dictionary=adjusted_settings,+ landmark_names=get_landmark_names(model_info=recording_processing_parameter_model.tracking_model_info),)return processed_skeleton_array+++def get_landmark_names(model_info: ModelInfo) -> list:+ if hasattr(model_info, "body_landmark_names"):+ return model_info.body_landmark_names+ elif hasattr(model_info, "landmark_names"):+ return model_info.landmark_names+ else:+ raise AttributeError("Model does not have landmark names")diff --git a/freemocap/core_processes/post_process_skeleton_data/process_single_camera_skeleton_data.py b/freemocap/core_processes/post_process_skeleton_data/process_single_camera_skeleton_data.pyindex 7c7a6fb2..a3498918 100644--- a/freemocap/core_processes/post_process_skeleton_data/process_single_camera_skeleton_data.py+++ b/freemocap/core_processes/post_process_skeleton_data/process_single_camera_skeleton_data.py@@ -4,8 +4,8 @@ from typing import Tuple, Unionimport numpy as np-from freemocap.core_processes.capture_volume_calibration.save_mediapipe_3d_data_to_npy import (- save_mediapipe_3d_data_to_npy,+from freemocap.core_processes.capture_volume_calibration.save_3d_data_to_npy import (+ save_3d_data_to_npy,)from freemocap.utilities.geometry.project_3d_data_to_z_plane import project_3d_data_to_z_plane@@ -15,6 +15,7 @@ logger = logging.getLogger(__name__)def process_single_camera_skeleton_data(input_image_data_frame_marker_xyz: np.ndarray,raw_data_folder_path: Union[str, Path],+ file_prefix: str = "",project_to_z_plane: bool = True,) -> Tuple[np.ndarray, np.ndarray]:skeleton_reprojection_error_fr_mar = np.zeros(input_image_data_frame_marker_xyz.shape[0:2])@@ -30,12 +31,13 @@ def process_single_camera_skeleton_data(else:raw_skel3d_frame_marker_xyz = input_image_data_frame_marker_xyz- save_mediapipe_3d_data_to_npy(+ save_3d_data_to_npy(data3d_numFrames_numTrackedPoints_XYZ=raw_skel3d_frame_marker_xyz,data3d_numFrames_numTrackedPoints_reprojectionError=skeleton_reprojection_error_fr_mar,data3d_numCams_numFrames_numTrackedPoints_reprojectionError=skeleton_reprojection_error_cam_fr_mar,path_to_folder_where_data_will_be_saved=raw_data_folder_path,processing_level="raw",+ file_prefix=file_prefix,)return raw_skel3d_frame_marker_xyz, skeleton_reprojection_error_fr_mardiff --git a/freemocap/core_processes/post_process_skeleton_data/split_and_save.py b/freemocap/core_processes/post_process_skeleton_data/split_and_save.pynew file mode 100644index 00000000..8c2d6bec--- /dev/null+++ b/freemocap/core_processes/post_process_skeleton_data/split_and_save.py@@ -0,0 +1,199 @@+from pathlib import Path+from typing import Dict, List+import logging+import numpy as np+import pandas as pd++from skellytracker.trackers.base_tracker.model_info import ModelInfo++from freemocap.system.paths_and_filenames.file_and_folder_names import (+ BODY_3D_DATAFRAME_CSV_FILE_NAME,+ FACE_3D_DATAFRAME_CSV_FILE_NAME,+ LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME,+ RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME,+)++logger = logging.getLogger(__name__)+++# TODO: This is dependent on the ordering we currently use for mediapipe, maybe it should live in model_info+category_info = {+ "body": BODY_3D_DATAFRAME_CSV_FILE_NAME,+ "right_hand": RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME,+ "left_hand": LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME,+ "face": FACE_3D_DATAFRAME_CSV_FILE_NAME,+}++# TODO: come up with a sensible way of splitting this for different types of trackers+++def split_data(+ skeleton_3d_data: np.ndarray,+ model_info: ModelInfo,+) -> Dict[str, np.ndarray]:+ """+ Splits the skeleton data into separate arrays for each data category using the provided model info.+ Examples categories are body, left_hand, right_hand, and face.++ Parameters:+ - skeleton: The skeleton data to be split.+ - model_info: The model info used in tracking.++ Returns:+ - A dictionary where each key is a category and the value is the corresponding data array.+ """++ split_data = {}++ # TODO: this function probably shouldn't exist, and its existence implies we should be separately tracking body, hands, and face++ current_index = 0+ for category in category_info.keys():+ tracked_points_name = "num_tracked_points_" + category+ if hasattr(model_info, tracked_points_name):+ prev_index = current_index+ current_index += getattr(model_info, tracked_points_name)+ split_data[category] = skeleton_3d_data[:, prev_index:current_index, :]++ if split_data == {}:+ logger.debug("No data categories found in model info, skipping data splitting")++ return split_data+++def create_column_names(model_info: ModelInfo) -> Dict[str, List[str]]:+ """+ Creates a list of column names for the skeleton data based on the model info.++ Parameters:+ - model_info: The model info used in tracking.++ Returns:+ - A list of column names for the skeleton data.+ """++ column_names = {}+ for category in category_info.keys():+ tracked_points_name = "num_tracked_points_" + category+ landmark_names = category + "_landmark_names"+ if hasattr(model_info, tracked_points_name):+ category_column_names = []+ if hasattr(model_info, landmark_names) and len(getattr(model_info, landmark_names)) == getattr(+ model_info, tracked_points_name+ ):+ for name in getattr(model_info, landmark_names):+ category_column_names.append(f"{category}_{name}_x")+ category_column_names.append(f"{category}_{name}_y")+ category_column_names.append(f"{category}_{name}_z")+ else:+ for i in range(getattr(model_info, tracked_points_name)):+ category_column_names.append(f"{category}_{str(i).zfill(4)}_x")+ category_column_names.append(f"{category}_{str(i).zfill(4)}_y")+ category_column_names.append(f"{category}_{str(i).zfill(4)}_z")++ column_names[category] = category_column_names++ # TODO: this is a quick attempt and probably wrong somehow+ if column_names == {}:+ landmark_names = "landmark_names"+ tracked_points_name = "num_tracked_points"+ category_column_names = []+ if hasattr(model_info, landmark_names) and len(getattr(model_info, landmark_names)) == getattr(+ model_info, tracked_points_name+ ):+ for name in getattr(model_info, landmark_names):+ category_column_names.append(f"{category}_{name}_x")+ category_column_names.append(f"{category}_{name}_y")+ category_column_names.append(f"{category}_{name}_z")+ else:+ for i in range(getattr(model_info, tracked_points_name)):+ category_column_names.append(f"{category}_{str(i).zfill(4)}_x")+ category_column_names.append(f"{category}_{str(i).zfill(4)}_y")+ category_column_names.append(f"{category}_{str(i).zfill(4)}_z")++ column_names["total"] = category_column_names++ return column_names+++def save_split_csv(+ output_data_folder_path: str,+ split_data: Dict[str, np.ndarray],+ column_names: Dict[str, List[str]],+ model_info: ModelInfo,+) -> None:+ """+ Saves the split skeleton data to CSV files using column names.++ Parameters:+ - split_data: A dictionary where each key is a category and the value is the corresponding data array.+ - column_names: A dictionary where each key is a category and the value is a list of column names for the data.+ """+ if split_data == {}:+ logger.debug("No data categories found in model info, skipping data splitting")+ return++ number_of_frames = next(iter(split_data.values())).shape[0]++ file_prefix = model_info.name + "_"++ for category, data in split_data.items():+ column_name_list = column_names[category]+ tracked_points_name = "num_tracked_points_" + category++ data_flat = data.reshape(number_of_frames, getattr(model_info, tracked_points_name) * 3)+ dataframe = pd.DataFrame(data_flat, columns=column_name_list)+ dataframe.to_csv(str(Path(output_data_folder_path) / (file_prefix + category_info[category])), index=False)++ logger.info(f"Saved split skeleton data csvs to {output_data_folder_path}")+++def save_split_npy(output_data_folder_path: str, split_data: Dict[str, np.ndarray], file_prefix: str = "") -> None:+ """+ Saves the split skeleton data to numpy files.++ Parameters:+ - split_data: A dictionary where each key is a category and the value is the corresponding data array.+ """++ if split_data == {}:+ logger.debug("No data categories found in model info, skipping data splitting")+ return++ if file_prefix != "" and file_prefix[-1] != "_":+ file_prefix += "_"++ for category, data in split_data.items():+ np.save(str(Path(output_data_folder_path) / (file_prefix + category + "_3d_xyz.npy")), data)++ logger.info(f"Saved split skeleton data npys to {output_data_folder_path}")+++def split_and_save(+ skeleton_3d_data: np.ndarray,+ model_info: ModelInfo,+ output_data_folder_path: str,+) -> None:+ """+ Splits the skeleton data into separate arrays for each data category using the provided model info.+ Examples categories are body, left_hand, right_hand, and face.+ Then saves the split skeleton data to CSV files using column names.+ Then saves the split skeleton data to numpy files.++ Parameters:+ - skeleton: The skeleton data to be split.+ - model_info: The model info used in tracking.+ - output_data_folder_path: The path to save the split skeleton data to.+ """++ split_data_dict = split_data(skeleton_3d_data=skeleton_3d_data, model_info=model_info)+ column_names = create_column_names(model_info=model_info)+ save_split_csv(+ output_data_folder_path=output_data_folder_path,+ split_data=split_data_dict,+ column_names=column_names,+ model_info=model_info,+ )+ save_split_npy(+ output_data_folder_path=output_data_folder_path, split_data=split_data_dict, file_prefix=model_info.name+ )diff --git a/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py b/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.pyindex 04cc5328..a5493cab 100644--- a/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py+++ b/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py@@ -1,5 +1,6 @@import loggingimport multiprocessing+from typing import Optionalfrom freemocap.core_processes.post_process_skeleton_data.post_process_skeleton import post_process_data@@ -32,9 +33,9 @@ logger = logging.getLogger(__name__)def process_recording_folder(recording_processing_parameter_model: ProcessingParameterModel,kill_event: multiprocessing.Event = None,- logging_queue: multiprocessing.Queue = None,+ logging_queue: Optional[multiprocessing.Queue] = None,use_tqdm: bool = True,-) -> bool:+) -> None:"""Parameters@@ -72,7 +73,7 @@ def process_recording_folder(queue=logging_queue,use_tqdm=use_tqdm,)- except (RuntimeError, FileNotFoundError) as e:+ except (RuntimeError, ValueError) as e:logger.error("2D skeleton detection failed, cannot continue processing")if logging_queue:logging_queue.put(e)@@ -103,14 +104,20 @@ def process_recording_folder(logging_queue.put(exception)raise exception- # TODO: move the rotate by 90 function into skellyforge to skip duplication of responsibility- rotated_raw_skel3d_frame_marker_xyz = rotate_by_90_degrees_around_x_axis(raw_skel3d_frame_marker_xyz)- # TODO: find out if skellyforge does all the error handling we need - if not add it to post_process_data- skel3d_frame_marker_xyz = post_process_data(- recording_processing_parameter_model=recording_processing_parameter_model,- raw_skel3d_frame_marker_xyz=rotated_raw_skel3d_frame_marker_xyz,- queue=logging_queue,- )+ try:+ # TODO: move the rotate by 90 function into skellyforge to skip duplication of responsibility+ rotated_raw_skel3d_frame_marker_xyz = rotate_by_90_degrees_around_x_axis(raw_skel3d_frame_marker_xyz)+ # TODO: find out if skellyforge does all the error handling we need - if not add it to post_process_data+ skel3d_frame_marker_xyz = post_process_data(+ recording_processing_parameter_model=recording_processing_parameter_model,+ raw_skel3d_frame_marker_xyz=rotated_raw_skel3d_frame_marker_xyz,+ queue=logging_queue,+ )+ except (RuntimeError, ValueError, AttributeError) as e:+ logger.error("Post processing failed, cannot continue processing")+ if logging_queue:+ logging_queue.put(e)+ raise eif kill_event is not None and kill_event.is_set():exception = KillEventException("Process was killed")@@ -118,11 +125,17 @@ def process_recording_folder(logging_queue.put(exception)raise exception- anatomical_data_dict = calculate_anatomical_data(- processing_parameters=recording_processing_parameter_model,- skel3d_frame_marker_xyz=skel3d_frame_marker_xyz,- queue=logging_queue,- )+ try:+ anatomical_data_dict = calculate_anatomical_data(+ processing_parameters=recording_processing_parameter_model,+ skel3d_frame_marker_xyz=skel3d_frame_marker_xyz,+ queue=logging_queue,+ )+ except (RuntimeError, ValueError, TypeError, KeyError, AttributeError) as e:+ logger.error("Anatomical data calculation failed, cannot continue processing")+ if logging_queue:+ logging_queue.put(e)+ raise eif kill_event is not None and kill_event.is_set():exception = KillEventException("Process was killed")@@ -135,9 +148,13 @@ def process_recording_folder(skel3d_frame_marker_xyz=skel3d_frame_marker_xyz,segment_COM_frame_imgPoint_XYZ=anatomical_data_dict["segment_COM"],totalBodyCOM_frame_XYZ=anatomical_data_dict["total_body_COM"],+ rigid_bones_data=anatomical_data_dict["rigid_bones_data"],processing_parameters=recording_processing_parameter_model,queue=logging_queue,)- DataSaver(recording_folder_path=recording_processing_parameter_model.recording_info_model.path).save_all()+ DataSaver(+ recording_folder_path=recording_processing_parameter_model.recording_info_model.path,+ model_info=recording_processing_parameter_model.tracking_model_info,+ ).save_all()logger.info(f"Done processing {recording_processing_parameter_model.recording_info_model.path}")diff --git a/freemocap/core_processes/process_motion_capture_videos/process_recording_headless.py b/freemocap/core_processes/process_motion_capture_videos/process_recording_headless.pyindex 269eb8a4..e7f1ea95 100644--- a/freemocap/core_processes/process_motion_capture_videos/process_recording_headless.py+++ b/freemocap/core_processes/process_motion_capture_videos/process_recording_headless.py@@ -24,21 +24,28 @@ def process_recording_headless(path_to_camera_calibration_toml: Optional[Union[str, Path]] = None,path_to_blender_executable: Optional[Union[str, Path]] = None,recording_processing_parameter_model: Optional[ProcessingParameterModel] = None,+ recording_info_model: Optional[RecordingInfoModel] = None,+ run_blender: bool = True,+ make_jupyter_notebook: bool = True,use_tqdm: bool = True,):if path_to_blender_executable is None:path_to_blender_executable = get_best_guess_of_blender_path()if recording_processing_parameter_model is None:recording_processing_parameter_model = ProcessingParameterModel()-rec = recording_processing_parameter_model+ if recording_info_model is None:+ rec.recording_info_model = RecordingInfoModel(recording_folder_path=Path(recording_path))+ else:+ rec.recording_info_model = recording_info_model+logger.info(f"Processing recording:\n"f"Recording path: {recording_path}\n"f"Camera calibration toml path: {path_to_camera_calibration_toml}\n"f"Blender executable path: {path_to_blender_executable}\n"- f"Recording processing parameter model: {rec.dict()}"+ f"Recording processing parameter model: {rec.model_dump()}")rec.recording_info_model = RecordingInfoModel(recording_folder_path=Path(recording_path))@@ -66,10 +73,11 @@ def process_recording_headless(logger.info("Starting core processing pipeline...")process_recording_folder(recording_processing_parameter_model=rec, use_tqdm=use_tqdm)- logger.info("Generating jupyter notebook...")- generate_jupyter_notebook(path_to_recording=recording_path)+ if make_jupyter_notebook:+ logger.info("Generating jupyter notebook...")+ generate_jupyter_notebook(path_to_recording=recording_path)- if path_to_blender_executable:+ if path_to_blender_executable and run_blender:blender_file_path = get_blender_file_path(recording_folder_path=recording_path)logger.info(f"Exporting to {blender_file_path}")export_to_blender(@@ -77,6 +85,8 @@ def process_recording_headless(blender_file_path=blender_file_path,blender_exe_path=Path(path_to_blender_executable),)+ elif not run_blender:+ logger.info("Skipping blender export")else:logger.warning("No blender executable provided. Blender file will not be exported.")diff --git a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/anatomical_data_pipeline_functions.py b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/anatomical_data_pipeline_functions.pyindex 2bf721bb..4b9b5547 100644--- a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/anatomical_data_pipeline_functions.py+++ b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/anatomical_data_pipeline_functions.py@@ -1,11 +1,14 @@import loggingimport multiprocessing-from typing import Dict+from typing import Dict, Unionimport numpy as np-from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.convert_mediapipe_npy_to_csv import (- convert_mediapipe_npy_to_csv,+++from freemocap.core_processes.post_process_skeleton_data.calculate_center_of_mass import (+ calculate_center_of_mass_from_skeleton,)-from freemocap.core_processes.post_process_skeleton_data.calculate_center_of_mass import run_center_of_mass_calculations+from freemocap.core_processes.post_process_skeleton_data.create_skeleton import create_skeleton_model+from freemocap.core_processes.post_process_skeleton_data.enforce_rigid_bones import enforce_rigid_bones_from_skeletonfrom freemocap.data_layer.recording_models.post_processing_parameter_models import ProcessingParameterModelfrom freemocap.system.logging.queue_logger import DirectQueueHandlerfrom freemocap.system.logging.configure_logging import log_view_logging_format_string@@ -17,23 +20,40 @@ def calculate_anatomical_data(processing_parameters: ProcessingParameterModel,skel3d_frame_marker_xyz: np.ndarray,queue: multiprocessing.Queue,-) -> Dict[str, np.ndarray]:+) -> Dict[str, Union[np.ndarray, None]]:if queue:handler = DirectQueueHandler(queue)handler.setFormatter(logging.Formatter(fmt=log_view_logging_format_string, datefmt="%Y-%m-%dT%H:%M:%S"))logger.addHandler(handler)- logger.info("Calculating center of mass...")- segment_COM_frame_imgPoint_XYZ, totalBodyCOM_frame_XYZ = run_center_of_mass_calculations(- processed_skel3d_frame_marker_xyz=skel3d_frame_marker_xyz+ logger.info("Creating skeleton model...")+ skeleton = create_skeleton_model(+ actual_markers=processing_parameters.tracking_model_info.landmark_names,+ segment_connections=processing_parameters.tracking_model_info.segment_connections,+ virtual_markers=processing_parameters.tracking_model_info.virtual_markers_definitions,+ joint_hierarchy=processing_parameters.tracking_model_info.joint_hierarchy,+ center_of_mass_info=processing_parameters.tracking_model_info.center_of_mass_definitions,+ num_tracked_points=processing_parameters.tracking_model_info.num_tracked_points,)- convert_mediapipe_npy_to_csv(- mediapipe_3d_frame_trackedPoint_xyz=skel3d_frame_marker_xyz,- output_data_folder_path=processing_parameters.recording_info_model.output_data_folder_path,- ) # TODO: separate functionality of splitting into CSVs and savings to disk+ skeleton.integrate_freemocap_3d_data(skel3d_frame_marker_xyz)++ logger.info("Calculating center of mass...")+ try:+ segment_COM_frame_imgPoint_XYZ, totalBodyCOM_frame_XYZ = calculate_center_of_mass_from_skeleton(+ skeleton=skeleton+ )+ except ValueError:+ logger.warning("Center of mass cannot be calculated for this tracking model")+ segment_COM_frame_imgPoint_XYZ = totalBodyCOM_frame_XYZ = None+ except AttributeError as e:+ raise e++ logger.info("Enforcing rigid bones...")+ rigid_bones_data = enforce_rigid_bones_from_skeleton(skeleton=skeleton)return {"segment_COM": segment_COM_frame_imgPoint_XYZ,"total_body_COM": totalBodyCOM_frame_XYZ,+ "rigid_bones_data": rigid_bones_data,}diff --git a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/data_saving_pipeline_functions.py b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/data_saving_pipeline_functions.pyindex 83a1b6f8..97f2f961 100644--- a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/data_saving_pipeline_functions.py+++ b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/data_saving_pipeline_functions.py@@ -1,31 +1,31 @@import loggingimport multiprocessingfrom pathlib import Path+from typing import Optionalimport numpy as npfrom freemocap.core_processes.post_process_skeleton_data.post_process_skeleton import save_numpy_array_to_disk-from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.data_models.mediapipe_skeleton_names_and_connections import (- mediapipe_names_and_connections_dict,-)+from freemocap.core_processes.post_process_skeleton_data.split_and_save import split_and_savefrom freemocap.data_layer.recording_models.post_processing_parameter_models import ProcessingParameterModelfrom freemocap.system.logging.queue_logger import DirectQueueHandlerfrom freemocap.system.logging.configure_logging import log_view_logging_format_stringfrom freemocap.system.paths_and_filenames.file_and_folder_names import (CENTER_OF_MASS_FOLDER_NAME,+ DATA_3D_NPY_FILE_NAME,+ RIGID_BONES_NPY_FILE_NAME,SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME,TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME,)-from freemocap.utilities.save_dictionary_to_json import save_dictionary_to_json-logger = logging.getLogger(__name__)def save_data(skel3d_frame_marker_xyz: np.ndarray,- segment_COM_frame_imgPoint_XYZ: np.ndarray,- totalBodyCOM_frame_XYZ: np.ndarray,+ segment_COM_frame_imgPoint_XYZ: Optional[np.ndarray],+ totalBodyCOM_frame_XYZ: Optional[np.ndarray],+ rigid_bones_data: Optional[np.ndarray],processing_parameters: ProcessingParameterModel,queue: multiprocessing.Queue,):@@ -39,26 +39,43 @@ def save_data(logger.info("Saving post-processed data")save_numpy_array_to_disk(array_to_save=skel3d_frame_marker_xyz,- file_name="mediaPipeSkel_3d_body_hands_face.npy",- save_directory=path_to_folder_where_we_will_save_this_data,+ file_name=f"{processing_parameters.tracking_model_info.name}_{DATA_3D_NPY_FILE_NAME}",+ save_directory=Path(path_to_folder_where_we_will_save_this_data),)- logger.info("Saving segment center of mass data")- save_numpy_array_to_disk(- array_to_save=segment_COM_frame_imgPoint_XYZ,- file_name=SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME,- save_directory=Path(path_to_folder_where_we_will_save_this_data) / CENTER_OF_MASS_FOLDER_NAME,- )+ if segment_COM_frame_imgPoint_XYZ is not None:+ logger.info("Saving segment center of mass data")+ save_numpy_array_to_disk(+ array_to_save=segment_COM_frame_imgPoint_XYZ,+ file_name=f"{processing_parameters.tracking_model_info.name}_{SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME}",+ save_directory=Path(path_to_folder_where_we_will_save_this_data) / CENTER_OF_MASS_FOLDER_NAME,+ )+ else:+ logger.debug("segment_COM_frame_imgPoint_XYZ is None, could not save")- logger.info("Saving total body center of mass data")- save_numpy_array_to_disk(- array_to_save=totalBodyCOM_frame_XYZ,- file_name=TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME,- save_directory=Path(path_to_folder_where_we_will_save_this_data) / CENTER_OF_MASS_FOLDER_NAME,- )+ if totalBodyCOM_frame_XYZ is not None:+ logger.info("Saving total body center of mass data")+ save_numpy_array_to_disk(+ array_to_save=totalBodyCOM_frame_XYZ,+ file_name=f"{processing_parameters.tracking_model_info.name}_{TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME}",+ save_directory=Path(path_to_folder_where_we_will_save_this_data) / CENTER_OF_MASS_FOLDER_NAME,+ )+ else:+ logger.debug("totalBodyCOM_frame_XYZ is None, could not save")++ if rigid_bones_data is not None:+ logger.info("Saving rigid bones data")+ save_numpy_array_to_disk(+ array_to_save=rigid_bones_data,+ file_name=f"{processing_parameters.tracking_model_info.name}_{RIGID_BONES_NPY_FILE_NAME}",+ save_directory=path_to_folder_where_we_will_save_this_data,+ )+ else:+ logger.debug("rigid_bones_data is None, could not save")- save_dictionary_to_json(- save_path=processing_parameters.recording_info_model.output_data_folder_path,- file_name="mediapipe_names_and_connections_dict.json",- dictionary=mediapipe_names_and_connections_dict,+ logger.info("Splitting and saving data...")+ split_and_save(+ skeleton_3d_data=skel3d_frame_marker_xyz,+ model_info=processing_parameters.tracking_model_info,+ output_data_folder_path=processing_parameters.recording_info_model.output_data_folder_path,)diff --git a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/image_tracking_pipeline_functions.py b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/image_tracking_pipeline_functions.pyindex 50c9883d..4b89769e 100644--- a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/image_tracking_pipeline_functions.py+++ b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/image_tracking_pipeline_functions.py@@ -4,9 +4,10 @@ from pathlib import Pathimport numpy as npfrom skellytracker.process_folder_of_videos import process_folder_of_videos-from skellytracker.trackers.mediapipe_tracker.mediapipe_model_info import (- MediapipeTrackingParams,+from skellytracker.trackers.base_tracker.base_tracking_params import (+ BaseTrackingParams,)+from skellytracker.trackers.base_tracker.model_info import ModelInfofrom freemocap.data_layer.recording_models.post_processing_parameter_models import (ProcessingParameterModel,@@ -14,6 +15,7 @@ from freemocap.data_layer.recording_models.post_processing_parameter_models impofrom freemocap.system.logging.configure_logging import log_view_logging_format_stringfrom freemocap.system.logging.queue_logger import DirectQueueHandlerfrom freemocap.system.paths_and_filenames.file_and_folder_names import LOG_VIEW_PROGRESS_BAR_STRING, RAW_DATA_FOLDER_NAME+from freemocap.tests.test_image_tracking_data_shape import test_image_tracking_data_shapelogger = logging.getLogger(__name__)@@ -29,13 +31,13 @@ def run_image_tracking_pipeline(handler.setFormatter(logging.Formatter(fmt=log_view_logging_format_string, datefmt="%Y-%m-%dT%H:%M:%S"))logger.addHandler(handler)- if not processing_parameters.mediapipe_parameters_model.run_image_tracking:+ if not processing_parameters.tracking_parameters_model.run_image_tracking:logger.info(- f"Skipping 2d skeleton detection and loading data from: {processing_parameters.recording_info_model.mediapipe_2d_data_npy_file_path}"+ f"Skipping 2d skeleton detection and loading data from: {processing_parameters.recording_info_model.data_2d_npy_file_path}")try:image_data_numCams_numFrames_numTrackedPts_XYZ = np.load(- processing_parameters.recording_info_model.mediapipe_2d_data_npy_file_path+ processing_parameters.recording_info_model.data_2d_npy_file_path)except Exception as e:logger.error(e)@@ -45,7 +47,8 @@ def run_image_tracking_pipeline(logger.info(LOG_VIEW_PROGRESS_BAR_STRING)# 2d skeleton detectionimage_data_numCams_numFrames_numTrackedPts_XYZ = run_image_tracking(- tracking_params=processing_parameters.mediapipe_parameters_model,+ model_info=processing_parameters.tracking_model_info,+ tracking_params=processing_parameters.tracking_parameters_model,synchronized_videos_folder_path=Path(processing_parameters.recording_info_model.synchronized_videos_folder_path),@@ -55,28 +58,33 @@ def run_image_tracking_pipeline(use_tqdm=use_tqdm,)- if not processing_parameters.recording_info_model.data2d_status_check:- raise FileNotFoundError(- f"No mediapipe 2d data found at: {processing_parameters.recording_info_model.mediapipe_2d_data_npy_file_path}"+ # TODO: check that skellytracker saved out data correctly+ try:+ test_image_tracking_data_shape(+ synchronized_video_folder_path=processing_parameters.recording_info_model.synchronized_videos_folder_path,+ image_tracking_data=image_data_numCams_numFrames_numTrackedPts_XYZ,)+ except (ValueError, AssertionError) as e:+ logger.error(e)+ raise ValueError("image tracking data is not valid, cannot continue processing")return image_data_numCams_numFrames_numTrackedPts_XYZdef run_image_tracking(- tracking_params: MediapipeTrackingParams,+ model_info: ModelInfo,+ tracking_params: BaseTrackingParams,synchronized_videos_folder_path: Path,output_data_folder_path: Path,kill_event: multiprocessing.Event = None,use_tqdm: bool = True,):- if tracking_params.use_yolo_crop_method:- tracker_type = "YOLOMediapipeComboTracker"- else:- tracker_type = "MediapipeHolisticTracker"+ if hasattr(tracking_params, "use_yolo_crop_method") and tracking_params.use_yolo_crop_method:+ model_info.tracker_name = "YOLOMediapipeComboTracker"+ # TODO: this shouldn't be handled hereimage_data_numCams_numFrames_numTrackedPts_XYZ = process_folder_of_videos(- tracker_name=tracker_type,+ model_info=model_info,tracking_params=tracking_params,synchronized_video_path=synchronized_videos_folder_path,output_folder_path=output_data_folder_path,diff --git a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/pipeline_check.py b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/pipeline_check.pyindex 7f789d9b..f06f9959 100644--- a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/pipeline_check.py+++ b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/pipeline_check.py@@ -3,23 +3,26 @@ from freemocap.data_layer.recording_models.recording_info_model import Recordingdef processing_pipeline_check(processing_parameters: ProcessingParameterModel) -> None:- """Performs checks on the processing parameters to make sure they are valid with the existing data before running the pipeline"""+ """+ Performs checks on the processing parameters to make sure they are valid with the existing data before running the pipeline.+ Raises FileNotFoundError if any of the checks fail.+ """status_check_dict = processing_parameters.recording_info_model.status_checkif not status_check_dict["synchronized_videos_status_check"]:raise FileNotFoundError(f"Could not find synchronized_videos folder at {processing_parameters.recording_info_model.synchronized_videos_folder_path}")- if not processing_parameters.mediapipe_parameters_model.run_image_tracking:+ if not processing_parameters.tracking_parameters_model.run_image_tracking:if not status_check_dict["data2d_status_check"]:raise FileNotFoundError(- f"No mediapipe 2d data found at: {processing_parameters.recording_info_model.mediapipe_2d_data_npy_file_path}"+ f"No 2d data found at: {processing_parameters.recording_info_model.data_2d_npy_file_path}")if not processing_parameters.anipose_triangulate_3d_parameters_model.run_3d_triangulation:if not status_check_dict["data3d_status_check"] and not status_check_dict["single_video_check"]:raise FileNotFoundError(- f"No mediapipe 3d data found at: {processing_parameters.recording_info_model.mediapipe_3d_data_npy_file_path}"+ f"No 3d data found at: {processing_parameters.recording_info_model.data_3d_npy_file_path}")else:if not status_check_dict["calibration_toml_check"] and not status_check_dict["single_video_check"]:diff --git a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/triangulation_pipeline_functions.py b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/triangulation_pipeline_functions.pyindex 652fecb2..d864ac66 100644--- a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/triangulation_pipeline_functions.py+++ b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/triangulation_pipeline_functions.py@@ -1,14 +1,15 @@import loggingimport multiprocessingfrom pathlib import Path+from typing import Optionalimport numpy as npfrom freemocap.core_processes.capture_volume_calibration.by_camera_reprojection_filtering import (run_reprojection_error_filtering,)-from freemocap.core_processes.capture_volume_calibration.save_mediapipe_3d_data_to_npy import (- save_mediapipe_3d_data_to_npy,+from freemocap.core_processes.capture_volume_calibration.save_3d_data_to_npy import (+ save_3d_data_to_npy,)from freemocap.core_processes.capture_volume_calibration.triangulate_3d_data import triangulate_3d_datafrom freemocap.core_processes.capture_volume_calibration.anipose_camera_calibration.get_anipose_calibration_object import (@@ -28,8 +29,8 @@ logger = logging.getLogger(__name__)def get_triangulated_data(image_data_numCams_numFrames_numTrackedPts_XYZ: np.ndarray,processing_parameters: ProcessingParameterModel,- kill_event: multiprocessing.Event = None,- queue: multiprocessing.Queue = None,+ kill_event: Optional[multiprocessing.Event] = None,+ queue: Optional[multiprocessing.Queue] = None,) -> np.ndarray:if queue:handler = DirectQueueHandler(queue)@@ -39,23 +40,24 @@ def get_triangulated_data(if image_data_numCams_numFrames_numTrackedPts_XYZ.shape[0] == 1:logger.info("Skipping 3d triangulation for single camera data")logger.info(LOG_VIEW_PROGRESS_BAR_STRING)- (skel3d_frame_marker_xyz, skeleton_reprojection_error_fr_mar) = process_single_camera_skeleton_data(+ (skel3d_frame_marker_xyz, skeleton_reprojection_error_fr_mar) = process_single_camera_skeleton_data(input_image_data_frame_marker_xyz=image_data_numCams_numFrames_numTrackedPts_XYZ[0],raw_data_folder_path=Path(processing_parameters.recording_info_model.raw_data_folder_path),+ file_prefix=processing_parameters.tracking_model_info.name,project_to_z_plane=processing_parameters.anipose_triangulate_3d_parameters_model.flatten_single_camera_data,)elif not processing_parameters.anipose_triangulate_3d_parameters_model.run_3d_triangulation:logger.info(- f"Skipping 3d triangulation and loading data from: {processing_parameters.recording_info_model.raw_mediapipe_3d_data_npy_file_path}"+ f"Skipping 3d triangulation and loading data from: {processing_parameters.recording_info_model.raw_data_3d_npy_file_path}")logger.info(LOG_VIEW_PROGRESS_BAR_STRING)try:skel3d_frame_marker_xyz = np.load(- processing_parameters.recording_info_model.raw_mediapipe_3d_data_npy_file_path+ processing_parameters.recording_info_model.raw_data_3d_npy_file_path)skeleton_reprojection_error_fr_mar = np.load(- processing_parameters.recording_info_model.mediapipe_reprojection_error_data_npy_file_path+ processing_parameters.recording_info_model.reprojection_error_data_npy_file_path)except Exception as e:logger.error(e)@@ -69,11 +71,6 @@ def get_triangulated_data(f"No calibration file found at: {processing_parameters.recording_info_model.calibration_toml_path}")- if not processing_parameters.recording_info_model.data2d_status_check:- raise FileNotFoundError(- f"No mediapipe 2d data found at: {processing_parameters.recording_info_model.mediapipe_2d_data_npy_file_path}"- )-anipose_calibration_object = load_anipose_calibration_toml_from_path(camera_calibration_data_toml_path=processing_parameters.recording_info_model.calibration_toml_path,save_copy_of_calibration_to_this_path=processing_parameters.recording_info_model.path,@@ -84,16 +81,17 @@ def get_triangulated_data(skeleton_reprojection_error_cam_fr_mar,) = triangulate_3d_data(anipose_calibration_object=anipose_calibration_object,- mediapipe_2d_data=image_data_numCams_numFrames_numTrackedPts_XYZ[:, :, :, :2],+ image_2d_data=image_data_numCams_numFrames_numTrackedPts_XYZ[:, :, :, :2],use_triangulate_ransac=processing_parameters.anipose_triangulate_3d_parameters_model.use_triangulate_ransac_method,kill_event=kill_event,)- save_mediapipe_3d_data_to_npy(+ save_3d_data_to_npy(data3d_numFrames_numTrackedPoints_XYZ=skel3d_frame_marker_xyz,data3d_numFrames_numTrackedPoints_reprojectionError=skeleton_reprojection_error_fr_mar,data3d_numCams_numFrames_numTrackedPoints_reprojectionError=skeleton_reprojection_error_cam_fr_mar,path_to_folder_where_data_will_be_saved=processing_parameters.recording_info_model.raw_data_folder_path,processing_level="raw",+ file_prefix=processing_parameters.tracking_model_info.name,)if processing_parameters.anipose_triangulate_3d_parameters_model.run_reprojection_error_filtering:diff --git a/freemocap/data_layer/data_saver/data_loader.py b/freemocap/data_layer/data_saver/data_loader.pyindex fc07ed70..679d4fbe 100644--- a/freemocap/data_layer/data_saver/data_loader.py+++ b/freemocap/data_layer/data_saver/data_loader.py@@ -1,47 +1,55 @@-import jsonimport loggingfrom pathlib import Pathfrom typing import Union, Dict, Anyimport numpy as npimport pandas as pd--from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.data_models.mediapipe_skeleton_names_and_connections import (- mediapipe_skeleton_schema,+from skellytracker.trackers.mediapipe_tracker.mediapipe_model_info import (+ MediapipeModelInfo,)-from freemocap.core_processes.post_process_skeleton_data.calculate_center_of_mass import BODY_SEGMENT_NAMES+from skellytracker.trackers.base_tracker.model_info import ModelInfo+from freemocap.data_layer.data_saver.data_models import FrameData, Timestamps, Point, SkeletonSchemafrom freemocap.system.paths_and_filenames.file_and_folder_names import (- MEDIAPIPE_BODY_3D_DATAFRAME_CSV_FILE_NAME,- MEDIAPIPE_RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME,- MEDIAPIPE_LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME,- MEDIAPIPE_FACE_3D_DATAFRAME_CSV_FILE_NAME,+ BODY_3D_DATAFRAME_CSV_FILE_NAME,+ CENTER_OF_MASS_FOLDER_NAME,+ DATA_3D_NPY_FILE_NAME,+ RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME,+ LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME,+ FACE_3D_DATAFRAME_CSV_FILE_NAME,+ SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME,+ TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME,)from freemocap.system.paths_and_filenames.path_getters import (get_output_data_folder_path,get_timestamps_directory,- get_total_body_center_of_mass_file_path,- get_segment_center_of_mass_file_path,- get_full_npy_file_path,)logger = logging.getLogger(__name__)+# TODO: Need to generalize this beyond mediapipe, and make COM data optional+mediapipe_model_info = MediapipeModelInfo()++class DataLoader:def __init__(self,recording_folder_path: Union[str, Path],include_hands: bool = True,include_face: bool = True,+ model_info: ModelInfo = mediapipe_model_info,):self._recording_folder_path = Path(recording_folder_path)self.include_hands = include_handsself.include_face = include_face+ self._model_info = model_infoself._recording_name = self._recording_folder_path.nameself._output_folder_path = Path(get_output_data_folder_path(self._recording_folder_path))+ self.number_of_frames = None+ self._set_file_prefix()self._load_data()def _load_data(self):@@ -50,18 +58,17 @@ class DataLoader:self._load_full_npy_data()self._load_center_of_mass_data()# self._load_segment_lengths()- self._load_names_and_connections()- self._load_skeleton_schema()+ # self._load_skeleton_schema()self._validate_data()def _load_data_frames(self):- self.body_dataframe = self._load_dataframe(MEDIAPIPE_BODY_3D_DATAFRAME_CSV_FILE_NAME)+ self.body_dataframe = self._load_dataframe(self._file_prefix + BODY_3D_DATAFRAME_CSV_FILE_NAME)self.number_of_frames = len(self.body_dataframe)if self.include_hands:- self.right_hand_dataframe = self._load_dataframe(MEDIAPIPE_RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME)- self.left_hand_dataframe = self._load_dataframe(MEDIAPIPE_LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME)+ self.right_hand_dataframe = self._load_dataframe(self._file_prefix + RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME)+ self.left_hand_dataframe = self._load_dataframe(self._file_prefix + LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME)if self.include_face:- self.face_dataframe = self._load_dataframe(MEDIAPIPE_FACE_3D_DATAFRAME_CSV_FILE_NAME)+ self.face_dataframe = self._load_dataframe(self._file_prefix + FACE_3D_DATAFRAME_CSV_FILE_NAME)def _load_dataframe(self, filename):return pd.read_csv(self._output_folder_path / filename)@@ -76,13 +83,14 @@ class DataLoader:def _validate_dataframe(self, df, df_name):if df is not None and len(df) != self.number_of_frames:raise ValueError(- f"The number of frames in the {df_name} dataframe is different from the number of frames in the body dataframe."+ f"The number of frames in the {df_name} dataframe ({len(df)}) is different from the expected number of frames ({self.number_of_frames}).")def _validate_numpy_array(self, np_array, np_array_name):if np_array is not None and np_array.shape[0] != self.number_of_frames:+ print(f"\n\n {np_array.shape} \n\n")raise ValueError(- f"The number of frames in the {np_array_name} data is different from the number of frames in the body dataframe."+ f"The number of frames in the {np_array_name} data ({np_array.shape[0]}) is different from the expected number of frames ({self.number_of_frames}).")def _load_timestamps(self):@@ -105,22 +113,17 @@ class DataLoader:Load additional data like center of mass and segment lengths."""self.center_of_mass_xyz = np.load(- get_total_body_center_of_mass_file_path(output_data_folder=self._output_folder_path)+ self._output_folder_path / CENTER_OF_MASS_FOLDER_NAME / (self._file_prefix + TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME))self.segment_center_of_mass_segment_xyz = np.load(- get_segment_center_of_mass_file_path(output_data_folder=self._output_folder_path)+ self._output_folder_path / CENTER_OF_MASS_FOLDER_NAME / (self._file_prefix + SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME))def _load_full_npy_data(self):- self.data_frame_name_xyz = np.load(get_full_npy_file_path(output_data_folder=self._output_folder_path))+ self.data_frame_name_xyz = np.load(self._output_folder_path / (self._file_prefix + DATA_3D_NPY_FILE_NAME))- def _load_names_and_connections(self):- with open(self._output_folder_path / "mediapipe_names_and_connections_dict.json", "r") as file:- self.names_and_connections = json.loads(file.read())-- # def _load_segment_lengths(self):- # with open(self._output_folder_path / "mediapipe_skeleton_segment_lengths.json", "r") as file:- # self.segment_lengths = json.loads(file.read())+ if not self.number_of_frames:+ self.number_of_frames = self.data_frame_name_xyz.shape[0]def load_frame_data(self, frame_number: int) -> FrameData:return FrameData(@@ -137,7 +140,7 @@ class DataLoader:},)except AttributeError:- return Timestamps()+ return Timestamps(mean=None)def get_tracked_points(self, frame_number) -> Dict[str, Point]:right_hand_points = {}@@ -178,7 +181,10 @@ class DataLoader:z=self.center_of_mass_xyz[frame_number, 2],)- for segment_number, segment_name in enumerate(BODY_SEGMENT_NAMES):+ if self._model_info.center_of_mass_definitions is None:+ raise ValueError("center of mass definitions are not provided.")++ for segment_number, segment_name in enumerate(self._model_info.center_of_mass_definitions.keys()):com_data[segment_name] = Point(x=self.segment_center_of_mass_segment_xyz[frame_number, segment_number, 0],y=self.segment_center_of_mass_segment_xyz[frame_number, segment_number, 1],@@ -214,5 +220,11 @@ class DataLoader:recording_data_by_frame_number[frame_number] = self.load_frame_data(frame_number).to_dict()return recording_data_by_frame_number- def _load_skeleton_schema(self):- self.skeleton_schema = SkeletonSchema(schema_dict=mediapipe_skeleton_schema)+ def _load_skeleton_schema(self): # TODO: replace this with updated Skeleton class+ self.skeleton_schema = SkeletonSchema(schema_dict=MediapipeModelInfo.skeleton_schema)++ def _set_file_prefix(self) -> None:+ self._file_prefix = self._model_info.name++ if self._file_prefix[-1] != "_":+ self._file_prefix += "_"diff --git a/freemocap/data_layer/data_saver/data_models.py b/freemocap/data_layer/data_saver/data_models.pyindex 7d2e1775..e31059ac 100644--- a/freemocap/data_layer/data_saver/data_models.py+++ b/freemocap/data_layer/data_saver/data_models.py@@ -4,8 +4,8 @@ from typing import Optional, Anyfrom pydantic import BaseModel, Field, model_validator-from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.data_models.mediapipe_skeleton_names_and_connections import (- mediapipe_skeleton_schema,+from skellytracker.trackers.mediapipe_tracker.mediapipe_model_info import (+ MediapipeModelInfo,)@@ -110,11 +110,11 @@ class FrameData(BaseModel):class InfoDict(BaseModel):"""- A dictionary of information about this recording, such as the measured segement lengths and the schemas that we can use to interpret the tracked points (i.e./e.g. how to connect the dots of skeleton)+ A dictionary of information about this recording, such as the measured segement lengths and the segment connections that we can use to interpret the tracked points (i.e./e.g. how to connect the dots of skeleton)"""# segment_lengths: Dict[str, Any] = Field(default_factory=dict, description="The lengths of the segments of the body")- schemas: List[BaseModel] = Field(default_factory=list, description="The schemas for the tracked points")+ schemas: Optional[Dict[str, Any]] = Field(default_factory=dict, description="The segment connections for the tracked points")if __name__ == "__main__":@@ -123,6 +123,6 @@ if __name__ == "__main__":print("=====================================\n\n===============================")- skeleton_schema = SkeletonSchema(schema_dict=mediapipe_skeleton_schema)+ skeleton_schema = SkeletonSchema(schema_dict=MediapipeModelInfo.skeleton_schema)pprint.pp(skeleton_schema.dict())diff --git a/freemocap/data_layer/data_saver/data_saver.py b/freemocap/data_layer/data_saver/data_saver.pyindex 1e35cb78..bb184494 100644--- a/freemocap/data_layer/data_saver/data_saver.py+++ b/freemocap/data_layer/data_saver/data_saver.py@@ -6,6 +6,11 @@ from typing import Unionimport numpy as npimport pandas as pd+from skellytracker.trackers.mediapipe_tracker.mediapipe_model_info import (+ MediapipeModelInfo,+)+from skellytracker.trackers.base_tracker.model_info import ModelInfo+from freemocap.data_layer.data_saver.data_loader import DataLoaderfrom freemocap.data_layer.data_saver.data_models import InfoDict@@ -13,9 +18,17 @@ logger = logging.getLogger(__name__)from typing import Any, Dict+mediapipe_model_info = MediapipeModelInfo()+class DataSaver:- def __init__(self, recording_folder_path: Union[Path, str], include_hands: bool = True, include_face: bool = True):+ def __init__(+ self,+ recording_folder_path: Union[Path, str],+ include_hands: bool = True,+ include_face: bool = True,+ model_info: ModelInfo = mediapipe_model_info,+ ):"""Initialize DataFrameManager with the given recording_folder_path.@@ -28,7 +41,10 @@ class DataSaver:self._recording_name = self.recording_folder_path.nameself._data_loader = DataLoader(- recording_folder_path=self.recording_folder_path, include_hands=include_hands, include_face=include_face+ recording_folder_path=self.recording_folder_path,+ include_hands=include_hands,+ include_face=include_face,+ model_info=model_info,)self.recording_data_by_frame = None@@ -50,7 +66,7 @@ class DataSaver:def save_to_json(self, save_path: Union[str, Path] = None):dict_to_save = {}- dict_to_save["info"] = self._get_info_dict().dict()+ dict_to_save["info"] = self._get_info_dict().model_dump()dict_to_save["data_by_frame"] = self.recording_data_by_frameif save_path is None:@@ -100,7 +116,7 @@ class DataSaver:def _get_info_dict(self):return InfoDict(# segment_lengths=self._data_loader.segment_lengths,- schemas=[self._data_loader.skeleton_schema.dict()],+ schemas=self._data_loader._model_info.segment_connections,)diff --git a/freemocap/data_layer/recording_models/post_processing_parameter_models.py b/freemocap/data_layer/recording_models/post_processing_parameter_models.pyindex 84fdda2e..dca03214 100644--- a/freemocap/data_layer/recording_models/post_processing_parameter_models.py+++ b/freemocap/data_layer/recording_models/post_processing_parameter_models.py@@ -1,9 +1,9 @@import logging-from pydantic import ConfigDict, BaseModel-from skellytracker.trackers.mediapipe_tracker.mediapipe_model_info import (- MediapipeTrackingParams-)+from pydantic import BaseModel, ConfigDict+from skellytracker.trackers.mediapipe_tracker.mediapipe_model_info import MediapipeTrackingParams, MediapipeModelInfo+from skellytracker.trackers.base_tracker.base_tracking_params import BaseTrackingParams+from skellytracker.trackers.base_tracker.model_info import ModelInfofrom freemocap.data_layer.recording_models.recording_info_model import (RecordingInfoModel,@@ -37,6 +37,7 @@ class PostProcessingParametersModel(BaseModel):class ProcessingParameterModel(BaseModel):model_config = ConfigDict(arbitrary_types_allowed=True)recording_info_model: RecordingInfoModel = None- mediapipe_parameters_model: MediapipeTrackingParams = MediapipeTrackingParams()+ tracking_parameters_model: BaseTrackingParams = MediapipeTrackingParams()anipose_triangulate_3d_parameters_model: AniposeTriangulate3DParametersModel = AniposeTriangulate3DParametersModel()post_processing_parameters_model: PostProcessingParametersModel = PostProcessingParametersModel()+ tracking_model_info: ModelInfo = MediapipeModelInfo()diff --git a/freemocap/data_layer/recording_models/recording_info_model.py b/freemocap/data_layer/recording_models/recording_info_model.pyindex aa6c3c8c..5c34650b 100644--- a/freemocap/data_layer/recording_models/recording_info_model.py+++ b/freemocap/data_layer/recording_models/recording_info_model.py@@ -6,21 +6,32 @@ import numpy as npfrom freemocap.system.paths_and_filenames.file_and_folder_names import (CENTER_OF_MASS_FOLDER_NAME,- MEDIAPIPE_2D_NPY_FILE_NAME,- RAW_MEDIAPIPE_3D_NPY_FILE_NAME,+ DATA_2D_NPY_FILE_NAME,+ RAW_3D_NPY_FILE_NAME,OUTPUT_DATA_FOLDER_NAME,RAW_DATA_FOLDER_NAME,TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME,- RAW_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME,+ REPROJECTION_ERROR_NPY_FILE_NAME,SYNCHRONIZED_VIDEOS_FOLDER_NAME,ANNOTATED_VIDEOS_FOLDER_NAME,- MEDIAPIPE_3D_NPY_FILE_NAME,+ DATA_3D_NPY_FILE_NAME,+ OLD_DATA_2D_NPY_FILE_NAME,+ OLD_DATA_3D_NPY_FILE_NAME,+ OLD_RAW_3D_NPY_FILE_NAME,+ OLD_REPROJECTION_ERROR_NPY_FILE_NAME,+ OLD_TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME,)from freemocap.system.paths_and_filenames.path_getters import create_camera_calibration_file_name, get_blender_file_path-from freemocap.tests.test_image_tracking_data_shape import test_image_tracking_data_shape-from freemocap.tests.test_mediapipe_skeleton_data_shape import test_mediapipe_skeleton_data_shape+from freemocap.tests.test_image_tracking_data_shape import (+ test_image_tracking_data_exists,+ test_image_tracking_data_shape,+)+from freemocap.tests.test_skeleton_data_shape import test_skeleton_data_exists, test_skeleton_data_shapefrom freemocap.tests.test_synchronized_video_frame_counts import test_synchronized_video_frame_counts-from freemocap.tests.test_total_body_center_of_mass_data_shape import test_total_body_center_of_mass_data_shape+from freemocap.tests.test_total_body_center_of_mass_data_shape import (+ test_total_body_center_of_mass_data_exists,+ test_total_body_center_of_mass_data_shape,+)from freemocap.utilities.get_number_of_frames_of_videos_in_a_folder import get_number_of_frames_of_videos_in_a_folderfrom freemocap.utilities.get_video_paths import get_video_paths@@ -28,10 +39,7 @@ logger = logging.getLogger(__name__)class RecordingInfoModel:- def __init__(- self,- recording_folder_path: Union[Path, str],- ):+ def __init__(self, recording_folder_path: Union[Path, str], active_tracker: str = "mediapipe"):if any([Path(recording_folder_path).name == SYNCHRONIZED_VIDEOS_FOLDER_NAME,@@ -48,8 +56,26 @@ class RecordingInfoModel:Path(self._path) / create_camera_calibration_file_name(recording_name=self._name))+ self._active_tracker = active_tracker+self._recording_folder_status_checker = RecordingFolderStatusChecker(recording_info_model=self)+ @property+ def active_tracker(self) -> str:+ return self._active_tracker++ @active_tracker.setter+ def active_tracker(self, tracker_name: str):+ self._active_tracker = tracker_name++ @property+ def file_prefix(self) -> str:+ if self.active_tracker != "" and self.active_tracker[-1] != "_":+ return self.active_tracker + "_"+ else:+ return self.active_tracker++ # TODO: Create setters for the path names that depend on tracker name, and an overall setter that sets them all at once@propertydef path(self) -> str:return str(self._path)@@ -59,7 +85,7 @@ class RecordingInfoModel:return self._name@property- def status_check(self) -> Dict[str, bool]:+ def status_check(self) -> Dict[str, Union[bool, str, float, dict]]:return self._recording_folder_status_checker.status_check@property@@ -87,34 +113,75 @@ class RecordingInfoModel:return str(Path(self._path) / ANNOTATED_VIDEOS_FOLDER_NAME)@property- def mediapipe_2d_data_npy_file_path(self):- return str(Path(self._path) / OUTPUT_DATA_FOLDER_NAME / RAW_DATA_FOLDER_NAME / MEDIAPIPE_2D_NPY_FILE_NAME)+ def data_2d_npy_file_path(self):+ data_2d_path = Path(self.raw_data_folder_path) / (self.file_prefix + DATA_2D_NPY_FILE_NAME)+ old_data_2d_path = Path(self.raw_data_folder_path) / OLD_DATA_2D_NPY_FILE_NAME+ if self.active_tracker == "mediapipe" and old_data_2d_path.exists() and not data_2d_path.exists():+ return str(old_data_2d_path)+ else:+ return str(data_2d_path)@property- def mediapipe_3d_data_npy_file_path(self):- return str(Path(self._path) / OUTPUT_DATA_FOLDER_NAME / MEDIAPIPE_3D_NPY_FILE_NAME)+ def data_3d_npy_file_path(self):+ data_3d_path = Path(self._path) / OUTPUT_DATA_FOLDER_NAME / (self.file_prefix + DATA_3D_NPY_FILE_NAME)+ old_data_3d_path = Path(self._path) / OUTPUT_DATA_FOLDER_NAME / OLD_DATA_3D_NPY_FILE_NAME+ if self.active_tracker == "mediapipe" and old_data_3d_path.exists() and not data_3d_path.exists():+ return str(old_data_3d_path)+ else:+ return str(data_3d_path)@property- def raw_mediapipe_3d_data_npy_file_path(self):- return str(Path(self._path) / OUTPUT_DATA_FOLDER_NAME / RAW_DATA_FOLDER_NAME / RAW_MEDIAPIPE_3D_NPY_FILE_NAME)+ def raw_data_3d_npy_file_path(self):+ raw_data_path = (+ Path(self._path)+ / OUTPUT_DATA_FOLDER_NAME+ / RAW_DATA_FOLDER_NAME+ / (self.file_prefix + RAW_3D_NPY_FILE_NAME)+ )+ old_raw_data_path = Path(self._path) / OUTPUT_DATA_FOLDER_NAME / RAW_DATA_FOLDER_NAME / OLD_RAW_3D_NPY_FILE_NAME+ if self.active_tracker == "mediapipe" and old_raw_data_path.exists() and not raw_data_path.exists():+ return str(old_raw_data_path)+ else:+ return str(raw_data_path)@property- def mediapipe_reprojection_error_data_npy_file_path(self):- return str(+ def reprojection_error_data_npy_file_path(self):+ reprojection_error_path = (Path(self._path)/ OUTPUT_DATA_FOLDER_NAME/ RAW_DATA_FOLDER_NAME- / RAW_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME+ / (self.file_prefix + REPROJECTION_ERROR_NPY_FILE_NAME))+ raw_reprojection_error_path = (+ Path(self._path) / OUTPUT_DATA_FOLDER_NAME / RAW_DATA_FOLDER_NAME / OLD_REPROJECTION_ERROR_NPY_FILE_NAME+ )+ if (+ self.active_tracker == "mediapipe"+ and raw_reprojection_error_path.exists()+ and not reprojection_error_path.exists()+ ):+ return str(raw_reprojection_error_path)+ else:+ return str(reprojection_error_path)@propertydef total_body_center_of_mass_npy_file_path(self):- return str(+ total_body_com_path = (Path(self._path)/ OUTPUT_DATA_FOLDER_NAME/ CENTER_OF_MASS_FOLDER_NAME- / TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME+ / (self.file_prefix + TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME))+ raw_total_body_com_path = (+ Path(self._path)+ / OUTPUT_DATA_FOLDER_NAME+ / CENTER_OF_MASS_FOLDER_NAME+ / OLD_TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME+ )+ if self.active_tracker == "mediapipe" and raw_total_body_com_path.exists() and not total_body_com_path.exists():+ return str(raw_total_body_com_path)+ else:+ return str(total_body_com_path)@propertydef blender_file_path(self):@@ -185,21 +252,28 @@ class RecordingFolderStatusChecker:def check_data2d_status(self) -> bool:try:+ test_image_tracking_data_exists(+ synchronized_video_folder_path=self.recording_info_model.synchronized_videos_folder_path,+ image_tracking_data=self.recording_info_model.data_2d_npy_file_path,+ )test_image_tracking_data_shape(synchronized_video_folder_path=self.recording_info_model.synchronized_videos_folder_path,- image_tracking_data_file_path=self.recording_info_model.mediapipe_2d_data_npy_file_path,+ image_tracking_data=self.recording_info_model.data_2d_npy_file_path,)-return Trueexcept AssertionError:return Falsedef check_data3d_status(self) -> bool:try:- test_mediapipe_skeleton_data_shape(+ test_skeleton_data_exists(+ raw_skeleton_data=self.recording_info_model.data_3d_npy_file_path,+ reprojection_error_data=self.recording_info_model.reprojection_error_data_npy_file_path,+ )+ test_skeleton_data_shape(synchronized_video_folder_path=self.recording_info_model.synchronized_videos_folder_path,- raw_skeleton_npy_file_path=self.recording_info_model.mediapipe_3d_data_npy_file_path,- reprojection_error_file_path=self.recording_info_model.mediapipe_reprojection_error_data_npy_file_path,+ raw_skeleton_data=self.recording_info_model.data_3d_npy_file_path,+ reprojection_error_data=self.recording_info_model.reprojection_error_data_npy_file_path,)return Trueexcept AssertionError:@@ -207,9 +281,12 @@ class RecordingFolderStatusChecker:def check_center_of_mass_data_status(self) -> bool:try:+ test_total_body_center_of_mass_data_exists(+ total_body_center_of_mass_data=self.recording_info_model.total_body_center_of_mass_npy_file_path,+ )test_total_body_center_of_mass_data_shape(synchronized_video_folder_path=self.recording_info_model.synchronized_videos_folder_path,- total_body_center_of_mass_file_path=self.recording_info_model.total_body_center_of_mass_npy_file_path,+ total_body_center_of_mass_data=self.recording_info_model.total_body_center_of_mass_npy_file_path,)return Trueexcept AssertionError:diff --git a/freemocap/data_layer/skeleton_models/marker_info.py b/freemocap/data_layer/skeleton_models/marker_info.pynew file mode 100644index 00000000..c2e3c24b--- /dev/null+++ b/freemocap/data_layer/skeleton_models/marker_info.py@@ -0,0 +1,66 @@+from pydantic import BaseModel, model_validator, Field, field_validator+from typing import Dict, List, Optional, Union+++class VirtualMarkerInfo(BaseModel):+ virtual_markers: Dict[str, Dict[str, List[Union[float, str]]]]++ @field_validator("virtual_markers")+ @classmethod+ def validate_virtual_marker(cls, virtual_marker: Dict[str, List[Union[float, str]]]):+ for virtual_marker_name, virtual_marker_values in virtual_marker.items():+ marker_names = virtual_marker_values.get("marker_names", []) # TODO: make sure this .get is valid+ marker_weights = virtual_marker_values.get("marker_weights", [])++ if len(marker_names) != len(marker_weights):+ raise ValueError(+ f"The number of marker names must match the number of marker weights for {virtual_marker_name}. Currently there are {len(marker_names)} names and {len(marker_weights)} weights."+ )++ if not isinstance(marker_names, list) or not all(isinstance(name, str) for name in marker_names):+ raise ValueError(f"Marker names must be a list of strings for {marker_names}.")++ if not isinstance(marker_weights, list) or not all(+ isinstance(weight, (int, float)) for weight in marker_weights+ ):+ raise ValueError(f"Marker weights must be a list of numbers for {virtual_marker_name}.")++ weight_sum = sum(marker_weights)+ if not 0.99 <= weight_sum <= 1.01: # Allowing a tiny bit of floating-point leniency+ raise ValueError(+ f"Marker weights must sum to approximately 1 for {virtual_marker_name} Current sum is {weight_sum}."+ )++ return virtual_marker+++class MarkerInfo(BaseModel):+ original_marker_names: List[str]+ virtual_marker_definition: Optional[VirtualMarkerInfo] = None+ all_markers_list: List[str] = Field(default_factory=list)++ @model_validator(mode="before")+ def copy_markers_to_all(cls, values):+ """Copy markers to all_markers_list at initialization."""+ original_marker_names = values.get("original_marker_names")+ if original_marker_names:+ # Directly initializing all_markers_list with a copy of original_marker_names+ values["all_markers_list"] = original_marker_names.copy()+ return values++ def add_virtual_markers(self, virtual_markers_dict: Dict[str, Dict[str, List[Union[float, str]]]]):+ """Add virtual markers and update all_markers_list."""+ self.virtual_marker_definition = VirtualMarkerInfo(virtual_markers=virtual_markers_dict)+ for virtual_marker_name in self.virtual_marker_definition.virtual_markers.keys():+ if virtual_marker_name not in self.all_markers_list:+ self.all_markers_list.append(virtual_marker_name)++ @property+ def all_markers(self) -> List[str]:+ """Publicly expose the combined list of markers."""+ return self.all_markers_list++ @classmethod+ def create(cls, marker_list: List[str]):+ """Class method to create an instance with initial marker names."""+ return cls(original_marker_names=marker_list)diff --git a/freemocap/data_layer/skeleton_models/segments.py b/freemocap/data_layer/skeleton_models/segments.pynew file mode 100644index 00000000..add6f6cd--- /dev/null+++ b/freemocap/data_layer/skeleton_models/segments.py@@ -0,0 +1,36 @@+from typing import Dict+from pydantic import BaseModel, model_validator++from freemocap.data_layer.skeleton_models.marker_info import MarkerInfo+++class Segment(BaseModel):+ proximal: str+ distal: str+++class SegmentAnthropometry(BaseModel):+ segment_com_length: float+ segment_com_percentage: float+++class Segments(BaseModel):+ markers: MarkerInfo+ segment_connections: Dict[str, Segment]++ @model_validator(mode="before")+ def check_that_all_markers_exist(cls, values):+ markers = values.get("markers").all_markers+ segment_connections = values.get("segment_connections")++ for segment_name, segment_connection in segment_connections.items():+ if segment_connection.get("proximal") not in markers:+ raise ValueError(+ f"The proximal marker {segment_connection.proximal} for {segment_name} is not in the list of markers or virtual markers."+ )+ if segment_connection.get("distal") not in markers:+ raise ValueError(+ f"The distal marker {segment_connection.distal} for {segment_name} is not in the list of markers or virtual markers."+ )++ return valuesdiff --git a/freemocap/data_layer/skeleton_models/skeleton.py b/freemocap/data_layer/skeleton_models/skeleton.pynew file mode 100644index 00000000..108d6f8d--- /dev/null+++ b/freemocap/data_layer/skeleton_models/skeleton.py@@ -0,0 +1,133 @@+import logging+from typing import Dict, List, Optional+from pydantic import BaseModel, ConfigDict+import numpy as np+++from freemocap.data_layer.skeleton_models.marker_info import MarkerInfo+from freemocap.data_layer.skeleton_models.segments import Segment, Segments, SegmentAnthropometry++logger = logging.getLogger(__name__)+++class Skeleton(BaseModel):+ model_config = ConfigDict(arbitrary_types_allowed=True)+ markers: MarkerInfo+ num_tracked_points: int+ segments: Optional[Dict[str, Segment]] = None+ marker_data: Dict[str, np.ndarray] = {}+ virtual_marker_data: Dict[str, np.ndarray] = {}+ joint_hierarchy: Optional[Dict[str, List[str]]] = None+ center_of_mass_definitions: Optional[Dict[str, SegmentAnthropometry]] = None+ num_frames: Optional[int] = None++ def add_segments(self, segment_connections: Dict[str, Segment]) -> None:+ """+ Adds segment connection data to the skeleton model.++ Parameters:+ - segment_connections: A dictionary where each key is a segment name and its value is a dictionary+ with information about that segment (e.g., 'proximal', 'distal' marker names).+ """+ segments_model = Segments(+ markers=self.markers,+ segment_connections={name: segment for name, segment in segment_connections.items()},+ )+ self.segments = segments_model.segment_connections++ def add_joint_hierarchy(self, joint_hierarchy: Dict[str, List[str]]) -> None:+ """+ Adds joint hierarchy data to the skeleton model.++ Parameters:+ - joint_hierarchy: A dictionary with joint names as keys and lists of connected marker names as values.+ """+ for joint_name, joint_connections in joint_hierarchy.items():+ if joint_name not in self.markers.all_markers:+ raise ValueError(f"The joint {joint_name} is not in the list of markers or virtual markers.")+ for connected_marker in joint_connections:+ if connected_marker not in self.markers.all_markers:+ raise ValueError(+ f"The connected marker {connected_marker} for {joint_name} is not in the list of markers or virtual markers."+ )++ self.joint_hierarchy = joint_hierarchy++ def add_center_of_mass_definitions(self, center_of_mass_definitions: Dict[str, Dict[str, float]]) -> None:+ """+ Adds anthropometric center of mass definitions to the skeleton model.++ Parameters:+ - center_of_mass_definitions: A dictionary containing segment mass percentages.+ """+ if self.segments is None:+ raise ValueError("Segments must be defined before center of mass definitions can be added.")+ for com_segment_name in center_of_mass_definitions.keys():+ if com_segment_name not in self.segments.keys():+ raise ValueError(f"Segment {com_segment_name} is not in the list of segments.")++ self.center_of_mass_definitions = {+ name: SegmentAnthropometry(**values) for name, values in center_of_mass_definitions.items()+ }++ def integrate_freemocap_3d_data(self, freemocap_3d_data: np.ndarray) -> None:+ self.num_frames = freemocap_3d_data.shape[0]+ num_markers_in_data = freemocap_3d_data.shape[1]+ original_marker_names_list = self.markers.original_marker_names++ if num_markers_in_data != self.num_tracked_points:+ raise ValueError(+ f"The number of markers in the 3D data ({num_markers_in_data}) does not match "+ f"the expected number of tracked points ({self.num_tracked_points})."+ )++ self.marker_data = {+ marker_name: freemocap_3d_data[:, i, :] for i, marker_name in enumerate(original_marker_names_list)+ }++ try:+ self.calculate_virtual_markers()+ except ValueError:+ logger.info(+ "Freemocap data integrated without virtual markers, as no virtual marker definition was provided"+ )++ def calculate_virtual_markers(self) -> None:+ if not self.marker_data:+ raise ValueError(+ "3d marker data must be integrated before calculating virtual markers. Run `integrate_freemocap_3d_data()` first."+ )++ if not self.markers.virtual_marker_definition:+ raise ValueError(+ "Virtual marker info must be defined before calculating virtual markers. Run `add_virtual_markers()` first."+ )++ for vm_name, vm_info in self.markers.virtual_marker_definition.virtual_markers.items():+ vm_positions = np.zeros((self.marker_data[next(iter(self.marker_data))].shape[0], 3))+ for marker_name, weight in zip(vm_info["marker_names"], vm_info["marker_weights"]):+ vm_positions += self.marker_data[str(marker_name)] * weight+ self.virtual_marker_data[vm_name] = vm_positions++ self.marker_data.update(self.virtual_marker_data)++ def get_segment_markers(self, segment_name: str) -> Dict[str, np.ndarray]:+ """Returns a dictionary with the positions of the proximal and distal markers for a segment."""+ if not self.segments:+ raise ValueError("Segments must be defined before getting segment markers.")++ if not self.trajectories:+ raise ValueError("Trajectories must be defined before getting segment markers.")++ segment = self.segments.get(segment_name)+ if not segment:+ raise ValueError(f"Segment '{segment_name}' is not defined in the skeleton.")++ proximal_trajectories = self.trajectories.get(segment.proximal)+ distal_trajectories = self.trajectories.get(segment.distal)++ return {"proximal": proximal_trajectories, "distal": distal_trajectories}++ @property+ def trajectories(self) -> Dict[str, np.ndarray]:+ return self.marker_datadiff --git a/freemocap/gui/qt/main_window/freemocap_main_window.py b/freemocap/gui/qt/main_window/freemocap_main_window.pyindex b7783f02..0a179b40 100644--- a/freemocap/gui/qt/main_window/freemocap_main_window.py+++ b/freemocap/gui/qt/main_window/freemocap_main_window.py@@ -294,12 +294,16 @@ class MainWindow(QMainWindow):def _export_active_recording_to_blender(self):logger.debug("Exporting active recording to Blender...")- recording_path = self._active_recording_info_widget.get_active_recording_info(return_path=True)+ recording_path = self._active_recording_info_widget.get_active_recording_path()if self._visualization_control_panel.blender_executable_path is None:logger.error("Blender executable path is None!")return+ if not recording_path:+ logger.error("Recording path is None!")+ return+self._export_to_blender_thread_worker = ExportToBlenderThreadWorker(recording_path=recording_path,blender_file_path=Path(get_blender_file_path(recording_path)),@@ -323,9 +327,10 @@ class MainWindow(QMainWindow):def _generate_jupyter_notebook(self):logger.info("Exporting active recording to a Jupyter notebook...")- recording_path = self._active_recording_info_widget.get_active_recording_info(return_path=True)+ recording_path = self._active_recording_info_widget.get_active_recording_path()# TODO: Need to include jupyter notebook in recording files that we keep track of (2023-05-15)- generate_jupyter_notebook(path_to_recording=recording_path)+ if recording_path:+ generate_jupyter_notebook(path_to_recording=recording_path)def _handle_new_active_recording_selected(self, recording_info_model: RecordingInfoModel):logger.info(f"New active recording selected: {recording_info_model.path}")@@ -368,7 +373,7 @@ class MainWindow(QMainWindow):if active_recording_info.data3d_status_check:self._skelly_viewer_widget.load_skeleton_data(- mediapipe_skeleton_npy_path=active_recording_info.mediapipe_3d_data_npy_file_path+ mediapipe_skeleton_npy_path=active_recording_info.data_3d_npy_file_path)if active_recording_info.data2d_status_check:diff --git a/freemocap/gui/qt/widgets/active_recording_widget.py b/freemocap/gui/qt/widgets/active_recording_widget.pyindex 605bc9fc..1d0d48c8 100644--- a/freemocap/gui/qt/widgets/active_recording_widget.py+++ b/freemocap/gui/qt/widgets/active_recording_widget.py@@ -40,7 +40,7 @@ class ActiveRecordingInfoWidget(QWidget):def active_recording_view_widget(self):return self._active_recording_view_widget- def get_active_recording_info(self, return_path: bool = False) -> Union[RecordingInfoModel, Path, None]:+ def get_active_recording_info(self) -> Union[RecordingInfoModel, None]:# this is redundant to the `active_recording_info` property,# but it will be more intuitive to send this down as a callable# rather than relying on 'pass-by-reference' magic lol@@ -50,11 +50,12 @@ class ActiveRecordingInfoWidget(QWidget):if most_recent_path is not None:self.set_active_recording(most_recent_path)- if return_path:- return self._active_recording_info.path-return self._active_recording_info+ def get_active_recording_path(self) -> Optional[Path]:+ if self._active_recording_info:+ return Path(self._active_recording_info.path)+def set_active_recording(self,recording_folder_path: Union[str, Path, None],diff --git a/freemocap/gui/qt/widgets/control_panel/calibration_control_panel.py b/freemocap/gui/qt/widgets/control_panel/calibration_control_panel.pyindex 6a9daffd..eb303d64 100644--- a/freemocap/gui/qt/widgets/control_panel/calibration_control_panel.py+++ b/freemocap/gui/qt/widgets/control_panel/calibration_control_panel.py@@ -2,9 +2,9 @@ import loggingimport osfrom pathlib import Pathimport threading-from typing import Callable, Union+from typing import Callable, Optional, Union-from PySide6.QtCore import Qt, Slot+from PySide6.QtCore import Qt, Slot, QObjectfrom PySide6.QtGui import QDoubleValidatorfrom PySide6.QtWidgets import (QFileDialog,@@ -38,7 +38,7 @@ class CalibrationControlPanel(QWidget):get_active_recording_info: Callable[..., Union[RecordingInfoModel, Path]],kill_thread_event: threading.Event,gui_state: GuiState,- parent=None,+ parent: Optional[QObject] = None,):super().__init__(parent=parent)self.gui_state = gui_statediff --git a/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/parameter_groups/create_parameter_groups.py b/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/parameter_groups/create_parameter_groups.pyindex 52eba4ad..b4bb222a 100644--- a/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/parameter_groups/create_parameter_groups.py+++ b/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/parameter_groups/create_parameter_groups.py@@ -63,6 +63,7 @@ RUN_BUTTERWORTH_FILTER_NAME = "Run butterworth filter?"NUMBER_OF_PROCESSES_PARAMETER_NAME = "Max Number of Processes to Use"+# TODO: figure out how to generalize thisdef create_mediapipe_parameter_group(parameter_model: MediapipeTrackingParams,) -> Parameter:@@ -239,7 +240,7 @@ def extract_parameter_model_from_parameter_tree(parameter_values_dictionary = extract_processing_parameter_model_from_tree(parameter_object=parameter_object)return ProcessingParameterModel(- mediapipe_parameters_model=MediapipeTrackingParams(+ tracking_parameters_model=MediapipeTrackingParams(mediapipe_model_complexity=get_integer_from_mediapipe_model_complexity(parameter_values_dictionary[MEDIAPIPE_MODEL_COMPLEXITY]),diff --git a/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/process_motion_capture_data_panel.py b/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/process_motion_capture_data_panel.pyindex 8bb757f8..e08fa7bb 100644--- a/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/process_motion_capture_data_panel.py+++ b/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/process_motion_capture_data_panel.py@@ -3,7 +3,7 @@ import multiprocessingimport shutilimport threadingfrom pathlib import Path-from typing import Callable, Union+from typing import Callablefrom PySide6.QtCore import Signal, Slot, Qtfrom PySide6.QtGui import QFont@@ -39,7 +39,7 @@ class ProcessMotionCaptureDataPanel(QWidget):def __init__(self,recording_processing_parameters: ProcessingParameterModel,- get_active_recording_info: Callable[..., Union[RecordingInfoModel, Path]],+ get_active_recording_info: Callable[..., RecordingInfoModel],kill_thread_event: threading.Event,log_update: Callable,gui_state: GuiState,@@ -126,7 +126,7 @@ class ProcessMotionCaptureDataPanel(QWidget):children=[self._create_new_run_this_step_parameter(run_step_name=RUN_IMAGE_TRACKING_NAME),self._create_num_processes_parameter(),- create_mediapipe_parameter_group(session_processing_parameter_model.mediapipe_parameters_model),+ create_mediapipe_parameter_group(session_processing_parameter_model.tracking_parameters_model),],tip="Methods for tracking 2d points in images (e.g. mediapipe, deeplabcut(TODO), openpose(TODO), etc ...)",),@@ -249,6 +249,11 @@ class ProcessMotionCaptureDataPanel(QWidget):)shutil.copyfile(selected_camera_calibration_toml_path, copied_toml_path)+ # set active tracker in recording model to the currently selected tracker+ session_parameter_model.recording_info_model.active_tracker = (+ session_parameter_model.tracking_model_info.name+ )+self._process_motion_capture_data_thread_worker = ProcessMotionCaptureDataThreadWorker(session_parameter_model, kill_event=self._kill_thread_event)diff --git a/freemocap/gui/qt/workers/process_motion_capture_data_thread_worker.py b/freemocap/gui/qt/workers/process_motion_capture_data_thread_worker.pyindex 134ef5d2..528102f3 100644--- a/freemocap/gui/qt/workers/process_motion_capture_data_thread_worker.py+++ b/freemocap/gui/qt/workers/process_motion_capture_data_thread_worker.py@@ -21,29 +21,29 @@ class ProcessMotionCaptureDataThreadWorker(QThread):self, post_processing_parameters: ProcessingParameterModel, kill_event: multiprocessing.Event, parent=None):super().__init__()- self._post_processing_parameters = post_processing_parameters+ self._processing_parameters = post_processing_parametersself._kill_event = kill_eventself._queue = multiprocessing.Queue()self._process = multiprocessing.Process(- target=process_recording_folder, args=(self._post_processing_parameters, self._kill_event, self._queue)+ target=process_recording_folder, args=(self._processing_parameters, self._kill_event, self._queue))self._success = Nonedef run(self):logger.info(- f"Beginning processing of motion capture data with parameters: {self._post_processing_parameters.__dict__}"+ f"Beginning processing of motion capture data with parameters: {self._processing_parameters.dict(exclude={'tracking_model_info'})}")self._kill_event.clear()- recording_info_dict = self._post_processing_parameters.dict(exclude={"recording_info_model"})- Path(self._post_processing_parameters.recording_info_model.output_data_folder_path).mkdir(+ recording_info_dict = self._processing_parameters.dict(exclude={"recording_info_model"})+ Path(self._processing_parameters.recording_info_model.output_data_folder_path).mkdir(parents=True, exist_ok=True)save_dictionary_to_json(- save_path=self._post_processing_parameters.recording_info_model.output_data_folder_path,+ save_path=self._processing_parameters.recording_info_model.output_data_folder_path,file_name=RECORDING_PARAMETERS_JSON_FILE_NAME,dictionary=recording_info_dict,)diff --git a/freemocap/system/paths_and_filenames/file_and_folder_names.py b/freemocap/system/paths_and_filenames/file_and_folder_names.pyindex dcb8bd9a..82cd36c1 100644--- a/freemocap/system/paths_and_filenames/file_and_folder_names.py+++ b/freemocap/system/paths_and_filenames/file_and_folder_names.py@@ -21,26 +21,30 @@ CENTER_OF_MASS_FOLDER_NAME = "center_of_mass"MOST_RECENT_RECORDING_TOML_FILENAME = "most_recent_recording.toml"LAST_SUCCESSFUL_CALIBRATION_TOML_FILENAME = "last_successful_calibration.toml"GUI_STATE_JSON_FILENAME = "gui_state.json"-MEDIAPIPE_2D_NPY_FILE_NAME = "mediapipe2dData_numCams_numFrames_numTrackedPoints_pixelXY.npy"-MEDIAPIPE_BODY_WORLD_FILE_NAME = "mediapipeBodyWorld_numCams_numFrames_numTrackedPoints_XYZ.npy"+DATA_2D_NPY_FILE_NAME = "2dData_numCams_numFrames_numTrackedPoints_pixelXY.npy"-RAW_MEDIAPIPE_3D_NPY_FILE_NAME = "mediapipe3dData_numFrames_numTrackedPoints_spatialXYZ.npy"-RAW_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME = "mediapipe3dData_numFrames_numTrackedPoints_reprojectionError.npy"-RAW_MEDIAPIPE_FULL_REPROJECTION_ERROR_NPY_FILE_NAME = "mediapipe3dData_numCams_numFrames_numTrackedPoints_reprojectionError.npy"+RAW_3D_NPY_FILE_NAME = "3dData_numFrames_numTrackedPoints_spatialXYZ.npy"+REPROJECTION_ERROR_NPY_FILE_NAME = "3dData_numFrames_numTrackedPoints_reprojectionError.npy"+FULL_REPROJECTION_ERROR_NPY_FILE_NAME = "3dData_numCams_numFrames_numTrackedPoints_reprojectionError.npy"+REPROJECTION_FILTERED_PREFIX = "reprojection_filtered_"-REPROJECTION_FILTERED_MEDIAPIPE_3D_NPY_FILE_NAME = "reprojection_filtered_mediapipe3dData_numFrames_numTrackedPoints_spatialXYZ.npy"-REPROJECTION_FILTERED_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME = "reprojection_filtered_mediapipe3dData_numFrames_numTrackedPoints_reprojectionError.npy"-REPROJECTION_FILTERED_MEDIAPIPE_FULL_REPROJECTION_ERROR_NPY_FILE_NAME = "reprojection_filtered_mediapipe3dData_numCams_numFrames_numTrackedPoints_reprojectionError.npy"--MEDIAPIPE_3D_NPY_FILE_NAME = "mediaPipeSkel_3d_body_hands_face.npy"-MEDIAPIPE_BODY_3D_DATAFRAME_CSV_FILE_NAME = "mediapipe_body_3d_xyz.csv"-MEDIAPIPE_RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME = "mediapipe_right_hand_3d_xyz.csv"-MEDIAPIPE_LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME = "mediapipe_left_hand_3d_xyz.csv"-MEDIAPIPE_FACE_3D_DATAFRAME_CSV_FILE_NAME = "mediapipe_face_3d_xyz.csv"+DATA_3D_NPY_FILE_NAME = "skeleton_3d.npy"+RIGID_BONES_NPY_FILE_NAME = "rigid_bones_3d.npy"+BODY_3D_DATAFRAME_CSV_FILE_NAME = "body_3d_xyz.csv"+RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME = "right_hand_3d_xyz.csv"+LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME = "left_hand_3d_xyz.csv"+FACE_3D_DATAFRAME_CSV_FILE_NAME = "face_3d_xyz.csv"TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME = "total_body_center_of_mass_xyz.npy"SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME = "segmentCOM_frame_joint_xyz.npy"+OLD_DATA_2D_NPY_FILE_NAME = "mediapipe2dData_numCams_numFrames_numTrackedPoints_pixelXY.npy"+OLD_RAW_3D_NPY_FILE_NAME = "mediapipe3dData_numFrames_numTrackedPoints_spatialXYZ.npy"+OLD_REPROJECTION_ERROR_NPY_FILE_NAME = "mediapipe3dData_numFrames_numTrackedPoints_reprojectionError.npy"+OLD_DATA_3D_NPY_FILE_NAME = "mediaPipeSkel_3d_body_hands_face.npy"+OLD_TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME = "total_body_center_of_mass_xyz.npy"+OLD_SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME = "segmentCOM_frame_joint_xyz.npy"+RECORDING_PARAMETERS_JSON_FILE_NAME = "recording_parameters.json"# Figshare infodiff --git a/freemocap/system/paths_and_filenames/path_getters.py b/freemocap/system/paths_and_filenames/path_getters.pyindex be3b589f..8c719b79 100644--- a/freemocap/system/paths_and_filenames/path_getters.py+++ b/freemocap/system/paths_and_filenames/path_getters.py@@ -18,13 +18,12 @@ from freemocap.system.paths_and_filenames.file_and_folder_names import (OUTPUT_DATA_FOLDER_NAME,SYNCHRONIZED_VIDEOS_FOLDER_NAME,RAW_DATA_FOLDER_NAME,- RAW_MEDIAPIPE_3D_NPY_FILE_NAME,+ RAW_3D_NPY_FILE_NAME,CENTER_OF_MASS_FOLDER_NAME,TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME,- MEDIAPIPE_2D_NPY_FILE_NAME,- RAW_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME,- SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME,- MEDIAPIPE_3D_NPY_FILE_NAME,+ DATA_2D_NPY_FILE_NAME,+ REPROJECTION_ERROR_NPY_FILE_NAME,+ DATA_3D_NPY_FILE_NAME,GUI_STATE_JSON_FILENAME,)@@ -118,7 +117,7 @@ def session_time_tag_format():return "%Y-%m-%d_%H_%M_%S"-def default_session_name(string_tag: str = None):+def default_session_name(string_tag: Optional[str] = None):if string_tag is not None:string_tag = f"_{string_tag}"else:@@ -245,19 +244,20 @@ def get_synchronized_videos_folder_path(recording_folder_path: Union[str, Path])raise Exception(f"Could not find a videos folder in path {str(recording_folder_path)}")-def get_timestamps_directory(recording_directory: Union[str, Path]) -> str:+def get_timestamps_directory(recording_directory: Union[str, Path]) -> Optional[str]:synchronized_videos_path = get_synchronized_videos_folder_path(recording_folder_path=recording_directory)if "timestamps" in [path.name for path in Path(synchronized_videos_path).iterdir()]:return str(Path(synchronized_videos_path) / "timestamps")logger.warning(f"Could not find timestamps directory in {synchronized_videos_path}")+# TODO: figure out how to deal with different file prefixes for all of these belowdef get_raw_skeleton_npy_file_name(data_folder_name: Union[str, Path]) -> str:raw_data_subfolder_path = Path(data_folder_name) / RAW_DATA_FOLDER_NAMEif raw_data_subfolder_path.exists:raw_data_npy_path_list = [path.name for path in raw_data_subfolder_path.glob("*.npy")]- if RAW_MEDIAPIPE_3D_NPY_FILE_NAME in raw_data_npy_path_list:- return str(raw_data_subfolder_path / RAW_MEDIAPIPE_3D_NPY_FILE_NAME)+ if RAW_3D_NPY_FILE_NAME in raw_data_npy_path_list:+ return str(raw_data_subfolder_path / RAW_3D_NPY_FILE_NAME)logger.debug(list(Path(data_folder_name).glob("*")))logger.debug(list(Path(raw_data_subfolder_path).glob("*")))@@ -265,7 +265,7 @@ def get_raw_skeleton_npy_file_name(data_folder_name: Union[str, Path]) -> str:def get_full_npy_file_path(output_data_folder: Union[str, Path]) -> str:- path = Path(output_data_folder) / MEDIAPIPE_3D_NPY_FILE_NAME+ path = Path(output_data_folder) / DATA_3D_NPY_FILE_NAMEreturn str(path)@@ -280,24 +280,13 @@ def get_total_body_center_of_mass_file_path(output_data_folder: Union[str, Path]raise Exception(f"Could not find a total body center of mass npy file in path {str(output_data_folder)}")-def get_segment_center_of_mass_file_path(output_data_folder: Union[str, Path]) -> str:- center_of_mass_subfolder_path = Path(output_data_folder) / CENTER_OF_MASS_FOLDER_NAME- if center_of_mass_subfolder_path.exists:- center_of_mass_npy_path_list = [path.name for path in center_of_mass_subfolder_path.glob("*.npy")]-- if SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME in center_of_mass_npy_path_list:- return str(center_of_mass_subfolder_path / SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME)-- raise Exception(f"Could not find a total body center of mass npy file in path {str(output_data_folder)}")--def get_image_tracking_data_file_name(data_folder_name: Union[str, Path]) -> str:raw_data_subfolder_path = Path(data_folder_name) / RAW_DATA_FOLDER_NAMEif raw_data_subfolder_path.exists:raw_data_npy_path_list = [path.name for path in raw_data_subfolder_path.glob("*.npy")]- if MEDIAPIPE_2D_NPY_FILE_NAME in raw_data_npy_path_list:- return str(raw_data_subfolder_path / MEDIAPIPE_2D_NPY_FILE_NAME)+ if DATA_2D_NPY_FILE_NAME in raw_data_npy_path_list:+ return str(raw_data_subfolder_path / DATA_2D_NPY_FILE_NAME)raise Exception(f"Could not find a 2d data file in path {str(data_folder_name)}")@@ -307,7 +296,7 @@ def get_reprojection_error_file_path(data_folder_name: Union[str, Path]) -> str:if raw_data_subfolder_path.exists:raw_data_npy_path_list = [path.name for path in raw_data_subfolder_path.glob("*.npy")]- if RAW_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME in raw_data_npy_path_list:- return str(raw_data_subfolder_path / RAW_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME)+ if REPROJECTION_ERROR_NPY_FILE_NAME in raw_data_npy_path_list:+ return str(raw_data_subfolder_path / REPROJECTION_ERROR_NPY_FILE_NAME)raise Exception(f"Could not find reprojection error data file in path {str(data_folder_name)}")diff --git a/freemocap/tests/conftest.py b/freemocap/tests/conftest.pyindex af899a90..97faa3c5 100644--- a/freemocap/tests/conftest.py+++ b/freemocap/tests/conftest.py@@ -1,65 +1,69 @@+from typing import Optionalimport pytestfrom freemocap.core_processes.process_motion_capture_videos.process_recording_headless import (process_recording_headless,find_calibration_toml_path,)-from freemocap.system.paths_and_filenames.path_getters import (- get_output_data_folder_path,- get_synchronized_videos_folder_path,- get_raw_skeleton_npy_file_name,- get_total_body_center_of_mass_file_path,- get_image_tracking_data_file_name,- get_reprojection_error_file_path,-)+from freemocap.data_layer.recording_models.recording_info_model import RecordingInfoModelfrom freemocap.utilities.download_sample_data import (download_sample_data,)+class SessionInfo:+ sample_session_folder_path: str+ recording_info_model: RecordingInfoModeldef pytest_sessionstart():- pytest.sample_session_folder_path = download_sample_data()- calibration_toml_path = find_calibration_toml_path(pytest.sample_session_folder_path)+ SessionInfo.sample_session_folder_path = download_sample_data()+ calibration_toml_path = find_calibration_toml_path(SessionInfo.sample_session_folder_path)+ SessionInfo.recording_info_model = RecordingInfoModel(+ recording_folder_path=SessionInfo.sample_session_folder_path, active_tracker="mediapipe"+ )+ # TODO: make this configurable to be able to take different trackersprocess_recording_headless(- recording_path=pytest.sample_session_folder_path,+ recording_path=SessionInfo.sample_session_folder_path,path_to_camera_calibration_toml=calibration_toml_path,+ recording_info_model=SessionInfo.recording_info_model,+ run_blender=False,+ make_jupyter_notebook=False,use_tqdm=False,)@pytest.fixturedef sample_session_path():- return pytest.sample_session_folder_path+ return SessionInfo.sample_session_folder_path@pytest.fixturedef synchronized_video_folder_path():- return get_synchronized_videos_folder_path(pytest.sample_session_folder_path)+ return SessionInfo.recording_info_model.synchronized_videos_folder_path@pytest.fixturedef data_folder_path():- return get_output_data_folder_path(pytest.sample_session_folder_path)+ return SessionInfo.recording_info_model.output_data_folder_path@pytest.fixture-def raw_skeleton_npy_file_path():- return get_raw_skeleton_npy_file_name(get_output_data_folder_path(pytest.sample_session_folder_path))+def raw_skeleton_data():+ return SessionInfo.recording_info_model.raw_data_3d_npy_file_path@pytest.fixture-def total_body_center_of_mass_file_path():- return get_total_body_center_of_mass_file_path(get_output_data_folder_path(pytest.sample_session_folder_path))+def total_body_center_of_mass_data():+ return SessionInfo.recording_info_model.total_body_center_of_mass_npy_file_path@pytest.fixture-def image_tracking_data_file_path():- return get_image_tracking_data_file_name(get_output_data_folder_path(pytest.sample_session_folder_path))+def image_tracking_data():+ return SessionInfo.recording_info_model.data_2d_npy_file_path@pytest.fixture-def reprojection_error_file_path():- return get_reprojection_error_file_path(get_output_data_folder_path(pytest.sample_session_folder_path))+def reprojection_error_data():+ return SessionInfo.recording_info_model.reprojection_error_data_npy_file_pathif __name__ == "__main__":diff --git a/freemocap/tests/test_image_tracking_data_shape.py b/freemocap/tests/test_image_tracking_data_shape.pyindex 3f49b2a6..03cd5931 100644--- a/freemocap/tests/test_image_tracking_data_shape.py+++ b/freemocap/tests/test_image_tracking_data_shape.py@@ -1,5 +1,5 @@from pathlib import Path-from typing import Union+from typing import Optional, Unionimport numpy as npimport pytest@@ -10,29 +10,29 @@ from freemocap.utilities.get_number_of_frames_of_videos_in_a_folder import (from freemocap.utilities.get_video_paths import get_video_paths[email protected]("synchronized_video_folder_path", "image_tracking_data_file_path")-def test_image_tracking_data_shape([email protected]("synchronized_video_folder_path", "image_tracking_data")+def test_image_tracking_data_exists(synchronized_video_folder_path: Union[str, Path],- image_tracking_data_file_path,+ image_tracking_data: Union[str, Path],):"""- test that the `mediapipe 2d detection` process worked correctly by checking:+ test that the `2d detection` process worked correctly by checking:- 1. There is an `.npy` file containing the mediapipe 2d data in the `output_data_folder`+ There is an `.npy` file containing the 2d data in the `output_data_folder`2. The dimensions of that `npy` (number of cameras, number of frames, [ need to do - number of tracked points], [pixelX, pixelY] matches the videos in the `synchronized videos` folderTODO - check number of tracked points vs 'expected' number of tracked points"""- assert Path(image_tracking_data_file_path).is_file(), f"{image_tracking_data_file_path} is not a file"+ assert Path(image_tracking_data).is_file(), f"{image_tracking_data} is not a file"- image_tracking_data = np.load(image_tracking_data_file_path)+ image_tracking_data = np.load(image_tracking_data)list_of_video_paths = get_video_paths(Path(synchronized_video_folder_path))number_of_videos = len(list_of_video_paths)assert (image_tracking_data.shape[0] == number_of_videos- ), f"Number of videos in {image_tracking_data_file_path} does not match number of videos in synchronized videos folder"+ ), f"Number of videos in {image_tracking_data} does not match number of videos in synchronized videos folder"frame_counts = list(get_number_of_frames_of_videos_in_a_folder(synchronized_video_folder_path).values())@@ -41,10 +41,48 @@ def test_image_tracking_data_shape(), f"Videos in {synchronized_video_folder_path} have different frame counts: {frame_counts}"assert (image_tracking_data.shape[1] == frame_counts[0]- ), f"Number of frames in {image_tracking_data_file_path} does not match number of frames of videos in {synchronized_video_folder_path}"+ ), f"Number of frames in {image_tracking_data} does not match number of frames of videos in {synchronized_video_folder_path}"# TODO - check number of tracked points vs 'expected' number of tracked pointsassert (image_tracking_data.shape[3] == 3), f"Data has {image_tracking_data.shape[3]} dimensions, expected 3 dimensions"+[email protected]("synchronized_video_folder_path", "image_tracking_data")+def test_image_tracking_data_shape(+ synchronized_video_folder_path: Union[str, Path],+ image_tracking_data: Union[str, Path, np.ndarray],+):+ """+ test that the `2d detection` process worked correctly by checking:++ The dimensions of that `npy` (number of cameras, number of frames, [ need to do - number of tracked points], [pixelX, pixelY] matches the videos in the `synchronized videos` folder++ TODO - check number of tracked points vs 'expected' number of tracked points+ """+ if isinstance(image_tracking_data, (str, Path)):+ image2d_camera_frame_marker_xyz = np.load(image_tracking_data)+ else:+ image2d_camera_frame_marker_xyz = image_tracking_data++ list_of_video_paths = get_video_paths(Path(synchronized_video_folder_path))+ number_of_videos = len(list_of_video_paths)+ assert (+ image2d_camera_frame_marker_xyz.shape[0] == number_of_videos+ ), f"Number of videos in image_tracking_data does not match number of videos in synchronized videos folder"++ frame_counts = list(get_number_of_frames_of_videos_in_a_folder(synchronized_video_folder_path).values())++ assert (+ len(set(frame_counts)) == 1+ ), f"Videos in {synchronized_video_folder_path} have different frame counts: {frame_counts}"+ assert (+ image2d_camera_frame_marker_xyz.shape[1] == frame_counts[0]+ ), f"Number of frames in image_tracking_data does not match number of frames of videos in {synchronized_video_folder_path}"++ # TODO - check number of tracked points vs 'expected' number of tracked points++ assert (+ image2d_camera_frame_marker_xyz.shape[3] == 3+ ), f"Data has {image2d_camera_frame_marker_xyz.shape[3]} dimensions, expected 3 dimensions"diff --git a/freemocap/tests/test_mediapipe_skeleton_data_shape.py b/freemocap/tests/test_mediapipe_skeleton_data_shape.pydeleted file mode 100644index e26e79a3..00000000--- a/freemocap/tests/test_mediapipe_skeleton_data_shape.py+++ /dev/null@@ -1,61 +0,0 @@-from pathlib import Path-from typing import Union--import numpy as np-import pytest--from freemocap.utilities.get_number_of_frames_of_videos_in_a_folder import (- get_number_of_frames_of_videos_in_a_folder,-)--[email protected]("synchronized_video_folder_path", "raw_skeleton_npy_file_path", "reprojection_error_file_path")-def test_mediapipe_skeleton_data_shape(- synchronized_video_folder_path: Union[str, Path],- raw_skeleton_npy_file_path: Union[str, Path],- reprojection_error_file_path,-):- """- test that the `mediapipe 3d detection` process worked correctly by checking:-- 1. There is an `.npy` file containing the mediapipe 3d data in the `output_data_folder`- 2. The dimensions of that `npy` ( number of frames, [ need to do - number of tracked points], [X,Y,Z] matches the videos in the `synchronized videos` folder-- TODO - check number of tracked points vs 'expected' number of tracked points- """-- assert Path(- raw_skeleton_npy_file_path- ).is_file(), f"3d skeleton data file does not exist at {raw_skeleton_npy_file_path}"-- skel3d_frame_marker_xyz = np.load(raw_skeleton_npy_file_path)-- assert (- len(skel3d_frame_marker_xyz.shape) == 3- ), f"3d skeleton data file should have 3 dimensions - {raw_skeleton_npy_file_path}"-- assert Path(- reprojection_error_file_path- ).is_file(), f"3d skeleton reprojection error data file does not exist at {reprojection_error_file_path}"-- skeleton_reprojection_error_fr_mar = np.load(reprojection_error_file_path)-- assert (- len(skeleton_reprojection_error_fr_mar.shape) == 2- ), f"3d skeleton reprojection error data file should have 2 dimensions {reprojection_error_file_path}"-- frame_counts = list(get_number_of_frames_of_videos_in_a_folder(synchronized_video_folder_path).values())- assert (- len(set(frame_counts)) == 1- ), f"Videos in {synchronized_video_folder_path} have different frame counts: {frame_counts}"-- number_of_frames = frame_counts[0]-- assert skel3d_frame_marker_xyz.shape[0] == number_of_frames- assert skeleton_reprojection_error_fr_mar.shape[0] == number_of_frames-- # TODO - check number of tracked points vs 'expected' number of tracked points-- assert (- skel3d_frame_marker_xyz.shape[2] == 3- ), f"3d skeleton data file does not have 3 dimensions for X,Y,Z at {raw_skeleton_npy_file_path}"diff --git a/freemocap/tests/test_skeleton_data_shape.py b/freemocap/tests/test_skeleton_data_shape.pynew file mode 100644index 00000000..e3a155ae--- /dev/null+++ b/freemocap/tests/test_skeleton_data_shape.py@@ -0,0 +1,73 @@+from pathlib import Path+from typing import Optional, Union++import numpy as np+import pytest++from freemocap.utilities.get_number_of_frames_of_videos_in_a_folder import (+ get_number_of_frames_of_videos_in_a_folder,+)++[email protected]("raw_skeleton_data", "reprojection_error_data")+def test_skeleton_data_exists(+ raw_skeleton_data: Union[str, Path],+ reprojection_error_data: Union[str, Path],+):+ """+ test that the `3d detection` process worked correctly by checking that there is an `.npy` file containing the 3d data in the `output_data_folder`+ """++ assert Path(+ raw_skeleton_data+ ).is_file(), f"3d skeleton data file does not exist at {raw_skeleton_data}"++ assert Path(+ reprojection_error_data+ ).is_file(), f"3d skeleton reprojection error data file does not exist at {reprojection_error_data}"++[email protected]("synchronized_video_folder_path", "raw_skeleton_data", "reprojection_error_data")+def test_skeleton_data_shape(+ synchronized_video_folder_path: Union[str, Path],+ raw_skeleton_data: Union[str, Path, np.ndarray],+ reprojection_error_data: Union[str, Path, np.ndarray],+):+ """+ test that the `3d detection` process worked correctly by checking:+ 1. Dimensions of the raw skeleton numpy array are correct+ 2. Dimensions of the reprojection error numpy array are correct+ """+ if isinstance(raw_skeleton_data, (str, Path)):+ skel3d_frame_marker_xyz = np.load(raw_skeleton_data)+ else:+ skel3d_frame_marker_xyz = raw_skeleton_data++ assert (+ len(skel3d_frame_marker_xyz.shape) == 3+ ), f"3d skeleton data file should have 3 dimensions, but has shape {skel3d_frame_marker_xyz.shape}"++ if isinstance(reprojection_error_data, (str, Path)):+ skeleton_reprojection_error_fr_mar = np.load(reprojection_error_data)+ else:+ skeleton_reprojection_error_fr_mar = reprojection_error_data++ assert (+ len(skeleton_reprojection_error_fr_mar.shape) == 2+ ), f"3d skeleton reprojection error data file should have 2 dimensions but has shape {skeleton_reprojection_error_fr_mar.shape}"++ frame_counts = list(get_number_of_frames_of_videos_in_a_folder(synchronized_video_folder_path).values())+ assert (+ len(set(frame_counts)) == 1+ ), f"Videos in {synchronized_video_folder_path} have different frame counts: {frame_counts}"++ number_of_frames = frame_counts[0]++ assert skel3d_frame_marker_xyz.shape[0] == number_of_frames+ assert skeleton_reprojection_error_fr_mar.shape[0] == number_of_frames++ # TODO - check number of tracked points vs 'expected' number of tracked points++ assert (+ skel3d_frame_marker_xyz.shape[2] == 3+ ), f"3d skeleton data file should have 3 dimensions for X,Y,Z, but instead has {skel3d_frame_marker_xyz.shape[2]}"diff --git a/freemocap/tests/test_total_body_center_of_mass_data_shape.py b/freemocap/tests/test_total_body_center_of_mass_data_shape.pyindex 285d0530..5f82674d 100644--- a/freemocap/tests/test_total_body_center_of_mass_data_shape.py+++ b/freemocap/tests/test_total_body_center_of_mass_data_shape.py@@ -1,5 +1,5 @@from pathlib import Path-from typing import Union+from typing import Optional, Unionimport numpy as npimport pytest@@ -9,16 +9,24 @@ from freemocap.utilities.get_number_of_frames_of_videos_in_a_folder import ()[email protected]("synchronized_video_folder_path", "total_body_center_of_mass_file_path")-def test_total_body_center_of_mass_data_shape(- synchronized_video_folder_path: Union[str, Path],- total_body_center_of_mass_file_path: Union[str, Path],[email protected]("total_body_center_of_mass_data")+def test_total_body_center_of_mass_data_exists(+ total_body_center_of_mass_data: Union[str, Path],):assert Path(- total_body_center_of_mass_file_path- ).is_file(), f"No file found at {total_body_center_of_mass_file_path}"+ total_body_center_of_mass_data+ ).is_file(), f"No file found at {total_body_center_of_mass_data}"- total_body_center_of_mass_fr_xyz = np.load(total_body_center_of_mass_file_path)+[email protected]("synchronized_video_folder_path", "total_body_center_of_mass_data")+def test_total_body_center_of_mass_data_shape(+ synchronized_video_folder_path: Union[str, Path],+ total_body_center_of_mass_data: Union[str, Path, np.ndarray],+):+ if isinstance(total_body_center_of_mass_data, (str, Path)):+ total_body_center_of_mass_fr_xyz = np.load(total_body_center_of_mass_data)+ else:+ total_body_center_of_mass_fr_xyz = total_body_center_of_mass_dataframe_counts = list(get_number_of_frames_of_videos_in_a_folder(synchronized_video_folder_path).values())assert (@@ -29,8 +37,9 @@ def test_total_body_center_of_mass_data_shape(assert (total_body_center_of_mass_fr_xyz.shape[0] == number_of_frames- ), f"Number of frames in {total_body_center_of_mass_file_path} does not match number of frames of videos in {synchronized_video_folder_path}"+ ), f"Number of frames in {total_body_center_of_mass_fr_xyz.shape} does not match number of frames of videos in {synchronized_video_folder_path}"assert (total_body_center_of_mass_fr_xyz.shape[1] == 3- ), "`total_body_center_of_mass_fr_xyz.shape[1]` should have 3 dimensions (X,Y,Z) "+ ), f"total body center of mass should have 3 dimensions (X,Y,Z) but has {total_body_center_of_mass_fr_xyz.shape[1]}"+diff --git a/freemocap/utilities/create_nested_dict_from_pydantic.py b/freemocap/utilities/create_nested_dict_from_pydantic.pyindex d96490fd..6d829ae7 100644--- a/freemocap/utilities/create_nested_dict_from_pydantic.py+++ b/freemocap/utilities/create_nested_dict_from_pydantic.py@@ -27,10 +27,10 @@ def create_nested_dict(pydantic_model):if __name__ == "__main__":- from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.data_models.mediapipe_skeleton_names_and_connections import (- mediapipe_skeleton_schema,+ from skellytracker.trackers.mediapipe_tracker.mediapipe_model_info import (+ MediapipeModelInfo,)from freemocap.data_layer.data_saver import SkeletonSchema- skeleton_schema = SkeletonSchema(schema_dict=mediapipe_skeleton_schema)+ skeleton_schema = SkeletonSchema(schema_dict=MediapipeModelInfo.skeleton_schema)print(create_nested_dict(skeleton_schema))diff --git a/freemocap/utilities/update_1_4_path_names.py b/freemocap/utilities/update_1_4_path_names.pynew file mode 100644index 00000000..2c1ad9c0--- /dev/null+++ b/freemocap/utilities/update_1_4_path_names.py@@ -0,0 +1,89 @@+from typing import Optional, Union+from pathlib import Path+++from freemocap.system.paths_and_filenames.file_and_folder_names import (+ RAW_DATA_FOLDER_NAME,+ OUTPUT_DATA_FOLDER_NAME,+ CENTER_OF_MASS_FOLDER_NAME,+ RECORDING_SESSIONS_FOLDER_NAME,+)+from freemocap.system.paths_and_filenames.path_getters import get_freemocap_data_folder_path+++def update_1_4_path_names(freemocap_data_folder_path: Optional[Union[str, Path]]) -> None:+ if freemocap_data_folder_path is None:+ freemocap_data_folder_path = get_freemocap_data_folder_path()++ print(f"Searching for recording folders in {freemocap_data_folder_path}")++ for session_folder in (Path(freemocap_data_folder_path) / RECORDING_SESSIONS_FOLDER_NAME).iterdir():+ if not session_folder.is_dir():+ continue++ for recording_folder in session_folder.iterdir():+ if not recording_folder.is_dir():+ continue++ update_recording_folder(recording_folder)+++def update_recording_folder(recording_folder: Path) -> None:+ if (+ recording_folder.name == OUTPUT_DATA_FOLDER_NAME+ ): # this covers cases where the recording folder isn't a subfolder of a session+ rename_skeleton_file_path(recording_folder)+ if (recording_folder / RAW_DATA_FOLDER_NAME).exists():+ rename_raw_data_paths(recording_folder / RAW_DATA_FOLDER_NAME)+ if (recording_folder / CENTER_OF_MASS_FOLDER_NAME).exists():+ rename_COM_paths(recording_folder / CENTER_OF_MASS_FOLDER_NAME)+ elif (recording_folder / OUTPUT_DATA_FOLDER_NAME).exists():+ rename_skeleton_file_path(recording_folder)+ if (recording_folder / OUTPUT_DATA_FOLDER_NAME / RAW_DATA_FOLDER_NAME).exists():+ rename_raw_data_paths(recording_folder / OUTPUT_DATA_FOLDER_NAME / RAW_DATA_FOLDER_NAME)+ if (recording_folder / OUTPUT_DATA_FOLDER_NAME / CENTER_OF_MASS_FOLDER_NAME).exists():+ rename_COM_paths(recording_folder / CENTER_OF_MASS_FOLDER_NAME)+++def rename_raw_data_paths(raw_data_folder_path: Path) -> None:+ print(f"renaming raw data files in {raw_data_folder_path}")+ for file in raw_data_folder_path.iterdir():+ if file.name == "mediapipe2dData_numCams_numFrames_numTrackedPoints_pixelXY.npy":+ file.rename(file.parent / "mediapipe_2dData_numCams_numFrames_numTrackedPoints_pixelXY.npy")+ elif file.name == "mediapipe3dData_numCams_numFrames_numTrackedPoints_reprojectionError.npy":+ file.rename(file.parent / "mediapipe_3dData_numCams_numFrames_numTrackedPoints_reprojectionError.npy")+ elif file.name == "mediapipe3dData_numFrames_numTrackedPoints_reprojectionError.npy":+ file.rename(file.parent / "mediapipe_3dData_numFrames_numTrackedPoints_reprojectionError.npy")+ elif file.name == "mediapipe3dData_numFrames_numTrackedPoints_spatialXYZ.npy":+ file.rename(file.parent / "mediapipe_3dData_numFrames_numTrackedPoints_spatialXYZ.npy")++ print(f"successfully renamed raw data files in {raw_data_folder_path}")+++def rename_skeleton_file_path(output_data_folder_path: Path) -> None:+ print(f"renaming skeleton 3D files in {output_data_folder_path}")+ for file in output_data_folder_path.iterdir():+ if file.name == "mediaPipeSkel_3d_body_hands_face.npy":+ file.rename(file.parent / "mediapipe_skeleton_3d.npy")++ print(f"successfully renamed skeleton 3D files in {output_data_folder_path}")+++def rename_COM_paths(COM_data_folder_path: Path) -> None:+ print(f"renaming center of mass files in {COM_data_folder_path}")+ for file in COM_data_folder_path.iterdir():+ if file.name == "segmentCOM_frame_joint_xyz.npy":+ file.rename(file.parent / "mediapipe_segmentCOM_frame_joint_xyz.npy")+ elif file.name == "total_body_center_of_mass_xyz.npy":+ file.rename(file.parent / "mediapipe_total_body_center_of_mass_xyz.npy")++ print(f"successfully renamed center of mass files in {COM_data_folder_path}")+++if __name__ == "__main__":+ import sys++ if len(sys.argv) > 1:+ update_1_4_path_names(Path(sys.argv[1]))+ else:+ update_1_4_path_names(None)diff --git a/pyproject.toml b/pyproject.tomlindex 44dfd45c..8dcf7933 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -69,10 +69,10 @@ keywords = [dependencies = ["skellycam==2024.07.1092","skelly_viewer==2024.07.1023",- "skellyforge==2024.07.1006",+ "skellyforge==2024.8.1008","skelly_synchronize==2024.07.1032",- "skellytracker[all]==2024.06.1016",- "ajc27_freemocap_blender_addon==v2024.07.1022",+ "skellytracker[all]==2024.08.1018",+ "ajc27_freemocap_blender_addon==v2024.09.1024","opencv-contrib-python==4.8.*","toml==0.10.2","aniposelib==0.4.3",