Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

Generalize mediapipe code to other trackers (#536)

philipqueenc83c8bd99c7d
diff --git a/.flake8 b/.flake8
index b3d0a521..a1263f6d 100644
--- a/.flake8
+++ b/.flake8
@@ -12,12 +12,10 @@ exclude =
per-file-ignores =
./freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py:C901
- ./freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/mediapipe_skeleton_detector.py:C901
- ./freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/convert_mediapipe_npy_to_csv.py:E203
+ ./freemocap/core_processes/export_data/blender_stuff/export_to_blender/methods/ajc_addon/run_ajc_addon_main.py:C901
./freemocap/gui/qt/widgets/control_panel/calibration_control_panel.py:C901
./experimental/batch_process/batch_process.py:E226
# C901: function is too complex
-# E203: whitespace before ':' (PEP8 says this doesn't apply to slicing)
# E226: missing whitespace around arithmetic operator (file slashes misread as arithmetic)
max-line-length = 88
diff --git a/experimental/openpose/run_openpose.py b/experimental/openpose/run_openpose.py
new file mode 100644
index 00000000..2d0a5531
--- /dev/null
+++ b/experimental/openpose/run_openpose.py
@@ -0,0 +1,43 @@
+from pathlib import Path
+
+def main():
+ try:
+ from freemocap.data_layer.recording_models.post_processing_parameter_models import ProcessingParameterModel, AniposeTriangulate3DParametersModel, PostProcessingParametersModel
+ from freemocap.data_layer.recording_models.recording_info_model import RecordingInfoModel
+ from freemocap.core_processes.process_motion_capture_videos.process_recording_folder import process_recording_folder
+ from skellytracker.trackers.openpose_tracker.openpose_model_info import OpenPoseModelInfo, OpenPoseTrackingParams
+ except ImportError as e:
+ if 'ProcessingParameterModel' not in dir() or 'process_folder_of_videos' not in dir() or 'OpenPoseModelInfo' not in dir():
+ raise e
+
+
+
+ recording_folder = Path(r'D:\mdn_treadmill_for_testing')
+ openpose_root_folder_path = r'C:\openpose'
+
+
+ recording_model = RecordingInfoModel(
+ recording_folder_path=recording_folder,
+ active_tracker="openpose"
+ )
+
+ openpose_processing_parameters = ProcessingParameterModel(
+ recording_info_model= recording_model,
+ tracking_parameters_model=OpenPoseTrackingParams(
+ # net_resolution="-1x656",
+ openpose_root_folder_path=str(openpose_root_folder_path),
+ run_image_tracking=True
+ ),
+ anipose_triangulate_3d_parameters_model=AniposeTriangulate3DParametersModel(),
+ post_processing_parameters_model=PostProcessingParametersModel(),
+ tracking_model_info=OpenPoseModelInfo()
+ )
+
+ process_recording_folder(
+ recording_processing_parameter_model=openpose_processing_parameters,
+ kill_event=None,
+ logging_queue=None,
+ )
+
+if __name__ == "__main__":
+ main()
diff --git a/experimental/react_fastapi/api/routes/session/session_router.py b/experimental/react_fastapi/api/routes/session/session_router.py
index fa70caaa..04fb0a98 100644
--- a/experimental/react_fastapi/api/routes/session/session_router.py
+++ b/experimental/react_fastapi/api/routes/session/session_router.py
@@ -6,7 +6,7 @@ from fastapi import APIRouter
from jon_scratch.pupil_calibration_pipeline.qt_gl_laser_skeleton_visualizer import (
QtGlLaserSkeletonVisualizer,
)
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
from src.cameras.launch_camera_frame_loop import launch_camera_frame_loop
from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManager
from src.config.home_dir import (
@@ -33,8 +33,7 @@ session_router = APIRouter()
class TweakedPydanticBaseModel(BaseModel):
- class Config:
- arbitrary_types_allowed = True
+ model_config = ConfigDict(arbitrary_types_allowed = True)
class SessionCreateModel(BaseModel):
diff --git a/freemocap/core_processes/capture_volume_calibration/by_camera_reprojection_filtering.py b/freemocap/core_processes/capture_volume_calibration/by_camera_reprojection_filtering.py
index 72822de5..94b375b1 100644
--- a/freemocap/core_processes/capture_volume_calibration/by_camera_reprojection_filtering.py
+++ b/freemocap/core_processes/capture_volume_calibration/by_camera_reprojection_filtering.py
@@ -5,14 +5,10 @@ from matplotlib import pyplot as plt
import numpy as np
-from freemocap.core_processes.capture_volume_calibration.save_mediapipe_3d_data_to_npy import (
- save_mediapipe_3d_data_to_npy,
+from freemocap.core_processes.capture_volume_calibration.save_3d_data_to_npy import (
+ save_3d_data_to_npy,
)
from freemocap.core_processes.capture_volume_calibration.triangulate_3d_data import triangulate_3d_data
-
-from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.data_models.mediapipe_skeleton_names_and_connections import (
- NUMBER_OF_MEDIAPIPE_BODY_MARKERS,
-)
from freemocap.data_layer.recording_models.post_processing_parameter_models import ProcessingParameterModel
logger = logging.getLogger(__name__)
@@ -38,6 +34,13 @@ def run_reprojection_error_filtering(
:return: filtered 3d data
"""
+ if hasattr(
+ processing_parameters.tracking_model_info, "num_tracked_points_body"
+ ): # we don't want to reproject hand and face data
+ num_tracked_points = processing_parameters.tracking_model_info.num_tracked_points_body
+ else:
+ num_tracked_points = processing_parameters.tracking_model_info.num_tracked_points
+
(
reprojection_filtered_skel3d_frame_marker_xyz,
reprojection_filtered_skeleton_reprojection_error_fr_mar,
@@ -46,19 +49,20 @@ def run_reprojection_error_filtering(
reprojection_error_camera_frame_marker=skeleton_reprojection_error_cam_fr_mar,
reprojection_error_frame_marker=skeleton_reprojection_error_fr_mar,
reprojection_error_confidence_threshold=processing_parameters.anipose_triangulate_3d_parameters_model.reprojection_error_confidence_cutoff,
- mediapipe_2d_data=image_data_numCams_numFrames_numTrackedPts_XYZ[:, :, :, :2],
+ image_2d_data=image_data_numCams_numFrames_numTrackedPts_XYZ[:, :, :, :2],
raw_skel3d_frame_marker_xyz=raw_skel3d_frame_marker_xyz,
anipose_calibration_object=anipose_calibration_object,
- output_data_folder_path=processing_parameters.recording_info_model.raw_data_folder_path,
+ num_tracked_points=num_tracked_points,
use_triangulate_ransac=processing_parameters.anipose_triangulate_3d_parameters_model.use_triangulate_ransac_method,
minimum_cameras_to_reproject=processing_parameters.anipose_triangulate_3d_parameters_model.minimum_cameras_to_reproject,
)
- save_mediapipe_3d_data_to_npy(
+ save_3d_data_to_npy(
data3d_numFrames_numTrackedPoints_XYZ=reprojection_filtered_skel3d_frame_marker_xyz,
data3d_numFrames_numTrackedPoints_reprojectionError=reprojection_filtered_skeleton_reprojection_error_fr_mar,
data3d_numCams_numFrames_numTrackedPoints_reprojectionError=reprojection_filtered_skeleton_reprojection_error_cam_fr_mar,
path_to_folder_where_data_will_be_saved=processing_parameters.recording_info_model.raw_data_folder_path,
processing_level="reprojection_filtered",
+ file_prefix=processing_parameters.tracking_model_info.name,
)
plot_reprojection_error(
raw_reprojection_error_frame_marker=skeleton_reprojection_error_fr_mar,
@@ -76,14 +80,14 @@ def filter_by_reprojection_error(
reprojection_error_camera_frame_marker: np.ndarray,
reprojection_error_frame_marker: np.ndarray,
reprojection_error_confidence_threshold: float,
- mediapipe_2d_data: np.ndarray,
+ image_2d_data: np.ndarray,
raw_skel3d_frame_marker_xyz: np.ndarray,
anipose_calibration_object,
- output_data_folder_path: Union[str, Path],
+ num_tracked_points: int,
use_triangulate_ransac: bool = False,
minimum_cameras_to_reproject: int = 3,
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
- total_cameras = mediapipe_2d_data.shape[0]
+ total_cameras = image_2d_data.shape[0]
num_cameras_to_remove = 1
if total_cameras <= minimum_cameras_to_reproject:
@@ -96,8 +100,8 @@ def filter_by_reprojection_error(
reprojection_error_camera_frame_marker,
)
- body2d_camera_frame_marker_xy = mediapipe_2d_data[:, :, :NUMBER_OF_MEDIAPIPE_BODY_MARKERS, :2]
- bodyReprojErr_camera_frame_marker = reprojection_error_camera_frame_marker[:, :, :NUMBER_OF_MEDIAPIPE_BODY_MARKERS]
+ body2d_camera_frame_marker_xy = image_2d_data[:, :, :num_tracked_points, :2]
+ bodyReprojErr_camera_frame_marker = reprojection_error_camera_frame_marker[:, :, :num_tracked_points]
reprojection_error_confidence_threshold = min(max(reprojection_error_confidence_threshold, 0), 100)
@@ -121,7 +125,7 @@ def filter_by_reprojection_error(
new_reprojError_cam_frame_marker,
) = triangulate_3d_data(
anipose_calibration_object=anipose_calibration_object,
- mediapipe_2d_data=data_to_reproject_camera_frame_marker_xy,
+ image_2d_data=data_to_reproject_camera_frame_marker_xy,
use_triangulate_ransac=use_triangulate_ransac,
)
@@ -139,15 +143,13 @@ def filter_by_reprojection_error(
# put retriangulated data back in place
filtered_skel3d_frame_marker_xyz = raw_skel3d_frame_marker_xyz.copy()
- filtered_skel3d_frame_marker_xyz[:, :NUMBER_OF_MEDIAPIPE_BODY_MARKERS, :] = retriangulated_data_frame_marker_xyz
+ filtered_skel3d_frame_marker_xyz[:, :num_tracked_points, :] = retriangulated_data_frame_marker_xyz
filtered_reprojection_error_frame_marker = reprojection_error_frame_marker.copy()
- filtered_reprojection_error_frame_marker[:, :NUMBER_OF_MEDIAPIPE_BODY_MARKERS] = new_reprojection_error_flat
+ filtered_reprojection_error_frame_marker[:, :num_tracked_points] = new_reprojection_error_flat
filtered_reprojection_error_camera_frame_marker = reprojection_error_camera_frame_marker.copy()
- filtered_reprojection_error_camera_frame_marker[
- :, :, :NUMBER_OF_MEDIAPIPE_BODY_MARKERS
- ] = new_reprojError_cam_frame_marker
+ filtered_reprojection_error_camera_frame_marker[:, :, :num_tracked_points] = new_reprojError_cam_frame_marker
return (
filtered_skel3d_frame_marker_xyz,
@@ -172,7 +174,7 @@ def _get_data_to_reproject(
input_2d_data_camera_frame_marker_xy: np.ndarray,
) -> tuple[np.ndarray, list]:
indices_above_threshold = np.nonzero(reprojError_cam_frame_marker > reprojection_error_threshold)
- logger.info(f"SHAPE OF INDICES ABOVE THRESHOLD: {indices_above_threshold[0].shape}")
+ logger.debug(f"SHAPE OF INDICES ABOVE THRESHOLD: {indices_above_threshold[0].shape}")
total_frame_marker_combos = (
input_2d_data_camera_frame_marker_xy.shape[1] * input_2d_data_camera_frame_marker_xy.shape[2]
diff --git a/freemocap/core_processes/capture_volume_calibration/save_3d_data_to_npy.py b/freemocap/core_processes/capture_volume_calibration/save_3d_data_to_npy.py
new file mode 100644
index 00000000..c82ffc2f
--- /dev/null
+++ b/freemocap/core_processes/capture_volume_calibration/save_3d_data_to_npy.py
@@ -0,0 +1,67 @@
+import logging
+from pathlib import Path
+from typing import Union
+
+import numpy as np
+
+from freemocap.system.paths_and_filenames.file_and_folder_names import (
+ RAW_3D_NPY_FILE_NAME,
+ FULL_REPROJECTION_ERROR_NPY_FILE_NAME,
+ REPROJECTION_ERROR_NPY_FILE_NAME,
+ REPROJECTION_FILTERED_PREFIX,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def save_3d_data_to_npy(
+ data3d_numFrames_numTrackedPoints_XYZ: np.ndarray,
+ data3d_numFrames_numTrackedPoints_reprojectionError: np.ndarray,
+ data3d_numCams_numFrames_numTrackedPoints_reprojectionError: np.ndarray,
+ path_to_folder_where_data_will_be_saved: Union[str, Path],
+ processing_level: str,
+ file_prefix: str = "",
+):
+ path_to_folder_where_data_will_be_saved = Path(path_to_folder_where_data_will_be_saved)
+ Path(path_to_folder_where_data_will_be_saved).mkdir(parents=True, exist_ok=True) # save spatial XYZ data
+ if processing_level == "raw":
+ data_3d_save_path = path_to_folder_where_data_will_be_saved / f"{file_prefix}_{RAW_3D_NPY_FILE_NAME}"
+ reprojection_error_save_path = (
+ path_to_folder_where_data_will_be_saved / f"{file_prefix}_{REPROJECTION_ERROR_NPY_FILE_NAME}"
+ )
+ full_reprojection_error_save_path = (
+ path_to_folder_where_data_will_be_saved / f"{file_prefix}_{FULL_REPROJECTION_ERROR_NPY_FILE_NAME}"
+ )
+ elif processing_level == "reprojection_filtered":
+ data_3d_save_path = (
+ path_to_folder_where_data_will_be_saved
+ / f"{REPROJECTION_FILTERED_PREFIX}{file_prefix}_{RAW_3D_NPY_FILE_NAME}"
+ )
+ reprojection_error_save_path = (
+ path_to_folder_where_data_will_be_saved
+ / f"{REPROJECTION_FILTERED_PREFIX}{file_prefix}_{REPROJECTION_ERROR_NPY_FILE_NAME}"
+ )
+ full_reprojection_error_save_path = (
+ path_to_folder_where_data_will_be_saved
+ / f"{REPROJECTION_FILTERED_PREFIX}{file_prefix}_{FULL_REPROJECTION_ERROR_NPY_FILE_NAME}"
+ )
+ else:
+ logger.exception(f"processing_level: {processing_level} not recognized")
+ raise Exception
+
+ logger.info(f"saving: {data_3d_save_path}")
+ np.save(str(data_3d_save_path), data3d_numFrames_numTrackedPoints_XYZ)
+
+ # save reprojection error
+
+ logger.info(f"saving: {reprojection_error_save_path}")
+ np.save(
+ str(reprojection_error_save_path),
+ data3d_numFrames_numTrackedPoints_reprojectionError,
+ )
+
+ logger.info(f"saving: {full_reprojection_error_save_path}")
+ np.save(
+ str(full_reprojection_error_save_path),
+ data3d_numCams_numFrames_numTrackedPoints_reprojectionError,
+ )
diff --git a/freemocap/core_processes/capture_volume_calibration/save_mediapipe_3d_data_to_npy.py b/freemocap/core_processes/capture_volume_calibration/save_mediapipe_3d_data_to_npy.py
deleted file mode 100644
index 710b7dae..00000000
--- a/freemocap/core_processes/capture_volume_calibration/save_mediapipe_3d_data_to_npy.py
+++ /dev/null
@@ -1,65 +0,0 @@
-import logging
-from pathlib import Path
-from typing import Union
-
-import numpy as np
-
-from freemocap.system.paths_and_filenames.file_and_folder_names import (
- RAW_MEDIAPIPE_3D_NPY_FILE_NAME,
- RAW_MEDIAPIPE_FULL_REPROJECTION_ERROR_NPY_FILE_NAME,
- RAW_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME,
- REPROJECTION_FILTERED_MEDIAPIPE_3D_NPY_FILE_NAME,
- REPROJECTION_FILTERED_MEDIAPIPE_FULL_REPROJECTION_ERROR_NPY_FILE_NAME,
- REPROJECTION_FILTERED_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME,
-)
-
-logger = logging.getLogger(__name__)
-
-
-def save_mediapipe_3d_data_to_npy(
- data3d_numFrames_numTrackedPoints_XYZ: np.ndarray,
- data3d_numFrames_numTrackedPoints_reprojectionError: np.ndarray,
- data3d_numCams_numFrames_numTrackedPoints_reprojectionError: np.ndarray,
- path_to_folder_where_data_will_be_saved: Union[str, Path],
- processing_level: str,
-):
- path_to_folder_where_data_will_be_saved = Path(path_to_folder_where_data_will_be_saved)
- Path(path_to_folder_where_data_will_be_saved).mkdir(parents=True, exist_ok=True) # save spatial XYZ data
- if processing_level == "raw":
- mediapipe_3dData_save_path = path_to_folder_where_data_will_be_saved / RAW_MEDIAPIPE_3D_NPY_FILE_NAME
- mediapipe_reprojection_error_save_path = (
- path_to_folder_where_data_will_be_saved / RAW_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME
- )
- mediapipe_full_reprojection_error_save_path = (
- path_to_folder_where_data_will_be_saved / RAW_MEDIAPIPE_FULL_REPROJECTION_ERROR_NPY_FILE_NAME
- )
- elif processing_level == "reprojection_filtered":
- mediapipe_3dData_save_path = (
- path_to_folder_where_data_will_be_saved / REPROJECTION_FILTERED_MEDIAPIPE_3D_NPY_FILE_NAME
- )
- mediapipe_reprojection_error_save_path = (
- path_to_folder_where_data_will_be_saved / REPROJECTION_FILTERED_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME
- )
- mediapipe_full_reprojection_error_save_path = (
- path_to_folder_where_data_will_be_saved / REPROJECTION_FILTERED_MEDIAPIPE_FULL_REPROJECTION_ERROR_NPY_FILE_NAME
- )
- else:
- logger.exception(f"processing_level: {processing_level} not recognized")
- raise Exception
-
- logger.info(f"saving: {mediapipe_3dData_save_path}")
- np.save(str(mediapipe_3dData_save_path), data3d_numFrames_numTrackedPoints_XYZ)
-
- # save reprojection error
-
- logger.info(f"saving: {mediapipe_reprojection_error_save_path}")
- np.save(
- str(mediapipe_reprojection_error_save_path),
- data3d_numFrames_numTrackedPoints_reprojectionError,
- )
-
- logger.info(f"saving: {mediapipe_full_reprojection_error_save_path}")
- np.save(
- str(mediapipe_full_reprojection_error_save_path),
- data3d_numCams_numFrames_numTrackedPoints_reprojectionError,
- )
diff --git a/freemocap/core_processes/capture_volume_calibration/triangulate_3d_data.py b/freemocap/core_processes/capture_volume_calibration/triangulate_3d_data.py
index b6089da3..a411bec0 100644
--- a/freemocap/core_processes/capture_volume_calibration/triangulate_3d_data.py
+++ b/freemocap/core_processes/capture_volume_calibration/triangulate_3d_data.py
@@ -4,8 +4,8 @@ from pathlib import Path
import numpy as np
-from freemocap.core_processes.capture_volume_calibration.save_mediapipe_3d_data_to_npy import (
- save_mediapipe_3d_data_to_npy,
+from freemocap.core_processes.capture_volume_calibration.save_3d_data_to_npy import (
+ save_3d_data_to_npy,
)
logger = logging.getLogger(__name__)
@@ -13,14 +13,14 @@ logger = logging.getLogger(__name__)
def triangulate_3d_data(
anipose_calibration_object,
- mediapipe_2d_data: np.ndarray,
+ image_2d_data: np.ndarray,
use_triangulate_ransac: bool = False,
kill_event: multiprocessing.Event = None,
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
- number_of_cameras = mediapipe_2d_data.shape[0]
- number_of_frames = mediapipe_2d_data.shape[1]
- number_of_tracked_points = mediapipe_2d_data.shape[2]
- number_of_spatial_dimensions = mediapipe_2d_data.shape[3]
+ number_of_cameras = image_2d_data.shape[0]
+ number_of_frames = image_2d_data.shape[1]
+ number_of_tracked_points = image_2d_data.shape[2]
+ number_of_spatial_dimensions = image_2d_data.shape[3]
if not number_of_spatial_dimensions == 2:
logger.error(
@@ -30,7 +30,7 @@ def triangulate_3d_data(
# reshape data to collapse across 'frames' so it becomes [number_of_cameras,
# number_of_2d_points(numFrames*numPoints), XY]
- data2d_flat = mediapipe_2d_data.reshape(number_of_cameras, -1, 2)
+ data2d_flat = image_2d_data.reshape(number_of_cameras, -1, 2)
logger.info(
f"Reconstructing 3d points from 2d points with shape: \n"
@@ -74,9 +74,9 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
- "--mediapipe_2d_data_path",
+ "--data_2d_path",
type=str,
- help="path to mediapipe 2d data",
+ help="path to 2d data",
required=True,
)
parser.add_argument(
@@ -103,12 +103,18 @@ if __name__ == "__main__":
help="use triangulate ransac anipose method ",
required=False,
)
+ parser.add_argument(
+ "--file_prefix",
+ type=str,
+ help="file prefix",
+ required=False,
+ )
args = parser.parse_args()
- mediapipe_2d_data = np.load(args.mediapipe_2d_data_path)
+ data_2d = np.load(args.data_2d_path)
if args.output_data_folder_path is None:
- args.output_data_folder_path = Path(args.mediapipe_2d_data_path).parent
+ args.output_data_folder_path = Path(args.data_2d_path).parent
if args.save_data_as_csv is None:
args.save_data_as_csv = True # default
@@ -116,11 +122,14 @@ if __name__ == "__main__":
if args.use_triangulate_ransac is None:
args.use_triangulate_ransac = True
+ if args.file_prefix is None:
+ args.file_prefix = ""
+
if args.calibration_file_path:
anipose_calibration_object = load_anipose_calibration_toml_from_path(args.calibration_file_path)
else:
anipose_calibration_object = load_anipose_calibration_toml_from_path(
- Path(args.mediapipe_2d_data_path).parent.parent / "camera_calibration_data.toml"
+ Path(args.data_2d_path).parent.parent / "camera_calibration_data.toml"
)
(
@@ -129,13 +138,14 @@ if __name__ == "__main__":
skeleton_reprojection_error_cam_fr_mar,
) = triangulate_3d_data(
anipose_calibration_object=anipose_calibration_object,
- mediapipe_2d_data=mediapipe_2d_data,
+ image_2d_data=data_2d,
use_triangulate_ransac=args.use_triangulate_ransac,
)
- save_mediapipe_3d_data_to_npy(
+ save_3d_data_to_npy(
data3d_numFrames_numTrackedPoints_XYZ=skel3d_frame_marker_xyz,
data3d_numFrames_numTrackedPoints_reprojectionError=skeleton_reprojection_error_fr_mar,
data3d_numCams_numFrames_numTrackedPoints_reprojectionError=skeleton_reprojection_error_cam_fr_mar,
path_to_folder_where_data_will_be_saved=args.output_data_folder_path,
processing_level="raw",
+ file_prefix=args.file_prefix,
)
diff --git a/freemocap/core_processes/detecting_things_in_2d_images/__init__.py b/freemocap/core_processes/detecting_things_in_2d_images/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/__init__.py b/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/convert_mediapipe_npy_to_csv.py b/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/convert_mediapipe_npy_to_csv.py
deleted file mode 100644
index ee7dcf76..00000000
--- a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/convert_mediapipe_npy_to_csv.py
+++ /dev/null
@@ -1,331 +0,0 @@
-# %%
-import logging
-from pathlib import Path
-from typing import Union
-
-import numpy as np
-import pandas as pd
-from mediapipe.python.solutions import holistic as mp_holistic
-
-from freemocap.system.paths_and_filenames.file_and_folder_names import (
- MEDIAPIPE_RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME,
- MEDIAPIPE_BODY_3D_DATAFRAME_CSV_FILE_NAME,
- MEDIAPIPE_LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME,
-)
-
-logger = logging.getLogger(__name__)
-
-
-def convert_mediapipe_npy_to_csv(
- mediapipe_3d_frame_trackedPoint_xyz: np.ndarray,
- output_data_folder_path: Union[str, Path],
-):
- logger.info(
- f"Converting npy data with shape: {mediapipe_3d_frame_trackedPoint_xyz.shape} into `csv` and smaller `npy` files"
- )
-
- # %%
- mediapipe_pose_landmark_names = [landmark.name.lower() for landmark in mp_holistic.PoseLandmark]
- mediapipe_hand_landmark_names = [landmark.name.lower() for landmark in mp_holistic.HandLandmark]
- # face_landmark_names = [landmark.name.lower() for landmark in mp_holistic.PoseLandmark] #gonna have the clever for the face
- # logger.info(f"Body tracked point names: {mediapipe_pose_landmark_names}")
- # logger.info(mediapipe_hand_landmark_names)
-
- # %%
- # get number of points in body, hands, face
-
- number_of_body_points = len(mediapipe_pose_landmark_names)
- number_of_hand_points = len(mediapipe_hand_landmark_names)
-
- first_body_marker_index = 0
- last_body_marker_index = number_of_body_points - 1
-
- first_right_hand_marker_index = last_body_marker_index + 1
- last_right_hand_marker_index = number_of_body_points + number_of_hand_points - 1
-
- first_left_hand_marker_index = last_right_hand_marker_index + 1
- last_left_hand_marker_index = last_right_hand_marker_index + 1 + number_of_hand_points - 1
-
- first_face_marker_index = last_left_hand_marker_index + 1
- last_face_marker_index = mediapipe_3d_frame_trackedPoint_xyz.shape[1]
-
- number_of_face_points = last_face_marker_index - first_face_marker_index
-
- # logger.info(
- # f"body tracked point indices: {first_body_marker_index}:{last_body_marker_index}"
- # )
- # logger.info(
- # f"right hand tracked point indices: {first_right_hand_marker_index}:{last_right_hand_marker_index}"
- # )
- # logger.info(
- # f"left hand tracked point indices: {first_left_hand_marker_index}:{last_left_hand_marker_index}"
- # )
- # logger.info(
- # f"face tracked point indices: {first_face_marker_index}:{last_face_marker_index}"
- # )
- #
- # logger.info(
- # f"number of body points: {last_body_marker_index - first_body_marker_index + 1}"
- # )
- # logger.info(
- # f"number of right hand points: {last_right_hand_marker_index - first_right_hand_marker_index + 1}"
- # )
- # logger.info(
- # f"number of left hand points: {last_left_hand_marker_index - first_left_hand_marker_index + 1}"
- # )
- # logger.info(
- # f"number of face points: {last_face_marker_index - first_face_marker_index + 1}"
- # )
-
- # %%
- body_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[:, first_body_marker_index : last_body_marker_index + 1, :]
- right_hand_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[
- :, first_right_hand_marker_index : last_right_hand_marker_index + 1, :
- ]
- left_hand_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[
- :, first_left_hand_marker_index : last_left_hand_marker_index + 1, :
- ]
- face_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[:, first_face_marker_index : last_face_marker_index + 1, :]
-
- # logger.info(f"body 3d xyz shape: {body_3d_xyz.shape}")
- # logger.info(f"right hand 3d xyz shape: {right_hand_3d_xyz.shape}")
- # logger.info(f"left hand 3d xyz shape: {left_hand_3d_xyz.shape}")
- # logger.info(f"face 3d xyz shape: {face_3d_xyz.shape}")
-
- # %%
- # save broken up npy files
- np.save(str(Path(output_data_folder_path) / "mediapipe_body_3d_xyz.npy"), body_3d_xyz)
- np.save(
- str(Path(output_data_folder_path) / "mediapipe_right_hand_3d_xyz.npy"),
- right_hand_3d_xyz,
- )
- np.save(
- str(Path(output_data_folder_path) / "mediapipe_left_hand_3d_xyz.npy"),
- left_hand_3d_xyz,
- )
- np.save(str(Path(output_data_folder_path) / "mediapipe_face_3d_xyz.npy"), face_3d_xyz)
-
- # %%
- # create pandas data frame headers
-
- body_3d_xyz_header = []
- for landmark_name in mediapipe_pose_landmark_names:
- body_3d_xyz_header.append(f"{landmark_name}_x")
- body_3d_xyz_header.append(f"{landmark_name}_y")
- body_3d_xyz_header.append(f"{landmark_name}_z")
-
- right_hand_3d_xyz_header = []
- for landmark_name in mediapipe_hand_landmark_names:
- right_hand_3d_xyz_header.append(f"right_hand_{landmark_name}_x")
- right_hand_3d_xyz_header.append(f"right_hand_{landmark_name}_y")
- right_hand_3d_xyz_header.append(f"right_hand_{landmark_name}_z")
-
- left_hand_3d_xyz_header = []
- for landmark_name in mediapipe_hand_landmark_names:
- left_hand_3d_xyz_header.append(f"left_hand_{landmark_name}_x")
- left_hand_3d_xyz_header.append(f"left_hand_{landmark_name}_y")
- left_hand_3d_xyz_header.append(f"left_hand_{landmark_name}_z")
-
- face_3d_xyz_header = []
- for landmark_number in range(last_face_marker_index - first_face_marker_index):
- face_3d_xyz_header.append(f"face_{str(landmark_number).zfill(4)}_x")
- face_3d_xyz_header.append(f"face_{str(landmark_number).zfill(4)}_y")
- face_3d_xyz_header.append(f"face_{str(landmark_number).zfill(4)}_z")
- #
- # logger.debug(
- # f"length of body 3d xyz header: {len(body_3d_xyz_header)}, should be: {number_of_body_points * 3}"
- # )
- # logger.debug(
- # f"length of right hand 3d xyz header: {len(right_hand_3d_xyz_header)}, should be: {number_of_hand_points * 3}"
- # )
- # logger.debug(
- # f"length of left hand 3d xyz header: {len(left_hand_3d_xyz_header)}, should be: {number_of_hand_points * 3}"
- # )
- # logger.debug(
- # f"length of face 3d xyz header: {len(face_3d_xyz_header)}, should be: {number_of_face_points * 3}"
- # )
-
- # %%
- number_of_frames = mediapipe_3d_frame_trackedPoint_xyz.shape[0]
- body_flat = body_3d_xyz.reshape(number_of_frames, number_of_body_points * 3)
-
- body_dataframe = pd.DataFrame(body_flat, columns=body_3d_xyz_header)
- body_dataframe.to_csv(str(Path(output_data_folder_path) / MEDIAPIPE_BODY_3D_DATAFRAME_CSV_FILE_NAME), index=False)
-
- right_hand_flat = right_hand_3d_xyz.reshape(number_of_frames, number_of_hand_points * 3)
- right_hand_dataframe = pd.DataFrame(right_hand_flat, columns=right_hand_3d_xyz_header)
- right_hand_dataframe.to_csv(
- str(Path(output_data_folder_path) / MEDIAPIPE_RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME),
- index=False,
- )
-
- left_hand_flat = left_hand_3d_xyz.reshape(number_of_frames, number_of_hand_points * 3)
- left_hand_dataframe = pd.DataFrame(left_hand_flat, columns=left_hand_3d_xyz_header)
- left_hand_dataframe.to_csv(
- str(Path(output_data_folder_path) / MEDIAPIPE_LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME),
- index=False,
- )
-
- face_flat = face_3d_xyz.reshape(number_of_frames, number_of_face_points * 3)
- face_dataframe = pd.DataFrame(face_flat, columns=face_3d_xyz_header)
- face_dataframe.to_csv(str(Path(output_data_folder_path) / "mediapipe_face_3d_xyz.csv"), index=False)
-
- # %%
-
- # %%
- mediapipe_pose_landmark_names = [landmark.name.lower() for landmark in mp_holistic.PoseLandmark]
- mediapipe_hand_landmark_names = [landmark.name.lower() for landmark in mp_holistic.HandLandmark]
- # face_landmark_names = [landmark.name.lower() for landmark in mp_holistic.PoseLandmark] #gonna have the clever for the face
-
- # logger.debug(f"Body tracked point names: {mediapipe_pose_landmark_names}")
- # logger.debug(mediapipe_hand_landmark_names)
-
- # %%
- # get number of points in body, hands, face
-
- number_of_body_points = len(mediapipe_pose_landmark_names)
- number_of_hand_points = len(mediapipe_hand_landmark_names)
-
- first_body_marker_index = 0
- last_body_marker_index = number_of_body_points - 1
-
- first_right_hand_marker_index = last_body_marker_index + 1
- last_right_hand_marker_index = number_of_body_points + number_of_hand_points - 1
-
- first_left_hand_marker_index = last_right_hand_marker_index + 1
- last_left_hand_marker_index = last_right_hand_marker_index + 1 + number_of_hand_points - 1
-
- first_face_marker_index = last_left_hand_marker_index + 1
- last_face_marker_index = mediapipe_3d_frame_trackedPoint_xyz.shape[1]
-
- number_of_face_points = last_face_marker_index - first_face_marker_index
-
- # logger.debug(
- # f"body tracked point indices: {first_body_marker_index}:{last_body_marker_index}"
- # )
- # logger.debug(
- # f"right hand tracked point indices: {first_right_hand_marker_index}:{last_right_hand_marker_index}"
- # )
- # logger.debug(
- # f"left hand tracked point indices: {first_left_hand_marker_index}:{last_left_hand_marker_index}"
- # )
- # logger.debug(
- # f"face tracked point indices: {first_face_marker_index}:{last_face_marker_index}"
- # )
- #
- # logger.debug(
- # f"number of body points: {last_body_marker_index - first_body_marker_index + 1}"
- # )
- # logger.debug(
- # f"number of right hand points: {last_right_hand_marker_index - first_right_hand_marker_index + 1}"
- # )
- # logger.debug(
- # f"number of left hand points: {last_left_hand_marker_index - first_left_hand_marker_index + 1}"
- # )
- # logger.debug(
- # f"number of face points: {last_face_marker_index - first_face_marker_index + 1}"
- # )
-
- # %%
- body_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[:, first_body_marker_index : last_body_marker_index + 1, :]
- right_hand_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[
- :, first_right_hand_marker_index : last_right_hand_marker_index + 1, :
- ]
- left_hand_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[
- :, first_left_hand_marker_index : last_left_hand_marker_index + 1, :
- ]
- face_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[:, first_face_marker_index : last_face_marker_index + 1, :]
-
- logger.debug(f"body 3d xyz shape: {body_3d_xyz.shape}")
- logger.debug(f"right hand 3d xyz shape: {right_hand_3d_xyz.shape}")
- logger.debug(f"left hand 3d xyz shape: {left_hand_3d_xyz.shape}")
- logger.debug(f"face 3d xyz shape: {face_3d_xyz.shape}")
-
- # %%
- # save broken up npy files
- np.save(str(Path(output_data_folder_path) / "mediapipe_body_3d_xyz.npy"), body_3d_xyz)
- np.save(
- str(Path(output_data_folder_path) / "mediapipe_right_hand_3d_xyz.npy"),
- right_hand_3d_xyz,
- )
- np.save(
- str(Path(output_data_folder_path) / "mediapipe_left_hand_3d_xyz.npy"),
- left_hand_3d_xyz,
- )
- np.save(str(Path(output_data_folder_path) / "mediapipe_face_3d_xyz.npy"), face_3d_xyz)
-
- # %%
- # create pandas data frame headers
-
- body_3d_xyz_header = []
- for landmark_name in mediapipe_pose_landmark_names:
- body_3d_xyz_header.append(f"{landmark_name}_x")
- body_3d_xyz_header.append(f"{landmark_name}_y")
- body_3d_xyz_header.append(f"{landmark_name}_z")
-
- right_hand_3d_xyz_header = []
- for landmark_name in mediapipe_hand_landmark_names:
- right_hand_3d_xyz_header.append(f"right_hand_{landmark_name}_x")
- right_hand_3d_xyz_header.append(f"right_hand_{landmark_name}_y")
- right_hand_3d_xyz_header.append(f"right_hand_{landmark_name}_z")
-
- left_hand_3d_xyz_header = []
- for landmark_name in mediapipe_hand_landmark_names:
- left_hand_3d_xyz_header.append(f"left_hand_{landmark_name}_x")
- left_hand_3d_xyz_header.append(f"left_hand_{landmark_name}_y")
- left_hand_3d_xyz_header.append(f"left_hand_{landmark_name}_z")
-
- face_3d_xyz_header = []
- for landmark_number in range(last_face_marker_index - first_face_marker_index):
- face_3d_xyz_header.append(f"face_{str(landmark_number).zfill(4)}_x")
- face_3d_xyz_header.append(f"face_{str(landmark_number).zfill(4)}_y")
- face_3d_xyz_header.append(f"face_{str(landmark_number).zfill(4)}_z")
-
- logger.debug(
- f"length of body 3d xyz `.csv` header: {len(body_3d_xyz_header)}, should be: {number_of_body_points * 3}"
- )
- logger.debug(
- f"length of right hand 3d xyz `.csv` header: {len(right_hand_3d_xyz_header)}, should be: {number_of_hand_points * 3}"
- )
- logger.debug(
- f"length of left hand 3d xyz `.csv` header: {len(left_hand_3d_xyz_header)}, should be: {number_of_hand_points * 3}"
- )
- logger.debug(
- f"length of face 3d xyz `.csv` header: {len(face_3d_xyz_header)}, should be: {number_of_face_points * 3}"
- )
-
- # %%
- number_of_frames = mediapipe_3d_frame_trackedPoint_xyz.shape[0]
- body_flat = body_3d_xyz.reshape(number_of_frames, number_of_body_points * 3)
-
- body_dataframe = pd.DataFrame(body_flat, columns=body_3d_xyz_header)
- body_dataframe.to_csv(str(Path(output_data_folder_path) / MEDIAPIPE_BODY_3D_DATAFRAME_CSV_FILE_NAME), index=False)
-
- right_hand_flat = right_hand_3d_xyz.reshape(number_of_frames, number_of_hand_points * 3)
- right_hand_dataframe = pd.DataFrame(right_hand_flat, columns=right_hand_3d_xyz_header)
- right_hand_dataframe.to_csv(
- str(Path(output_data_folder_path) / MEDIAPIPE_RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME),
- index=False,
- )
-
- left_hand_flat = left_hand_3d_xyz.reshape(number_of_frames, number_of_hand_points * 3)
- left_hand_dataframe = pd.DataFrame(left_hand_flat, columns=left_hand_3d_xyz_header)
- left_hand_dataframe.to_csv(
- str(Path(output_data_folder_path) / MEDIAPIPE_LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME),
- index=False,
- )
-
- face_flat = face_3d_xyz.reshape(number_of_frames, number_of_face_points * 3)
- face_dataframe = pd.DataFrame(face_flat, columns=face_3d_xyz_header)
- face_dataframe.to_csv(str(Path(output_data_folder_path) / "mediapipe_face_3d_xyz.csv"), index=False)
-
- logger.info("Done saving out `csv` and broken up `npy` files")
-
-
-if __name__ == "__main__":
- mediapipe_3d_frame_trackedPoint_xyz = np.load(
- r"C:\Users\jonma\freemocap_data\session_10-15-2022-09_50_10\output_data\post_processed_data\mediaPipeSkel_3d_origin_aligned.npy"
- )
- output_data_folder_path = r"C:\Users\jonma\freemocap_data\session_10-15-2022-09_50_10\output_data"
-
- convert_mediapipe_npy_to_csv(mediapipe_3d_frame_trackedPoint_xyz, output_data_folder_path)
diff --git a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/data_models/__init__.py b/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/data_models/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/data_models/mediapipe_dataclasses.py b/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/data_models/mediapipe_dataclasses.py
deleted file mode 100644
index 14b88588..00000000
--- a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/data_models/mediapipe_dataclasses.py
+++ /dev/null
@@ -1,60 +0,0 @@
-import logging
-from dataclasses import dataclass
-from typing import Any
-
-import numpy as np
-from skellycam.detection.models.frame_payload import FramePayload
-
-logger = logging.getLogger(__name__)
-
-
-@dataclass
-class Mediapipe2dNumpyArrays:
- body_frameNumber_trackedPointNumber_XYZ: np.ndarray = None
- body_world_frameNumber_trackedPointNumber_XYZ: np.ndarray = None
- rightHand_frameNumber_trackedPointNumber_XYZ: np.ndarray = None
- leftHand_frameNumber_trackedPointNumber_XYZ: np.ndarray = None
- face_frameNumber_trackedPointNumber_XYZ: np.ndarray = None
-
- body_frameNumber_trackedPointNumber_confidence: np.ndarray = None
-
- @property
- def has_data(self):
- return not np.isnan(self.body_frameNumber_trackedPointNumber_XYZ).all()
-
- @property
- def all_data2d_nFrames_nTrackedPts_XY(self):
- """dimensions will be [number_of_frames , number_of_markers, XY]"""
-
- if self.body_frameNumber_trackedPointNumber_XYZ is None:
- # if there's no body data, there's no hand or face data either
- return
-
- if len(self.body_frameNumber_trackedPointNumber_XYZ.shape) == 3: # multiple frames
- return np.hstack(
- [
- self.body_frameNumber_trackedPointNumber_XYZ,
- self.rightHand_frameNumber_trackedPointNumber_XYZ,
- self.leftHand_frameNumber_trackedPointNumber_XYZ,
- self.face_frameNumber_trackedPointNumber_XYZ,
- ]
- )
- elif len(self.body_frameNumber_trackedPointNumber_XYZ.shape) == 2: # single frame
- return np.vstack(
- [
- self.body_frameNumber_trackedPointNumber_XYZ,
- self.rightHand_frameNumber_trackedPointNumber_XYZ,
- self.leftHand_frameNumber_trackedPointNumber_XYZ,
- self.face_frameNumber_trackedPointNumber_XYZ,
- ]
- )
- else:
- logger.error("data should have either 2 or 3 dimensions")
-
-
-@dataclass
-class Mediapipe2dDataPayload:
- raw_frame_payload: FramePayload = None
- mediapipe_results: Any = None
- annotated_image: np.ndarray = None
- pixel_data_numpy_arrays: Mediapipe2dNumpyArrays = None
diff --git a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/data_models/mediapipe_skeleton_names_and_connections.py b/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/data_models/mediapipe_skeleton_names_and_connections.py
deleted file mode 100644
index 572ff38d..00000000
--- a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/data_models/mediapipe_skeleton_names_and_connections.py
+++ /dev/null
@@ -1,101 +0,0 @@
-from mediapipe.python.solutions import holistic as mp_holistic
-
-mediapipe_body_connections = [connection for connection in mp_holistic.POSE_CONNECTIONS]
-
-mediapipe_hand_connections = [connection for connection in mp_holistic.HAND_CONNECTIONS]
-
-mediapipe_face_connections = [connection for connection in mp_holistic.FACEMESH_CONTOURS]
-
-mediapipe_body_landmark_names = [landmark.name.lower() for landmark in mp_holistic.PoseLandmark]
-NUMBER_OF_MEDIAPIPE_BODY_MARKERS = len(mediapipe_body_landmark_names)
-
-mediapipe_hand_landmark_names = [landmark.name.lower() for landmark in mp_holistic.HandLandmark]
-
-mediapipe_face_landmark_names = [
- "right_eye",
- "left_eye",
- "nose_tip",
- "mouth_center",
- "right_ear_tragion",
- "left_ear_tragion",
-]
-
-mediapipe_tracked_point_names_dict = {
- "body": mediapipe_body_landmark_names,
- "right_hand": [f"right_hand_{name}" for name in mediapipe_hand_landmark_names],
- "left_hand": [f"left_hand_{name}" for name in mediapipe_hand_landmark_names],
- "face": mediapipe_face_landmark_names,
-}
-
-mediapipe_names_and_connections_dict = {
- "body": {
- "names": mediapipe_body_landmark_names,
- "connections": mediapipe_body_connections,
- },
- "right_hand": {
- "names": [f"right_hand_{name}" for name in mediapipe_hand_landmark_names],
- "connections": mediapipe_hand_connections,
- },
- "left_hand": {
- "names": [f"left_hand_{name}" for name in mediapipe_hand_landmark_names],
- "connections": mediapipe_hand_connections,
- },
- "face": {
- "names": mediapipe_face_landmark_names,
- "connections": mediapipe_face_connections,
- },
-}
-mediapipe_virtual_marker_definitions_dict = {
- "head_center": {
- "marker_names": ["left_ear", "right_ear"],
- "marker_weights": [0.5, 0.5],
- },
- "neck_center": {
- "marker_names": ["left_shoulder", "right_shoulder"],
- "marker_weights": [0.5, 0.5],
- },
- "trunk_center": {
- "marker_names": ["left_shoulder", "right_shoulder", "left_hip", "right_hip"],
- "marker_weights": [0.25, 0.25, 0.25, 0.25],
- },
- "hips_center": {
- "marker_names": ["left_hip", "right_hip"],
- "marker_weights": [0.5, 0.5],
- },
-}
-mediapipe_skeleton_schema = {
- "body": {
- "point_names": mediapipe_body_landmark_names,
- "connections": mediapipe_body_connections,
- "virtual_marker_definitions": mediapipe_virtual_marker_definitions_dict,
- "parent": "hips_center",
- },
- "hands": {
- "right": {
- "point_names": [name for name in mediapipe_hand_landmark_names],
- "connections": mediapipe_hand_connections,
- "parent": "right_wrist",
- },
- "left": {
- "point_names": [name for name in mediapipe_hand_landmark_names],
- "connections": mediapipe_hand_connections,
- "parent": "left_wrist",
- },
- },
- "face": {
- "point_names": mediapipe_face_landmark_names,
- "connections": mediapipe_face_connections,
- "parent": "nose",
- },
-}
-
-if __name__ == "__main__":
- import pprint
-
- #
- # print("mediapipe_body_connections:")
- # pprint.pp(mediapipe_tracked_point_names_dict)
-
- print("=====================================================")
- print("mediapipe_skeleton_schema:")
- pprint.pp(mediapipe_skeleton_schema)
diff --git a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/mediapipe_skeleton_detector.py b/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/mediapipe_skeleton_detector.py
deleted file mode 100644
index 70c7920d..00000000
--- a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/mediapipe_skeleton_detector.py
+++ /dev/null
@@ -1,507 +0,0 @@
-import logging
-import multiprocessing
-from pathlib import Path
-from typing import List, Optional, Union, Callable, Tuple
-
-import cv2
-import mediapipe as mp
-import numpy as np
-from skellycam.opencv.video_recorder.video_recorder import VideoRecorder
-from skellytracker.trackers.mediapipe_tracker.mediapipe_model_info import (
- MediapipeTrackingParams
-)
-from tqdm import tqdm
-
-from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.data_models.mediapipe_dataclasses import (
- Mediapipe2dNumpyArrays,
-)
-from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.data_models.mediapipe_skeleton_names_and_connections import (
- mediapipe_tracked_point_names_dict,
-)
-from freemocap.system.paths_and_filenames.file_and_folder_names import (
- MEDIAPIPE_2D_NPY_FILE_NAME,
- ANNOTATED_VIDEOS_FOLDER_NAME,
- MEDIAPIPE_BODY_WORLD_FILE_NAME,
-)
-from freemocap.utilities.get_video_paths import get_video_paths
-
-logger = logging.getLogger(__name__)
-
-mp_drawing = mp.solutions.drawing_utils
-mp_drawing_styles = mp.solutions.drawing_styles
-mp_holistic = mp.solutions.holistic
-
-body_drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
-hand_drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
-face_drawing_spec = mp_drawing.DrawingSpec(thickness=1, circle_radius=1)
-
-
-class MediaPipeSkeletonDetector:
- def __init__(
- self,
- parameter_model: Optional[MediapipeTrackingParams] = None,
- use_tqdm: bool = True,
- ):
- if parameter_model is None:
- parameter_model = MediapipeTrackingParams()
-
- self._parameter_model = parameter_model
- self._use_tqdm = use_tqdm
- self._mediapipe_payload_list = []
-
- self._mediapipe_tracked_point_names_dict = mediapipe_tracked_point_names_dict
-
- self.body_names_list = self._mediapipe_tracked_point_names_dict["body"]
- self.right_hand_names_list = self._mediapipe_tracked_point_names_dict["right_hand"]
- self.left_hand_names_list = self._mediapipe_tracked_point_names_dict["left_hand"]
- self.face_names_list = self._mediapipe_tracked_point_names_dict["face"]
-
- # TODO - build a better iterator and list of `face_marker_names` that will only pull out the face_counters & iris edges (mp.python.solutions.face_mesh_connections.FACEMESH_CONTOURS, FACE_MESH_IRISES)
-
- self.number_of_body_tracked_points = len(self.body_names_list)
- self.number_of_right_hand_tracked_points = len(self.right_hand_names_list)
- self.number_of_left_hand_tracked_points = len(self.left_hand_names_list)
- self.number_of_face_tracked_points = mp.solutions.face_mesh.FACEMESH_NUM_LANDMARKS_WITH_IRISES
-
- self.number_of_tracked_points_total = (
- self.number_of_body_tracked_points
- + self.number_of_left_hand_tracked_points
- + self.number_of_right_hand_tracked_points
- + self.number_of_face_tracked_points
- )
-
- def process_folder_full_of_videos(
- self,
- path_to_folder_of_videos_to_process: Union[Path, str],
- output_data_folder_path: Union[str, Path],
- kill_event: multiprocessing.Event = None,
- use_multiprocessing: bool = True,
- ) -> Union[np.ndarray, None]:
- path_to_folder_of_videos_to_process = Path(path_to_folder_of_videos_to_process)
- logger.info(f"processing videos from: {path_to_folder_of_videos_to_process}")
-
- tasks = self._create_video_processing_tasks(
- output_data_folder_path=output_data_folder_path,
- path_to_folder_of_videos_to_process=path_to_folder_of_videos_to_process,
- )
-
- if use_multiprocessing:
- with multiprocessing.Pool() as pool:
- mediapipe2d_single_camera_npy_arrays_list = pool.starmap(self.process_single_video, tasks)
- else:
- mediapipe2d_single_camera_npy_arrays_list = []
- for task in tasks:
- if kill_event is not None:
- if kill_event.is_set():
- break
- mediapipe2d_single_camera_npy_arrays_list.append(self.process_single_video(*task))
-
- (
- body_world_numCams_numFrames_numTrackedPts_XYZ,
- data2d_numCams_numFrames_numTrackedPts_XY,
- ) = self._build_output_numpy_array(mediapipe2d_single_camera_npy_arrays_list)
-
- self._save_mediapipe2d_data_to_npy(
- data2d_numCams_numFrames_numTrackedPts_XY=data2d_numCams_numFrames_numTrackedPts_XY,
- body_world_numCams_numFrames_numTrackedPts_XYZ=body_world_numCams_numFrames_numTrackedPts_XYZ,
- output_data_folder_path=Path(output_data_folder_path),
- )
- return data2d_numCams_numFrames_numTrackedPts_XY
-
- @staticmethod
- def process_single_video(
- synchronized_video_file_path: Path,
- output_data_folder_path: Path,
- mediapipe_parameters_model: MediapipeTrackingParams,
- annotate_image: Callable,
- list_of_mediapipe_results_to_npy_arrays: Callable,
- use_tqdm: bool = True,
- ):
- logger.info(f"Running `mediapipe` skeleton detection on video: {str(synchronized_video_file_path)}")
-
- holistic_tracker = mp_holistic.Holistic(
- model_complexity=mediapipe_parameters_model.mediapipe_model_complexity,
- min_detection_confidence=mediapipe_parameters_model.min_detection_confidence,
- min_tracking_confidence=mediapipe_parameters_model.min_tracking_confidence,
- )
-
- video_capture_object = cv2.VideoCapture(str(synchronized_video_file_path))
-
- video_width = video_capture_object.get(cv2.CAP_PROP_FRAME_WIDTH)
- video_height = video_capture_object.get(cv2.CAP_PROP_FRAME_HEIGHT)
- video_framerate = video_capture_object.get(cv2.CAP_PROP_FPS)
-
- video_mediapipe_results_list = []
- video_annotated_images_list = []
-
- success, image = video_capture_object.read()
-
- number_of_frames = int(video_capture_object.get(cv2.CAP_PROP_FRAME_COUNT))
-
- if use_tqdm:
- iterator = tqdm(
- range(number_of_frames),
- desc=f"mediapiping video: {synchronized_video_file_path.name}",
- total=number_of_frames,
- colour="magenta",
- unit="frames",
- dynamic_ncols=True,
- )
- else:
- iterator = range(number_of_frames)
-
- for _frame_number in iterator:
- if not success or image is None:
- logger.error(f"Failed to load an image from: {str(synchronized_video_file_path)}")
- raise Exception
-
- mediapipe_results = holistic_tracker.process(
- image
- ) # <-this is where the magic happens, i.e. where the raw image is processed by a convolutional neural network to provide an estimate of joint position in pixel coordinates. Please don't forget that this is insane and should not be possible lol
-
- mediapipe_single_frame_npy_data = list_of_mediapipe_results_to_npy_arrays(
- [mediapipe_results],
- image_width=image.shape[0],
- image_height=image.shape[1],
- )
-
- confidence_threshold = 0.5
-
- threshold_mask = (
- mediapipe_single_frame_npy_data.body_frameNumber_trackedPointNumber_confidence < confidence_threshold
- )
- mediapipe_single_frame_npy_data.body_frameNumber_trackedPointNumber_XYZ[threshold_mask, :] = np.nan
-
- video_mediapipe_results_list.append(mediapipe_results)
- annotated_image = annotate_image(image, mediapipe_results)
- video_annotated_images_list.append(annotated_image)
-
- success, image = video_capture_object.read()
-
- try:
- annotated_video_path = output_data_folder_path.parent.parent / ANNOTATED_VIDEOS_FOLDER_NAME
- annotated_video_path.mkdir(exist_ok=True, parents=True)
- annotated_video_name = synchronized_video_file_path.stem + "_mediapipe.mp4"
- annotated_video_save_path = annotated_video_path / annotated_video_name
-
- logger.info(f"Saving mediapipe annotated video to : {annotated_video_save_path}")
-
- video_recorder = VideoRecorder()
-
- video_recorder.save_image_list_to_disk(
- image_list=video_annotated_images_list,
- path_to_save_video_file=annotated_video_save_path,
- frames_per_second=video_framerate,
- )
- except Exception as e:
- logger.error(f"Failed to save annotated video to disk: {e}")
- raise e
-
- camera_mediapipe_2d_single_camera_npy_arrays = list_of_mediapipe_results_to_npy_arrays(
- video_mediapipe_results_list,
- image_width=video_width,
- image_height=video_height,
- )
-
- # return the numpy array for this video
- return camera_mediapipe_2d_single_camera_npy_arrays
-
- def _build_output_numpy_array(self, mediapipe2d_single_camera_npy_arrays_list):
- all_cameras_data2d_list = [
- m2d.all_data2d_nFrames_nTrackedPts_XY for m2d in mediapipe2d_single_camera_npy_arrays_list
- ]
- all_cameras_pose_world_data_list = [
- m2d.body_world_frameNumber_trackedPointNumber_XYZ for m2d in mediapipe2d_single_camera_npy_arrays_list
- ]
- all_cameras_right_hand_world_data_list = [
- m2d.rightHand_frameNumber_trackedPointNumber_XYZ for m2d in mediapipe2d_single_camera_npy_arrays_list
- ]
- all_cameras_left_hand_world_data_list = [
- m2d.leftHand_frameNumber_trackedPointNumber_XYZ for m2d in mediapipe2d_single_camera_npy_arrays_list
- ]
- all_cameras_face_world_data_list = [
- m2d.face_frameNumber_trackedPointNumber_XYZ for m2d in mediapipe2d_single_camera_npy_arrays_list
- ]
- number_of_cameras = len(all_cameras_data2d_list)
- number_of_frames = all_cameras_data2d_list[0].shape[0]
- number_of_tracked_points = all_cameras_data2d_list[0].shape[1]
- number_of_spatial_dimensions = all_cameras_data2d_list[0].shape[2]
- number_of_body_points = all_cameras_pose_world_data_list[0].shape[1] # noqa
- data2d_numCams_numFrames_numTrackedPts_XY = np.empty(
- (
- number_of_cameras,
- number_of_frames,
- number_of_tracked_points,
- number_of_spatial_dimensions,
- )
- )
- body_world_numCams_numFrames_numTrackedPts_XYZ = np.empty(
- (
- number_of_cameras,
- number_of_frames,
- number_of_tracked_points,
- number_of_spatial_dimensions,
- )
- )
- for cam_num in range(number_of_cameras):
- data2d_numCams_numFrames_numTrackedPts_XY[cam_num, :, :, :] = all_cameras_data2d_list[cam_num]
-
- pose_3d = all_cameras_pose_world_data_list[cam_num]
- right_hand_3d = all_cameras_right_hand_world_data_list[cam_num]
- left_hand_3d = all_cameras_left_hand_world_data_list[cam_num]
- face_3d = all_cameras_face_world_data_list[cam_num]
-
- body_world_numCams_numFrames_numTrackedPts_XYZ[cam_num, :, :, :] = np.concatenate(
- (pose_3d, right_hand_3d, left_hand_3d, face_3d), axis=1
- )
-
- logger.info(
- f"The shape of body_world_numCams_numFrames_numTrackedPts_XYZ is "
- f"{body_world_numCams_numFrames_numTrackedPts_XYZ.shape}"
- )
- return body_world_numCams_numFrames_numTrackedPts_XYZ, data2d_numCams_numFrames_numTrackedPts_XY
-
- def _create_video_processing_tasks(
- self,
- output_data_folder_path: Union[str, Path],
- path_to_folder_of_videos_to_process: Union[Path, str],
- ) -> List[Tuple]:
- video_paths = get_video_paths(path_to_folder_of_videos_to_process)
- tasks = [
- (
- video_path,
- output_data_folder_path,
- self._parameter_model,
- self._annotate_image,
- self._list_of_mediapipe_results_to_npy_arrays,
- self._use_tqdm,
- )
- for video_path in video_paths
- ]
- return tasks
-
- def _save_mediapipe2d_data_to_npy(
- self,
- data2d_numCams_numFrames_numTrackedPts_XY: np.ndarray,
- body_world_numCams_numFrames_numTrackedPts_XYZ: np.ndarray,
- output_data_folder_path: Union[str, Path],
- ):
- mediapipe_2dData_save_path = Path(output_data_folder_path) / MEDIAPIPE_2D_NPY_FILE_NAME
- mediapipe_2dData_save_path.parent.mkdir(exist_ok=True, parents=True)
- logger.info(f"saving mediapipe image npy file: {mediapipe_2dData_save_path}")
- np.save(str(mediapipe_2dData_save_path), data2d_numCams_numFrames_numTrackedPts_XY)
-
- mediapipe_body_world_save_path = Path(output_data_folder_path) / MEDIAPIPE_BODY_WORLD_FILE_NAME
- mediapipe_body_world_save_path.parent.mkdir(exist_ok=True, parents=True)
- logger.info(f"saving mediapipe body world npy xyz: {mediapipe_body_world_save_path}")
- np.save(str(mediapipe_body_world_save_path), body_world_numCams_numFrames_numTrackedPts_XYZ)
-
- @staticmethod
- def _annotate_image(image, mediapipe_results):
- mp_drawing.draw_landmarks(
- image=image,
- landmark_list=mediapipe_results.face_landmarks,
- connections=mp_holistic.FACEMESH_CONTOURS,
- landmark_drawing_spec=None,
- connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_contours_style(),
- )
- mp_drawing.draw_landmarks(
- image=image,
- landmark_list=mediapipe_results.face_landmarks,
- connections=mp_holistic.FACEMESH_TESSELATION,
- landmark_drawing_spec=None,
- connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_tesselation_style(),
- )
- mp_drawing.draw_landmarks(
- image=image,
- landmark_list=mediapipe_results.pose_landmarks,
- connections=mp_holistic.POSE_CONNECTIONS,
- landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style(),
- )
- mp_drawing.draw_landmarks(
- image=image,
- landmark_list=mediapipe_results.left_hand_landmarks,
- connections=mp_holistic.HAND_CONNECTIONS,
- landmark_drawing_spec=None,
- connection_drawing_spec=mp_drawing_styles.get_default_hand_connections_style(),
- )
-
- mp_drawing.draw_landmarks(
- image=image,
- landmark_list=mediapipe_results.right_hand_landmarks,
- connections=mp_holistic.HAND_CONNECTIONS,
- landmark_drawing_spec=None,
- connection_drawing_spec=mp_drawing_styles.get_default_hand_connections_style(),
- )
- return image
-
- def _list_of_mediapipe_results_to_npy_arrays(
- self,
- mediapipe_results_list: List,
- image_width: Union[int, float],
- image_height: Union[int, float],
- ) -> Mediapipe2dNumpyArrays:
- number_of_frames = len(mediapipe_results_list)
- number_of_spatial_dimensions = 3 # this will be 2d XY pixel data, with mediapipe's estimate of Z
-
- body_frameNumber_trackedPointNumber_XYZ = np.zeros(
- (
- number_of_frames,
- self.number_of_body_tracked_points,
- number_of_spatial_dimensions,
- )
- )
- body_frameNumber_trackedPointNumber_XYZ[:] = np.nan
-
- body_world_frameNumber_trackedPointNumber_XYZ = np.zeros(
- (
- number_of_frames,
- self.number_of_body_tracked_points,
- number_of_spatial_dimensions,
- )
- )
- body_world_frameNumber_trackedPointNumber_XYZ[:] = np.nan
-
- body_frameNumber_trackedPointNumber_confidence = np.zeros(
- (number_of_frames, self.number_of_body_tracked_points)
- )
- body_frameNumber_trackedPointNumber_confidence[:] = np.nan # only body markers get a 'confidence' value
-
- rightHand_frameNumber_trackedPointNumber_XYZ = np.zeros(
- (
- number_of_frames,
- self.number_of_right_hand_tracked_points,
- number_of_spatial_dimensions,
- )
- )
- rightHand_frameNumber_trackedPointNumber_XYZ[:] = np.nan
-
- leftHand_frameNumber_trackedPointNumber_XYZ = np.zeros(
- (
- number_of_frames,
- self.number_of_left_hand_tracked_points,
- number_of_spatial_dimensions,
- )
- )
- leftHand_frameNumber_trackedPointNumber_XYZ[:] = np.nan
-
- face_frameNumber_trackedPointNumber_XYZ = np.zeros(
- (
- number_of_frames,
- self.number_of_face_tracked_points,
- number_of_spatial_dimensions,
- )
- )
- face_frameNumber_trackedPointNumber_XYZ[:] = np.nan
-
- all_body_tracked_points_visible_on_frame_bool_list = []
- all_right_hand_points_visible_on_frame_bool_list = []
- all_left_hand_points_visible_on_frame_bool_list = []
- all_face_points_visible_on_frame_bool_list = []
- all_tracked_points_visible_on_frame_list = []
-
- for frame_number, frame_results in enumerate(mediapipe_results_list):
- # get the Body data (aka 'pose')
- if frame_results.pose_landmarks is not None:
- for landmark_number, landmark_data in enumerate(frame_results.pose_landmarks.landmark):
- body_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (
- landmark_data.x * image_width
- )
- body_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (
- landmark_data.y * image_height
- )
- body_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (
- landmark_data.z
- * image_width
- # z is on roughly the same scale as x, according to mediapipe docs
- )
- body_frameNumber_trackedPointNumber_confidence[
- frame_number, landmark_number
- ] = landmark_data.visibility # mediapipe calls their 'confidence' score 'visibility'
-
- for landmark_number, landmark_data in enumerate(frame_results.pose_world_landmarks.landmark):
- body_world_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (
- landmark_data.x * image_width
- )
- body_world_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (
- landmark_data.y * image_height
- )
- body_world_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (
- landmark_data.z * image_width
- )
-
- # get Right Hand data
- if frame_results.right_hand_landmarks is not None:
- for landmark_number, landmark_data in enumerate(frame_results.right_hand_landmarks.landmark):
- rightHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (
- landmark_data.x * image_width
- )
- rightHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (
- landmark_data.y * image_height
- )
- rightHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (
- landmark_data.z * image_width
- )
-
- # get Left Hand data
- if frame_results.left_hand_landmarks is not None:
- for landmark_number, landmark_data in enumerate(frame_results.left_hand_landmarks.landmark):
- leftHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (
- landmark_data.x * image_width
- )
- leftHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (
- landmark_data.y * image_height
- )
- leftHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (
- landmark_data.z * image_width
- )
-
- # get Face data
- if frame_results.face_landmarks is not None:
- for landmark_number, landmark_data in enumerate(frame_results.face_landmarks.landmark):
- face_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (
- landmark_data.x * image_width
- )
- face_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (
- landmark_data.y * image_height
- )
- face_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (
- landmark_data.z * image_width
- )
-
- # check if all tracked points are visible on this frame
- all_body_visible = all(sum(np.isnan(body_frameNumber_trackedPointNumber_XYZ[frame_number, :, :])) == 0)
- all_body_tracked_points_visible_on_frame_bool_list.append(all_body_visible)
-
- all_right_hand_visible = all(
- sum(np.isnan(rightHand_frameNumber_trackedPointNumber_XYZ[frame_number, :, :])) == 0
- )
- all_right_hand_points_visible_on_frame_bool_list.append(all_right_hand_visible)
-
- all_left_hand_visible = all(
- sum(np.isnan(leftHand_frameNumber_trackedPointNumber_XYZ[frame_number, :, :])) == 0
- )
- all_left_hand_points_visible_on_frame_bool_list.append(all_left_hand_visible)
-
- all_face_visible = all(sum(np.isnan(face_frameNumber_trackedPointNumber_XYZ[frame_number, :, :])) == 0)
- all_face_points_visible_on_frame_bool_list.append(all_face_visible)
-
- all_points_visible = all(
- [
- all_body_visible,
- all_right_hand_visible,
- all_left_hand_visible,
- all_face_visible,
- ],
- )
-
- all_tracked_points_visible_on_frame_list.append(all_points_visible)
-
- return Mediapipe2dNumpyArrays(
- body_frameNumber_trackedPointNumber_XYZ=np.squeeze(body_frameNumber_trackedPointNumber_XYZ),
- body_world_frameNumber_trackedPointNumber_XYZ=np.squeeze(body_world_frameNumber_trackedPointNumber_XYZ),
- rightHand_frameNumber_trackedPointNumber_XYZ=np.squeeze(rightHand_frameNumber_trackedPointNumber_XYZ),
- leftHand_frameNumber_trackedPointNumber_XYZ=np.squeeze(leftHand_frameNumber_trackedPointNumber_XYZ),
- face_frameNumber_trackedPointNumber_XYZ=np.squeeze(face_frameNumber_trackedPointNumber_XYZ),
- body_frameNumber_trackedPointNumber_confidence=np.squeeze(body_frameNumber_trackedPointNumber_confidence),
- )
diff --git a/freemocap/core_processes/export_data/blender_stuff/export_to_blender/methods/ajc_addon/run_ajc_addon_main.py b/freemocap/core_processes/export_data/blender_stuff/export_to_blender/methods/ajc_addon/run_ajc_addon_main.py
index d5789cd7..cacb92e9 100644
--- a/freemocap/core_processes/export_data/blender_stuff/export_to_blender/methods/ajc_addon/run_ajc_addon_main.py
+++ b/freemocap/core_processes/export_data/blender_stuff/export_to_blender/methods/ajc_addon/run_ajc_addon_main.py
@@ -13,6 +13,14 @@ from freemocap.core_processes.export_data.blender_stuff.export_to_blender.method
install_ajc_addon,
)
from freemocap.core_processes.export_data.blender_stuff.export_to_blender.methods.ajc_addon.run_simple import run_simple
+from freemocap.system.paths_and_filenames.file_and_folder_names import (
+ OLD_TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME,
+ OLD_SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME,
+ OLD_REPROJECTION_ERROR_NPY_FILE_NAME,
+ REPROJECTION_ERROR_NPY_FILE_NAME,
+ SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME,
+ TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME,
+)
logger = logging.getLogger(__name__)
@@ -37,6 +45,7 @@ def run_ajc_blender_addon_subprocess(
blender_exe_path: Union[str, Path],
):
try:
+ # TODO: pass active tracker into here
ajc_blender_addon_validator(recording_folder_path=recording_folder_path)
except FileNotFoundError as e:
logger.error("Missing required files to run AJC addon, did something go wrong during processing?")
@@ -114,42 +123,63 @@ def get_blenders_numpy(blender_exe_path: Union[str, Path]) -> str:
return numpy_path
-def ajc_blender_addon_validator(recording_folder_path: Union[str, Path]):
+def ajc_blender_addon_validator(recording_folder_path: Union[str, Path], active_tracker: str = "mediapipe"):
"""
Check if the required files exist in the recording folder
"""
recording_path = Path(recording_folder_path)
output_data_path = recording_path / "output_data"
- if not (output_data_path / "mediapipe_body_3d_xyz.npy").exists():
- raise FileNotFoundError(f"Could not find required file: {output_data_path / 'mediapipe_body_3d_xyz.npy'}")
-
- if not (output_data_path / "mediapipe_right_hand_3d_xyz.npy").exists():
- raise FileNotFoundError(f"Could not find required file: {output_data_path / 'mediapipe_right_hand_3d_xyz.npy'}")
-
- if not (output_data_path / "mediapipe_left_hand_3d_xyz.npy").exists():
- raise FileNotFoundError(f"Could not find required file: {output_data_path / 'mediapipe_left_hand_3d_xyz.npy'}")
+ if active_tracker[-1] != "_":
+ active_tracker = active_tracker + "_"
- if not (output_data_path / "mediapipe_face_3d_xyz.npy").exists():
- raise FileNotFoundError(f"Could not find required file: {output_data_path / 'mediapipe_face_3d_xyz.npy'}")
+ if not (output_data_path / (active_tracker + "body_3d_xyz.npy")).exists():
+ raise FileNotFoundError(
+ f"Could not find required file: {output_data_path / (active_tracker + 'body_3d_xyz.npy')}"
+ )
- if not (output_data_path / "center_of_mass" / "total_body_center_of_mass_xyz.npy").exists():
+ if not (output_data_path / (active_tracker + "right_hand_3d_xyz.npy")).exists():
raise FileNotFoundError(
- f"Could not find required file: {output_data_path / 'center_of_mass' / 'total_body_center_of_mass_xyz.npy'}"
+ f"Could not find required file: {output_data_path / (active_tracker + 'right_hand_3d_xyz.npy')}"
)
- if not (output_data_path / "center_of_mass" / "segmentCOM_frame_joint_xyz.npy").exists():
+ if not (output_data_path / (active_tracker + "left_hand_3d_xyz.npy")).exists():
raise FileNotFoundError(
- f"Could not find required file: {output_data_path / 'center_of_mass' / 'segmentCOM_frame_joint_xyz.npy'}"
+ f"Could not find required file: {output_data_path / (active_tracker + 'left_hand_3d_xyz.npy')}"
)
- if not (
- output_data_path / "raw_data" / "mediapipe3dData_numFrames_numTrackedPoints_reprojectionError.npy"
- ).exists():
+ if not (output_data_path / (active_tracker + "face_3d_xyz.npy")).exists():
raise FileNotFoundError(
- f"Could not find required file: {output_data_path / 'raw_data' / 'mediapipe3dData_numFrames_numTrackedPoints_reprojectionError.npy'}"
+ f"Could not find required file: {output_data_path / (active_tracker + 'face_3d_xyz.npy')}"
)
+ if not (output_data_path / "center_of_mass" / (active_tracker + TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME)).exists():
+ if (
+ active_tracker != "mediapipe_"
+ or not (output_data_path / "center_of_mass" / OLD_TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME).exists()
+ ):
+ raise FileNotFoundError(
+ f"Could not find required file: {output_data_path / 'center_of_mass' / (active_tracker + TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME)}"
+ )
+
+ if not (output_data_path / "center_of_mass" / (active_tracker + SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME)).exists():
+ if (
+ active_tracker != "mediapipe_"
+ or not (output_data_path / "center_of_mass" / OLD_SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME).exists()
+ ):
+ raise FileNotFoundError(
+ f"Could not find required file: {output_data_path / 'center_of_mass' / (active_tracker + SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME)}"
+ )
+
+ if not (output_data_path / "raw_data" / (active_tracker + REPROJECTION_ERROR_NPY_FILE_NAME)).exists():
+ if (
+ active_tracker != "mediapipe_"
+ or not (output_data_path / "raw_data" / OLD_REPROJECTION_ERROR_NPY_FILE_NAME).exists()
+ ):
+ raise FileNotFoundError(
+ f"Could not find required file: {output_data_path / 'raw_data' / (active_tracker + REPROJECTION_ERROR_NPY_FILE_NAME)}"
+ )
+
if __name__ == "__main__":
from freemocap.core_processes.export_data.blender_stuff.get_best_guess_of_blender_path import (
diff --git a/freemocap/core_processes/post_process_skeleton_data/calculate_center_of_mass.py b/freemocap/core_processes/post_process_skeleton_data/calculate_center_of_mass.py
index 187627eb..bdf8e02a 100644
--- a/freemocap/core_processes/post_process_skeleton_data/calculate_center_of_mass.py
+++ b/freemocap/core_processes/post_process_skeleton_data/calculate_center_of_mass.py
@@ -1,405 +1,140 @@
import logging
-from typing import List
-
+from typing import Dict, Tuple
import numpy as np
-import pandas as pd
-from rich.progress import track
-from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.data_models.mediapipe_skeleton_names_and_connections import (
- mediapipe_body_landmark_names,
-)
+from freemocap.data_layer.skeleton_models.segments import SegmentAnthropometry
+from freemocap.data_layer.skeleton_models.skeleton import Skeleton
logger = logging.getLogger(__name__)
-def mediapipe_body_names_match_expected(mediapipe_body_landmark_names: List[str]) -> bool:
+def calculate_all_segments_com(
+ segment_positions: Dict[str, Dict[str, np.ndarray]], center_of_mass_definitions: Dict[str, SegmentAnthropometry]
+) -> Dict[str, np.ndarray]:
+ """
+ Calculates the center of mass (COM) for each segment based on provided segment positions
+ and anthropometric data.
+
+ Parameters:
+ - segment_positions: A dictionary where each key is a segment name, and the value is another
+ dictionary with 'proximal' and 'distal' keys containing the respective marker positions
+ as numpy arrays.
+ - center_of_mass_definitions: A dictionary containing anthropometric information for each segment.
+ Each key is a segment name with a value that is a dictionary, which includes the
+ 'segment_com_length' key representing the percentage distance from the proximal marker
+ to the segment's COM.
+
+ Returns:
+ - A dictionary where each key is a segment name, and the value is the calculated COM
+ position as a numpy array.
"""
- Check if the mediapipe folks have changed their landmark names. If they have, then this function may need to be updated.
- Args:
- mediapipe_body_landmark_names: List of strings, each string is the name of a mediapipe landmark.
+ segment_com_data = {}
+
+ for segment_name, segment_info in center_of_mass_definitions.items():
+ proximal = segment_positions[segment_name]["proximal"]
+ distal = segment_positions[segment_name]["distal"]
+
+ com_length = segment_info.segment_com_length
+
+ segment_com = proximal + (distal - proximal) * com_length
+
+ segment_com_data[segment_name] = segment_com
+
+ return segment_com_data
+
+
+def calculate_total_body_center_of_mass(
+ segment_center_of_mass_data: Dict[str, np.ndarray],
+ center_of_mass_definitions: Dict[str, SegmentAnthropometry],
+ num_frames: int,
+) -> np.ndarray:
+ """
+ Calculates the total body center of mass for each frame based on segment COM positions and anthropometric data.
+
+ Parameters:
+ - segment_com_data: A dictionary with segment names as keys and COM positions as values for each frame.
+ - center_of_mass_definitions: A dictionary containing segment mass percentages.
+ - num_frames: The number of frames in the data.
Returns:
- bool: True if the mediapipe landmark names are as expected, False otherwise.
+ - A numpy array containing the position of the total body center of mass for each frame.
"""
- expected_mediapipe_body_landmark_names = [
- "nose",
- "left_eye_inner",
- "left_eye",
- "left_eye_outer",
- "right_eye_inner",
- "right_eye",
- "right_eye_outer",
- "left_ear",
- "right_ear",
- "mouth_left",
- "mouth_right",
- "left_shoulder",
- "right_shoulder",
- "left_elbow",
- "right_elbow",
- "left_wrist",
- "right_wrist",
- "left_pinky",
- "right_pinky",
- "left_index",
- "right_index",
- "left_thumb",
- "right_thumb",
- "left_hip",
- "right_hip",
- "left_knee",
- "right_knee",
- "left_ankle",
- "right_ankle",
- "left_heel",
- "right_heel",
- "left_foot_index",
- "right_foot_index",
- ]
- return mediapipe_body_landmark_names == expected_mediapipe_body_landmark_names
-
-
-def return_indices_of_joints(list_of_indices, list_of_joint_names):
- indices = []
- for name in list_of_joint_names:
- this_name_index = list_of_indices.index(name)
- indices.append(this_name_index)
-
- return indices
-
-
-# %%
-
-
-def return_XYZ_coordinates_of_markers(freemocap_data, indices_list, frame):
- XYZ_coordinates = []
- for index in indices_list:
- this_joint_coordinate = freemocap_data[frame, index, :]
- XYZ_coordinates.append(this_joint_coordinate)
-
- return XYZ_coordinates
-
-
-# %%
-
-
-def build_virtual_trunk_marker(freemocap_data, list_of_indices, trunk_joint_connection, frame):
- trunk_marker_indices = return_indices_of_joints(list_of_indices, trunk_joint_connection)
-
- trunk_XYZ_coordinates = return_XYZ_coordinates_of_markers(freemocap_data, trunk_marker_indices, frame)
-
- trunk_proximal = (trunk_XYZ_coordinates[0] + trunk_XYZ_coordinates[1]) / 2
- trunk_distal = (trunk_XYZ_coordinates[2] + trunk_XYZ_coordinates[3]) / 2
-
- return trunk_proximal, trunk_distal
-
-
-# %%
-
-
-def build_mediapipe_skeleton(mediapipe_pose_data, segment_dataframe, mediapipe_indices) -> list:
- """This function takes in the mediapipe pose data array and the segment_conn_len_perc_dataframe.
- For each frame of data, it loops through each segment we want to find and identifies the names
- of the proximal and distal joints of that segment. Then it searches the mediapipe_indices list
- to find the index that corresponds to the name of that segment. We plug the index into the
- mediapipe_pose_data array to find the proximal/distal joints' XYZ coordinates at that frame.
- The segment, its proximal joint and its distal joint gets thrown into a dictionary.
- And then that dictionary gets saved to a list for each frame. By the end of the function, you
- have a list that contains the skeleton segment XYZ coordinates for each frame."""
-
- num_frames = mediapipe_pose_data.shape[0]
- num_frame_range = range(num_frames)
-
- mediapipe_frame_segment_joint_XYZ = [] # empty list to hold all the skeleton XYZ coordinates/frame
-
- for frame in track(num_frame_range, description="Building a MediaPipe Skeleton"):
- trunk_joint_connection = [
- "left_shoulder",
- "right_shoulder",
- "left_hip",
- "right_hip",
- ]
- trunk_virtual_markers = build_virtual_trunk_marker(
- mediapipe_pose_data, mediapipe_indices, trunk_joint_connection, frame
- )
-
- mediapipe_pose_skeleton_coordinates = {}
- for (
- segment,
- segment_info,
- ) in (
- segment_dataframe.iterrows()
- ): # iterate through the data frame by the segment name and all the info for that segment
- if segment == "trunk":
- # based on index, extract coordinate data from fmc mediapipe data
- mediapipe_pose_skeleton_coordinates[segment] = [
- trunk_virtual_markers[0],
- trunk_virtual_markers[1],
- ]
- elif segment == "left_hand" or segment == "right_hand":
- proximal_joint_hand = segment_info["Joint_Connection"][0]
- if segment == "left_hand":
- distal_joint_hand = "left_index"
- else:
- distal_joint_hand = "right_index"
-
- proximal_joint_hand_index = mediapipe_indices.index(proximal_joint_hand)
- distal_joint_hand_index = mediapipe_indices.index(distal_joint_hand)
-
- mediapipe_pose_skeleton_coordinates[segment] = [
- mediapipe_pose_data[frame, proximal_joint_hand_index, :],
- mediapipe_pose_data[frame, distal_joint_hand_index, :],
- ]
-
- elif segment == "left_foot" or segment == "right_foot":
- if segment == "left_foot":
- proximal_joint_foot_name = "left_ankle"
- else:
- proximal_joint_foot_name = "right_ankle"
-
- proximal_joint_foot_index = mediapipe_indices.index(proximal_joint_foot_name)
-
- distal_joint_foot = segment_info["Joint_Connection"][1]
- distal_joint_foot_index = mediapipe_indices.index(distal_joint_foot)
- mediapipe_pose_skeleton_coordinates[segment] = [
- mediapipe_pose_data[frame, proximal_joint_foot_index, :],
- mediapipe_pose_data[frame, distal_joint_foot_index, :],
- ]
-
- else:
- proximal_joint_name = segment_info["Joint_Connection"][0]
- distal_joint_name = segment_info["Joint_Connection"][1]
-
- # get the mediapipe index for the proximal and distal joint for this segment
- proximal_joint_index = mediapipe_indices.index(proximal_joint_name)
- distal_joint_index = mediapipe_indices.index(distal_joint_name)
-
- # use the mediapipe indices to get the XYZ coordinates for the prox/distal joints and throw it in a dictionary
- # mediapipe_pose_skeleton_coordinates[segment] = {'proximal':mediapipe_pose_data[frame,proximal_joint_index,:],'distal':mediapipe_pose_data[frame,distal_joint_index,:]}
- mediapipe_pose_skeleton_coordinates[segment] = [
- mediapipe_pose_data[frame, proximal_joint_index, :],
- mediapipe_pose_data[frame, distal_joint_index, :],
- ]
-
- mediapipe_frame_segment_joint_XYZ.append(mediapipe_pose_skeleton_coordinates)
-
- return mediapipe_frame_segment_joint_XYZ
-
-
-# %%
-# values for segment weight and segment mass percentages taken from Winter anthropometry tables
-# https://imgur.com/a/aD74j
-# Winter, D.A. (2005) Biomechanics and Motor Control of Human Movement. 3rd Edition, John Wiley & Sons, Inc., Hoboken.
-
-
-BODY_SEGMENT_NAMES = [
- "head",
- "trunk",
- "right_upper_arm",
- "left_upper_arm",
- "right_forearm",
- "left_forearm",
- "right_hand",
- "left_hand",
- "right_thigh",
- "left_thigh",
- "right_shin",
- "left_shin",
- "right_foot",
- "left_foot",
-]
-
-joint_connections = [
- ["left_ear", "right_ear"],
- ["mid_chest_marker", "mid_hip_marker"],
- ["right_shoulder", "right_elbow"],
- ["left_shoulder", "left_elbow"],
- ["right_elbow", "right_wrist"],
- ["left_elbow", "left_wrist"],
- ["right_wrist", "right_hand_marker"],
- ["left_wrist", "left_hand_marker"],
- ["right_hip", "right_knee"],
- ["left_hip", "left_knee"],
- ["right_knee", "right_ankle"],
- ["left_knee", "left_ankle"],
- ["right_back_of_foot_marker", "right_foot_index"],
- ["left_back_of_foot_marker", "left_foot_index"],
-]
-
-segment_COM_lengths = [
- 0.5,
- 0.5,
- 0.436,
- 0.436,
- 0.430,
- 0.430,
- 0.506,
- 0.506,
- 0.433,
- 0.433,
- 0.433,
- 0.433,
- 0.5,
- 0.5,
-]
-
-segment_COM_percentages = [
- 0.081,
- 0.497,
- 0.028,
- 0.028,
- 0.016,
- 0.016,
- 0.006,
- 0.006,
- 0.1,
- 0.1,
- 0.0465,
- 0.0465,
- 0.0145,
- 0.0145,
-]
-
-
-# %%
-def calculate_segment_COM(
- segment_conn_len_perc_dataframe,
- skelcoordinates_frame_segment_joint_XYZ,
- num_frame_range,
-):
- segment_COM_frame_dict = []
- for frame in track(num_frame_range, description="Calculating Segment Center of Mass"):
- segment_COM_dict = {}
- for segment, segment_info in segment_conn_len_perc_dataframe.iterrows():
- this_segment_XYZ = skelcoordinates_frame_segment_joint_XYZ[frame][segment]
-
- # for mediapipe
- this_segment_proximal = this_segment_XYZ[0]
- this_segment_distal = this_segment_XYZ[1]
- this_segment_COM_length = segment_info["Segment_COM_Length"]
-
- this_segment_COM = this_segment_proximal + this_segment_COM_length * (
- this_segment_distal - this_segment_proximal
- )
- segment_COM_dict[segment] = this_segment_COM
- segment_COM_frame_dict.append(segment_COM_dict)
- return segment_COM_frame_dict
-
-
-# %%
-
-
-def reformat_segment_COM(segment_COM_frame_dict, num_frame_range, num_segments):
- segment_COM_frame_imgPoint_XYZ = np.empty([int(len(num_frame_range)), int(num_segments), 3])
- for frame in num_frame_range:
- this_frame_skeleton = segment_COM_frame_dict[frame]
- for joint_count, segment in enumerate(this_frame_skeleton.keys()):
- segment_COM_frame_imgPoint_XYZ[frame, joint_count, :] = this_frame_skeleton[segment]
- return segment_COM_frame_imgPoint_XYZ
-
-
-# %%
-
-
-def calculate_total_body_COM(segment_conn_len_perc_dataframe, segment_COM_frame_dict, num_frame_range):
- logger.info("Calculating Total Body Center of Mass")
- totalBodyCOM_frame_XYZ = np.empty([int(len(num_frame_range)), 3])
-
- for frame in track(num_frame_range, description="Calculating Total Body Center of Mass..."):
- this_frame_total_body_percentages = []
- this_frame_skeleton = segment_COM_frame_dict[frame]
-
- for segment, segment_info in segment_conn_len_perc_dataframe.iterrows():
- this_segment_COM = this_frame_skeleton[segment]
- this_segment_COM_percentage = segment_info["Segment_COM_Percentage"]
-
- this_segment_total_body_percentage = this_segment_COM * this_segment_COM_percentage
- this_frame_total_body_percentages.append(this_segment_total_body_percentage)
-
- this_frame_total_body_COM = np.nansum(this_frame_total_body_percentages, axis=0)
-
- totalBodyCOM_frame_XYZ[frame, :] = this_frame_total_body_COM
-
- return totalBodyCOM_frame_XYZ
-
-
-# %%
-
-
-def build_anthropometric_dataframe(
- segments: list,
- joint_connections: list,
- segment_COM_lengths: list,
- segment_COM_percentages: list,
-) -> pd.DataFrame:
- # load anthropometric data into a pandas dataframe
- df = pd.DataFrame(
- list(
- zip(
- segments,
- joint_connections,
- segment_COM_lengths,
- segment_COM_percentages,
- )
- ),
- columns=[
- "Segment_Name",
- "Joint_Connection",
- "Segment_COM_Length",
- "Segment_COM_Percentage",
- ],
- )
- segment_conn_len_perc_dataframe = df.set_index("Segment_Name")
- return segment_conn_len_perc_dataframe
+ total_body_com = np.zeros((num_frames, 3))
+ for segment_name, segment_info in center_of_mass_definitions.items():
-def calculate_center_of_mass(
- freemocap_marker_data_array: np.ndarray,
- pose_estimation_skeleton: list,
- anthropometric_info_dataframe: pd.DataFrame,
-):
- num_frames = freemocap_marker_data_array.shape[0]
- num_frame_range = range(num_frames)
- num_segments = len(anthropometric_info_dataframe)
+ segment_com = segment_center_of_mass_data.get(segment_name)
+ if segment_com is None:
+ raise ValueError(f"Segment {segment_name} not found in segment center of mass data.")
- segment_COM_frame_dict = calculate_segment_COM(
- anthropometric_info_dataframe, pose_estimation_skeleton, num_frame_range
- )
- segment_COM_frame_imgPoint_XYZ = reformat_segment_COM(segment_COM_frame_dict, num_frame_range, num_segments)
- totalBodyCOM_frame_XYZ = calculate_total_body_COM(
- anthropometric_info_dataframe, segment_COM_frame_dict, num_frame_range
- )
+ segment_mass_percentage = segment_info.segment_com_percentage
- return (
- segment_COM_frame_dict,
- segment_COM_frame_imgPoint_XYZ,
- totalBodyCOM_frame_XYZ,
- )
+ total_body_com += segment_com * segment_mass_percentage
+ return total_body_com
-def run_center_of_mass_calculations(processed_skel3d_frame_marker_xyz: np.ndarray):
- anthropometric_info_dataframe = build_anthropometric_dataframe(
- BODY_SEGMENT_NAMES, joint_connections, segment_COM_lengths, segment_COM_percentages
- )
- if not mediapipe_body_names_match_expected(mediapipe_body_landmark_names):
- raise ValueError(
- "Mediapipe body landmark names do not match expected names - Perhaps they altered the names in a new version? This code will need to be updated"
- )
-
- skelcoordinates_frame_segment_joint_XYZ = build_mediapipe_skeleton(
- processed_skel3d_frame_marker_xyz,
- anthropometric_info_dataframe,
- mediapipe_body_landmark_names,
- )
- (
- segment_COM_frame_dict,
- segment_COM_frame_imgPoint_XYZ,
- totalBodyCOM_frame_XYZ,
- ) = calculate_center_of_mass(
- processed_skel3d_frame_marker_xyz,
- skelcoordinates_frame_segment_joint_XYZ,
- anthropometric_info_dataframe,
+
+def get_all_segment_markers(skeleton: Skeleton) -> Dict[str, Dict[str, np.ndarray]]:
+ """
+ Retrieves the proximal and distal marker positions for all segments within a skeleton.
+
+ Parameters:
+ - skeleton: The Skeleton instance containing segment information and marker data.
+
+ Returns:
+ - A dictionary where each key is a segment name, and the value is another dictionary
+ with 'proximal' and 'distal' keys containing the respective marker positions as numpy arrays.
+ """
+ segment_positions = {}
+
+ if skeleton.segments is None:
+ raise ValueError("Segments must be defined before getting segment markers.")
+
+ for segment_name in skeleton.segments.keys():
+ segment_positions[segment_name] = skeleton.get_segment_markers(segment_name)
+
+ return segment_positions
+
+
+def merge_segment_com_data(segment_com_data: Dict[str, np.ndarray]) -> np.ndarray:
+ """
+ Merges the center of mass data from multiple segments into a single array.
+
+ Parameters:
+ - segment_com_data: A dictionary where each key is a segment name and the value is the center of mass data for that segment.
+
+ Returns:
+ - A numpy array containing the merged center of mass data.
+ """
+ com_data_list = list(segment_com_data.values())
+
+ return np.stack(com_data_list, axis=1)
+
+
+def calculate_center_of_mass_from_skeleton(skeleton: Skeleton) -> Tuple[np.ndarray, np.ndarray]:
+ """
+ Calculates the segment and total body center of mass for a skeleton based on anthropometric data.
+
+ Parameters:
+ - skeleton: The Skeleton instance containing marker data and segment information.
+ - anthropometric_data: A dictionary containing segment mass percentages
+
+ Returns:
+ - A tuple containing the segment center of mass data and the total body center of mass.
+ """
+ if skeleton.center_of_mass_definitions is None:
+ raise ValueError("Segment center of mass definitions must be defined before calculating center of mass.")
+
+ if skeleton.segments is None:
+ raise ValueError("Segments must be defined before calculating center of mass.")
+
+ segment_3d_positions = get_all_segment_markers(skeleton) # TODO: Should this be a method of the skeleton model?
+ segment_com_data = calculate_all_segments_com(segment_3d_positions, skeleton.center_of_mass_definitions)
+ total_body_com = calculate_total_body_center_of_mass(
+ segment_com_data, skeleton.center_of_mass_definitions, skeleton.num_frames
)
- return segment_COM_frame_imgPoint_XYZ, totalBodyCOM_frame_XYZ
+ merged_segment_com_data = merge_segment_com_data(segment_com_data)
+
+ return merged_segment_com_data, total_body_com
diff --git a/freemocap/core_processes/post_process_skeleton_data/create_skeleton.py b/freemocap/core_processes/post_process_skeleton_data/create_skeleton.py
new file mode 100644
index 00000000..5e329c9c
--- /dev/null
+++ b/freemocap/core_processes/post_process_skeleton_data/create_skeleton.py
@@ -0,0 +1,67 @@
+from typing import List, Dict, Optional
+
+
+from freemocap.data_layer.skeleton_models.marker_info import MarkerInfo
+from freemocap.data_layer.skeleton_models.segments import Segment, SegmentAnthropometry
+from freemocap.data_layer.skeleton_models.skeleton import Skeleton
+
+
+def create_marker_info(
+ marker_list: List[str], virtual_markers: Optional[Dict[str, Dict[str, List]]] = None
+) -> MarkerInfo:
+ """
+ Creates a MarkerInfo instance from a list of actual marker names and optional virtual markers.
+
+ Parameters:
+ - marker_list: A list of strings representing the names of actual markers.
+ - virtual_markers: A dictionary defining virtual markers and their related data. Each key is a virtual marker name,
+ and its value is another dictionary with 'marker_names' and 'marker_weights' as keys.
+
+ Returns:
+ - An instance of MarkerInfo populated with actual and, if provided, virtual markers.
+ """
+ marker_hub = MarkerInfo.create(marker_list=marker_list)
+
+ if virtual_markers:
+ marker_hub.add_virtual_markers(virtual_markers)
+ return marker_hub
+
+
+def create_skeleton_model(
+ actual_markers: List[str],
+ num_tracked_points: int,
+ segment_connections: Optional[Dict[str, Segment]] = None,
+ virtual_markers: Optional[Dict[str, Dict[str, List]]] = None,
+ joint_hierarchy: Optional[Dict[str, List[str]]] = None,
+ center_of_mass_info: Optional[Dict[str, SegmentAnthropometry]] = None,
+) -> Skeleton:
+ """
+ Creates a Skeleton model that includes both actual and optionally virtual markers
+ Optionally integrates segment connections, joint hierarchy, and anthro data if needed
+ Parameters:
+ - actual_markers: A list of strings representing the names of actual markers.
+ - num_tracked_points: The number of tracked points expected in the 3D data.
+ - segment_connections: Optional; A dictionary where each key is a segment name and its value is a dictionary
+ with information about that segment (e.g., 'proximal', 'distal' marker names).
+ - virtual_markers: Optional; a dictionary with information necessary to compute virtual markers.
+ - joint_hierarchy: Optional; a dictionary with joint names as keys and lists of connected marker names as values.
+ - center_of_mass_info: Optional; a dictionary containing segment mass percentages
+
+ Returns:
+ - An instance of the Skeleton class that represents the complete skeletal model including markers, segments,
+ and optionally, joint hierarchy data.
+ """
+ marker_hub = create_marker_info(marker_list=actual_markers, virtual_markers=virtual_markers)
+
+ skeleton_model = Skeleton(markers=marker_hub, num_tracked_points=num_tracked_points)
+
+ if segment_connections:
+ skeleton_model.add_segments(segment_connections)
+
+ if joint_hierarchy:
+ skeleton_model.add_joint_hierarchy(joint_hierarchy)
+
+ if center_of_mass_info:
+ skeleton_model.add_center_of_mass_definitions(center_of_mass_info)
+
+ return skeleton_model
diff --git a/freemocap/core_processes/post_process_skeleton_data/enforce_rigid_bones.py b/freemocap/core_processes/post_process_skeleton_data/enforce_rigid_bones.py
new file mode 100644
index 00000000..3dd7970b
--- /dev/null
+++ b/freemocap/core_processes/post_process_skeleton_data/enforce_rigid_bones.py
@@ -0,0 +1,149 @@
+from copy import deepcopy
+import numpy as np
+from typing import Dict, List, Union
+
+from freemocap.data_layer.skeleton_models.segments import Segment
+from freemocap.data_layer.skeleton_models.skeleton import Skeleton
+
+
+def calculate_bone_lengths_and_statistics(
+ marker_data: Dict[str, np.ndarray], segment_connections: Dict[str, Segment]
+) -> Dict[str, Dict[str, Union[np.ndarray, float]]]:
+ """
+ Calculates bone lengths for each frame and their statistics (median and standard deviation)
+ based on marker positions and segment connections.
+
+ Parameters:
+ - marker_data: A dictionary containing marker trajectories with marker names as keys and
+ 3D positions as values (numpy arrays).
+ - segment_connections: A dictionary defining the segments (bones) with segment names as keys
+ and dictionaries with 'proximal' and 'distal' markers as values.
+
+ Returns:
+ - A dictionary with segment names as keys and dictionaries with lengths, median lengths,
+ and standard deviations as values.
+ """
+ bone_statistics = {}
+
+ for segment_name, segment in segment_connections.items():
+ proximal_positions = marker_data[segment.proximal]
+ distal_positions = marker_data[segment.distal]
+
+ lengths = np.linalg.norm(distal_positions - proximal_positions, axis=1)
+ valid_lengths = lengths[~np.isnan(lengths)]
+
+ median_length = np.median(valid_lengths)
+ stdev_length = np.std(valid_lengths)
+
+ bone_statistics[segment_name] = {"lengths": lengths, "median": median_length, "stdev": stdev_length}
+
+ return bone_statistics
+
+
+def enforce_rigid_bones(
+ marker_data: Dict[str, np.ndarray],
+ segment_connections: Dict[str, Segment],
+ bone_lengths_and_statistics: Dict[str, Dict[str, Union[np.ndarray, float]]],
+ joint_hierarchy: Dict[str, List[str]],
+) -> Dict[str, np.ndarray]:
+ """
+ Enforces rigid bones by adjusting the distal joints of each segment to match the median length.
+
+ Parameters:
+ - marker_data: The original marker positions.
+ - segment_connections: Information about how segments (bones) are connected.
+ - bone_lengths_and_statistics: The desired bone lengths and statistics for each segment.
+ - joint_hierarchy: The hierarchy of joints, indicating parent-child relationships.
+
+ Returns:
+ - A dictionary of adjusted marker positions.
+ """
+ rigid_marker_data = deepcopy(marker_data)
+
+ for segment_name, stats in bone_lengths_and_statistics.items():
+ desired_length = stats["median"]
+ lengths = stats["lengths"]
+
+ segment = segment_connections[segment_name]
+ proximal_marker, distal_marker = segment.proximal, segment.distal
+
+ for frame_index, current_length in enumerate(lengths):
+ if current_length != desired_length:
+ proximal_position = marker_data[proximal_marker][frame_index]
+ distal_position = marker_data[distal_marker][frame_index]
+ direction = distal_position - proximal_position
+ try:
+ direction /= np.linalg.norm(direction) # Normalize to unit vector
+ except ZeroDivisionError:
+ direction /= 1e-5 # Set to a small value if the direction is zero
+ adjustment = (desired_length - current_length) * direction
+
+ rigid_marker_data[distal_marker][frame_index] += adjustment
+
+ adjust_children(distal_marker, frame_index, adjustment, rigid_marker_data, joint_hierarchy)
+
+ return rigid_marker_data
+
+
+def adjust_children(
+ parent_marker: str,
+ frame_index: int,
+ adjustment: np.ndarray,
+ marker_data: Dict[str, np.ndarray],
+ joint_hierarchy: Dict[str, List[str]],
+):
+ """
+ Recursively adjusts the positions of child markers based on the adjustment of the parent marker.
+ """
+ if parent_marker in joint_hierarchy:
+ for child_marker in joint_hierarchy[parent_marker]:
+ marker_data[child_marker][frame_index] += adjustment
+ adjust_children(child_marker, frame_index, adjustment, marker_data, joint_hierarchy)
+
+
+def merge_rigid_marker_data(rigid_marker_data: Dict[str, np.ndarray]) -> np.ndarray:
+ """
+ Merges the center of mass data from multiple segments into a single array.
+
+ Parameters:
+ - segment_com_data: A dictionary where each key is a segment name and the value is the center of mass data for that segment.
+
+ Returns:
+ - A numpy array containing the merged center of mass data.
+ """
+ # TODO: We could use this more broadly as a skeleton method
+
+ rigid_marker_data_list = list(rigid_marker_data.values())
+
+ return np.stack(rigid_marker_data_list, axis=1)
+
+
+def enforce_rigid_bones_from_skeleton(skeleton: Skeleton) -> np.ndarray:
+ """
+ Calculates bone lengths and statistics from skeleton data and enforces rigid bones.
+
+ Parameters:
+ - skeleton: The Skeleton instance containing segment information, joint hierarchy, and marker data.
+
+ Returns:
+ - A numpy array of adjusted marker positions.
+ """
+ # TODO: Should this be a method of Skeleton?
+ if not skeleton.segments:
+ raise ValueError("Segments must be defined before rigid bones can be enforced.")
+
+ if not skeleton.joint_hierarchy:
+ raise ValueError("Joint hierarchy must be defined before rigid bones can be enforced.")
+
+ bone_lengths_and_statistcs = calculate_bone_lengths_and_statistics(
+ marker_data=skeleton.marker_data, segment_connections=skeleton.segments
+ )
+
+ rigid_marker_data = enforce_rigid_bones(
+ marker_data=skeleton.marker_data,
+ segment_connections=skeleton.segments,
+ bone_lengths_and_statistics=bone_lengths_and_statistcs,
+ joint_hierarchy=skeleton.joint_hierarchy,
+ )
+
+ return merge_rigid_marker_data(rigid_marker_data=rigid_marker_data)
diff --git a/freemocap/core_processes/post_process_skeleton_data/post_process_skeleton.py b/freemocap/core_processes/post_process_skeleton_data/post_process_skeleton.py
index 535f0c0f..f238d121 100644
--- a/freemocap/core_processes/post_process_skeleton_data/post_process_skeleton.py
+++ b/freemocap/core_processes/post_process_skeleton_data/post_process_skeleton.py
@@ -16,6 +16,7 @@ from skellyforge.freemocap_utils.constants import (
TASK_FINDING_GOOD_FRAME,
)
from skellyforge.freemocap_utils.postprocessing_widgets.task_worker_thread import TaskWorkerThread
+from skellytracker.trackers.base_tracker.model_info import ModelInfo
from freemocap.system.logging.configure_logging import log_view_logging_format_string
from freemocap.system.logging.queue_logger import DirectQueueHandler
@@ -32,11 +33,7 @@ class PostProcessedDataHandler:
self.processed_skeleton = processed_skeleton
-def save_numpy_array_to_disk(
- array_to_save: np.ndarray,
- file_name: str,
- save_directory: Union[str, Path]
-):
+def save_numpy_array_to_disk(array_to_save: np.ndarray, file_name: str, save_directory: Union[str, Path]):
if not file_name.endswith(".npy"):
file_name += ".npy"
Path(save_directory).mkdir(parents=True, exist_ok=True)
@@ -67,7 +64,9 @@ def adjust_default_settings(filter_sampling_rate, filter_cutoff_frequency, filte
return adjusted_settings
-def run_post_processing_worker(raw_skel3d_frame_marker_xyz: np.ndarray, settings_dictionary: dict):
+def run_post_processing_worker(
+ raw_skel3d_frame_marker_xyz: np.ndarray, settings_dictionary: dict, landmark_names: list
+) -> np.ndarray:
def handle_thread_finished(results, post_processed_data_handler: PostProcessedDataHandler):
processed_skeleton = results[TASK_FILTERING]["result"]
post_processed_data_handler.data_callback(processed_skeleton=processed_skeleton)
@@ -84,6 +83,7 @@ def run_post_processing_worker(raw_skel3d_frame_marker_xyz: np.ndarray, settings
task_list=task_list,
settings=settings_dictionary,
all_tasks_finished_callback=lambda results: handle_thread_finished(results, post_processed_data_handler),
+ landmark_names=landmark_names,
)
worker_thread.start()
worker_thread.join()
@@ -93,7 +93,7 @@ def run_post_processing_worker(raw_skel3d_frame_marker_xyz: np.ndarray, settings
def post_process_data(
- recording_processing_parameter_model, raw_skel3d_frame_marker_xyz: np.ndarray, queue: multiprocessing.Queue
+ recording_processing_parameter_model, raw_skel3d_frame_marker_xyz: np.ndarray, queue: multiprocessing.Queue
) -> np.ndarray:
if queue:
handler = DirectQueueHandler(queue)
@@ -104,7 +104,18 @@ def post_process_data(
)
adjusted_settings = adjust_default_settings(filter_sampling_rate, filter_cutoff_frequency, filter_order)
processed_skeleton_array = run_post_processing_worker(
- raw_skel3d_frame_marker_xyz=raw_skel3d_frame_marker_xyz, settings_dictionary=adjusted_settings
+ raw_skel3d_frame_marker_xyz=raw_skel3d_frame_marker_xyz,
+ settings_dictionary=adjusted_settings,
+ landmark_names=get_landmark_names(model_info=recording_processing_parameter_model.tracking_model_info),
)
return processed_skeleton_array
+
+
+def get_landmark_names(model_info: ModelInfo) -> list:
+ if hasattr(model_info, "body_landmark_names"):
+ return model_info.body_landmark_names
+ elif hasattr(model_info, "landmark_names"):
+ return model_info.landmark_names
+ else:
+ raise AttributeError("Model does not have landmark names")
diff --git a/freemocap/core_processes/post_process_skeleton_data/process_single_camera_skeleton_data.py b/freemocap/core_processes/post_process_skeleton_data/process_single_camera_skeleton_data.py
index 7c7a6fb2..a3498918 100644
--- a/freemocap/core_processes/post_process_skeleton_data/process_single_camera_skeleton_data.py
+++ b/freemocap/core_processes/post_process_skeleton_data/process_single_camera_skeleton_data.py
@@ -4,8 +4,8 @@ from typing import Tuple, Union
import numpy as np
-from freemocap.core_processes.capture_volume_calibration.save_mediapipe_3d_data_to_npy import (
- save_mediapipe_3d_data_to_npy,
+from freemocap.core_processes.capture_volume_calibration.save_3d_data_to_npy import (
+ save_3d_data_to_npy,
)
from freemocap.utilities.geometry.project_3d_data_to_z_plane import project_3d_data_to_z_plane
@@ -15,6 +15,7 @@ logger = logging.getLogger(__name__)
def process_single_camera_skeleton_data(
input_image_data_frame_marker_xyz: np.ndarray,
raw_data_folder_path: Union[str, Path],
+ file_prefix: str = "",
project_to_z_plane: bool = True,
) -> Tuple[np.ndarray, np.ndarray]:
skeleton_reprojection_error_fr_mar = np.zeros(input_image_data_frame_marker_xyz.shape[0:2])
@@ -30,12 +31,13 @@ def process_single_camera_skeleton_data(
else:
raw_skel3d_frame_marker_xyz = input_image_data_frame_marker_xyz
- save_mediapipe_3d_data_to_npy(
+ save_3d_data_to_npy(
data3d_numFrames_numTrackedPoints_XYZ=raw_skel3d_frame_marker_xyz,
data3d_numFrames_numTrackedPoints_reprojectionError=skeleton_reprojection_error_fr_mar,
data3d_numCams_numFrames_numTrackedPoints_reprojectionError=skeleton_reprojection_error_cam_fr_mar,
path_to_folder_where_data_will_be_saved=raw_data_folder_path,
processing_level="raw",
+ file_prefix=file_prefix,
)
return raw_skel3d_frame_marker_xyz, skeleton_reprojection_error_fr_mar
diff --git a/freemocap/core_processes/post_process_skeleton_data/split_and_save.py b/freemocap/core_processes/post_process_skeleton_data/split_and_save.py
new file mode 100644
index 00000000..8c2d6bec
--- /dev/null
+++ b/freemocap/core_processes/post_process_skeleton_data/split_and_save.py
@@ -0,0 +1,199 @@
+from pathlib import Path
+from typing import Dict, List
+import logging
+import numpy as np
+import pandas as pd
+
+from skellytracker.trackers.base_tracker.model_info import ModelInfo
+
+from freemocap.system.paths_and_filenames.file_and_folder_names import (
+ BODY_3D_DATAFRAME_CSV_FILE_NAME,
+ FACE_3D_DATAFRAME_CSV_FILE_NAME,
+ LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME,
+ RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# TODO: This is dependent on the ordering we currently use for mediapipe, maybe it should live in model_info
+category_info = {
+ "body": BODY_3D_DATAFRAME_CSV_FILE_NAME,
+ "right_hand": RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME,
+ "left_hand": LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME,
+ "face": FACE_3D_DATAFRAME_CSV_FILE_NAME,
+}
+
+# TODO: come up with a sensible way of splitting this for different types of trackers
+
+
+def split_data(
+ skeleton_3d_data: np.ndarray,
+ model_info: ModelInfo,
+) -> Dict[str, np.ndarray]:
+ """
+ Splits the skeleton data into separate arrays for each data category using the provided model info.
+ Examples categories are body, left_hand, right_hand, and face.
+
+ Parameters:
+ - skeleton: The skeleton data to be split.
+ - model_info: The model info used in tracking.
+
+ Returns:
+ - A dictionary where each key is a category and the value is the corresponding data array.
+ """
+
+ split_data = {}
+
+ # TODO: this function probably shouldn't exist, and its existence implies we should be separately tracking body, hands, and face
+
+ current_index = 0
+ for category in category_info.keys():
+ tracked_points_name = "num_tracked_points_" + category
+ if hasattr(model_info, tracked_points_name):
+ prev_index = current_index
+ current_index += getattr(model_info, tracked_points_name)
+ split_data[category] = skeleton_3d_data[:, prev_index:current_index, :]
+
+ if split_data == {}:
+ logger.debug("No data categories found in model info, skipping data splitting")
+
+ return split_data
+
+
+def create_column_names(model_info: ModelInfo) -> Dict[str, List[str]]:
+ """
+ Creates a list of column names for the skeleton data based on the model info.
+
+ Parameters:
+ - model_info: The model info used in tracking.
+
+ Returns:
+ - A list of column names for the skeleton data.
+ """
+
+ column_names = {}
+ for category in category_info.keys():
+ tracked_points_name = "num_tracked_points_" + category
+ landmark_names = category + "_landmark_names"
+ if hasattr(model_info, tracked_points_name):
+ category_column_names = []
+ if hasattr(model_info, landmark_names) and len(getattr(model_info, landmark_names)) == getattr(
+ model_info, tracked_points_name
+ ):
+ for name in getattr(model_info, landmark_names):
+ category_column_names.append(f"{category}_{name}_x")
+ category_column_names.append(f"{category}_{name}_y")
+ category_column_names.append(f"{category}_{name}_z")
+ else:
+ for i in range(getattr(model_info, tracked_points_name)):
+ category_column_names.append(f"{category}_{str(i).zfill(4)}_x")
+ category_column_names.append(f"{category}_{str(i).zfill(4)}_y")
+ category_column_names.append(f"{category}_{str(i).zfill(4)}_z")
+
+ column_names[category] = category_column_names
+
+ # TODO: this is a quick attempt and probably wrong somehow
+ if column_names == {}:
+ landmark_names = "landmark_names"
+ tracked_points_name = "num_tracked_points"
+ category_column_names = []
+ if hasattr(model_info, landmark_names) and len(getattr(model_info, landmark_names)) == getattr(
+ model_info, tracked_points_name
+ ):
+ for name in getattr(model_info, landmark_names):
+ category_column_names.append(f"{category}_{name}_x")
+ category_column_names.append(f"{category}_{name}_y")
+ category_column_names.append(f"{category}_{name}_z")
+ else:
+ for i in range(getattr(model_info, tracked_points_name)):
+ category_column_names.append(f"{category}_{str(i).zfill(4)}_x")
+ category_column_names.append(f"{category}_{str(i).zfill(4)}_y")
+ category_column_names.append(f"{category}_{str(i).zfill(4)}_z")
+
+ column_names["total"] = category_column_names
+
+ return column_names
+
+
+def save_split_csv(
+ output_data_folder_path: str,
+ split_data: Dict[str, np.ndarray],
+ column_names: Dict[str, List[str]],
+ model_info: ModelInfo,
+) -> None:
+ """
+ Saves the split skeleton data to CSV files using column names.
+
+ Parameters:
+ - split_data: A dictionary where each key is a category and the value is the corresponding data array.
+ - column_names: A dictionary where each key is a category and the value is a list of column names for the data.
+ """
+ if split_data == {}:
+ logger.debug("No data categories found in model info, skipping data splitting")
+ return
+
+ number_of_frames = next(iter(split_data.values())).shape[0]
+
+ file_prefix = model_info.name + "_"
+
+ for category, data in split_data.items():
+ column_name_list = column_names[category]
+ tracked_points_name = "num_tracked_points_" + category
+
+ data_flat = data.reshape(number_of_frames, getattr(model_info, tracked_points_name) * 3)
+ dataframe = pd.DataFrame(data_flat, columns=column_name_list)
+ dataframe.to_csv(str(Path(output_data_folder_path) / (file_prefix + category_info[category])), index=False)
+
+ logger.info(f"Saved split skeleton data csvs to {output_data_folder_path}")
+
+
+def save_split_npy(output_data_folder_path: str, split_data: Dict[str, np.ndarray], file_prefix: str = "") -> None:
+ """
+ Saves the split skeleton data to numpy files.
+
+ Parameters:
+ - split_data: A dictionary where each key is a category and the value is the corresponding data array.
+ """
+
+ if split_data == {}:
+ logger.debug("No data categories found in model info, skipping data splitting")
+ return
+
+ if file_prefix != "" and file_prefix[-1] != "_":
+ file_prefix += "_"
+
+ for category, data in split_data.items():
+ np.save(str(Path(output_data_folder_path) / (file_prefix + category + "_3d_xyz.npy")), data)
+
+ logger.info(f"Saved split skeleton data npys to {output_data_folder_path}")
+
+
+def split_and_save(
+ skeleton_3d_data: np.ndarray,
+ model_info: ModelInfo,
+ output_data_folder_path: str,
+) -> None:
+ """
+ Splits the skeleton data into separate arrays for each data category using the provided model info.
+ Examples categories are body, left_hand, right_hand, and face.
+ Then saves the split skeleton data to CSV files using column names.
+ Then saves the split skeleton data to numpy files.
+
+ Parameters:
+ - skeleton: The skeleton data to be split.
+ - model_info: The model info used in tracking.
+ - output_data_folder_path: The path to save the split skeleton data to.
+ """
+
+ split_data_dict = split_data(skeleton_3d_data=skeleton_3d_data, model_info=model_info)
+ column_names = create_column_names(model_info=model_info)
+ save_split_csv(
+ output_data_folder_path=output_data_folder_path,
+ split_data=split_data_dict,
+ column_names=column_names,
+ model_info=model_info,
+ )
+ save_split_npy(
+ output_data_folder_path=output_data_folder_path, split_data=split_data_dict, file_prefix=model_info.name
+ )
diff --git a/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py b/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py
index 04cc5328..a5493cab 100644
--- a/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py
+++ b/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py
@@ -1,5 +1,6 @@
import logging
import multiprocessing
+from typing import Optional
from freemocap.core_processes.post_process_skeleton_data.post_process_skeleton import post_process_data
@@ -32,9 +33,9 @@ logger = logging.getLogger(__name__)
def process_recording_folder(
recording_processing_parameter_model: ProcessingParameterModel,
kill_event: multiprocessing.Event = None,
- logging_queue: multiprocessing.Queue = None,
+ logging_queue: Optional[multiprocessing.Queue] = None,
use_tqdm: bool = True,
-) -> bool:
+) -> None:
"""
Parameters
@@ -72,7 +73,7 @@ def process_recording_folder(
queue=logging_queue,
use_tqdm=use_tqdm,
)
- except (RuntimeError, FileNotFoundError) as e:
+ except (RuntimeError, ValueError) as e:
logger.error("2D skeleton detection failed, cannot continue processing")
if logging_queue:
logging_queue.put(e)
@@ -103,14 +104,20 @@ def process_recording_folder(
logging_queue.put(exception)
raise exception
- # TODO: move the rotate by 90 function into skellyforge to skip duplication of responsibility
- rotated_raw_skel3d_frame_marker_xyz = rotate_by_90_degrees_around_x_axis(raw_skel3d_frame_marker_xyz)
- # TODO: find out if skellyforge does all the error handling we need - if not add it to post_process_data
- skel3d_frame_marker_xyz = post_process_data(
- recording_processing_parameter_model=recording_processing_parameter_model,
- raw_skel3d_frame_marker_xyz=rotated_raw_skel3d_frame_marker_xyz,
- queue=logging_queue,
- )
+ try:
+ # TODO: move the rotate by 90 function into skellyforge to skip duplication of responsibility
+ rotated_raw_skel3d_frame_marker_xyz = rotate_by_90_degrees_around_x_axis(raw_skel3d_frame_marker_xyz)
+ # TODO: find out if skellyforge does all the error handling we need - if not add it to post_process_data
+ skel3d_frame_marker_xyz = post_process_data(
+ recording_processing_parameter_model=recording_processing_parameter_model,
+ raw_skel3d_frame_marker_xyz=rotated_raw_skel3d_frame_marker_xyz,
+ queue=logging_queue,
+ )
+ except (RuntimeError, ValueError, AttributeError) as e:
+ logger.error("Post processing failed, cannot continue processing")
+ if logging_queue:
+ logging_queue.put(e)
+ raise e
if kill_event is not None and kill_event.is_set():
exception = KillEventException("Process was killed")
@@ -118,11 +125,17 @@ def process_recording_folder(
logging_queue.put(exception)
raise exception
- anatomical_data_dict = calculate_anatomical_data(
- processing_parameters=recording_processing_parameter_model,
- skel3d_frame_marker_xyz=skel3d_frame_marker_xyz,
- queue=logging_queue,
- )
+ try:
+ anatomical_data_dict = calculate_anatomical_data(
+ processing_parameters=recording_processing_parameter_model,
+ skel3d_frame_marker_xyz=skel3d_frame_marker_xyz,
+ queue=logging_queue,
+ )
+ except (RuntimeError, ValueError, TypeError, KeyError, AttributeError) as e:
+ logger.error("Anatomical data calculation failed, cannot continue processing")
+ if logging_queue:
+ logging_queue.put(e)
+ raise e
if kill_event is not None and kill_event.is_set():
exception = KillEventException("Process was killed")
@@ -135,9 +148,13 @@ def process_recording_folder(
skel3d_frame_marker_xyz=skel3d_frame_marker_xyz,
segment_COM_frame_imgPoint_XYZ=anatomical_data_dict["segment_COM"],
totalBodyCOM_frame_XYZ=anatomical_data_dict["total_body_COM"],
+ rigid_bones_data=anatomical_data_dict["rigid_bones_data"],
processing_parameters=recording_processing_parameter_model,
queue=logging_queue,
)
- DataSaver(recording_folder_path=recording_processing_parameter_model.recording_info_model.path).save_all()
+ DataSaver(
+ recording_folder_path=recording_processing_parameter_model.recording_info_model.path,
+ model_info=recording_processing_parameter_model.tracking_model_info,
+ ).save_all()
logger.info(f"Done processing {recording_processing_parameter_model.recording_info_model.path}")
diff --git a/freemocap/core_processes/process_motion_capture_videos/process_recording_headless.py b/freemocap/core_processes/process_motion_capture_videos/process_recording_headless.py
index 269eb8a4..e7f1ea95 100644
--- a/freemocap/core_processes/process_motion_capture_videos/process_recording_headless.py
+++ b/freemocap/core_processes/process_motion_capture_videos/process_recording_headless.py
@@ -24,21 +24,28 @@ def process_recording_headless(
path_to_camera_calibration_toml: Optional[Union[str, Path]] = None,
path_to_blender_executable: Optional[Union[str, Path]] = None,
recording_processing_parameter_model: Optional[ProcessingParameterModel] = None,
+ recording_info_model: Optional[RecordingInfoModel] = None,
+ run_blender: bool = True,
+ make_jupyter_notebook: bool = True,
use_tqdm: bool = True,
):
if path_to_blender_executable is None:
path_to_blender_executable = get_best_guess_of_blender_path()
if recording_processing_parameter_model is None:
recording_processing_parameter_model = ProcessingParameterModel()
-
rec = recording_processing_parameter_model
+ if recording_info_model is None:
+ rec.recording_info_model = RecordingInfoModel(recording_folder_path=Path(recording_path))
+ else:
+ rec.recording_info_model = recording_info_model
+
logger.info(
f"Processing recording:\n"
f"Recording path: {recording_path}\n"
f"Camera calibration toml path: {path_to_camera_calibration_toml}\n"
f"Blender executable path: {path_to_blender_executable}\n"
- f"Recording processing parameter model: {rec.dict()}"
+ f"Recording processing parameter model: {rec.model_dump()}"
)
rec.recording_info_model = RecordingInfoModel(recording_folder_path=Path(recording_path))
@@ -66,10 +73,11 @@ def process_recording_headless(
logger.info("Starting core processing pipeline...")
process_recording_folder(recording_processing_parameter_model=rec, use_tqdm=use_tqdm)
- logger.info("Generating jupyter notebook...")
- generate_jupyter_notebook(path_to_recording=recording_path)
+ if make_jupyter_notebook:
+ logger.info("Generating jupyter notebook...")
+ generate_jupyter_notebook(path_to_recording=recording_path)
- if path_to_blender_executable:
+ if path_to_blender_executable and run_blender:
blender_file_path = get_blender_file_path(recording_folder_path=recording_path)
logger.info(f"Exporting to {blender_file_path}")
export_to_blender(
@@ -77,6 +85,8 @@ def process_recording_headless(
blender_file_path=blender_file_path,
blender_exe_path=Path(path_to_blender_executable),
)
+ elif not run_blender:
+ logger.info("Skipping blender export")
else:
logger.warning("No blender executable provided. Blender file will not be exported.")
diff --git a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/anatomical_data_pipeline_functions.py b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/anatomical_data_pipeline_functions.py
index 2bf721bb..4b9b5547 100644
--- a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/anatomical_data_pipeline_functions.py
+++ b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/anatomical_data_pipeline_functions.py
@@ -1,11 +1,14 @@
import logging
import multiprocessing
-from typing import Dict
+from typing import Dict, Union
import numpy as np
-from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.convert_mediapipe_npy_to_csv import (
- convert_mediapipe_npy_to_csv,
+
+
+from freemocap.core_processes.post_process_skeleton_data.calculate_center_of_mass import (
+ calculate_center_of_mass_from_skeleton,
)
-from freemocap.core_processes.post_process_skeleton_data.calculate_center_of_mass import run_center_of_mass_calculations
+from freemocap.core_processes.post_process_skeleton_data.create_skeleton import create_skeleton_model
+from freemocap.core_processes.post_process_skeleton_data.enforce_rigid_bones import enforce_rigid_bones_from_skeleton
from freemocap.data_layer.recording_models.post_processing_parameter_models import ProcessingParameterModel
from freemocap.system.logging.queue_logger import DirectQueueHandler
from freemocap.system.logging.configure_logging import log_view_logging_format_string
@@ -17,23 +20,40 @@ def calculate_anatomical_data(
processing_parameters: ProcessingParameterModel,
skel3d_frame_marker_xyz: np.ndarray,
queue: multiprocessing.Queue,
-) -> Dict[str, np.ndarray]:
+) -> Dict[str, Union[np.ndarray, None]]:
if queue:
handler = DirectQueueHandler(queue)
handler.setFormatter(logging.Formatter(fmt=log_view_logging_format_string, datefmt="%Y-%m-%dT%H:%M:%S"))
logger.addHandler(handler)
- logger.info("Calculating center of mass...")
- segment_COM_frame_imgPoint_XYZ, totalBodyCOM_frame_XYZ = run_center_of_mass_calculations(
- processed_skel3d_frame_marker_xyz=skel3d_frame_marker_xyz
+ logger.info("Creating skeleton model...")
+ skeleton = create_skeleton_model(
+ actual_markers=processing_parameters.tracking_model_info.landmark_names,
+ segment_connections=processing_parameters.tracking_model_info.segment_connections,
+ virtual_markers=processing_parameters.tracking_model_info.virtual_markers_definitions,
+ joint_hierarchy=processing_parameters.tracking_model_info.joint_hierarchy,
+ center_of_mass_info=processing_parameters.tracking_model_info.center_of_mass_definitions,
+ num_tracked_points=processing_parameters.tracking_model_info.num_tracked_points,
)
- convert_mediapipe_npy_to_csv(
- mediapipe_3d_frame_trackedPoint_xyz=skel3d_frame_marker_xyz,
- output_data_folder_path=processing_parameters.recording_info_model.output_data_folder_path,
- ) # TODO: separate functionality of splitting into CSVs and savings to disk
+ skeleton.integrate_freemocap_3d_data(skel3d_frame_marker_xyz)
+
+ logger.info("Calculating center of mass...")
+ try:
+ segment_COM_frame_imgPoint_XYZ, totalBodyCOM_frame_XYZ = calculate_center_of_mass_from_skeleton(
+ skeleton=skeleton
+ )
+ except ValueError:
+ logger.warning("Center of mass cannot be calculated for this tracking model")
+ segment_COM_frame_imgPoint_XYZ = totalBodyCOM_frame_XYZ = None
+ except AttributeError as e:
+ raise e
+
+ logger.info("Enforcing rigid bones...")
+ rigid_bones_data = enforce_rigid_bones_from_skeleton(skeleton=skeleton)
return {
"segment_COM": segment_COM_frame_imgPoint_XYZ,
"total_body_COM": totalBodyCOM_frame_XYZ,
+ "rigid_bones_data": rigid_bones_data,
}
diff --git a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/data_saving_pipeline_functions.py b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/data_saving_pipeline_functions.py
index 83a1b6f8..97f2f961 100644
--- a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/data_saving_pipeline_functions.py
+++ b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/data_saving_pipeline_functions.py
@@ -1,31 +1,31 @@
import logging
import multiprocessing
from pathlib import Path
+from typing import Optional
import numpy as np
from freemocap.core_processes.post_process_skeleton_data.post_process_skeleton import save_numpy_array_to_disk
-from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.data_models.mediapipe_skeleton_names_and_connections import (
- mediapipe_names_and_connections_dict,
-)
+from freemocap.core_processes.post_process_skeleton_data.split_and_save import split_and_save
from freemocap.data_layer.recording_models.post_processing_parameter_models import ProcessingParameterModel
from freemocap.system.logging.queue_logger import DirectQueueHandler
from freemocap.system.logging.configure_logging import log_view_logging_format_string
from freemocap.system.paths_and_filenames.file_and_folder_names import (
CENTER_OF_MASS_FOLDER_NAME,
+ DATA_3D_NPY_FILE_NAME,
+ RIGID_BONES_NPY_FILE_NAME,
SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME,
TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME,
)
-from freemocap.utilities.save_dictionary_to_json import save_dictionary_to_json
-
logger = logging.getLogger(__name__)
def save_data(
skel3d_frame_marker_xyz: np.ndarray,
- segment_COM_frame_imgPoint_XYZ: np.ndarray,
- totalBodyCOM_frame_XYZ: np.ndarray,
+ segment_COM_frame_imgPoint_XYZ: Optional[np.ndarray],
+ totalBodyCOM_frame_XYZ: Optional[np.ndarray],
+ rigid_bones_data: Optional[np.ndarray],
processing_parameters: ProcessingParameterModel,
queue: multiprocessing.Queue,
):
@@ -39,26 +39,43 @@ def save_data(
logger.info("Saving post-processed data")
save_numpy_array_to_disk(
array_to_save=skel3d_frame_marker_xyz,
- file_name="mediaPipeSkel_3d_body_hands_face.npy",
- save_directory=path_to_folder_where_we_will_save_this_data,
+ file_name=f"{processing_parameters.tracking_model_info.name}_{DATA_3D_NPY_FILE_NAME}",
+ save_directory=Path(path_to_folder_where_we_will_save_this_data),
)
- logger.info("Saving segment center of mass data")
- save_numpy_array_to_disk(
- array_to_save=segment_COM_frame_imgPoint_XYZ,
- file_name=SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME,
- save_directory=Path(path_to_folder_where_we_will_save_this_data) / CENTER_OF_MASS_FOLDER_NAME,
- )
+ if segment_COM_frame_imgPoint_XYZ is not None:
+ logger.info("Saving segment center of mass data")
+ save_numpy_array_to_disk(
+ array_to_save=segment_COM_frame_imgPoint_XYZ,
+ file_name=f"{processing_parameters.tracking_model_info.name}_{SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME}",
+ save_directory=Path(path_to_folder_where_we_will_save_this_data) / CENTER_OF_MASS_FOLDER_NAME,
+ )
+ else:
+ logger.debug("segment_COM_frame_imgPoint_XYZ is None, could not save")
- logger.info("Saving total body center of mass data")
- save_numpy_array_to_disk(
- array_to_save=totalBodyCOM_frame_XYZ,
- file_name=TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME,
- save_directory=Path(path_to_folder_where_we_will_save_this_data) / CENTER_OF_MASS_FOLDER_NAME,
- )
+ if totalBodyCOM_frame_XYZ is not None:
+ logger.info("Saving total body center of mass data")
+ save_numpy_array_to_disk(
+ array_to_save=totalBodyCOM_frame_XYZ,
+ file_name=f"{processing_parameters.tracking_model_info.name}_{TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME}",
+ save_directory=Path(path_to_folder_where_we_will_save_this_data) / CENTER_OF_MASS_FOLDER_NAME,
+ )
+ else:
+ logger.debug("totalBodyCOM_frame_XYZ is None, could not save")
+
+ if rigid_bones_data is not None:
+ logger.info("Saving rigid bones data")
+ save_numpy_array_to_disk(
+ array_to_save=rigid_bones_data,
+ file_name=f"{processing_parameters.tracking_model_info.name}_{RIGID_BONES_NPY_FILE_NAME}",
+ save_directory=path_to_folder_where_we_will_save_this_data,
+ )
+ else:
+ logger.debug("rigid_bones_data is None, could not save")
- save_dictionary_to_json(
- save_path=processing_parameters.recording_info_model.output_data_folder_path,
- file_name="mediapipe_names_and_connections_dict.json",
- dictionary=mediapipe_names_and_connections_dict,
+ logger.info("Splitting and saving data...")
+ split_and_save(
+ skeleton_3d_data=skel3d_frame_marker_xyz,
+ model_info=processing_parameters.tracking_model_info,
+ output_data_folder_path=processing_parameters.recording_info_model.output_data_folder_path,
)
diff --git a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/image_tracking_pipeline_functions.py b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/image_tracking_pipeline_functions.py
index 50c9883d..4b89769e 100644
--- a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/image_tracking_pipeline_functions.py
+++ b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/image_tracking_pipeline_functions.py
@@ -4,9 +4,10 @@ from pathlib import Path
import numpy as np
from skellytracker.process_folder_of_videos import process_folder_of_videos
-from skellytracker.trackers.mediapipe_tracker.mediapipe_model_info import (
- MediapipeTrackingParams,
+from skellytracker.trackers.base_tracker.base_tracking_params import (
+ BaseTrackingParams,
)
+from skellytracker.trackers.base_tracker.model_info import ModelInfo
from freemocap.data_layer.recording_models.post_processing_parameter_models import (
ProcessingParameterModel,
@@ -14,6 +15,7 @@ from freemocap.data_layer.recording_models.post_processing_parameter_models impo
from freemocap.system.logging.configure_logging import log_view_logging_format_string
from freemocap.system.logging.queue_logger import DirectQueueHandler
from freemocap.system.paths_and_filenames.file_and_folder_names import LOG_VIEW_PROGRESS_BAR_STRING, RAW_DATA_FOLDER_NAME
+from freemocap.tests.test_image_tracking_data_shape import test_image_tracking_data_shape
logger = logging.getLogger(__name__)
@@ -29,13 +31,13 @@ def run_image_tracking_pipeline(
handler.setFormatter(logging.Formatter(fmt=log_view_logging_format_string, datefmt="%Y-%m-%dT%H:%M:%S"))
logger.addHandler(handler)
- if not processing_parameters.mediapipe_parameters_model.run_image_tracking:
+ if not processing_parameters.tracking_parameters_model.run_image_tracking:
logger.info(
- f"Skipping 2d skeleton detection and loading data from: {processing_parameters.recording_info_model.mediapipe_2d_data_npy_file_path}"
+ f"Skipping 2d skeleton detection and loading data from: {processing_parameters.recording_info_model.data_2d_npy_file_path}"
)
try:
image_data_numCams_numFrames_numTrackedPts_XYZ = np.load(
- processing_parameters.recording_info_model.mediapipe_2d_data_npy_file_path
+ processing_parameters.recording_info_model.data_2d_npy_file_path
)
except Exception as e:
logger.error(e)
@@ -45,7 +47,8 @@ def run_image_tracking_pipeline(
logger.info(LOG_VIEW_PROGRESS_BAR_STRING)
# 2d skeleton detection
image_data_numCams_numFrames_numTrackedPts_XYZ = run_image_tracking(
- tracking_params=processing_parameters.mediapipe_parameters_model,
+ model_info=processing_parameters.tracking_model_info,
+ tracking_params=processing_parameters.tracking_parameters_model,
synchronized_videos_folder_path=Path(
processing_parameters.recording_info_model.synchronized_videos_folder_path
),
@@ -55,28 +58,33 @@ def run_image_tracking_pipeline(
use_tqdm=use_tqdm,
)
- if not processing_parameters.recording_info_model.data2d_status_check:
- raise FileNotFoundError(
- f"No mediapipe 2d data found at: {processing_parameters.recording_info_model.mediapipe_2d_data_npy_file_path}"
+ # TODO: check that skellytracker saved out data correctly
+ try:
+ test_image_tracking_data_shape(
+ synchronized_video_folder_path=processing_parameters.recording_info_model.synchronized_videos_folder_path,
+ image_tracking_data=image_data_numCams_numFrames_numTrackedPts_XYZ,
)
+ except (ValueError, AssertionError) as e:
+ logger.error(e)
+ raise ValueError("image tracking data is not valid, cannot continue processing")
return image_data_numCams_numFrames_numTrackedPts_XYZ
def run_image_tracking(
- tracking_params: MediapipeTrackingParams,
+ model_info: ModelInfo,
+ tracking_params: BaseTrackingParams,
synchronized_videos_folder_path: Path,
output_data_folder_path: Path,
kill_event: multiprocessing.Event = None,
use_tqdm: bool = True,
):
- if tracking_params.use_yolo_crop_method:
- tracker_type = "YOLOMediapipeComboTracker"
- else:
- tracker_type = "MediapipeHolisticTracker"
+ if hasattr(tracking_params, "use_yolo_crop_method") and tracking_params.use_yolo_crop_method:
+ model_info.tracker_name = "YOLOMediapipeComboTracker"
+ # TODO: this shouldn't be handled here
image_data_numCams_numFrames_numTrackedPts_XYZ = process_folder_of_videos(
- tracker_name=tracker_type,
+ model_info=model_info,
tracking_params=tracking_params,
synchronized_video_path=synchronized_videos_folder_path,
output_folder_path=output_data_folder_path,
diff --git a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/pipeline_check.py b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/pipeline_check.py
index 7f789d9b..f06f9959 100644
--- a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/pipeline_check.py
+++ b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/pipeline_check.py
@@ -3,23 +3,26 @@ from freemocap.data_layer.recording_models.recording_info_model import Recording
def processing_pipeline_check(processing_parameters: ProcessingParameterModel) -> None:
- """Performs checks on the processing parameters to make sure they are valid with the existing data before running the pipeline"""
+ """
+ Performs checks on the processing parameters to make sure they are valid with the existing data before running the pipeline.
+ Raises FileNotFoundError if any of the checks fail.
+ """
status_check_dict = processing_parameters.recording_info_model.status_check
if not status_check_dict["synchronized_videos_status_check"]:
raise FileNotFoundError(
f"Could not find synchronized_videos folder at {processing_parameters.recording_info_model.synchronized_videos_folder_path}"
)
- if not processing_parameters.mediapipe_parameters_model.run_image_tracking:
+ if not processing_parameters.tracking_parameters_model.run_image_tracking:
if not status_check_dict["data2d_status_check"]:
raise FileNotFoundError(
- f"No mediapipe 2d data found at: {processing_parameters.recording_info_model.mediapipe_2d_data_npy_file_path}"
+ f"No 2d data found at: {processing_parameters.recording_info_model.data_2d_npy_file_path}"
)
if not processing_parameters.anipose_triangulate_3d_parameters_model.run_3d_triangulation:
if not status_check_dict["data3d_status_check"] and not status_check_dict["single_video_check"]:
raise FileNotFoundError(
- f"No mediapipe 3d data found at: {processing_parameters.recording_info_model.mediapipe_3d_data_npy_file_path}"
+ f"No 3d data found at: {processing_parameters.recording_info_model.data_3d_npy_file_path}"
)
else:
if not status_check_dict["calibration_toml_check"] and not status_check_dict["single_video_check"]:
diff --git a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/triangulation_pipeline_functions.py b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/triangulation_pipeline_functions.py
index 652fecb2..d864ac66 100644
--- a/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/triangulation_pipeline_functions.py
+++ b/freemocap/core_processes/process_motion_capture_videos/processing_pipeline_functions/triangulation_pipeline_functions.py
@@ -1,14 +1,15 @@
import logging
import multiprocessing
from pathlib import Path
+from typing import Optional
import numpy as np
from freemocap.core_processes.capture_volume_calibration.by_camera_reprojection_filtering import (
run_reprojection_error_filtering,
)
-from freemocap.core_processes.capture_volume_calibration.save_mediapipe_3d_data_to_npy import (
- save_mediapipe_3d_data_to_npy,
+from freemocap.core_processes.capture_volume_calibration.save_3d_data_to_npy import (
+ save_3d_data_to_npy,
)
from freemocap.core_processes.capture_volume_calibration.triangulate_3d_data import triangulate_3d_data
from freemocap.core_processes.capture_volume_calibration.anipose_camera_calibration.get_anipose_calibration_object import (
@@ -28,8 +29,8 @@ logger = logging.getLogger(__name__)
def get_triangulated_data(
image_data_numCams_numFrames_numTrackedPts_XYZ: np.ndarray,
processing_parameters: ProcessingParameterModel,
- kill_event: multiprocessing.Event = None,
- queue: multiprocessing.Queue = None,
+ kill_event: Optional[multiprocessing.Event] = None,
+ queue: Optional[multiprocessing.Queue] = None,
) -> np.ndarray:
if queue:
handler = DirectQueueHandler(queue)
@@ -39,23 +40,24 @@ def get_triangulated_data(
if image_data_numCams_numFrames_numTrackedPts_XYZ.shape[0] == 1:
logger.info("Skipping 3d triangulation for single camera data")
logger.info(LOG_VIEW_PROGRESS_BAR_STRING)
- (skel3d_frame_marker_xyz, skeleton_reprojection_error_fr_mar) = process_single_camera_skeleton_data(
+ (skel3d_frame_marker_xyz, skeleton_reprojection_error_fr_mar) = process_single_camera_skeleton_data(
input_image_data_frame_marker_xyz=image_data_numCams_numFrames_numTrackedPts_XYZ[0],
raw_data_folder_path=Path(processing_parameters.recording_info_model.raw_data_folder_path),
+ file_prefix=processing_parameters.tracking_model_info.name,
project_to_z_plane=processing_parameters.anipose_triangulate_3d_parameters_model.flatten_single_camera_data,
)
elif not processing_parameters.anipose_triangulate_3d_parameters_model.run_3d_triangulation:
logger.info(
- f"Skipping 3d triangulation and loading data from: {processing_parameters.recording_info_model.raw_mediapipe_3d_data_npy_file_path}"
+ f"Skipping 3d triangulation and loading data from: {processing_parameters.recording_info_model.raw_data_3d_npy_file_path}"
)
logger.info(LOG_VIEW_PROGRESS_BAR_STRING)
try:
skel3d_frame_marker_xyz = np.load(
- processing_parameters.recording_info_model.raw_mediapipe_3d_data_npy_file_path
+ processing_parameters.recording_info_model.raw_data_3d_npy_file_path
)
skeleton_reprojection_error_fr_mar = np.load(
- processing_parameters.recording_info_model.mediapipe_reprojection_error_data_npy_file_path
+ processing_parameters.recording_info_model.reprojection_error_data_npy_file_path
)
except Exception as e:
logger.error(e)
@@ -69,11 +71,6 @@ def get_triangulated_data(
f"No calibration file found at: {processing_parameters.recording_info_model.calibration_toml_path}"
)
- if not processing_parameters.recording_info_model.data2d_status_check:
- raise FileNotFoundError(
- f"No mediapipe 2d data found at: {processing_parameters.recording_info_model.mediapipe_2d_data_npy_file_path}"
- )
-
anipose_calibration_object = load_anipose_calibration_toml_from_path(
camera_calibration_data_toml_path=processing_parameters.recording_info_model.calibration_toml_path,
save_copy_of_calibration_to_this_path=processing_parameters.recording_info_model.path,
@@ -84,16 +81,17 @@ def get_triangulated_data(
skeleton_reprojection_error_cam_fr_mar,
) = triangulate_3d_data(
anipose_calibration_object=anipose_calibration_object,
- mediapipe_2d_data=image_data_numCams_numFrames_numTrackedPts_XYZ[:, :, :, :2],
+ image_2d_data=image_data_numCams_numFrames_numTrackedPts_XYZ[:, :, :, :2],
use_triangulate_ransac=processing_parameters.anipose_triangulate_3d_parameters_model.use_triangulate_ransac_method,
kill_event=kill_event,
)
- save_mediapipe_3d_data_to_npy(
+ save_3d_data_to_npy(
data3d_numFrames_numTrackedPoints_XYZ=skel3d_frame_marker_xyz,
data3d_numFrames_numTrackedPoints_reprojectionError=skeleton_reprojection_error_fr_mar,
data3d_numCams_numFrames_numTrackedPoints_reprojectionError=skeleton_reprojection_error_cam_fr_mar,
path_to_folder_where_data_will_be_saved=processing_parameters.recording_info_model.raw_data_folder_path,
processing_level="raw",
+ file_prefix=processing_parameters.tracking_model_info.name,
)
if processing_parameters.anipose_triangulate_3d_parameters_model.run_reprojection_error_filtering:
diff --git a/freemocap/data_layer/data_saver/data_loader.py b/freemocap/data_layer/data_saver/data_loader.py
index fc07ed70..679d4fbe 100644
--- a/freemocap/data_layer/data_saver/data_loader.py
+++ b/freemocap/data_layer/data_saver/data_loader.py
@@ -1,47 +1,55 @@
-import json
import logging
from pathlib import Path
from typing import Union, Dict, Any
import numpy as np
import pandas as pd
-
-from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.data_models.mediapipe_skeleton_names_and_connections import (
- mediapipe_skeleton_schema,
+from skellytracker.trackers.mediapipe_tracker.mediapipe_model_info import (
+ MediapipeModelInfo,
)
-from freemocap.core_processes.post_process_skeleton_data.calculate_center_of_mass import BODY_SEGMENT_NAMES
+from skellytracker.trackers.base_tracker.model_info import ModelInfo
+
from freemocap.data_layer.data_saver.data_models import FrameData, Timestamps, Point, SkeletonSchema
from freemocap.system.paths_and_filenames.file_and_folder_names import (
- MEDIAPIPE_BODY_3D_DATAFRAME_CSV_FILE_NAME,
- MEDIAPIPE_RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME,
- MEDIAPIPE_LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME,
- MEDIAPIPE_FACE_3D_DATAFRAME_CSV_FILE_NAME,
+ BODY_3D_DATAFRAME_CSV_FILE_NAME,
+ CENTER_OF_MASS_FOLDER_NAME,
+ DATA_3D_NPY_FILE_NAME,
+ RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME,
+ LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME,
+ FACE_3D_DATAFRAME_CSV_FILE_NAME,
+ SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME,
+ TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME,
)
from freemocap.system.paths_and_filenames.path_getters import (
get_output_data_folder_path,
get_timestamps_directory,
- get_total_body_center_of_mass_file_path,
- get_segment_center_of_mass_file_path,
- get_full_npy_file_path,
)
logger = logging.getLogger(__name__)
+# TODO: Need to generalize this beyond mediapipe, and make COM data optional
+mediapipe_model_info = MediapipeModelInfo()
+
+
class DataLoader:
def __init__(
self,
recording_folder_path: Union[str, Path],
include_hands: bool = True,
include_face: bool = True,
+ model_info: ModelInfo = mediapipe_model_info,
):
self._recording_folder_path = Path(recording_folder_path)
self.include_hands = include_hands
self.include_face = include_face
+ self._model_info = model_info
self._recording_name = self._recording_folder_path.name
self._output_folder_path = Path(get_output_data_folder_path(self._recording_folder_path))
+ self.number_of_frames = None
+ self._set_file_prefix()
self._load_data()
def _load_data(self):
@@ -50,18 +58,17 @@ class DataLoader:
self._load_full_npy_data()
self._load_center_of_mass_data()
# self._load_segment_lengths()
- self._load_names_and_connections()
- self._load_skeleton_schema()
+ # self._load_skeleton_schema()
self._validate_data()
def _load_data_frames(self):
- self.body_dataframe = self._load_dataframe(MEDIAPIPE_BODY_3D_DATAFRAME_CSV_FILE_NAME)
+ self.body_dataframe = self._load_dataframe(self._file_prefix + BODY_3D_DATAFRAME_CSV_FILE_NAME)
self.number_of_frames = len(self.body_dataframe)
if self.include_hands:
- self.right_hand_dataframe = self._load_dataframe(MEDIAPIPE_RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME)
- self.left_hand_dataframe = self._load_dataframe(MEDIAPIPE_LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME)
+ self.right_hand_dataframe = self._load_dataframe(self._file_prefix + RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME)
+ self.left_hand_dataframe = self._load_dataframe(self._file_prefix + LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME)
if self.include_face:
- self.face_dataframe = self._load_dataframe(MEDIAPIPE_FACE_3D_DATAFRAME_CSV_FILE_NAME)
+ self.face_dataframe = self._load_dataframe(self._file_prefix + FACE_3D_DATAFRAME_CSV_FILE_NAME)
def _load_dataframe(self, filename):
return pd.read_csv(self._output_folder_path / filename)
@@ -76,13 +83,14 @@ class DataLoader:
def _validate_dataframe(self, df, df_name):
if df is not None and len(df) != self.number_of_frames:
raise ValueError(
- f"The number of frames in the {df_name} dataframe is different from the number of frames in the body dataframe."
+ f"The number of frames in the {df_name} dataframe ({len(df)}) is different from the expected number of frames ({self.number_of_frames})."
)
def _validate_numpy_array(self, np_array, np_array_name):
if np_array is not None and np_array.shape[0] != self.number_of_frames:
+ print(f"\n\n {np_array.shape} \n\n")
raise ValueError(
- f"The number of frames in the {np_array_name} data is different from the number of frames in the body dataframe."
+ f"The number of frames in the {np_array_name} data ({np_array.shape[0]}) is different from the expected number of frames ({self.number_of_frames})."
)
def _load_timestamps(self):
@@ -105,22 +113,17 @@ class DataLoader:
Load additional data like center of mass and segment lengths.
"""
self.center_of_mass_xyz = np.load(
- get_total_body_center_of_mass_file_path(output_data_folder=self._output_folder_path)
+ self._output_folder_path / CENTER_OF_MASS_FOLDER_NAME / (self._file_prefix + TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME)
)
self.segment_center_of_mass_segment_xyz = np.load(
- get_segment_center_of_mass_file_path(output_data_folder=self._output_folder_path)
+ self._output_folder_path / CENTER_OF_MASS_FOLDER_NAME / (self._file_prefix + SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME)
)
def _load_full_npy_data(self):
- self.data_frame_name_xyz = np.load(get_full_npy_file_path(output_data_folder=self._output_folder_path))
+ self.data_frame_name_xyz = np.load(self._output_folder_path / (self._file_prefix + DATA_3D_NPY_FILE_NAME))
- def _load_names_and_connections(self):
- with open(self._output_folder_path / "mediapipe_names_and_connections_dict.json", "r") as file:
- self.names_and_connections = json.loads(file.read())
-
- # def _load_segment_lengths(self):
- # with open(self._output_folder_path / "mediapipe_skeleton_segment_lengths.json", "r") as file:
- # self.segment_lengths = json.loads(file.read())
+ if not self.number_of_frames:
+ self.number_of_frames = self.data_frame_name_xyz.shape[0]
def load_frame_data(self, frame_number: int) -> FrameData:
return FrameData(
@@ -137,7 +140,7 @@ class DataLoader:
},
)
except AttributeError:
- return Timestamps()
+ return Timestamps(mean=None)
def get_tracked_points(self, frame_number) -> Dict[str, Point]:
right_hand_points = {}
@@ -178,7 +181,10 @@ class DataLoader:
z=self.center_of_mass_xyz[frame_number, 2],
)
- for segment_number, segment_name in enumerate(BODY_SEGMENT_NAMES):
+ if self._model_info.center_of_mass_definitions is None:
+ raise ValueError("center of mass definitions are not provided.")
+
+ for segment_number, segment_name in enumerate(self._model_info.center_of_mass_definitions.keys()):
com_data[segment_name] = Point(
x=self.segment_center_of_mass_segment_xyz[frame_number, segment_number, 0],
y=self.segment_center_of_mass_segment_xyz[frame_number, segment_number, 1],
@@ -214,5 +220,11 @@ class DataLoader:
recording_data_by_frame_number[frame_number] = self.load_frame_data(frame_number).to_dict()
return recording_data_by_frame_number
- def _load_skeleton_schema(self):
- self.skeleton_schema = SkeletonSchema(schema_dict=mediapipe_skeleton_schema)
+ def _load_skeleton_schema(self): # TODO: replace this with updated Skeleton class
+ self.skeleton_schema = SkeletonSchema(schema_dict=MediapipeModelInfo.skeleton_schema)
+
+ def _set_file_prefix(self) -> None:
+ self._file_prefix = self._model_info.name
+
+ if self._file_prefix[-1] != "_":
+ self._file_prefix += "_"
diff --git a/freemocap/data_layer/data_saver/data_models.py b/freemocap/data_layer/data_saver/data_models.py
index 7d2e1775..e31059ac 100644
--- a/freemocap/data_layer/data_saver/data_models.py
+++ b/freemocap/data_layer/data_saver/data_models.py
@@ -4,8 +4,8 @@ from typing import Optional, Any
from pydantic import BaseModel, Field, model_validator
-from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.data_models.mediapipe_skeleton_names_and_connections import (
- mediapipe_skeleton_schema,
+from skellytracker.trackers.mediapipe_tracker.mediapipe_model_info import (
+ MediapipeModelInfo,
)
@@ -110,11 +110,11 @@ class FrameData(BaseModel):
class InfoDict(BaseModel):
"""
- A dictionary of information about this recording, such as the measured segement lengths and the schemas that we can use to interpret the tracked points (i.e./e.g. how to connect the dots of skeleton)
+ A dictionary of information about this recording, such as the measured segement lengths and the segment connections that we can use to interpret the tracked points (i.e./e.g. how to connect the dots of skeleton)
"""
# segment_lengths: Dict[str, Any] = Field(default_factory=dict, description="The lengths of the segments of the body")
- schemas: List[BaseModel] = Field(default_factory=list, description="The schemas for the tracked points")
+ schemas: Optional[Dict[str, Any]] = Field(default_factory=dict, description="The segment connections for the tracked points")
if __name__ == "__main__":
@@ -123,6 +123,6 @@ if __name__ == "__main__":
print("=====================================\n\n===============================")
- skeleton_schema = SkeletonSchema(schema_dict=mediapipe_skeleton_schema)
+ skeleton_schema = SkeletonSchema(schema_dict=MediapipeModelInfo.skeleton_schema)
pprint.pp(skeleton_schema.dict())
diff --git a/freemocap/data_layer/data_saver/data_saver.py b/freemocap/data_layer/data_saver/data_saver.py
index 1e35cb78..bb184494 100644
--- a/freemocap/data_layer/data_saver/data_saver.py
+++ b/freemocap/data_layer/data_saver/data_saver.py
@@ -6,6 +6,11 @@ from typing import Union
import numpy as np
import pandas as pd
+from skellytracker.trackers.mediapipe_tracker.mediapipe_model_info import (
+ MediapipeModelInfo,
+)
+from skellytracker.trackers.base_tracker.model_info import ModelInfo
+
from freemocap.data_layer.data_saver.data_loader import DataLoader
from freemocap.data_layer.data_saver.data_models import InfoDict
@@ -13,9 +18,17 @@ logger = logging.getLogger(__name__)
from typing import Any, Dict
+mediapipe_model_info = MediapipeModelInfo()
+
class DataSaver:
- def __init__(self, recording_folder_path: Union[Path, str], include_hands: bool = True, include_face: bool = True):
+ def __init__(
+ self,
+ recording_folder_path: Union[Path, str],
+ include_hands: bool = True,
+ include_face: bool = True,
+ model_info: ModelInfo = mediapipe_model_info,
+ ):
"""
Initialize DataFrameManager with the given recording_folder_path.
@@ -28,7 +41,10 @@ class DataSaver:
self._recording_name = self.recording_folder_path.name
self._data_loader = DataLoader(
- recording_folder_path=self.recording_folder_path, include_hands=include_hands, include_face=include_face
+ recording_folder_path=self.recording_folder_path,
+ include_hands=include_hands,
+ include_face=include_face,
+ model_info=model_info,
)
self.recording_data_by_frame = None
@@ -50,7 +66,7 @@ class DataSaver:
def save_to_json(self, save_path: Union[str, Path] = None):
dict_to_save = {}
- dict_to_save["info"] = self._get_info_dict().dict()
+ dict_to_save["info"] = self._get_info_dict().model_dump()
dict_to_save["data_by_frame"] = self.recording_data_by_frame
if save_path is None:
@@ -100,7 +116,7 @@ class DataSaver:
def _get_info_dict(self):
return InfoDict(
# segment_lengths=self._data_loader.segment_lengths,
- schemas=[self._data_loader.skeleton_schema.dict()],
+ schemas=self._data_loader._model_info.segment_connections,
)
diff --git a/freemocap/data_layer/recording_models/post_processing_parameter_models.py b/freemocap/data_layer/recording_models/post_processing_parameter_models.py
index 84fdda2e..dca03214 100644
--- a/freemocap/data_layer/recording_models/post_processing_parameter_models.py
+++ b/freemocap/data_layer/recording_models/post_processing_parameter_models.py
@@ -1,9 +1,9 @@
import logging
-from pydantic import ConfigDict, BaseModel
-from skellytracker.trackers.mediapipe_tracker.mediapipe_model_info import (
- MediapipeTrackingParams
-)
+from pydantic import BaseModel, ConfigDict
+from skellytracker.trackers.mediapipe_tracker.mediapipe_model_info import MediapipeTrackingParams, MediapipeModelInfo
+from skellytracker.trackers.base_tracker.base_tracking_params import BaseTrackingParams
+from skellytracker.trackers.base_tracker.model_info import ModelInfo
from freemocap.data_layer.recording_models.recording_info_model import (
RecordingInfoModel,
@@ -37,6 +37,7 @@ class PostProcessingParametersModel(BaseModel):
class ProcessingParameterModel(BaseModel):
model_config = ConfigDict(arbitrary_types_allowed=True)
recording_info_model: RecordingInfoModel = None
- mediapipe_parameters_model: MediapipeTrackingParams = MediapipeTrackingParams()
+ tracking_parameters_model: BaseTrackingParams = MediapipeTrackingParams()
anipose_triangulate_3d_parameters_model: AniposeTriangulate3DParametersModel = AniposeTriangulate3DParametersModel()
post_processing_parameters_model: PostProcessingParametersModel = PostProcessingParametersModel()
+ tracking_model_info: ModelInfo = MediapipeModelInfo()
diff --git a/freemocap/data_layer/recording_models/recording_info_model.py b/freemocap/data_layer/recording_models/recording_info_model.py
index aa6c3c8c..5c34650b 100644
--- a/freemocap/data_layer/recording_models/recording_info_model.py
+++ b/freemocap/data_layer/recording_models/recording_info_model.py
@@ -6,21 +6,32 @@ import numpy as np
from freemocap.system.paths_and_filenames.file_and_folder_names import (
CENTER_OF_MASS_FOLDER_NAME,
- MEDIAPIPE_2D_NPY_FILE_NAME,
- RAW_MEDIAPIPE_3D_NPY_FILE_NAME,
+ DATA_2D_NPY_FILE_NAME,
+ RAW_3D_NPY_FILE_NAME,
OUTPUT_DATA_FOLDER_NAME,
RAW_DATA_FOLDER_NAME,
TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME,
- RAW_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME,
+ REPROJECTION_ERROR_NPY_FILE_NAME,
SYNCHRONIZED_VIDEOS_FOLDER_NAME,
ANNOTATED_VIDEOS_FOLDER_NAME,
- MEDIAPIPE_3D_NPY_FILE_NAME,
+ DATA_3D_NPY_FILE_NAME,
+ OLD_DATA_2D_NPY_FILE_NAME,
+ OLD_DATA_3D_NPY_FILE_NAME,
+ OLD_RAW_3D_NPY_FILE_NAME,
+ OLD_REPROJECTION_ERROR_NPY_FILE_NAME,
+ OLD_TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME,
)
from freemocap.system.paths_and_filenames.path_getters import create_camera_calibration_file_name, get_blender_file_path
-from freemocap.tests.test_image_tracking_data_shape import test_image_tracking_data_shape
-from freemocap.tests.test_mediapipe_skeleton_data_shape import test_mediapipe_skeleton_data_shape
+from freemocap.tests.test_image_tracking_data_shape import (
+ test_image_tracking_data_exists,
+ test_image_tracking_data_shape,
+)
+from freemocap.tests.test_skeleton_data_shape import test_skeleton_data_exists, test_skeleton_data_shape
from freemocap.tests.test_synchronized_video_frame_counts import test_synchronized_video_frame_counts
-from freemocap.tests.test_total_body_center_of_mass_data_shape import test_total_body_center_of_mass_data_shape
+from freemocap.tests.test_total_body_center_of_mass_data_shape import (
+ test_total_body_center_of_mass_data_exists,
+ test_total_body_center_of_mass_data_shape,
+)
from freemocap.utilities.get_number_of_frames_of_videos_in_a_folder import get_number_of_frames_of_videos_in_a_folder
from freemocap.utilities.get_video_paths import get_video_paths
@@ -28,10 +39,7 @@ logger = logging.getLogger(__name__)
class RecordingInfoModel:
- def __init__(
- self,
- recording_folder_path: Union[Path, str],
- ):
+ def __init__(self, recording_folder_path: Union[Path, str], active_tracker: str = "mediapipe"):
if any(
[
Path(recording_folder_path).name == SYNCHRONIZED_VIDEOS_FOLDER_NAME,
@@ -48,8 +56,26 @@ class RecordingInfoModel:
Path(self._path) / create_camera_calibration_file_name(recording_name=self._name)
)
+ self._active_tracker = active_tracker
+
self._recording_folder_status_checker = RecordingFolderStatusChecker(recording_info_model=self)
+ @property
+ def active_tracker(self) -> str:
+ return self._active_tracker
+
+ @active_tracker.setter
+ def active_tracker(self, tracker_name: str):
+ self._active_tracker = tracker_name
+
+ @property
+ def file_prefix(self) -> str:
+ if self.active_tracker != "" and self.active_tracker[-1] != "_":
+ return self.active_tracker + "_"
+ else:
+ return self.active_tracker
+
+ # TODO: Create setters for the path names that depend on tracker name, and an overall setter that sets them all at once
@property
def path(self) -> str:
return str(self._path)
@@ -59,7 +85,7 @@ class RecordingInfoModel:
return self._name
@property
- def status_check(self) -> Dict[str, bool]:
+ def status_check(self) -> Dict[str, Union[bool, str, float, dict]]:
return self._recording_folder_status_checker.status_check
@property
@@ -87,34 +113,75 @@ class RecordingInfoModel:
return str(Path(self._path) / ANNOTATED_VIDEOS_FOLDER_NAME)
@property
- def mediapipe_2d_data_npy_file_path(self):
- return str(Path(self._path) / OUTPUT_DATA_FOLDER_NAME / RAW_DATA_FOLDER_NAME / MEDIAPIPE_2D_NPY_FILE_NAME)
+ def data_2d_npy_file_path(self):
+ data_2d_path = Path(self.raw_data_folder_path) / (self.file_prefix + DATA_2D_NPY_FILE_NAME)
+ old_data_2d_path = Path(self.raw_data_folder_path) / OLD_DATA_2D_NPY_FILE_NAME
+ if self.active_tracker == "mediapipe" and old_data_2d_path.exists() and not data_2d_path.exists():
+ return str(old_data_2d_path)
+ else:
+ return str(data_2d_path)
@property
- def mediapipe_3d_data_npy_file_path(self):
- return str(Path(self._path) / OUTPUT_DATA_FOLDER_NAME / MEDIAPIPE_3D_NPY_FILE_NAME)
+ def data_3d_npy_file_path(self):
+ data_3d_path = Path(self._path) / OUTPUT_DATA_FOLDER_NAME / (self.file_prefix + DATA_3D_NPY_FILE_NAME)
+ old_data_3d_path = Path(self._path) / OUTPUT_DATA_FOLDER_NAME / OLD_DATA_3D_NPY_FILE_NAME
+ if self.active_tracker == "mediapipe" and old_data_3d_path.exists() and not data_3d_path.exists():
+ return str(old_data_3d_path)
+ else:
+ return str(data_3d_path)
@property
- def raw_mediapipe_3d_data_npy_file_path(self):
- return str(Path(self._path) / OUTPUT_DATA_FOLDER_NAME / RAW_DATA_FOLDER_NAME / RAW_MEDIAPIPE_3D_NPY_FILE_NAME)
+ def raw_data_3d_npy_file_path(self):
+ raw_data_path = (
+ Path(self._path)
+ / OUTPUT_DATA_FOLDER_NAME
+ / RAW_DATA_FOLDER_NAME
+ / (self.file_prefix + RAW_3D_NPY_FILE_NAME)
+ )
+ old_raw_data_path = Path(self._path) / OUTPUT_DATA_FOLDER_NAME / RAW_DATA_FOLDER_NAME / OLD_RAW_3D_NPY_FILE_NAME
+ if self.active_tracker == "mediapipe" and old_raw_data_path.exists() and not raw_data_path.exists():
+ return str(old_raw_data_path)
+ else:
+ return str(raw_data_path)
@property
- def mediapipe_reprojection_error_data_npy_file_path(self):
- return str(
+ def reprojection_error_data_npy_file_path(self):
+ reprojection_error_path = (
Path(self._path)
/ OUTPUT_DATA_FOLDER_NAME
/ RAW_DATA_FOLDER_NAME
- / RAW_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME
+ / (self.file_prefix + REPROJECTION_ERROR_NPY_FILE_NAME)
)
+ raw_reprojection_error_path = (
+ Path(self._path) / OUTPUT_DATA_FOLDER_NAME / RAW_DATA_FOLDER_NAME / OLD_REPROJECTION_ERROR_NPY_FILE_NAME
+ )
+ if (
+ self.active_tracker == "mediapipe"
+ and raw_reprojection_error_path.exists()
+ and not reprojection_error_path.exists()
+ ):
+ return str(raw_reprojection_error_path)
+ else:
+ return str(reprojection_error_path)
@property
def total_body_center_of_mass_npy_file_path(self):
- return str(
+ total_body_com_path = (
Path(self._path)
/ OUTPUT_DATA_FOLDER_NAME
/ CENTER_OF_MASS_FOLDER_NAME
- / TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME
+ / (self.file_prefix + TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME)
)
+ raw_total_body_com_path = (
+ Path(self._path)
+ / OUTPUT_DATA_FOLDER_NAME
+ / CENTER_OF_MASS_FOLDER_NAME
+ / OLD_TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME
+ )
+ if self.active_tracker == "mediapipe" and raw_total_body_com_path.exists() and not total_body_com_path.exists():
+ return str(raw_total_body_com_path)
+ else:
+ return str(total_body_com_path)
@property
def blender_file_path(self):
@@ -185,21 +252,28 @@ class RecordingFolderStatusChecker:
def check_data2d_status(self) -> bool:
try:
+ test_image_tracking_data_exists(
+ synchronized_video_folder_path=self.recording_info_model.synchronized_videos_folder_path,
+ image_tracking_data=self.recording_info_model.data_2d_npy_file_path,
+ )
test_image_tracking_data_shape(
synchronized_video_folder_path=self.recording_info_model.synchronized_videos_folder_path,
- image_tracking_data_file_path=self.recording_info_model.mediapipe_2d_data_npy_file_path,
+ image_tracking_data=self.recording_info_model.data_2d_npy_file_path,
)
-
return True
except AssertionError:
return False
def check_data3d_status(self) -> bool:
try:
- test_mediapipe_skeleton_data_shape(
+ test_skeleton_data_exists(
+ raw_skeleton_data=self.recording_info_model.data_3d_npy_file_path,
+ reprojection_error_data=self.recording_info_model.reprojection_error_data_npy_file_path,
+ )
+ test_skeleton_data_shape(
synchronized_video_folder_path=self.recording_info_model.synchronized_videos_folder_path,
- raw_skeleton_npy_file_path=self.recording_info_model.mediapipe_3d_data_npy_file_path,
- reprojection_error_file_path=self.recording_info_model.mediapipe_reprojection_error_data_npy_file_path,
+ raw_skeleton_data=self.recording_info_model.data_3d_npy_file_path,
+ reprojection_error_data=self.recording_info_model.reprojection_error_data_npy_file_path,
)
return True
except AssertionError:
@@ -207,9 +281,12 @@ class RecordingFolderStatusChecker:
def check_center_of_mass_data_status(self) -> bool:
try:
+ test_total_body_center_of_mass_data_exists(
+ total_body_center_of_mass_data=self.recording_info_model.total_body_center_of_mass_npy_file_path,
+ )
test_total_body_center_of_mass_data_shape(
synchronized_video_folder_path=self.recording_info_model.synchronized_videos_folder_path,
- total_body_center_of_mass_file_path=self.recording_info_model.total_body_center_of_mass_npy_file_path,
+ total_body_center_of_mass_data=self.recording_info_model.total_body_center_of_mass_npy_file_path,
)
return True
except AssertionError:
diff --git a/freemocap/data_layer/skeleton_models/marker_info.py b/freemocap/data_layer/skeleton_models/marker_info.py
new file mode 100644
index 00000000..c2e3c24b
--- /dev/null
+++ b/freemocap/data_layer/skeleton_models/marker_info.py
@@ -0,0 +1,66 @@
+from pydantic import BaseModel, model_validator, Field, field_validator
+from typing import Dict, List, Optional, Union
+
+
+class VirtualMarkerInfo(BaseModel):
+ virtual_markers: Dict[str, Dict[str, List[Union[float, str]]]]
+
+ @field_validator("virtual_markers")
+ @classmethod
+ def validate_virtual_marker(cls, virtual_marker: Dict[str, List[Union[float, str]]]):
+ for virtual_marker_name, virtual_marker_values in virtual_marker.items():
+ marker_names = virtual_marker_values.get("marker_names", []) # TODO: make sure this .get is valid
+ marker_weights = virtual_marker_values.get("marker_weights", [])
+
+ if len(marker_names) != len(marker_weights):
+ raise ValueError(
+ f"The number of marker names must match the number of marker weights for {virtual_marker_name}. Currently there are {len(marker_names)} names and {len(marker_weights)} weights."
+ )
+
+ if not isinstance(marker_names, list) or not all(isinstance(name, str) for name in marker_names):
+ raise ValueError(f"Marker names must be a list of strings for {marker_names}.")
+
+ if not isinstance(marker_weights, list) or not all(
+ isinstance(weight, (int, float)) for weight in marker_weights
+ ):
+ raise ValueError(f"Marker weights must be a list of numbers for {virtual_marker_name}.")
+
+ weight_sum = sum(marker_weights)
+ if not 0.99 <= weight_sum <= 1.01: # Allowing a tiny bit of floating-point leniency
+ raise ValueError(
+ f"Marker weights must sum to approximately 1 for {virtual_marker_name} Current sum is {weight_sum}."
+ )
+
+ return virtual_marker
+
+
+class MarkerInfo(BaseModel):
+ original_marker_names: List[str]
+ virtual_marker_definition: Optional[VirtualMarkerInfo] = None
+ all_markers_list: List[str] = Field(default_factory=list)
+
+ @model_validator(mode="before")
+ def copy_markers_to_all(cls, values):
+ """Copy markers to all_markers_list at initialization."""
+ original_marker_names = values.get("original_marker_names")
+ if original_marker_names:
+ # Directly initializing all_markers_list with a copy of original_marker_names
+ values["all_markers_list"] = original_marker_names.copy()
+ return values
+
+ def add_virtual_markers(self, virtual_markers_dict: Dict[str, Dict[str, List[Union[float, str]]]]):
+ """Add virtual markers and update all_markers_list."""
+ self.virtual_marker_definition = VirtualMarkerInfo(virtual_markers=virtual_markers_dict)
+ for virtual_marker_name in self.virtual_marker_definition.virtual_markers.keys():
+ if virtual_marker_name not in self.all_markers_list:
+ self.all_markers_list.append(virtual_marker_name)
+
+ @property
+ def all_markers(self) -> List[str]:
+ """Publicly expose the combined list of markers."""
+ return self.all_markers_list
+
+ @classmethod
+ def create(cls, marker_list: List[str]):
+ """Class method to create an instance with initial marker names."""
+ return cls(original_marker_names=marker_list)
diff --git a/freemocap/data_layer/skeleton_models/segments.py b/freemocap/data_layer/skeleton_models/segments.py
new file mode 100644
index 00000000..add6f6cd
--- /dev/null
+++ b/freemocap/data_layer/skeleton_models/segments.py
@@ -0,0 +1,36 @@
+from typing import Dict
+from pydantic import BaseModel, model_validator
+
+from freemocap.data_layer.skeleton_models.marker_info import MarkerInfo
+
+
+class Segment(BaseModel):
+ proximal: str
+ distal: str
+
+
+class SegmentAnthropometry(BaseModel):
+ segment_com_length: float
+ segment_com_percentage: float
+
+
+class Segments(BaseModel):
+ markers: MarkerInfo
+ segment_connections: Dict[str, Segment]
+
+ @model_validator(mode="before")
+ def check_that_all_markers_exist(cls, values):
+ markers = values.get("markers").all_markers
+ segment_connections = values.get("segment_connections")
+
+ for segment_name, segment_connection in segment_connections.items():
+ if segment_connection.get("proximal") not in markers:
+ raise ValueError(
+ f"The proximal marker {segment_connection.proximal} for {segment_name} is not in the list of markers or virtual markers."
+ )
+ if segment_connection.get("distal") not in markers:
+ raise ValueError(
+ f"The distal marker {segment_connection.distal} for {segment_name} is not in the list of markers or virtual markers."
+ )
+
+ return values
diff --git a/freemocap/data_layer/skeleton_models/skeleton.py b/freemocap/data_layer/skeleton_models/skeleton.py
new file mode 100644
index 00000000..108d6f8d
--- /dev/null
+++ b/freemocap/data_layer/skeleton_models/skeleton.py
@@ -0,0 +1,133 @@
+import logging
+from typing import Dict, List, Optional
+from pydantic import BaseModel, ConfigDict
+import numpy as np
+
+
+from freemocap.data_layer.skeleton_models.marker_info import MarkerInfo
+from freemocap.data_layer.skeleton_models.segments import Segment, Segments, SegmentAnthropometry
+
+logger = logging.getLogger(__name__)
+
+
+class Skeleton(BaseModel):
+ model_config = ConfigDict(arbitrary_types_allowed=True)
+ markers: MarkerInfo
+ num_tracked_points: int
+ segments: Optional[Dict[str, Segment]] = None
+ marker_data: Dict[str, np.ndarray] = {}
+ virtual_marker_data: Dict[str, np.ndarray] = {}
+ joint_hierarchy: Optional[Dict[str, List[str]]] = None
+ center_of_mass_definitions: Optional[Dict[str, SegmentAnthropometry]] = None
+ num_frames: Optional[int] = None
+
+ def add_segments(self, segment_connections: Dict[str, Segment]) -> None:
+ """
+ Adds segment connection data to the skeleton model.
+
+ Parameters:
+ - segment_connections: A dictionary where each key is a segment name and its value is a dictionary
+ with information about that segment (e.g., 'proximal', 'distal' marker names).
+ """
+ segments_model = Segments(
+ markers=self.markers,
+ segment_connections={name: segment for name, segment in segment_connections.items()},
+ )
+ self.segments = segments_model.segment_connections
+
+ def add_joint_hierarchy(self, joint_hierarchy: Dict[str, List[str]]) -> None:
+ """
+ Adds joint hierarchy data to the skeleton model.
+
+ Parameters:
+ - joint_hierarchy: A dictionary with joint names as keys and lists of connected marker names as values.
+ """
+ for joint_name, joint_connections in joint_hierarchy.items():
+ if joint_name not in self.markers.all_markers:
+ raise ValueError(f"The joint {joint_name} is not in the list of markers or virtual markers.")
+ for connected_marker in joint_connections:
+ if connected_marker not in self.markers.all_markers:
+ raise ValueError(
+ f"The connected marker {connected_marker} for {joint_name} is not in the list of markers or virtual markers."
+ )
+
+ self.joint_hierarchy = joint_hierarchy
+
+ def add_center_of_mass_definitions(self, center_of_mass_definitions: Dict[str, Dict[str, float]]) -> None:
+ """
+ Adds anthropometric center of mass definitions to the skeleton model.
+
+ Parameters:
+ - center_of_mass_definitions: A dictionary containing segment mass percentages.
+ """
+ if self.segments is None:
+ raise ValueError("Segments must be defined before center of mass definitions can be added.")
+ for com_segment_name in center_of_mass_definitions.keys():
+ if com_segment_name not in self.segments.keys():
+ raise ValueError(f"Segment {com_segment_name} is not in the list of segments.")
+
+ self.center_of_mass_definitions = {
+ name: SegmentAnthropometry(**values) for name, values in center_of_mass_definitions.items()
+ }
+
+ def integrate_freemocap_3d_data(self, freemocap_3d_data: np.ndarray) -> None:
+ self.num_frames = freemocap_3d_data.shape[0]
+ num_markers_in_data = freemocap_3d_data.shape[1]
+ original_marker_names_list = self.markers.original_marker_names
+
+ if num_markers_in_data != self.num_tracked_points:
+ raise ValueError(
+ f"The number of markers in the 3D data ({num_markers_in_data}) does not match "
+ f"the expected number of tracked points ({self.num_tracked_points})."
+ )
+
+ self.marker_data = {
+ marker_name: freemocap_3d_data[:, i, :] for i, marker_name in enumerate(original_marker_names_list)
+ }
+
+ try:
+ self.calculate_virtual_markers()
+ except ValueError:
+ logger.info(
+ "Freemocap data integrated without virtual markers, as no virtual marker definition was provided"
+ )
+
+ def calculate_virtual_markers(self) -> None:
+ if not self.marker_data:
+ raise ValueError(
+ "3d marker data must be integrated before calculating virtual markers. Run `integrate_freemocap_3d_data()` first."
+ )
+
+ if not self.markers.virtual_marker_definition:
+ raise ValueError(
+ "Virtual marker info must be defined before calculating virtual markers. Run `add_virtual_markers()` first."
+ )
+
+ for vm_name, vm_info in self.markers.virtual_marker_definition.virtual_markers.items():
+ vm_positions = np.zeros((self.marker_data[next(iter(self.marker_data))].shape[0], 3))
+ for marker_name, weight in zip(vm_info["marker_names"], vm_info["marker_weights"]):
+ vm_positions += self.marker_data[str(marker_name)] * weight
+ self.virtual_marker_data[vm_name] = vm_positions
+
+ self.marker_data.update(self.virtual_marker_data)
+
+ def get_segment_markers(self, segment_name: str) -> Dict[str, np.ndarray]:
+ """Returns a dictionary with the positions of the proximal and distal markers for a segment."""
+ if not self.segments:
+ raise ValueError("Segments must be defined before getting segment markers.")
+
+ if not self.trajectories:
+ raise ValueError("Trajectories must be defined before getting segment markers.")
+
+ segment = self.segments.get(segment_name)
+ if not segment:
+ raise ValueError(f"Segment '{segment_name}' is not defined in the skeleton.")
+
+ proximal_trajectories = self.trajectories.get(segment.proximal)
+ distal_trajectories = self.trajectories.get(segment.distal)
+
+ return {"proximal": proximal_trajectories, "distal": distal_trajectories}
+
+ @property
+ def trajectories(self) -> Dict[str, np.ndarray]:
+ return self.marker_data
diff --git a/freemocap/gui/qt/main_window/freemocap_main_window.py b/freemocap/gui/qt/main_window/freemocap_main_window.py
index b7783f02..0a179b40 100644
--- a/freemocap/gui/qt/main_window/freemocap_main_window.py
+++ b/freemocap/gui/qt/main_window/freemocap_main_window.py
@@ -294,12 +294,16 @@ class MainWindow(QMainWindow):
def _export_active_recording_to_blender(self):
logger.debug("Exporting active recording to Blender...")
- recording_path = self._active_recording_info_widget.get_active_recording_info(return_path=True)
+ recording_path = self._active_recording_info_widget.get_active_recording_path()
if self._visualization_control_panel.blender_executable_path is None:
logger.error("Blender executable path is None!")
return
+ if not recording_path:
+ logger.error("Recording path is None!")
+ return
+
self._export_to_blender_thread_worker = ExportToBlenderThreadWorker(
recording_path=recording_path,
blender_file_path=Path(get_blender_file_path(recording_path)),
@@ -323,9 +327,10 @@ class MainWindow(QMainWindow):
def _generate_jupyter_notebook(self):
logger.info("Exporting active recording to a Jupyter notebook...")
- recording_path = self._active_recording_info_widget.get_active_recording_info(return_path=True)
+ recording_path = self._active_recording_info_widget.get_active_recording_path()
# TODO: Need to include jupyter notebook in recording files that we keep track of (2023-05-15)
- generate_jupyter_notebook(path_to_recording=recording_path)
+ if recording_path:
+ generate_jupyter_notebook(path_to_recording=recording_path)
def _handle_new_active_recording_selected(self, recording_info_model: RecordingInfoModel):
logger.info(f"New active recording selected: {recording_info_model.path}")
@@ -368,7 +373,7 @@ class MainWindow(QMainWindow):
if active_recording_info.data3d_status_check:
self._skelly_viewer_widget.load_skeleton_data(
- mediapipe_skeleton_npy_path=active_recording_info.mediapipe_3d_data_npy_file_path
+ mediapipe_skeleton_npy_path=active_recording_info.data_3d_npy_file_path
)
if active_recording_info.data2d_status_check:
diff --git a/freemocap/gui/qt/widgets/active_recording_widget.py b/freemocap/gui/qt/widgets/active_recording_widget.py
index 605bc9fc..1d0d48c8 100644
--- a/freemocap/gui/qt/widgets/active_recording_widget.py
+++ b/freemocap/gui/qt/widgets/active_recording_widget.py
@@ -40,7 +40,7 @@ class ActiveRecordingInfoWidget(QWidget):
def active_recording_view_widget(self):
return self._active_recording_view_widget
- def get_active_recording_info(self, return_path: bool = False) -> Union[RecordingInfoModel, Path, None]:
+ def get_active_recording_info(self) -> Union[RecordingInfoModel, None]:
# this is redundant to the `active_recording_info` property,
# but it will be more intuitive to send this down as a callable
# rather than relying on 'pass-by-reference' magic lol
@@ -50,11 +50,12 @@ class ActiveRecordingInfoWidget(QWidget):
if most_recent_path is not None:
self.set_active_recording(most_recent_path)
- if return_path:
- return self._active_recording_info.path
-
return self._active_recording_info
+ def get_active_recording_path(self) -> Optional[Path]:
+ if self._active_recording_info:
+ return Path(self._active_recording_info.path)
+
def set_active_recording(
self,
recording_folder_path: Union[str, Path, None],
diff --git a/freemocap/gui/qt/widgets/control_panel/calibration_control_panel.py b/freemocap/gui/qt/widgets/control_panel/calibration_control_panel.py
index 6a9daffd..eb303d64 100644
--- a/freemocap/gui/qt/widgets/control_panel/calibration_control_panel.py
+++ b/freemocap/gui/qt/widgets/control_panel/calibration_control_panel.py
@@ -2,9 +2,9 @@ import logging
import os
from pathlib import Path
import threading
-from typing import Callable, Union
+from typing import Callable, Optional, Union
-from PySide6.QtCore import Qt, Slot
+from PySide6.QtCore import Qt, Slot, QObject
from PySide6.QtGui import QDoubleValidator
from PySide6.QtWidgets import (
QFileDialog,
@@ -38,7 +38,7 @@ class CalibrationControlPanel(QWidget):
get_active_recording_info: Callable[..., Union[RecordingInfoModel, Path]],
kill_thread_event: threading.Event,
gui_state: GuiState,
- parent=None,
+ parent: Optional[QObject] = None,
):
super().__init__(parent=parent)
self.gui_state = gui_state
diff --git a/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/parameter_groups/create_parameter_groups.py b/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/parameter_groups/create_parameter_groups.py
index 52eba4ad..b4bb222a 100644
--- a/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/parameter_groups/create_parameter_groups.py
+++ b/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/parameter_groups/create_parameter_groups.py
@@ -63,6 +63,7 @@ RUN_BUTTERWORTH_FILTER_NAME = "Run butterworth filter?"
NUMBER_OF_PROCESSES_PARAMETER_NAME = "Max Number of Processes to Use"
+# TODO: figure out how to generalize this
def create_mediapipe_parameter_group(
parameter_model: MediapipeTrackingParams,
) -> Parameter:
@@ -239,7 +240,7 @@ def extract_parameter_model_from_parameter_tree(
parameter_values_dictionary = extract_processing_parameter_model_from_tree(parameter_object=parameter_object)
return ProcessingParameterModel(
- mediapipe_parameters_model=MediapipeTrackingParams(
+ tracking_parameters_model=MediapipeTrackingParams(
mediapipe_model_complexity=get_integer_from_mediapipe_model_complexity(
parameter_values_dictionary[MEDIAPIPE_MODEL_COMPLEXITY]
),
diff --git a/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/process_motion_capture_data_panel.py b/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/process_motion_capture_data_panel.py
index 8bb757f8..e08fa7bb 100644
--- a/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/process_motion_capture_data_panel.py
+++ b/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/process_motion_capture_data_panel.py
@@ -3,7 +3,7 @@ import multiprocessing
import shutil
import threading
from pathlib import Path
-from typing import Callable, Union
+from typing import Callable
from PySide6.QtCore import Signal, Slot, Qt
from PySide6.QtGui import QFont
@@ -39,7 +39,7 @@ class ProcessMotionCaptureDataPanel(QWidget):
def __init__(
self,
recording_processing_parameters: ProcessingParameterModel,
- get_active_recording_info: Callable[..., Union[RecordingInfoModel, Path]],
+ get_active_recording_info: Callable[..., RecordingInfoModel],
kill_thread_event: threading.Event,
log_update: Callable,
gui_state: GuiState,
@@ -126,7 +126,7 @@ class ProcessMotionCaptureDataPanel(QWidget):
children=[
self._create_new_run_this_step_parameter(run_step_name=RUN_IMAGE_TRACKING_NAME),
self._create_num_processes_parameter(),
- create_mediapipe_parameter_group(session_processing_parameter_model.mediapipe_parameters_model),
+ create_mediapipe_parameter_group(session_processing_parameter_model.tracking_parameters_model),
],
tip="Methods for tracking 2d points in images (e.g. mediapipe, deeplabcut(TODO), openpose(TODO), etc ...)",
),
@@ -249,6 +249,11 @@ class ProcessMotionCaptureDataPanel(QWidget):
)
shutil.copyfile(selected_camera_calibration_toml_path, copied_toml_path)
+ # set active tracker in recording model to the currently selected tracker
+ session_parameter_model.recording_info_model.active_tracker = (
+ session_parameter_model.tracking_model_info.name
+ )
+
self._process_motion_capture_data_thread_worker = ProcessMotionCaptureDataThreadWorker(
session_parameter_model, kill_event=self._kill_thread_event
)
diff --git a/freemocap/gui/qt/workers/process_motion_capture_data_thread_worker.py b/freemocap/gui/qt/workers/process_motion_capture_data_thread_worker.py
index 134ef5d2..528102f3 100644
--- a/freemocap/gui/qt/workers/process_motion_capture_data_thread_worker.py
+++ b/freemocap/gui/qt/workers/process_motion_capture_data_thread_worker.py
@@ -21,29 +21,29 @@ class ProcessMotionCaptureDataThreadWorker(QThread):
self, post_processing_parameters: ProcessingParameterModel, kill_event: multiprocessing.Event, parent=None
):
super().__init__()
- self._post_processing_parameters = post_processing_parameters
+ self._processing_parameters = post_processing_parameters
self._kill_event = kill_event
self._queue = multiprocessing.Queue()
self._process = multiprocessing.Process(
- target=process_recording_folder, args=(self._post_processing_parameters, self._kill_event, self._queue)
+ target=process_recording_folder, args=(self._processing_parameters, self._kill_event, self._queue)
)
self._success = None
def run(self):
logger.info(
- f"Beginning processing of motion capture data with parameters: {self._post_processing_parameters.__dict__}"
+ f"Beginning processing of motion capture data with parameters: {self._processing_parameters.dict(exclude={'tracking_model_info'})}"
)
self._kill_event.clear()
- recording_info_dict = self._post_processing_parameters.dict(exclude={"recording_info_model"})
- Path(self._post_processing_parameters.recording_info_model.output_data_folder_path).mkdir(
+ recording_info_dict = self._processing_parameters.dict(exclude={"recording_info_model"})
+ Path(self._processing_parameters.recording_info_model.output_data_folder_path).mkdir(
parents=True, exist_ok=True
)
save_dictionary_to_json(
- save_path=self._post_processing_parameters.recording_info_model.output_data_folder_path,
+ save_path=self._processing_parameters.recording_info_model.output_data_folder_path,
file_name=RECORDING_PARAMETERS_JSON_FILE_NAME,
dictionary=recording_info_dict,
)
diff --git a/freemocap/system/paths_and_filenames/file_and_folder_names.py b/freemocap/system/paths_and_filenames/file_and_folder_names.py
index dcb8bd9a..82cd36c1 100644
--- a/freemocap/system/paths_and_filenames/file_and_folder_names.py
+++ b/freemocap/system/paths_and_filenames/file_and_folder_names.py
@@ -21,26 +21,30 @@ CENTER_OF_MASS_FOLDER_NAME = "center_of_mass"
MOST_RECENT_RECORDING_TOML_FILENAME = "most_recent_recording.toml"
LAST_SUCCESSFUL_CALIBRATION_TOML_FILENAME = "last_successful_calibration.toml"
GUI_STATE_JSON_FILENAME = "gui_state.json"
-MEDIAPIPE_2D_NPY_FILE_NAME = "mediapipe2dData_numCams_numFrames_numTrackedPoints_pixelXY.npy"
-MEDIAPIPE_BODY_WORLD_FILE_NAME = "mediapipeBodyWorld_numCams_numFrames_numTrackedPoints_XYZ.npy"
+DATA_2D_NPY_FILE_NAME = "2dData_numCams_numFrames_numTrackedPoints_pixelXY.npy"
-RAW_MEDIAPIPE_3D_NPY_FILE_NAME = "mediapipe3dData_numFrames_numTrackedPoints_spatialXYZ.npy"
-RAW_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME = "mediapipe3dData_numFrames_numTrackedPoints_reprojectionError.npy"
-RAW_MEDIAPIPE_FULL_REPROJECTION_ERROR_NPY_FILE_NAME = "mediapipe3dData_numCams_numFrames_numTrackedPoints_reprojectionError.npy"
+RAW_3D_NPY_FILE_NAME = "3dData_numFrames_numTrackedPoints_spatialXYZ.npy"
+REPROJECTION_ERROR_NPY_FILE_NAME = "3dData_numFrames_numTrackedPoints_reprojectionError.npy"
+FULL_REPROJECTION_ERROR_NPY_FILE_NAME = "3dData_numCams_numFrames_numTrackedPoints_reprojectionError.npy"
+REPROJECTION_FILTERED_PREFIX = "reprojection_filtered_"
-REPROJECTION_FILTERED_MEDIAPIPE_3D_NPY_FILE_NAME = "reprojection_filtered_mediapipe3dData_numFrames_numTrackedPoints_spatialXYZ.npy"
-REPROJECTION_FILTERED_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME = "reprojection_filtered_mediapipe3dData_numFrames_numTrackedPoints_reprojectionError.npy"
-REPROJECTION_FILTERED_MEDIAPIPE_FULL_REPROJECTION_ERROR_NPY_FILE_NAME = "reprojection_filtered_mediapipe3dData_numCams_numFrames_numTrackedPoints_reprojectionError.npy"
-
-MEDIAPIPE_3D_NPY_FILE_NAME = "mediaPipeSkel_3d_body_hands_face.npy"
-MEDIAPIPE_BODY_3D_DATAFRAME_CSV_FILE_NAME = "mediapipe_body_3d_xyz.csv"
-MEDIAPIPE_RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME = "mediapipe_right_hand_3d_xyz.csv"
-MEDIAPIPE_LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME = "mediapipe_left_hand_3d_xyz.csv"
-MEDIAPIPE_FACE_3D_DATAFRAME_CSV_FILE_NAME = "mediapipe_face_3d_xyz.csv"
+DATA_3D_NPY_FILE_NAME = "skeleton_3d.npy"
+RIGID_BONES_NPY_FILE_NAME = "rigid_bones_3d.npy"
+BODY_3D_DATAFRAME_CSV_FILE_NAME = "body_3d_xyz.csv"
+RIGHT_HAND_3D_DATAFRAME_CSV_FILE_NAME = "right_hand_3d_xyz.csv"
+LEFT_HAND_3D_DATAFRAME_CSV_FILE_NAME = "left_hand_3d_xyz.csv"
+FACE_3D_DATAFRAME_CSV_FILE_NAME = "face_3d_xyz.csv"
TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME = "total_body_center_of_mass_xyz.npy"
SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME = "segmentCOM_frame_joint_xyz.npy"
+OLD_DATA_2D_NPY_FILE_NAME = "mediapipe2dData_numCams_numFrames_numTrackedPoints_pixelXY.npy"
+OLD_RAW_3D_NPY_FILE_NAME = "mediapipe3dData_numFrames_numTrackedPoints_spatialXYZ.npy"
+OLD_REPROJECTION_ERROR_NPY_FILE_NAME = "mediapipe3dData_numFrames_numTrackedPoints_reprojectionError.npy"
+OLD_DATA_3D_NPY_FILE_NAME = "mediaPipeSkel_3d_body_hands_face.npy"
+OLD_TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME = "total_body_center_of_mass_xyz.npy"
+OLD_SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME = "segmentCOM_frame_joint_xyz.npy"
+
RECORDING_PARAMETERS_JSON_FILE_NAME = "recording_parameters.json"
# Figshare info
diff --git a/freemocap/system/paths_and_filenames/path_getters.py b/freemocap/system/paths_and_filenames/path_getters.py
index be3b589f..8c719b79 100644
--- a/freemocap/system/paths_and_filenames/path_getters.py
+++ b/freemocap/system/paths_and_filenames/path_getters.py
@@ -18,13 +18,12 @@ from freemocap.system.paths_and_filenames.file_and_folder_names import (
OUTPUT_DATA_FOLDER_NAME,
SYNCHRONIZED_VIDEOS_FOLDER_NAME,
RAW_DATA_FOLDER_NAME,
- RAW_MEDIAPIPE_3D_NPY_FILE_NAME,
+ RAW_3D_NPY_FILE_NAME,
CENTER_OF_MASS_FOLDER_NAME,
TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME,
- MEDIAPIPE_2D_NPY_FILE_NAME,
- RAW_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME,
- SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME,
- MEDIAPIPE_3D_NPY_FILE_NAME,
+ DATA_2D_NPY_FILE_NAME,
+ REPROJECTION_ERROR_NPY_FILE_NAME,
+ DATA_3D_NPY_FILE_NAME,
GUI_STATE_JSON_FILENAME,
)
@@ -118,7 +117,7 @@ def session_time_tag_format():
return "%Y-%m-%d_%H_%M_%S"
-def default_session_name(string_tag: str = None):
+def default_session_name(string_tag: Optional[str] = None):
if string_tag is not None:
string_tag = f"_{string_tag}"
else:
@@ -245,19 +244,20 @@ def get_synchronized_videos_folder_path(recording_folder_path: Union[str, Path])
raise Exception(f"Could not find a videos folder in path {str(recording_folder_path)}")
-def get_timestamps_directory(recording_directory: Union[str, Path]) -> str:
+def get_timestamps_directory(recording_directory: Union[str, Path]) -> Optional[str]:
synchronized_videos_path = get_synchronized_videos_folder_path(recording_folder_path=recording_directory)
if "timestamps" in [path.name for path in Path(synchronized_videos_path).iterdir()]:
return str(Path(synchronized_videos_path) / "timestamps")
logger.warning(f"Could not find timestamps directory in {synchronized_videos_path}")
+# TODO: figure out how to deal with different file prefixes for all of these below
def get_raw_skeleton_npy_file_name(data_folder_name: Union[str, Path]) -> str:
raw_data_subfolder_path = Path(data_folder_name) / RAW_DATA_FOLDER_NAME
if raw_data_subfolder_path.exists:
raw_data_npy_path_list = [path.name for path in raw_data_subfolder_path.glob("*.npy")]
- if RAW_MEDIAPIPE_3D_NPY_FILE_NAME in raw_data_npy_path_list:
- return str(raw_data_subfolder_path / RAW_MEDIAPIPE_3D_NPY_FILE_NAME)
+ if RAW_3D_NPY_FILE_NAME in raw_data_npy_path_list:
+ return str(raw_data_subfolder_path / RAW_3D_NPY_FILE_NAME)
logger.debug(list(Path(data_folder_name).glob("*")))
logger.debug(list(Path(raw_data_subfolder_path).glob("*")))
@@ -265,7 +265,7 @@ def get_raw_skeleton_npy_file_name(data_folder_name: Union[str, Path]) -> str:
def get_full_npy_file_path(output_data_folder: Union[str, Path]) -> str:
- path = Path(output_data_folder) / MEDIAPIPE_3D_NPY_FILE_NAME
+ path = Path(output_data_folder) / DATA_3D_NPY_FILE_NAME
return str(path)
@@ -280,24 +280,13 @@ def get_total_body_center_of_mass_file_path(output_data_folder: Union[str, Path]
raise Exception(f"Could not find a total body center of mass npy file in path {str(output_data_folder)}")
-def get_segment_center_of_mass_file_path(output_data_folder: Union[str, Path]) -> str:
- center_of_mass_subfolder_path = Path(output_data_folder) / CENTER_OF_MASS_FOLDER_NAME
- if center_of_mass_subfolder_path.exists:
- center_of_mass_npy_path_list = [path.name for path in center_of_mass_subfolder_path.glob("*.npy")]
-
- if SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME in center_of_mass_npy_path_list:
- return str(center_of_mass_subfolder_path / SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME)
-
- raise Exception(f"Could not find a total body center of mass npy file in path {str(output_data_folder)}")
-
-
def get_image_tracking_data_file_name(data_folder_name: Union[str, Path]) -> str:
raw_data_subfolder_path = Path(data_folder_name) / RAW_DATA_FOLDER_NAME
if raw_data_subfolder_path.exists:
raw_data_npy_path_list = [path.name for path in raw_data_subfolder_path.glob("*.npy")]
- if MEDIAPIPE_2D_NPY_FILE_NAME in raw_data_npy_path_list:
- return str(raw_data_subfolder_path / MEDIAPIPE_2D_NPY_FILE_NAME)
+ if DATA_2D_NPY_FILE_NAME in raw_data_npy_path_list:
+ return str(raw_data_subfolder_path / DATA_2D_NPY_FILE_NAME)
raise Exception(f"Could not find a 2d data file in path {str(data_folder_name)}")
@@ -307,7 +296,7 @@ def get_reprojection_error_file_path(data_folder_name: Union[str, Path]) -> str:
if raw_data_subfolder_path.exists:
raw_data_npy_path_list = [path.name for path in raw_data_subfolder_path.glob("*.npy")]
- if RAW_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME in raw_data_npy_path_list:
- return str(raw_data_subfolder_path / RAW_MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME)
+ if REPROJECTION_ERROR_NPY_FILE_NAME in raw_data_npy_path_list:
+ return str(raw_data_subfolder_path / REPROJECTION_ERROR_NPY_FILE_NAME)
raise Exception(f"Could not find reprojection error data file in path {str(data_folder_name)}")
diff --git a/freemocap/tests/conftest.py b/freemocap/tests/conftest.py
index af899a90..97faa3c5 100644
--- a/freemocap/tests/conftest.py
+++ b/freemocap/tests/conftest.py
@@ -1,65 +1,69 @@
+from typing import Optional
import pytest
from freemocap.core_processes.process_motion_capture_videos.process_recording_headless import (
process_recording_headless,
find_calibration_toml_path,
)
-from freemocap.system.paths_and_filenames.path_getters import (
- get_output_data_folder_path,
- get_synchronized_videos_folder_path,
- get_raw_skeleton_npy_file_name,
- get_total_body_center_of_mass_file_path,
- get_image_tracking_data_file_name,
- get_reprojection_error_file_path,
-)
+from freemocap.data_layer.recording_models.recording_info_model import RecordingInfoModel
from freemocap.utilities.download_sample_data import (
download_sample_data,
)
+class SessionInfo:
+ sample_session_folder_path: str
+ recording_info_model: RecordingInfoModel
def pytest_sessionstart():
- pytest.sample_session_folder_path = download_sample_data()
- calibration_toml_path = find_calibration_toml_path(pytest.sample_session_folder_path)
+ SessionInfo.sample_session_folder_path = download_sample_data()
+ calibration_toml_path = find_calibration_toml_path(SessionInfo.sample_session_folder_path)
+ SessionInfo.recording_info_model = RecordingInfoModel(
+ recording_folder_path=SessionInfo.sample_session_folder_path, active_tracker="mediapipe"
+ )
+ # TODO: make this configurable to be able to take different trackers
process_recording_headless(
- recording_path=pytest.sample_session_folder_path,
+ recording_path=SessionInfo.sample_session_folder_path,
path_to_camera_calibration_toml=calibration_toml_path,
+ recording_info_model=SessionInfo.recording_info_model,
+ run_blender=False,
+ make_jupyter_notebook=False,
use_tqdm=False,
)
@pytest.fixture
def sample_session_path():
- return pytest.sample_session_folder_path
+ return SessionInfo.sample_session_folder_path
@pytest.fixture
def synchronized_video_folder_path():
- return get_synchronized_videos_folder_path(pytest.sample_session_folder_path)
+ return SessionInfo.recording_info_model.synchronized_videos_folder_path
@pytest.fixture
def data_folder_path():
- return get_output_data_folder_path(pytest.sample_session_folder_path)
+ return SessionInfo.recording_info_model.output_data_folder_path
@pytest.fixture
-def raw_skeleton_npy_file_path():
- return get_raw_skeleton_npy_file_name(get_output_data_folder_path(pytest.sample_session_folder_path))
+def raw_skeleton_data():
+ return SessionInfo.recording_info_model.raw_data_3d_npy_file_path
@pytest.fixture
-def total_body_center_of_mass_file_path():
- return get_total_body_center_of_mass_file_path(get_output_data_folder_path(pytest.sample_session_folder_path))
+def total_body_center_of_mass_data():
+ return SessionInfo.recording_info_model.total_body_center_of_mass_npy_file_path
@pytest.fixture
-def image_tracking_data_file_path():
- return get_image_tracking_data_file_name(get_output_data_folder_path(pytest.sample_session_folder_path))
+def image_tracking_data():
+ return SessionInfo.recording_info_model.data_2d_npy_file_path
@pytest.fixture
-def reprojection_error_file_path():
- return get_reprojection_error_file_path(get_output_data_folder_path(pytest.sample_session_folder_path))
+def reprojection_error_data():
+ return SessionInfo.recording_info_model.reprojection_error_data_npy_file_path
if __name__ == "__main__":
diff --git a/freemocap/tests/test_image_tracking_data_shape.py b/freemocap/tests/test_image_tracking_data_shape.py
index 3f49b2a6..03cd5931 100644
--- a/freemocap/tests/test_image_tracking_data_shape.py
+++ b/freemocap/tests/test_image_tracking_data_shape.py
@@ -1,5 +1,5 @@
from pathlib import Path
-from typing import Union
+from typing import Optional, Union
import numpy as np
import pytest
@@ -10,29 +10,29 @@ from freemocap.utilities.get_number_of_frames_of_videos_in_a_folder import (
from freemocap.utilities.get_video_paths import get_video_paths
[email protected]("synchronized_video_folder_path", "image_tracking_data_file_path")
-def test_image_tracking_data_shape(
[email protected]("synchronized_video_folder_path", "image_tracking_data")
+def test_image_tracking_data_exists(
synchronized_video_folder_path: Union[str, Path],
- image_tracking_data_file_path,
+ image_tracking_data: Union[str, Path],
):
"""
- test that the `mediapipe 2d detection` process worked correctly by checking:
+ test that the `2d detection` process worked correctly by checking:
- 1. There is an `.npy` file containing the mediapipe 2d data in the `output_data_folder`
+ There is an `.npy` file containing the 2d data in the `output_data_folder`
2. The dimensions of that `npy` (number of cameras, number of frames, [ need to do - number of tracked points], [pixelX, pixelY] matches the videos in the `synchronized videos` folder
TODO - check number of tracked points vs 'expected' number of tracked points
"""
- assert Path(image_tracking_data_file_path).is_file(), f"{image_tracking_data_file_path} is not a file"
+ assert Path(image_tracking_data).is_file(), f"{image_tracking_data} is not a file"
- image_tracking_data = np.load(image_tracking_data_file_path)
+ image_tracking_data = np.load(image_tracking_data)
list_of_video_paths = get_video_paths(Path(synchronized_video_folder_path))
number_of_videos = len(list_of_video_paths)
assert (
image_tracking_data.shape[0] == number_of_videos
- ), f"Number of videos in {image_tracking_data_file_path} does not match number of videos in synchronized videos folder"
+ ), f"Number of videos in {image_tracking_data} does not match number of videos in synchronized videos folder"
frame_counts = list(get_number_of_frames_of_videos_in_a_folder(synchronized_video_folder_path).values())
@@ -41,10 +41,48 @@ def test_image_tracking_data_shape(
), f"Videos in {synchronized_video_folder_path} have different frame counts: {frame_counts}"
assert (
image_tracking_data.shape[1] == frame_counts[0]
- ), f"Number of frames in {image_tracking_data_file_path} does not match number of frames of videos in {synchronized_video_folder_path}"
+ ), f"Number of frames in {image_tracking_data} does not match number of frames of videos in {synchronized_video_folder_path}"
# TODO - check number of tracked points vs 'expected' number of tracked points
assert (
image_tracking_data.shape[3] == 3
), f"Data has {image_tracking_data.shape[3]} dimensions, expected 3 dimensions"
+
[email protected]("synchronized_video_folder_path", "image_tracking_data")
+def test_image_tracking_data_shape(
+ synchronized_video_folder_path: Union[str, Path],
+ image_tracking_data: Union[str, Path, np.ndarray],
+):
+ """
+ test that the `2d detection` process worked correctly by checking:
+
+ The dimensions of that `npy` (number of cameras, number of frames, [ need to do - number of tracked points], [pixelX, pixelY] matches the videos in the `synchronized videos` folder
+
+ TODO - check number of tracked points vs 'expected' number of tracked points
+ """
+ if isinstance(image_tracking_data, (str, Path)):
+ image2d_camera_frame_marker_xyz = np.load(image_tracking_data)
+ else:
+ image2d_camera_frame_marker_xyz = image_tracking_data
+
+ list_of_video_paths = get_video_paths(Path(synchronized_video_folder_path))
+ number_of_videos = len(list_of_video_paths)
+ assert (
+ image2d_camera_frame_marker_xyz.shape[0] == number_of_videos
+ ), f"Number of videos in image_tracking_data does not match number of videos in synchronized videos folder"
+
+ frame_counts = list(get_number_of_frames_of_videos_in_a_folder(synchronized_video_folder_path).values())
+
+ assert (
+ len(set(frame_counts)) == 1
+ ), f"Videos in {synchronized_video_folder_path} have different frame counts: {frame_counts}"
+ assert (
+ image2d_camera_frame_marker_xyz.shape[1] == frame_counts[0]
+ ), f"Number of frames in image_tracking_data does not match number of frames of videos in {synchronized_video_folder_path}"
+
+ # TODO - check number of tracked points vs 'expected' number of tracked points
+
+ assert (
+ image2d_camera_frame_marker_xyz.shape[3] == 3
+ ), f"Data has {image2d_camera_frame_marker_xyz.shape[3]} dimensions, expected 3 dimensions"
diff --git a/freemocap/tests/test_mediapipe_skeleton_data_shape.py b/freemocap/tests/test_mediapipe_skeleton_data_shape.py
deleted file mode 100644
index e26e79a3..00000000
--- a/freemocap/tests/test_mediapipe_skeleton_data_shape.py
+++ /dev/null
@@ -1,61 +0,0 @@
-from pathlib import Path
-from typing import Union
-
-import numpy as np
-import pytest
-
-from freemocap.utilities.get_number_of_frames_of_videos_in_a_folder import (
- get_number_of_frames_of_videos_in_a_folder,
-)
-
-
[email protected]("synchronized_video_folder_path", "raw_skeleton_npy_file_path", "reprojection_error_file_path")
-def test_mediapipe_skeleton_data_shape(
- synchronized_video_folder_path: Union[str, Path],
- raw_skeleton_npy_file_path: Union[str, Path],
- reprojection_error_file_path,
-):
- """
- test that the `mediapipe 3d detection` process worked correctly by checking:
-
- 1. There is an `.npy` file containing the mediapipe 3d data in the `output_data_folder`
- 2. The dimensions of that `npy` ( number of frames, [ need to do - number of tracked points], [X,Y,Z] matches the videos in the `synchronized videos` folder
-
- TODO - check number of tracked points vs 'expected' number of tracked points
- """
-
- assert Path(
- raw_skeleton_npy_file_path
- ).is_file(), f"3d skeleton data file does not exist at {raw_skeleton_npy_file_path}"
-
- skel3d_frame_marker_xyz = np.load(raw_skeleton_npy_file_path)
-
- assert (
- len(skel3d_frame_marker_xyz.shape) == 3
- ), f"3d skeleton data file should have 3 dimensions - {raw_skeleton_npy_file_path}"
-
- assert Path(
- reprojection_error_file_path
- ).is_file(), f"3d skeleton reprojection error data file does not exist at {reprojection_error_file_path}"
-
- skeleton_reprojection_error_fr_mar = np.load(reprojection_error_file_path)
-
- assert (
- len(skeleton_reprojection_error_fr_mar.shape) == 2
- ), f"3d skeleton reprojection error data file should have 2 dimensions {reprojection_error_file_path}"
-
- frame_counts = list(get_number_of_frames_of_videos_in_a_folder(synchronized_video_folder_path).values())
- assert (
- len(set(frame_counts)) == 1
- ), f"Videos in {synchronized_video_folder_path} have different frame counts: {frame_counts}"
-
- number_of_frames = frame_counts[0]
-
- assert skel3d_frame_marker_xyz.shape[0] == number_of_frames
- assert skeleton_reprojection_error_fr_mar.shape[0] == number_of_frames
-
- # TODO - check number of tracked points vs 'expected' number of tracked points
-
- assert (
- skel3d_frame_marker_xyz.shape[2] == 3
- ), f"3d skeleton data file does not have 3 dimensions for X,Y,Z at {raw_skeleton_npy_file_path}"
diff --git a/freemocap/tests/test_skeleton_data_shape.py b/freemocap/tests/test_skeleton_data_shape.py
new file mode 100644
index 00000000..e3a155ae
--- /dev/null
+++ b/freemocap/tests/test_skeleton_data_shape.py
@@ -0,0 +1,73 @@
+from pathlib import Path
+from typing import Optional, Union
+
+import numpy as np
+import pytest
+
+from freemocap.utilities.get_number_of_frames_of_videos_in_a_folder import (
+ get_number_of_frames_of_videos_in_a_folder,
+)
+
+
[email protected]("raw_skeleton_data", "reprojection_error_data")
+def test_skeleton_data_exists(
+ raw_skeleton_data: Union[str, Path],
+ reprojection_error_data: Union[str, Path],
+):
+ """
+ test that the `3d detection` process worked correctly by checking that there is an `.npy` file containing the 3d data in the `output_data_folder`
+ """
+
+ assert Path(
+ raw_skeleton_data
+ ).is_file(), f"3d skeleton data file does not exist at {raw_skeleton_data}"
+
+ assert Path(
+ reprojection_error_data
+ ).is_file(), f"3d skeleton reprojection error data file does not exist at {reprojection_error_data}"
+
+
[email protected]("synchronized_video_folder_path", "raw_skeleton_data", "reprojection_error_data")
+def test_skeleton_data_shape(
+ synchronized_video_folder_path: Union[str, Path],
+ raw_skeleton_data: Union[str, Path, np.ndarray],
+ reprojection_error_data: Union[str, Path, np.ndarray],
+):
+ """
+ test that the `3d detection` process worked correctly by checking:
+ 1. Dimensions of the raw skeleton numpy array are correct
+ 2. Dimensions of the reprojection error numpy array are correct
+ """
+ if isinstance(raw_skeleton_data, (str, Path)):
+ skel3d_frame_marker_xyz = np.load(raw_skeleton_data)
+ else:
+ skel3d_frame_marker_xyz = raw_skeleton_data
+
+ assert (
+ len(skel3d_frame_marker_xyz.shape) == 3
+ ), f"3d skeleton data file should have 3 dimensions, but has shape {skel3d_frame_marker_xyz.shape}"
+
+ if isinstance(reprojection_error_data, (str, Path)):
+ skeleton_reprojection_error_fr_mar = np.load(reprojection_error_data)
+ else:
+ skeleton_reprojection_error_fr_mar = reprojection_error_data
+
+ assert (
+ len(skeleton_reprojection_error_fr_mar.shape) == 2
+ ), f"3d skeleton reprojection error data file should have 2 dimensions but has shape {skeleton_reprojection_error_fr_mar.shape}"
+
+ frame_counts = list(get_number_of_frames_of_videos_in_a_folder(synchronized_video_folder_path).values())
+ assert (
+ len(set(frame_counts)) == 1
+ ), f"Videos in {synchronized_video_folder_path} have different frame counts: {frame_counts}"
+
+ number_of_frames = frame_counts[0]
+
+ assert skel3d_frame_marker_xyz.shape[0] == number_of_frames
+ assert skeleton_reprojection_error_fr_mar.shape[0] == number_of_frames
+
+ # TODO - check number of tracked points vs 'expected' number of tracked points
+
+ assert (
+ skel3d_frame_marker_xyz.shape[2] == 3
+ ), f"3d skeleton data file should have 3 dimensions for X,Y,Z, but instead has {skel3d_frame_marker_xyz.shape[2]}"
diff --git a/freemocap/tests/test_total_body_center_of_mass_data_shape.py b/freemocap/tests/test_total_body_center_of_mass_data_shape.py
index 285d0530..5f82674d 100644
--- a/freemocap/tests/test_total_body_center_of_mass_data_shape.py
+++ b/freemocap/tests/test_total_body_center_of_mass_data_shape.py
@@ -1,5 +1,5 @@
from pathlib import Path
-from typing import Union
+from typing import Optional, Union
import numpy as np
import pytest
@@ -9,16 +9,24 @@ from freemocap.utilities.get_number_of_frames_of_videos_in_a_folder import (
)
[email protected]("synchronized_video_folder_path", "total_body_center_of_mass_file_path")
-def test_total_body_center_of_mass_data_shape(
- synchronized_video_folder_path: Union[str, Path],
- total_body_center_of_mass_file_path: Union[str, Path],
[email protected]("total_body_center_of_mass_data")
+def test_total_body_center_of_mass_data_exists(
+ total_body_center_of_mass_data: Union[str, Path],
):
assert Path(
- total_body_center_of_mass_file_path
- ).is_file(), f"No file found at {total_body_center_of_mass_file_path}"
+ total_body_center_of_mass_data
+ ).is_file(), f"No file found at {total_body_center_of_mass_data}"
- total_body_center_of_mass_fr_xyz = np.load(total_body_center_of_mass_file_path)
+
[email protected]("synchronized_video_folder_path", "total_body_center_of_mass_data")
+def test_total_body_center_of_mass_data_shape(
+ synchronized_video_folder_path: Union[str, Path],
+ total_body_center_of_mass_data: Union[str, Path, np.ndarray],
+):
+ if isinstance(total_body_center_of_mass_data, (str, Path)):
+ total_body_center_of_mass_fr_xyz = np.load(total_body_center_of_mass_data)
+ else:
+ total_body_center_of_mass_fr_xyz = total_body_center_of_mass_data
frame_counts = list(get_number_of_frames_of_videos_in_a_folder(synchronized_video_folder_path).values())
assert (
@@ -29,8 +37,9 @@ def test_total_body_center_of_mass_data_shape(
assert (
total_body_center_of_mass_fr_xyz.shape[0] == number_of_frames
- ), f"Number of frames in {total_body_center_of_mass_file_path} does not match number of frames of videos in {synchronized_video_folder_path}"
+ ), f"Number of frames in {total_body_center_of_mass_fr_xyz.shape} does not match number of frames of videos in {synchronized_video_folder_path}"
assert (
total_body_center_of_mass_fr_xyz.shape[1] == 3
- ), "`total_body_center_of_mass_fr_xyz.shape[1]` should have 3 dimensions (X,Y,Z) "
+ ), f"total body center of mass should have 3 dimensions (X,Y,Z) but has {total_body_center_of_mass_fr_xyz.shape[1]}"
+
diff --git a/freemocap/utilities/create_nested_dict_from_pydantic.py b/freemocap/utilities/create_nested_dict_from_pydantic.py
index d96490fd..6d829ae7 100644
--- a/freemocap/utilities/create_nested_dict_from_pydantic.py
+++ b/freemocap/utilities/create_nested_dict_from_pydantic.py
@@ -27,10 +27,10 @@ def create_nested_dict(pydantic_model):
if __name__ == "__main__":
- from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.data_models.mediapipe_skeleton_names_and_connections import (
- mediapipe_skeleton_schema,
+ from skellytracker.trackers.mediapipe_tracker.mediapipe_model_info import (
+ MediapipeModelInfo,
)
from freemocap.data_layer.data_saver import SkeletonSchema
- skeleton_schema = SkeletonSchema(schema_dict=mediapipe_skeleton_schema)
+ skeleton_schema = SkeletonSchema(schema_dict=MediapipeModelInfo.skeleton_schema)
print(create_nested_dict(skeleton_schema))
diff --git a/freemocap/utilities/update_1_4_path_names.py b/freemocap/utilities/update_1_4_path_names.py
new file mode 100644
index 00000000..2c1ad9c0
--- /dev/null
+++ b/freemocap/utilities/update_1_4_path_names.py
@@ -0,0 +1,89 @@
+from typing import Optional, Union
+from pathlib import Path
+
+
+from freemocap.system.paths_and_filenames.file_and_folder_names import (
+ RAW_DATA_FOLDER_NAME,
+ OUTPUT_DATA_FOLDER_NAME,
+ CENTER_OF_MASS_FOLDER_NAME,
+ RECORDING_SESSIONS_FOLDER_NAME,
+)
+from freemocap.system.paths_and_filenames.path_getters import get_freemocap_data_folder_path
+
+
+def update_1_4_path_names(freemocap_data_folder_path: Optional[Union[str, Path]]) -> None:
+ if freemocap_data_folder_path is None:
+ freemocap_data_folder_path = get_freemocap_data_folder_path()
+
+ print(f"Searching for recording folders in {freemocap_data_folder_path}")
+
+ for session_folder in (Path(freemocap_data_folder_path) / RECORDING_SESSIONS_FOLDER_NAME).iterdir():
+ if not session_folder.is_dir():
+ continue
+
+ for recording_folder in session_folder.iterdir():
+ if not recording_folder.is_dir():
+ continue
+
+ update_recording_folder(recording_folder)
+
+
+def update_recording_folder(recording_folder: Path) -> None:
+ if (
+ recording_folder.name == OUTPUT_DATA_FOLDER_NAME
+ ): # this covers cases where the recording folder isn't a subfolder of a session
+ rename_skeleton_file_path(recording_folder)
+ if (recording_folder / RAW_DATA_FOLDER_NAME).exists():
+ rename_raw_data_paths(recording_folder / RAW_DATA_FOLDER_NAME)
+ if (recording_folder / CENTER_OF_MASS_FOLDER_NAME).exists():
+ rename_COM_paths(recording_folder / CENTER_OF_MASS_FOLDER_NAME)
+ elif (recording_folder / OUTPUT_DATA_FOLDER_NAME).exists():
+ rename_skeleton_file_path(recording_folder)
+ if (recording_folder / OUTPUT_DATA_FOLDER_NAME / RAW_DATA_FOLDER_NAME).exists():
+ rename_raw_data_paths(recording_folder / OUTPUT_DATA_FOLDER_NAME / RAW_DATA_FOLDER_NAME)
+ if (recording_folder / OUTPUT_DATA_FOLDER_NAME / CENTER_OF_MASS_FOLDER_NAME).exists():
+ rename_COM_paths(recording_folder / CENTER_OF_MASS_FOLDER_NAME)
+
+
+def rename_raw_data_paths(raw_data_folder_path: Path) -> None:
+ print(f"renaming raw data files in {raw_data_folder_path}")
+ for file in raw_data_folder_path.iterdir():
+ if file.name == "mediapipe2dData_numCams_numFrames_numTrackedPoints_pixelXY.npy":
+ file.rename(file.parent / "mediapipe_2dData_numCams_numFrames_numTrackedPoints_pixelXY.npy")
+ elif file.name == "mediapipe3dData_numCams_numFrames_numTrackedPoints_reprojectionError.npy":
+ file.rename(file.parent / "mediapipe_3dData_numCams_numFrames_numTrackedPoints_reprojectionError.npy")
+ elif file.name == "mediapipe3dData_numFrames_numTrackedPoints_reprojectionError.npy":
+ file.rename(file.parent / "mediapipe_3dData_numFrames_numTrackedPoints_reprojectionError.npy")
+ elif file.name == "mediapipe3dData_numFrames_numTrackedPoints_spatialXYZ.npy":
+ file.rename(file.parent / "mediapipe_3dData_numFrames_numTrackedPoints_spatialXYZ.npy")
+
+ print(f"successfully renamed raw data files in {raw_data_folder_path}")
+
+
+def rename_skeleton_file_path(output_data_folder_path: Path) -> None:
+ print(f"renaming skeleton 3D files in {output_data_folder_path}")
+ for file in output_data_folder_path.iterdir():
+ if file.name == "mediaPipeSkel_3d_body_hands_face.npy":
+ file.rename(file.parent / "mediapipe_skeleton_3d.npy")
+
+ print(f"successfully renamed skeleton 3D files in {output_data_folder_path}")
+
+
+def rename_COM_paths(COM_data_folder_path: Path) -> None:
+ print(f"renaming center of mass files in {COM_data_folder_path}")
+ for file in COM_data_folder_path.iterdir():
+ if file.name == "segmentCOM_frame_joint_xyz.npy":
+ file.rename(file.parent / "mediapipe_segmentCOM_frame_joint_xyz.npy")
+ elif file.name == "total_body_center_of_mass_xyz.npy":
+ file.rename(file.parent / "mediapipe_total_body_center_of_mass_xyz.npy")
+
+ print(f"successfully renamed center of mass files in {COM_data_folder_path}")
+
+
+if __name__ == "__main__":
+ import sys
+
+ if len(sys.argv) > 1:
+ update_1_4_path_names(Path(sys.argv[1]))
+ else:
+ update_1_4_path_names(None)
diff --git a/pyproject.toml b/pyproject.toml
index 44dfd45c..8dcf7933 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -69,10 +69,10 @@ keywords = [
dependencies = [
"skellycam==2024.07.1092",
"skelly_viewer==2024.07.1023",
- "skellyforge==2024.07.1006",
+ "skellyforge==2024.8.1008",
"skelly_synchronize==2024.07.1032",
- "skellytracker[all]==2024.06.1016",
- "ajc27_freemocap_blender_addon==v2024.07.1022",
+ "skellytracker[all]==2024.08.1018",
+ "ajc27_freemocap_blender_addon==v2024.09.1024",
"opencv-contrib-python==4.8.*",
"toml==0.10.2",
"aniposelib==0.4.3",