Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

Trent/single camera 2 (#342)

trentwirth1bc2a2c635b5
diff --git a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/mediapipe_skeleton_detector.py b/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/mediapipe_skeleton_detector.py
index 92818239..0f65cb70 100644
--- a/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/mediapipe_skeleton_detector.py
+++ b/freemocap/core_processes/detecting_things_in_2d_images/mediapipe_stuff/mediapipe_skeleton_detector.py
@@ -1,6 +1,6 @@
import logging
-import multiprocessing
from dataclasses import dataclass
+import multiprocessing
from pathlib import Path
from typing import Any, List, Union
@@ -11,52 +11,54 @@ from skellycam.detection.models.frame_payload import FramePayload
from skellycam.opencv.video_recorder.video_recorder import VideoRecorder
from tqdm import tqdm
+from freemocap.system.paths_and_files_names import MEDIAPIPE_2D_NPY_FILE_NAME, ANNOTATED_VIDEOS_FOLDER_NAME, \
+ MEDIAPIPE_BODY_WORLD_FILE_NAME
from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.mediapipe_skeleton_names_and_connections import (
mediapipe_tracked_point_names_dict,
)
from freemocap.parameter_info_models.recording_processing_parameter_models import MediapipeParametersModel
-from freemocap.system.paths_and_files_names import MEDIAPIPE_2D_NPY_FILE_NAME, ANNOTATED_VIDEOS_FOLDER_NAME
logger = logging.getLogger(__name__)
@dataclass
class Mediapipe2dNumpyArrays:
- body2d_frameNumber_trackedPointNumber_XY: np.ndarray = None
- rightHand2d_frameNumber_trackedPointNumber_XY: np.ndarray = None
- leftHand2d_frameNumber_trackedPointNumber_XY: np.ndarray = None
- face2d_frameNumber_trackedPointNumber_XY: np.ndarray = None
+ body_frameNumber_trackedPointNumber_XYZ: np.ndarray = None
+ body_world_frameNumber_trackedPointNumber_XYZ: np.ndarray = None
+ rightHand_frameNumber_trackedPointNumber_XYZ: np.ndarray = None
+ leftHand_frameNumber_trackedPointNumber_XYZ: np.ndarray = None
+ face_frameNumber_trackedPointNumber_XYZ: np.ndarray = None
- body2d_frameNumber_trackedPointNumber_confidence: np.ndarray = None
+ body_frameNumber_trackedPointNumber_confidence: np.ndarray = None
@property
def has_data(self):
- return not np.isnan(self.body2d_frameNumber_trackedPointNumber_XY).all()
+ return not np.isnan(self.body_frameNumber_trackedPointNumber_XYZ).all()
@property
def all_data2d_nFrames_nTrackedPts_XY(self):
"""dimensions will be [number_of_frames , number_of_markers, XY]"""
- if self.body2d_frameNumber_trackedPointNumber_XY is None:
+ if self.body_frameNumber_trackedPointNumber_XYZ is None:
# if there's no body data, there's no hand or face data either
return
- if len(self.body2d_frameNumber_trackedPointNumber_XY.shape) == 3: # multiple frames
+ if len(self.body_frameNumber_trackedPointNumber_XYZ.shape) == 3: # multiple frames
return np.hstack(
[
- self.body2d_frameNumber_trackedPointNumber_XY,
- self.rightHand2d_frameNumber_trackedPointNumber_XY,
- self.leftHand2d_frameNumber_trackedPointNumber_XY,
- self.face2d_frameNumber_trackedPointNumber_XY,
+ self.body_frameNumber_trackedPointNumber_XYZ,
+ self.rightHand_frameNumber_trackedPointNumber_XYZ,
+ self.leftHand_frameNumber_trackedPointNumber_XYZ,
+ self.face_frameNumber_trackedPointNumber_XYZ,
]
)
- elif len(self.body2d_frameNumber_trackedPointNumber_XY.shape) == 2: # single frame
+ elif len(self.body_frameNumber_trackedPointNumber_XYZ.shape) == 2: # single frame
return np.vstack(
[
- self.body2d_frameNumber_trackedPointNumber_XY,
- self.rightHand2d_frameNumber_trackedPointNumber_XY,
- self.leftHand2d_frameNumber_trackedPointNumber_XY,
- self.face2d_frameNumber_trackedPointNumber_XY,
+ self.body_frameNumber_trackedPointNumber_XYZ,
+ self.rightHand_frameNumber_trackedPointNumber_XYZ,
+ self.leftHand_frameNumber_trackedPointNumber_XYZ,
+ self.face_frameNumber_trackedPointNumber_XYZ,
]
)
else:
@@ -133,9 +135,9 @@ class MediaPipeSkeletonDetector:
image_width=annotated_image.shape[0],
image_height=annotated_image.shape[1],
)
- mediapipe_single_frame_npy_data.body2d_frameNumber_trackedPointNumber_XY = self._threshold_by_confidence(
- mediapipe_single_frame_npy_data.body2d_frameNumber_trackedPointNumber_XY,
- mediapipe_single_frame_npy_data.body2d_frameNumber_trackedPointNumber_confidence,
+ mediapipe_single_frame_npy_data.body_frameNumber_trackedPointNumber_XYZ = self._threshold_by_confidence(
+ mediapipe_single_frame_npy_data.body_frameNumber_trackedPointNumber_XYZ,
+ mediapipe_single_frame_npy_data.body_frameNumber_trackedPointNumber_confidence,
confidence_threshold=0.5,
)
@@ -223,14 +225,28 @@ class MediaPipeSkeletonDetector:
m2d.all_data2d_nFrames_nTrackedPts_XY for m2d in mediapipe2d_single_camera_npy_arrays_list
]
+ all_cameras_pose_world_data_list = [
+ m2d.body_world_frameNumber_trackedPointNumber_XYZ for m2d in mediapipe2d_single_camera_npy_arrays_list
+ ]
+
+ all_cameras_right_hand_world_data_list = [
+ m2d.rightHand_frameNumber_trackedPointNumber_XYZ for m2d in mediapipe2d_single_camera_npy_arrays_list
+ ]
+
+ all_cameras_left_hand_world_data_list = [
+ m2d.leftHand_frameNumber_trackedPointNumber_XYZ for m2d in mediapipe2d_single_camera_npy_arrays_list
+ ]
+
+ all_cameras_face_world_data_list = [
+ m2d.face_frameNumber_trackedPointNumber_XYZ for m2d in mediapipe2d_single_camera_npy_arrays_list
+ ]
+
number_of_cameras = len(all_cameras_data2d_list)
number_of_frames = all_cameras_data2d_list[0].shape[0]
number_of_tracked_points = all_cameras_data2d_list[0].shape[1]
- number_of_spatial_dimensions = all_cameras_data2d_list[0].shape[2] # XY, 2d data
+ number_of_spatial_dimensions = all_cameras_data2d_list[0].shape[2]
- if not number_of_spatial_dimensions == 2:
- logger.error(f"this should be 2D data (XY pixel coordinates), but we founds {number_of_spatial_dimensions}")
- raise Exception
+ number_of_body_points = all_cameras_pose_world_data_list[0].shape[1]
data2d_numCams_numFrames_numTrackedPts_XY = np.empty(
(
@@ -241,11 +257,37 @@ class MediaPipeSkeletonDetector:
)
)
+ body_world_numCams_numFrames_numTrackedPts_XYZ = np.empty(
+ (
+ number_of_cameras,
+ number_of_frames,
+ number_of_tracked_points,
+ number_of_spatial_dimensions,
+ )
+ )
+
for cam_num in range(number_of_cameras):
data2d_numCams_numFrames_numTrackedPts_XY[cam_num, :, :, :] = all_cameras_data2d_list[cam_num]
- mediapipe_data_2d_npy_path = self._save_mediapipe2d_data_to_npy(
+ pose_3d = all_cameras_pose_world_data_list[cam_num]
+ right_hand_3d = all_cameras_right_hand_world_data_list[cam_num]
+ left_hand_3d = all_cameras_left_hand_world_data_list[cam_num]
+ face_3d = all_cameras_face_world_data_list[cam_num]
+
+ body_world_numCams_numFrames_numTrackedPts_XYZ[cam_num, :, :, :] = np.concatenate((
+ pose_3d,
+ right_hand_3d,
+ left_hand_3d,
+ face_3d),
+ axis=1)
+
+ logger.info(
+ f"The shape of body_world_numCams_numFrames_numTrackedPts_XYZ is "
+ f"{body_world_numCams_numFrames_numTrackedPts_XYZ.shape}")
+
+ self._save_mediapipe2d_data_to_npy(
data2d_numCams_numFrames_numTrackedPts_XY=data2d_numCams_numFrames_numTrackedPts_XY,
+ body_world_numCams_numFrames_numTrackedPts_XYZ=body_world_numCams_numFrames_numTrackedPts_XYZ,
output_data_folder_path=Path(output_data_folder_path),
)
return data2d_numCams_numFrames_numTrackedPts_XY
@@ -253,14 +295,18 @@ class MediaPipeSkeletonDetector:
def _save_mediapipe2d_data_to_npy(
self,
data2d_numCams_numFrames_numTrackedPts_XY: np.ndarray,
+ body_world_numCams_numFrames_numTrackedPts_XYZ: np.ndarray,
output_data_folder_path: Union[str, Path],
):
mediapipe_2dData_save_path = Path(output_data_folder_path) / MEDIAPIPE_2D_NPY_FILE_NAME
mediapipe_2dData_save_path.parent.mkdir(exist_ok=True, parents=True)
- logger.info(f"saving: {mediapipe_2dData_save_path}")
+ logger.info(f"saving mediapipe image npy file: {mediapipe_2dData_save_path}")
np.save(str(mediapipe_2dData_save_path), data2d_numCams_numFrames_numTrackedPts_XY)
- return mediapipe_2dData_save_path
+ mediapipe_body_world_save_path = Path(output_data_folder_path) / MEDIAPIPE_BODY_WORLD_FILE_NAME
+ mediapipe_body_world_save_path.parent.mkdir(exist_ok=True, parents=True)
+ logger.info(f"saving mediapipe body world npy xyz: {mediapipe_body_world_save_path}")
+ np.save(str(mediapipe_body_world_save_path), body_world_numCams_numFrames_numTrackedPts_XYZ)
def _annotate_image(self, image, mediapipe_results):
self._mp_drawing.draw_landmarks(
@@ -308,48 +354,57 @@ class MediaPipeSkeletonDetector:
) -> Mediapipe2dNumpyArrays:
number_of_frames = len(mediapipe_results_list)
- number_of_spatial_dimensions = 2 # this will be 2d XY pixel data
+ number_of_spatial_dimensions = 3 # this will be 2d XY pixel data, with mediapipe's estimate of Z
+
+ body_frameNumber_trackedPointNumber_XYZ = np.zeros(
+ (
+ number_of_frames,
+ self.number_of_body_tracked_points,
+ number_of_spatial_dimensions,
+ )
+ )
+ body_frameNumber_trackedPointNumber_XYZ[:] = np.nan
- body2d_frameNumber_trackedPointNumber_XY = np.zeros(
+ body_world_frameNumber_trackedPointNumber_XYZ = np.zeros(
(
number_of_frames,
self.number_of_body_tracked_points,
number_of_spatial_dimensions,
)
)
- body2d_frameNumber_trackedPointNumber_XY[:] = np.nan
+ body_world_frameNumber_trackedPointNumber_XYZ[:] = np.nan
- body2d_frameNumber_trackedPointNumber_confidence = np.zeros(
+ body_frameNumber_trackedPointNumber_confidence = np.zeros(
(number_of_frames, self.number_of_body_tracked_points)
)
- body2d_frameNumber_trackedPointNumber_confidence[:] = np.nan # only body markers get a 'confidence' value
+ body_frameNumber_trackedPointNumber_confidence[:] = np.nan # only body markers get a 'confidence' value
- rightHand2d_frameNumber_trackedPointNumber_XY = np.zeros(
+ rightHand_frameNumber_trackedPointNumber_XYZ = np.zeros(
(
number_of_frames,
self.number_of_right_hand_tracked_points,
number_of_spatial_dimensions,
)
)
- rightHand2d_frameNumber_trackedPointNumber_XY[:] = np.nan
+ rightHand_frameNumber_trackedPointNumber_XYZ[:] = np.nan
- leftHand2d_frameNumber_trackedPointNumber_XY = np.zeros(
+ leftHand_frameNumber_trackedPointNumber_XYZ = np.zeros(
(
number_of_frames,
self.number_of_left_hand_tracked_points,
number_of_spatial_dimensions,
)
)
- leftHand2d_frameNumber_trackedPointNumber_XY[:] = np.nan
+ leftHand_frameNumber_trackedPointNumber_XYZ[:] = np.nan
- face2d_frameNumber_trackedPointNumber_XY = np.zeros(
+ face_frameNumber_trackedPointNumber_XYZ = np.zeros(
(
number_of_frames,
self.number_of_face_tracked_points,
number_of_spatial_dimensions,
)
)
- face2d_frameNumber_trackedPointNumber_XY[:] = np.nan
+ face_frameNumber_trackedPointNumber_XYZ[:] = np.nan
all_body_tracked_points_visible_on_frame_bool_list = []
all_right_hand_points_visible_on_frame_bool_list = []
@@ -363,61 +418,84 @@ class MediaPipeSkeletonDetector:
if frame_results.pose_landmarks is not None:
for landmark_number, landmark_data in enumerate(frame_results.pose_landmarks.landmark):
- body2d_frameNumber_trackedPointNumber_XY[frame_number, landmark_number, 0] = (
+ body_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (
landmark_data.x * image_width
)
- body2d_frameNumber_trackedPointNumber_XY[frame_number, landmark_number, 1] = (
+ body_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (
landmark_data.y * image_height
)
- body2d_frameNumber_trackedPointNumber_confidence[
+ body_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (
+ landmark_data.z * image_width # z is on roughly the same scale as x, according to mediapipe docs
+ )
+ body_frameNumber_trackedPointNumber_confidence[
frame_number, landmark_number
] = landmark_data.visibility # mediapipe calls their 'confidence' score 'visibility'
+ for landmark_number, landmark_data in enumerate(frame_results.pose_world_landmarks.landmark):
+ body_world_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (
+ landmark_data.x * image_width
+ )
+ body_world_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (
+ landmark_data.y * image_height
+ )
+ body_world_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (
+ landmark_data.z * image_width
+ )
+
# get Right Hand data
if frame_results.right_hand_landmarks is not None:
for landmark_number, landmark_data in enumerate(frame_results.right_hand_landmarks.landmark):
- rightHand2d_frameNumber_trackedPointNumber_XY[frame_number, landmark_number, 0] = (
+ rightHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (
landmark_data.x * image_width
)
- rightHand2d_frameNumber_trackedPointNumber_XY[frame_number, landmark_number, 1] = (
+ rightHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (
landmark_data.y * image_height
)
+ rightHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (
+ landmark_data.z * image_width
+ )
# get Left Hand data
if frame_results.left_hand_landmarks is not None:
for landmark_number, landmark_data in enumerate(frame_results.left_hand_landmarks.landmark):
- leftHand2d_frameNumber_trackedPointNumber_XY[frame_number, landmark_number, 0] = (
+ leftHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (
landmark_data.x * image_width
)
- leftHand2d_frameNumber_trackedPointNumber_XY[frame_number, landmark_number, 1] = (
+ leftHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (
landmark_data.y * image_height
)
+ leftHand_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (
+ landmark_data.z * image_width
+ )
# get Face data
if frame_results.face_landmarks is not None:
for landmark_number, landmark_data in enumerate(frame_results.face_landmarks.landmark):
- face2d_frameNumber_trackedPointNumber_XY[frame_number, landmark_number, 0] = (
+ face_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 0] = (
landmark_data.x * image_width
)
- face2d_frameNumber_trackedPointNumber_XY[frame_number, landmark_number, 1] = (
+ face_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 1] = (
landmark_data.y * image_height
)
+ face_frameNumber_trackedPointNumber_XYZ[frame_number, landmark_number, 2] = (
+ landmark_data.z * image_width
+ )
# check if all tracked points are visible on this frame
- all_body_visible = all(sum(np.isnan(body2d_frameNumber_trackedPointNumber_XY[frame_number, :, :])) == 0)
+ all_body_visible = all(sum(np.isnan(body_frameNumber_trackedPointNumber_XYZ[frame_number, :, :])) == 0)
all_body_tracked_points_visible_on_frame_bool_list.append(all_body_visible)
all_right_hand_visible = all(
- sum(np.isnan(rightHand2d_frameNumber_trackedPointNumber_XY[frame_number, :, :])) == 0
+ sum(np.isnan(rightHand_frameNumber_trackedPointNumber_XYZ[frame_number, :, :])) == 0
)
all_right_hand_points_visible_on_frame_bool_list.append(all_right_hand_visible)
all_left_hand_visible = all(
- sum(np.isnan(leftHand2d_frameNumber_trackedPointNumber_XY[frame_number, :, :])) == 0
+ sum(np.isnan(leftHand_frameNumber_trackedPointNumber_XYZ[frame_number, :, :])) == 0
)
all_left_hand_points_visible_on_frame_bool_list.append(all_left_hand_visible)
- all_face_visible = all(sum(np.isnan(face2d_frameNumber_trackedPointNumber_XY[frame_number, :, :])) == 0)
+ all_face_visible = all(sum(np.isnan(face_frameNumber_trackedPointNumber_XYZ[frame_number, :, :])) == 0)
all_face_points_visible_on_frame_bool_list.append(all_face_visible)
all_points_visible = all(
@@ -432,12 +510,13 @@ class MediaPipeSkeletonDetector:
all_tracked_points_visible_on_frame_list.append(all_points_visible)
return Mediapipe2dNumpyArrays(
- body2d_frameNumber_trackedPointNumber_XY=np.squeeze(body2d_frameNumber_trackedPointNumber_XY),
- rightHand2d_frameNumber_trackedPointNumber_XY=np.squeeze(rightHand2d_frameNumber_trackedPointNumber_XY),
- leftHand2d_frameNumber_trackedPointNumber_XY=np.squeeze(leftHand2d_frameNumber_trackedPointNumber_XY),
- face2d_frameNumber_trackedPointNumber_XY=np.squeeze(face2d_frameNumber_trackedPointNumber_XY),
- body2d_frameNumber_trackedPointNumber_confidence=np.squeeze(
- body2d_frameNumber_trackedPointNumber_confidence
+ body_frameNumber_trackedPointNumber_XYZ=np.squeeze(body_frameNumber_trackedPointNumber_XYZ),
+ body_world_frameNumber_trackedPointNumber_XYZ=np.squeeze(body_world_frameNumber_trackedPointNumber_XYZ),
+ rightHand_frameNumber_trackedPointNumber_XYZ=np.squeeze(rightHand_frameNumber_trackedPointNumber_XYZ),
+ leftHand_frameNumber_trackedPointNumber_XYZ=np.squeeze(leftHand_frameNumber_trackedPointNumber_XYZ),
+ face_frameNumber_trackedPointNumber_XYZ=np.squeeze(face_frameNumber_trackedPointNumber_XYZ),
+ body_frameNumber_trackedPointNumber_confidence=np.squeeze(
+ body_frameNumber_trackedPointNumber_confidence
),
)
diff --git a/freemocap/core_processes/post_process_skeleton_data/process_single_camera_skeleton_data.py b/freemocap/core_processes/post_process_skeleton_data/process_single_camera_skeleton_data.py
new file mode 100644
index 00000000..265ce2fa
--- /dev/null
+++ b/freemocap/core_processes/post_process_skeleton_data/process_single_camera_skeleton_data.py
@@ -0,0 +1,33 @@
+from pathlib import Path
+import logging
+from typing import Union
+
+import numpy as np
+
+from freemocap.core_processes.capture_volume_calibration.triangulate_3d_data import save_mediapipe_3d_data_to_npy
+from freemocap.utilities.project_3d_data_to_z_plane import project_3d_data_to_z_plane
+
+logger = logging.getLogger(__name__)
+
+
+def process_single_camera_skeleton_data(input_image_data_frame_marker_xyz: np.ndarray,
+ raw_data_folder_path: Union[str, Path],
+ project_to_z_plane: bool = True) -> [np.ndarray, np.ndarray]:
+
+ if project_to_z_plane:
+
+ logger.info("Single camera detected - "
+ "Altering image-derived 3d data to resemble multi-camera reconstructed data.")
+
+ skeleton_reprojection_error_fr_mar = np.zeros(input_image_data_frame_marker_xyz.shape[0:2])
+
+ raw_skel3d_frame_marker_xyz = project_3d_data_to_z_plane(
+ skel3d_frame_marker_xyz=input_image_data_frame_marker_xyz)
+
+ save_mediapipe_3d_data_to_npy(
+ data3d_numFrames_numTrackedPoints_XYZ=raw_skel3d_frame_marker_xyz,
+ data3d_numFrames_numTrackedPoints_reprojectionError=skeleton_reprojection_error_fr_mar,
+ path_to_folder_where_data_will_be_saved=raw_data_folder_path,
+ )
+
+ return raw_skel3d_frame_marker_xyz, skeleton_reprojection_error_fr_mar
diff --git a/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py b/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py
index dae8634a..03ada531 100644
--- a/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py
+++ b/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py
@@ -4,6 +4,15 @@ from pathlib import Path
import pandas as pd
+from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.mediapipe_skeleton_names_and_connections import (
+ mediapipe_names_and_connections_dict,
+)
+from freemocap.core_processes.post_process_skeleton_data.process_single_camera_skeleton_data import \
+ process_single_camera_skeleton_data
+from freemocap.system.paths_and_files_names import (
+ MEDIAPIPE_BODY_3D_DATAFRAME_CSV_FILE_NAME,
+ RAW_DATA_FOLDER_NAME,
+)
from freemocap.core_processes.capture_volume_calibration.anipose_camera_calibration.get_anipose_calibration_object import (
load_anipose_calibration_toml_from_path,
)
@@ -16,9 +25,6 @@ from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.conv
from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.mediapipe_skeleton_detector import (
MediaPipeSkeletonDetector,
)
-from freemocap.core_processes.detecting_things_in_2d_images.mediapipe_stuff.mediapipe_skeleton_names_and_connections import (
- mediapipe_names_and_connections_dict,
-)
from freemocap.core_processes.post_process_skeleton_data.estimate_skeleton_segment_lengths import (
estimate_skeleton_segment_lengths,
mediapipe_skeleton_segment_definitions,
@@ -27,14 +33,12 @@ from freemocap.core_processes.post_process_skeleton_data.gap_fill_filter_and_ori
gap_fill_filter_origin_align_3d_data_and_then_calculate_center_of_mass,
)
from freemocap.parameter_info_models.recording_processing_parameter_models import RecordingProcessingParameterModel
-from freemocap.system.paths_and_files_names import (
- MEDIAPIPE_BODY_3D_DATAFRAME_CSV_FILE_NAME,
- RAW_DATA_FOLDER_NAME,
-)
+
from freemocap.tests.test_mediapipe_2d_data_shape import (
test_mediapipe_2d_data_shape,
)
from freemocap.tests.test_mediapipe_3d_data_shape import test_mediapipe_3d_data_shape
+from freemocap.utilities.rotate_by_90_degrees_around_x_axis import rotate_by_90_degrees_around_x_axis
from freemocap.utilities.save_dictionary_to_json import save_dictionary_to_json
logger = logging.getLogger(__name__)
@@ -66,7 +70,7 @@ def process_recording_folder(
parameter_model=rec.mediapipe_parameters_model,
)
- mediapipe_2d_data = mediapipe_skeleton_detector.process_folder_full_of_videos(
+ mediapipe_image_data_numCams_numFrames_numTrackedPts_XYZ = mediapipe_skeleton_detector.process_folder_full_of_videos(
rec.recording_info_model.synchronized_videos_folder_path,
Path(rec.recording_info_model.output_data_folder_path) / RAW_DATA_FOLDER_NAME,
kill_event=kill_event,
@@ -80,37 +84,49 @@ def process_recording_folder(
mediapipe_2d_data_file_path=rec.recording_info_model.mediapipe_2d_data_npy_file_path,
)
- logger.info("Triangulating 3d skeletons...")
+ # spoof 3D data if single camera
+ if mediapipe_image_data_numCams_numFrames_numTrackedPts_XYZ.shape[0] == 1:
- assert rec.recording_info_model.calibration_toml_check, f"No calibration file found at: {rec.recording_info_model.calibration_toml_path}"
- assert rec.recording_info_model.data2d_status_check, f"No mediapipe 2d data found at: {rec.recording_info_model.mediapipe_2d_data_npy_file_path}"
+ (raw_skel3d_frame_marker_xyz, skeleton_reprojection_error_fr_mar) = process_single_camera_skeleton_data(
+ input_image_data_frame_marker_xyz=mediapipe_image_data_numCams_numFrames_numTrackedPts_XYZ[0],
+ raw_data_folder_path=Path(rec.recording_info_model.raw_data_folder_path))
- anipose_calibration_object = load_anipose_calibration_toml_from_path(
- camera_calibration_data_toml_path=rec.recording_info_model.calibration_toml_path,
- save_copy_of_calibration_to_this_path=rec.recording_info_model.path,
- )
- (raw_skel3d_frame_marker_xyz, skeleton_reprojection_error_fr_mar,) = triangulate_3d_data(
- anipose_calibration_object=anipose_calibration_object,
- mediapipe_2d_data=mediapipe_2d_data,
- output_data_folder_path=rec.recording_info_model.raw_data_folder_path,
- mediapipe_confidence_cutoff_threshold=rec.anipose_triangulate_3d_parameters_model.confidence_threshold_cutoff,
- use_triangulate_ransac=rec.anipose_triangulate_3d_parameters_model.use_triangulate_ransac_method,
- kill_event=kill_event,
- )
+ else:
- if kill_event is not None and kill_event.is_set():
- return
+ logger.info("Triangulating 3d skeletons...")
- assert test_mediapipe_3d_data_shape(
- synchronized_videos_folder=rec.recording_info_model.synchronized_videos_folder_path,
- mediapipe_3d_data_npy_path=rec.recording_info_model.raw_mediapipe_3d_data_npy_file_path,
- medipipe_reprojection_error_data_npy_path=rec.recording_info_model.mediapipe_reprojection_error_data_npy_file_path,
- )
+ assert rec.recording_info_model.calibration_toml_check, f"No calibration file found at: {rec.recording_info_model.calibration_toml_path}"
+ assert rec.recording_info_model.data2d_status_check, f"No mediapipe 2d data found at: {rec.recording_info_model.mediapipe_2d_data_npy_file_path}"
+
+ anipose_calibration_object = load_anipose_calibration_toml_from_path(
+ camera_calibration_data_toml_path=rec.recording_info_model.calibration_toml_path,
+ save_copy_of_calibration_to_this_path=rec.recording_info_model.path,
+ )
+ (raw_skel3d_frame_marker_xyz, skeleton_reprojection_error_fr_mar,) = triangulate_3d_data(
+ anipose_calibration_object=anipose_calibration_object,
+ mediapipe_2d_data=mediapipe_image_data_numCams_numFrames_numTrackedPts_XYZ,
+ output_data_folder_path=rec.recording_info_model.raw_data_folder_path,
+ mediapipe_confidence_cutoff_threshold=rec.anipose_triangulate_3d_parameters_model.confidence_threshold_cutoff,
+ use_triangulate_ransac=rec.anipose_triangulate_3d_parameters_model.use_triangulate_ransac_method,
+ kill_event=kill_event,
+ )
+
+ if kill_event is not None and kill_event.is_set():
+ return
+
+ assert test_mediapipe_3d_data_shape(
+ synchronized_videos_folder=rec.recording_info_model.synchronized_videos_folder_path,
+ mediapipe_3d_data_npy_path=rec.recording_info_model.raw_mediapipe_3d_data_npy_file_path,
+ medipipe_reprojection_error_data_npy_path=rec.recording_info_model.mediapipe_reprojection_error_data_npy_file_path,
+ )
+
+ #rotate so skeleton is closer to 'vertical' in a z-up reference frame
+ rotated_raw_skel3d_frame_marker_xyz = rotate_by_90_degrees_around_x_axis(raw_skel3d_frame_marker_xyz)
logger.info("Gap-filling, butterworth filtering, origin aligning 3d skeletons, then calculating center of mass ...")
skel3d_frame_marker_xyz = gap_fill_filter_origin_align_3d_data_and_then_calculate_center_of_mass(
- raw_skel3d_frame_marker_xyz=raw_skel3d_frame_marker_xyz,
+ raw_skel3d_frame_marker_xyz=rotated_raw_skel3d_frame_marker_xyz,
skeleton_reprojection_error_fr_mar=skeleton_reprojection_error_fr_mar,
path_to_folder_where_we_will_save_this_data=rec.recording_info_model.output_data_folder_path,
sampling_rate=rec.post_processing_parameters_model.framerate,
diff --git a/freemocap/system/paths_and_files_names.py b/freemocap/system/paths_and_files_names.py
index c8b9a7ca..bf1d10a4 100644
--- a/freemocap/system/paths_and_files_names.py
+++ b/freemocap/system/paths_and_files_names.py
@@ -26,6 +26,7 @@ CENTER_OF_MASS_FOLDER_NAME = "center_of_mass"
MOST_RECENT_RECORDING_TOML_FILENAME = "most_recent_recording.toml"
LAST_SUCCESSFUL_CALIBRATION_TOML_FILENAME = "last_successful_calibration.toml"
MEDIAPIPE_2D_NPY_FILE_NAME = "mediapipe2dData_numCams_numFrames_numTrackedPoints_pixelXY.npy"
+MEDIAPIPE_BODY_WORLD_FILE_NAME = "mediapipeBodyWorld_numCams_numFrames_numTrackedPoints_XYZ.npy"
MEDIAPIPE_3D_NPY_FILE_NAME = "mediaPipeSkel_3d_body_hands_face.npy"
RAW_MEDIAPIPE_3D_NPY_FILE_NAME = "mediapipe3dData_numFrames_numTrackedPoints_spatialXYZ.npy"
MEDIAPIPE_REPROJECTION_ERROR_NPY_FILE_NAME = "mediapipe3dData_numFrames_numTrackedPoints_reprojectionError.npy"
diff --git a/freemocap/tests/test_mediapipe_2d_data_shape.py b/freemocap/tests/test_mediapipe_2d_data_shape.py
index aff8e3b5..6fd42488 100644
--- a/freemocap/tests/test_mediapipe_2d_data_shape.py
+++ b/freemocap/tests/test_mediapipe_2d_data_shape.py
@@ -43,6 +43,6 @@ def test_mediapipe_2d_data_shape(
# TODO - check number of tracked points vs 'expected' number of tracked points
- assert mediapipe_2d_data.shape[3] == 2
+ assert mediapipe_2d_data.shape[3] == 3 # x y z
return True
diff --git a/freemocap/utilities/project_3d_data_to_z_plane.py b/freemocap/utilities/project_3d_data_to_z_plane.py
new file mode 100644
index 00000000..273f34e8
--- /dev/null
+++ b/freemocap/utilities/project_3d_data_to_z_plane.py
@@ -0,0 +1,12 @@
+import numpy as np
+
+
+def project_3d_data_to_z_plane(skel3d_frame_marker_xyz: np.ndarray) -> np.ndarray:
+
+ projected_skel3d_frame_marker_xyz = np.zeros(skel3d_frame_marker_xyz.shape)
+
+ projected_skel3d_frame_marker_xyz[:, :, 0] = skel3d_frame_marker_xyz[:, :, 0]
+ projected_skel3d_frame_marker_xyz[:, :, 1] = skel3d_frame_marker_xyz[:, :, 1]
+ projected_skel3d_frame_marker_xyz[:, :, 2] = 0
+
+ return projected_skel3d_frame_marker_xyz
diff --git a/freemocap/utilities/rotate_by_90_degrees_around_x_axis.py b/freemocap/utilities/rotate_by_90_degrees_around_x_axis.py
new file mode 100644
index 00000000..93cac3ea
--- /dev/null
+++ b/freemocap/utilities/rotate_by_90_degrees_around_x_axis.py
@@ -0,0 +1,12 @@
+import numpy as np
+
+
+def rotate_by_90_degrees_around_x_axis(raw_skel3d_frame_marker_xyz: np.ndarray) -> np.ndarray:
+
+ swapped_skel3d_frame_marker_xyz = np.zeros(raw_skel3d_frame_marker_xyz.shape)
+
+ swapped_skel3d_frame_marker_xyz[:, :, 0] = raw_skel3d_frame_marker_xyz[:, :, 0]
+ swapped_skel3d_frame_marker_xyz[:, :, 1] = raw_skel3d_frame_marker_xyz[:, :, 2]
+ swapped_skel3d_frame_marker_xyz[:, :, 2] = -raw_skel3d_frame_marker_xyz[:, :, 1]
+
+ return swapped_skel3d_frame_marker_xyz