Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

Jon/convert to csv (#254)

jonmatthis0508a2c02398
diff --git a/src/core_processes/capture_volume_calibration/triangulate_3d_data.py b/src/core_processes/capture_volume_calibration/triangulate_3d_data.py
index 8b3becf7..92f77958 100644
--- a/src/core_processes/capture_volume_calibration/triangulate_3d_data.py
+++ b/src/core_processes/capture_volume_calibration/triangulate_3d_data.py
@@ -4,6 +4,10 @@ from typing import Union
import numpy as np
import logging
+from src.core_processes.mediapipe_stuff.convert_mediapipe_npy_to_csv import (
+ convert_mediapipe_npy_to_csv,
+)
+
logger = logging.getLogger(__name__)
@@ -31,7 +35,8 @@ def triangulate_3d_data(
anipose_calibration_object,
mediapipe_2d_data: np.ndarray,
output_data_folder_path: Union[str, Path],
- mediapipe_confidence_cutoff_threshold: float = 0.0,
+ mediapipe_confidence_cutoff_threshold: float,
+ save_data_as_csv: bool,
):
number_of_cameras = mediapipe_2d_data.shape[0]
number_of_frames = mediapipe_2d_data.shape[1]
@@ -82,6 +87,11 @@ def triangulate_3d_data(
output_data_folder_path=output_data_folder_path,
)
+ if save_data_as_csv:
+ convert_mediapipe_npy_to_csv(
+ data3d_numFrames_numTrackedPoints_XYZ, output_data_folder_path
+ )
+
return path_to_3d_data_npy
diff --git a/src/core_processes/mediapipe_stuff/convert_mediapipe_npy_to_csv.py b/src/core_processes/mediapipe_stuff/convert_mediapipe_npy_to_csv.py
new file mode 100644
index 00000000..237d7975
--- /dev/null
+++ b/src/core_processes/mediapipe_stuff/convert_mediapipe_npy_to_csv.py
@@ -0,0 +1,365 @@
+# %%
+import logging
+from pathlib import Path
+from typing import Union
+
+import pandas as pd
+import numpy as np
+from mediapipe.python.solutions import holistic as mp_holistic
+
+logger = logging.getLogger(__name__)
+
+
+def convert_mediapipe_npy_to_csv(
+ mediapipe_3d_frame_trackedPoint_xyz: np.ndarray,
+ output_data_folder_path: Union[str, Path],
+):
+
+ logger.info(
+ f"loaded npy data with shape: {mediapipe_3d_frame_trackedPoint_xyz.shape}"
+ )
+
+ # %%
+ mediapipe_pose_landmark_names = [
+ landmark.name.lower() for landmark in mp_holistic.PoseLandmark
+ ]
+ mediapipe_hand_landmark_names = [
+ landmark.name.lower() for landmark in mp_holistic.HandLandmark
+ ]
+ # face_landmark_names = [landmark.name.lower() for landmark in mp_holistic.PoseLandmark] #gonna have the clever for the face
+ logger.info(f"Body tracked point names: {mediapipe_pose_landmark_names}")
+ logger.info(mediapipe_hand_landmark_names)
+
+ # %%
+ # get number of points in body, hands, face
+
+ number_of_body_points = len(mediapipe_pose_landmark_names)
+ number_of_hand_points = len(mediapipe_hand_landmark_names)
+
+ first_body_marker_index = 0
+ last_body_marker_index = number_of_body_points - 1
+
+ first_right_hand_marker_index = last_body_marker_index + 1
+ last_right_hand_marker_index = number_of_body_points + number_of_hand_points - 1
+
+ first_left_hand_marker_index = last_right_hand_marker_index + 1
+ last_left_hand_marker_index = (
+ last_right_hand_marker_index + 1 + number_of_hand_points - 1
+ )
+
+ first_face_marker_index = last_left_hand_marker_index + 1
+ last_face_marker_index = mediapipe_3d_frame_trackedPoint_xyz.shape[1]
+
+ number_of_face_points = last_face_marker_index - first_face_marker_index
+
+ logger.info(
+ f"body tracked point indices: {first_body_marker_index}:{last_body_marker_index}"
+ )
+ logger.info(
+ f"right hand tracked point indices: {first_right_hand_marker_index}:{last_right_hand_marker_index}"
+ )
+ logger.info(
+ f"left hand tracked point indices: {first_left_hand_marker_index}:{last_left_hand_marker_index}"
+ )
+ logger.info(
+ f"face tracked point indices: {first_face_marker_index}:{last_face_marker_index}"
+ )
+
+ logger.info(
+ f"number of body points: {last_body_marker_index - first_body_marker_index + 1}"
+ )
+ logger.info(
+ f"number of right hand points: {last_right_hand_marker_index - first_right_hand_marker_index + 1}"
+ )
+ logger.info(
+ f"number of left hand points: {last_left_hand_marker_index - first_left_hand_marker_index + 1}"
+ )
+ logger.info(
+ f"number of face points: {last_face_marker_index - first_face_marker_index + 1}"
+ )
+
+ # %%
+ body_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[
+ :, first_body_marker_index : last_body_marker_index + 1, :
+ ]
+ right_hand_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[
+ :, first_right_hand_marker_index : last_right_hand_marker_index + 1, :
+ ]
+ left_hand_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[
+ :, first_left_hand_marker_index : last_left_hand_marker_index + 1, :
+ ]
+ face_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[
+ :, first_face_marker_index : last_face_marker_index + 1, :
+ ]
+
+ logger.info(f"body 3d xyz shape: {body_3d_xyz.shape}")
+ logger.info(f"right hand 3d xyz shape: {right_hand_3d_xyz.shape}")
+ logger.info(f"left hand 3d xyz shape: {left_hand_3d_xyz.shape}")
+ logger.info(f"face 3d xyz shape: {face_3d_xyz.shape}")
+
+ # %%
+ # save broken up npy files
+ np.save(
+ str(Path(output_data_folder_path) / "mediapipe_body_3d_xyz.npy"), body_3d_xyz
+ )
+ np.save(
+ str(Path(output_data_folder_path) / "mediapipe_right_hand_3d_xyz.npy"),
+ right_hand_3d_xyz,
+ )
+ np.save(
+ str(Path(output_data_folder_path) / "mediapipe_left_hand_3d_xyz.npy"),
+ left_hand_3d_xyz,
+ )
+ np.save(
+ str(Path(output_data_folder_path) / "mediapipe_face_3d_xyz.npy"), face_3d_xyz
+ )
+
+ # %%
+ # create pandas data frame headers
+
+ body_3d_xyz_header = []
+ for landmark_name in mediapipe_pose_landmark_names:
+ body_3d_xyz_header.append(f"{landmark_name}_x")
+ body_3d_xyz_header.append(f"{landmark_name}_y")
+ body_3d_xyz_header.append(f"{landmark_name}_z")
+
+ right_hand_3d_xyz_header = []
+ for landmark_name in mediapipe_hand_landmark_names:
+ right_hand_3d_xyz_header.append(f"right_hand_{landmark_name}_x")
+ right_hand_3d_xyz_header.append(f"right_hand_{landmark_name}_y")
+ right_hand_3d_xyz_header.append(f"right_hand_{landmark_name}_z")
+
+ left_hand_3d_xyz_header = []
+ for landmark_name in mediapipe_hand_landmark_names:
+ left_hand_3d_xyz_header.append(f"left_hand_{landmark_name}_x")
+ left_hand_3d_xyz_header.append(f"left_hand_{landmark_name}_y")
+ left_hand_3d_xyz_header.append(f"left_hand_{landmark_name}_z")
+
+ face_3d_xyz_header = []
+ for landmark_number in range(last_face_marker_index - first_face_marker_index):
+ face_3d_xyz_header.append(f"face_{str(landmark_number).zfill(4)}_x")
+ face_3d_xyz_header.append(f"face_{str(landmark_number).zfill(4)}_y")
+ face_3d_xyz_header.append(f"face_{str(landmark_number).zfill(4)}_z")
+
+ logger.info(
+ f"length of body 3d xyz header: {len(body_3d_xyz_header)}, should be: {number_of_body_points * 3}"
+ )
+ logger.info(
+ f"length of right hand 3d xyz header: {len(right_hand_3d_xyz_header)}, should be: {number_of_hand_points * 3}"
+ )
+ logger.info(
+ f"length of left hand 3d xyz header: {len(left_hand_3d_xyz_header)}, should be: {number_of_hand_points * 3}"
+ )
+ logger.info(
+ f"length of face 3d xyz header: {len(face_3d_xyz_header)}, should be: {number_of_face_points * 3}"
+ )
+
+ # %%
+ number_of_frames = mediapipe_3d_frame_trackedPoint_xyz.shape[0]
+ body_flat = body_3d_xyz.reshape(number_of_frames, number_of_body_points * 3)
+
+ body_dataframe = pd.DataFrame(body_flat, columns=body_3d_xyz_header)
+ body_dataframe.to_csv(
+ str(Path(output_data_folder_path) / "mediapipe_body_3d_xyz.csv"), index=False
+ )
+
+ right_hand_flat = right_hand_3d_xyz.reshape(
+ number_of_frames, number_of_hand_points * 3
+ )
+ right_hand_dataframe = pd.DataFrame(
+ right_hand_flat, columns=right_hand_3d_xyz_header
+ )
+ right_hand_dataframe.to_csv(
+ str(Path(output_data_folder_path) / "mediapipe_right_hand_3d_xyz.csv"),
+ index=False,
+ )
+
+ left_hand_flat = left_hand_3d_xyz.reshape(
+ number_of_frames, number_of_hand_points * 3
+ )
+ left_hand_dataframe = pd.DataFrame(left_hand_flat, columns=left_hand_3d_xyz_header)
+ left_hand_dataframe.to_csv(
+ str(Path(output_data_folder_path) / "mediapipe_left_hand_3d_xyz.csv"),
+ index=False,
+ )
+
+ face_flat = face_3d_xyz.reshape(number_of_frames, number_of_face_points * 3)
+ face_dataframe = pd.DataFrame(face_flat, columns=face_3d_xyz_header)
+ face_dataframe.to_csv(
+ str(Path(output_data_folder_path) / "mediapipe_face_3d_xyz.csv"), index=False
+ )
+
+ # %%
+
+ # %%
+ mediapipe_pose_landmark_names = [
+ landmark.name.lower() for landmark in mp_holistic.PoseLandmark
+ ]
+ mediapipe_hand_landmark_names = [
+ landmark.name.lower() for landmark in mp_holistic.HandLandmark
+ ]
+ # face_landmark_names = [landmark.name.lower() for landmark in mp_holistic.PoseLandmark] #gonna have the clever for the face
+ logger.info(f"Body tracked point names: {mediapipe_pose_landmark_names}")
+ logger.info(mediapipe_hand_landmark_names)
+
+ # %%
+ # get number of points in body, hands, face
+
+ number_of_body_points = len(mediapipe_pose_landmark_names)
+ number_of_hand_points = len(mediapipe_hand_landmark_names)
+
+ first_body_marker_index = 0
+ last_body_marker_index = number_of_body_points - 1
+
+ first_right_hand_marker_index = last_body_marker_index + 1
+ last_right_hand_marker_index = number_of_body_points + number_of_hand_points - 1
+
+ first_left_hand_marker_index = last_right_hand_marker_index + 1
+ last_left_hand_marker_index = (
+ last_right_hand_marker_index + 1 + number_of_hand_points - 1
+ )
+
+ first_face_marker_index = last_left_hand_marker_index + 1
+ last_face_marker_index = mediapipe_3d_frame_trackedPoint_xyz.shape[1]
+
+ number_of_face_points = last_face_marker_index - first_face_marker_index
+
+ logger.info(
+ f"body tracked point indices: {first_body_marker_index}:{last_body_marker_index}"
+ )
+ logger.info(
+ f"right hand tracked point indices: {first_right_hand_marker_index}:{last_right_hand_marker_index}"
+ )
+ logger.info(
+ f"left hand tracked point indices: {first_left_hand_marker_index}:{last_left_hand_marker_index}"
+ )
+ logger.info(
+ f"face tracked point indices: {first_face_marker_index}:{last_face_marker_index}"
+ )
+
+ logger.info(
+ f"number of body points: {last_body_marker_index - first_body_marker_index + 1}"
+ )
+ logger.info(
+ f"number of right hand points: {last_right_hand_marker_index - first_right_hand_marker_index + 1}"
+ )
+ logger.info(
+ f"number of left hand points: {last_left_hand_marker_index - first_left_hand_marker_index + 1}"
+ )
+ logger.info(
+ f"number of face points: {last_face_marker_index - first_face_marker_index + 1}"
+ )
+
+ # %%
+ body_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[
+ :, first_body_marker_index : last_body_marker_index + 1, :
+ ]
+ right_hand_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[
+ :, first_right_hand_marker_index : last_right_hand_marker_index + 1, :
+ ]
+ left_hand_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[
+ :, first_left_hand_marker_index : last_left_hand_marker_index + 1, :
+ ]
+ face_3d_xyz = mediapipe_3d_frame_trackedPoint_xyz[
+ :, first_face_marker_index : last_face_marker_index + 1, :
+ ]
+
+ logger.info(f"body 3d xyz shape: {body_3d_xyz.shape}")
+ logger.info(f"right hand 3d xyz shape: {right_hand_3d_xyz.shape}")
+ logger.info(f"left hand 3d xyz shape: {left_hand_3d_xyz.shape}")
+ logger.info(f"face 3d xyz shape: {face_3d_xyz.shape}")
+
+ # %%
+ # save broken up npy files
+ np.save(
+ str(Path(output_data_folder_path) / "mediapipe_body_3d_xyz.npy"), body_3d_xyz
+ )
+ np.save(
+ str(Path(output_data_folder_path) / "mediapipe_right_hand_3d_xyz.npy"),
+ right_hand_3d_xyz,
+ )
+ np.save(
+ str(Path(output_data_folder_path) / "mediapipe_left_hand_3d_xyz.npy"),
+ left_hand_3d_xyz,
+ )
+ np.save(
+ str(Path(output_data_folder_path) / "mediapipe_face_3d_xyz.npy"), face_3d_xyz
+ )
+
+ # %%
+ # create pandas data frame headers
+
+ body_3d_xyz_header = []
+ for landmark_name in mediapipe_pose_landmark_names:
+ body_3d_xyz_header.append(f"{landmark_name}_x")
+ body_3d_xyz_header.append(f"{landmark_name}_y")
+ body_3d_xyz_header.append(f"{landmark_name}_z")
+
+ right_hand_3d_xyz_header = []
+ for landmark_name in mediapipe_hand_landmark_names:
+ right_hand_3d_xyz_header.append(f"right_hand_{landmark_name}_x")
+ right_hand_3d_xyz_header.append(f"right_hand_{landmark_name}_y")
+ right_hand_3d_xyz_header.append(f"right_hand_{landmark_name}_z")
+
+ left_hand_3d_xyz_header = []
+ for landmark_name in mediapipe_hand_landmark_names:
+ left_hand_3d_xyz_header.append(f"left_hand_{landmark_name}_x")
+ left_hand_3d_xyz_header.append(f"left_hand_{landmark_name}_y")
+ left_hand_3d_xyz_header.append(f"left_hand_{landmark_name}_z")
+
+ face_3d_xyz_header = []
+ for landmark_number in range(last_face_marker_index - first_face_marker_index):
+ face_3d_xyz_header.append(f"face_{str(landmark_number).zfill(4)}_x")
+ face_3d_xyz_header.append(f"face_{str(landmark_number).zfill(4)}_y")
+ face_3d_xyz_header.append(f"face_{str(landmark_number).zfill(4)}_z")
+
+ logger.info(
+ f"length of body 3d xyz header: {len(body_3d_xyz_header)}, should be: {number_of_body_points * 3}"
+ )
+ logger.info(
+ f"length of right hand 3d xyz header: {len(right_hand_3d_xyz_header)}, should be: {number_of_hand_points * 3}"
+ )
+ logger.info(
+ f"length of left hand 3d xyz header: {len(left_hand_3d_xyz_header)}, should be: {number_of_hand_points * 3}"
+ )
+ logger.info(
+ f"length of face 3d xyz header: {len(face_3d_xyz_header)}, should be: {number_of_face_points * 3}"
+ )
+
+ # %%
+ number_of_frames = mediapipe_3d_frame_trackedPoint_xyz.shape[0]
+ body_flat = body_3d_xyz.reshape(number_of_frames, number_of_body_points * 3)
+ body_flat.shape
+
+ body_dataframe = pd.DataFrame(body_flat, columns=body_3d_xyz_header)
+ body_dataframe.to_csv(
+ str(Path(output_data_folder_path) / "mediapipe_body_3d_xyz.csv"), index=False
+ )
+
+ right_hand_flat = right_hand_3d_xyz.reshape(
+ number_of_frames, number_of_hand_points * 3
+ )
+ right_hand_dataframe = pd.DataFrame(
+ right_hand_flat, columns=right_hand_3d_xyz_header
+ )
+ right_hand_dataframe.to_csv(
+ str(Path(output_data_folder_path) / "mediapipe_right_hand_3d_xyz.csv"),
+ index=False,
+ )
+
+ left_hand_flat = left_hand_3d_xyz.reshape(
+ number_of_frames, number_of_hand_points * 3
+ )
+ left_hand_dataframe = pd.DataFrame(left_hand_flat, columns=left_hand_3d_xyz_header)
+ left_hand_dataframe.to_csv(
+ str(Path(output_data_folder_path) / "mediapipe_left_hand_3d_xyz.csv"),
+ index=False,
+ )
+
+ face_flat = face_3d_xyz.reshape(number_of_frames, number_of_face_points * 3)
+ face_dataframe = pd.DataFrame(face_flat, columns=face_3d_xyz_header)
+ face_dataframe.to_csv(
+ str(Path(output_data_folder_path) / "mediapipe_face_3d_xyz.csv"), index=False
+ )
+
+ # %%
diff --git a/src/gui/main/main_window/left_panel_controls/toolbox_widgets/process_session_data_panel.py b/src/gui/main/main_window/left_panel_controls/toolbox_widgets/process_session_data_panel.py
index 6be7679e..174f3689 100644
--- a/src/gui/main/main_window/left_panel_controls/toolbox_widgets/process_session_data_panel.py
+++ b/src/gui/main/main_window/left_panel_controls/toolbox_widgets/process_session_data_panel.py
@@ -53,19 +53,15 @@ class ProcessSessionDataPanel(QWidget):
)
self._layout.addLayout(self._mediapipe_confidence_cutoff_form_layout)
- self._visualize_freemocap_session_button = QPushButton(
- "TO DO - Visualize Freemocap Session"
+ self._convert_npy_to_csv_checkbox = QCheckBox(
+ "Convert 3D npy data to csv (experimental)"
)
- self._visualize_freemocap_session_button.setEnabled(False)
- self._layout.addWidget(self._visualize_freemocap_session_button)
+ self._convert_npy_to_csv_checkbox.setChecked(True)
+ self._layout.addWidget(self._convert_npy_to_csv_checkbox)
self._blender_path_form_layout = self._make_blender_path_layout()
self._layout.addLayout(self._blender_path_form_layout)
- # self._visualize_freemocap_session_button.clicked.connect(
- # self._visualize_freemocap_session
- # )
-
self._open_in_blender_button = QPushButton(
"Export to Blender (Freezes GUI, sorry!)"
)
@@ -86,6 +82,10 @@ class ProcessSessionDataPanel(QWidget):
def mediapipe_confidence_cutoff_threshold(self) -> float:
return float(self._mediapipe_confidence_cutoff_line_edit_widget.text())
+ @property
+ def convert_npy_to_csv_checkbox(self):
+ return self._convert_npy_to_csv_checkbox
+
@property
def triangulate_3d_data_button(self):
return self._triangulate_3d_data_button
diff --git a/src/gui/main/main_window/main_window.py b/src/gui/main/main_window/main_window.py
index 40eaa05f..513b83c6 100644
--- a/src/gui/main/main_window/main_window.py
+++ b/src/gui/main/main_window/main_window.py
@@ -708,6 +708,7 @@ class MainWindow(QMainWindow):
mediapipe_2d_data=mediapipe_2d_data,
output_data_folder_path=output_data_folder_path,
mediapipe_confidence_cutoff_threshold=self._control_panel.process_session_data_panel.mediapipe_confidence_cutoff_threshold,
+ save_data_as_csv=self._control_panel.process_session_data_panel.convert_npy_to_csv_checkbox.isChecked(),
auto_process_next_stage=auto_process_next_stage,
)
diff --git a/src/gui/main/workers/thread_worker_manager.py b/src/gui/main/workers/thread_worker_manager.py
index d78a855c..ced1fbeb 100644
--- a/src/gui/main/workers/thread_worker_manager.py
+++ b/src/gui/main/workers/thread_worker_manager.py
@@ -113,7 +113,8 @@ class ThreadWorkerManager(QWidget):
anipose_calibration_object,
mediapipe_2d_data: np.ndarray,
output_data_folder_path: Union[str, Path],
- mediapipe_confidence_cutoff_threshold: float = 0.0,
+ mediapipe_confidence_cutoff_threshold: float,
+ save_data_as_csv: bool,
auto_process_next_stage: bool = False,
):
logger.info("Launching `Triangulate 3d Data` thread worker...")
@@ -123,6 +124,7 @@ class ThreadWorkerManager(QWidget):
mediapipe_2d_data=mediapipe_2d_data,
output_data_folder_path=output_data_folder_path,
mediapipe_confidence_cutoff_threshold=mediapipe_confidence_cutoff_threshold,
+ save_data_as_csv=save_data_as_csv,
)
self._triangulate_3d_data_thread_worker.start()
diff --git a/src/gui/main/workers/triangulate_3d_data_thread_worker.py b/src/gui/main/workers/triangulate_3d_data_thread_worker.py
index e12c06ab..b17b3963 100644
--- a/src/gui/main/workers/triangulate_3d_data_thread_worker.py
+++ b/src/gui/main/workers/triangulate_3d_data_thread_worker.py
@@ -21,7 +21,8 @@ class Triangulate3dDataThreadWorker(QThread):
anipose_calibration_object,
mediapipe_2d_data: np.ndarray,
output_data_folder_path: Union[str, Path],
- mediapipe_confidence_cutoff_threshold: float = 0.0,
+ mediapipe_confidence_cutoff_threshold: float,
+ save_data_as_csv: bool,
):
super().__init__()
self._anipose_calibration_object = anipose_calibration_object
@@ -30,6 +31,7 @@ class Triangulate3dDataThreadWorker(QThread):
self._mediapipe_confidence_cutoff_threshold = (
mediapipe_confidence_cutoff_threshold
)
+ self._save_data_as_csv = save_data_as_csv
def run(self):
logger.info(
@@ -41,6 +43,7 @@ class Triangulate3dDataThreadWorker(QThread):
mediapipe_2d_data=self._mediapipe_2d_data,
output_data_folder_path=self._output_data_folder_path,
mediapipe_confidence_cutoff_threshold=self._mediapipe_confidence_cutoff_threshold,
+ save_data_as_csv=self._save_data_as_csv,
)
self.finished.emit()