Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

mediapipe is in there but its not doing anything and apparently the frames are still out of sync lol

Jond4cc1a04b8ec
diff --git a/src/api/routes/session/session_router.py b/src/api/routes/session/session_router.py
index 2736e769..677967df 100644
--- a/src/api/routes/session/session_router.py
+++ b/src/api/routes/session/session_router.py
@@ -65,7 +65,8 @@ def calibrate_session(session_calibrate_model: SessionCalibrateModel = SessionCa
show_camera_views_in_windows=True,
)
calibration_orchestrator.run_anipose_camera_calibration(
- charuco_square_size=session_calibrate_model.charuco_square_size)
+ charuco_square_size=session_calibrate_model.charuco_square_size,
+ pin_camera_0_to_origin = True)
@session_router.post("/session/record")
diff --git a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
index eae4f6b4..d16a0cc9 100644
--- a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
+++ b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
@@ -1,7 +1,7 @@
import logging
from dataclasses import dataclass
from pathlib import Path
-from typing import List, Dict, Union
+from typing import List, Dict, Union, Any
import cv2
import numpy as np
@@ -20,7 +20,7 @@ logger = logging.getLogger(__name__)
@dataclass
-class Mediapipe2dSingleCameraNpyArrays:
+class Mediapipe2dNumpyArrays:
body2d_frameNumber_trackedPointNumber_XY: np.ndarray = None
rightHand2d_frameNumber_trackedPointNumber_XY: np.ndarray = None
leftHand2d_frameNumber_trackedPointNumber_XY: np.ndarray = None
@@ -37,6 +37,14 @@ class Mediapipe2dSingleCameraNpyArrays:
self.face2d_frameNumber_trackedPointNumber_XY])
+@dataclass
+class Mediapipe2dDataPayload:
+ raw_frame_payload: FramePayload = None
+ mediapipe_results: Any = None
+ annotated_image: np.ndarray = None
+ pixel_data_numpy_arrays: Mediapipe2dNumpyArrays = None
+
+
class MediaPipeSkeletonDetector:
def __init__(self, session_id: str = None):
self._session_id = session_id
@@ -59,9 +67,39 @@ class MediaPipeSkeletonDetector:
min_tracking_confidence=self.min_tracking_confidence)
self._mediapipe_tracked_point_names_dict = mediapipe_tracked_point_names_dict
- def detect_skeleton_in_image(self, raw_image, annotate_image=True):
- mediapipe_results = self._holistic_tracker.process(raw_image) # <-this is where the magic happens
- return mediapipe_results
+ self.body_names_list = self._mediapipe_tracked_point_names_dict["body"]
+ self.right_hand_names_list = self._mediapipe_tracked_point_names_dict["right_hand"]
+ self.left_hand_names_list = self._mediapipe_tracked_point_names_dict["left_hand"]
+ self.face_names_list = self._mediapipe_tracked_point_names_dict["face"]
+
+ # TODO - build a better iterator and list of `face_marker_names` that will only pull out the face_counters & iris edges (mp.python.solutions.face_mesh_connections.FACEMESH_CONTOURS, FACE_MESH_IRISES)
+
+ self.number_of_body_tracked_points = len(self.body_names_list)
+ self.number_of_right_hand_tracked_points = len(self.right_hand_names_list)
+ self.number_of_left_hand_tracked_points = len(self.left_hand_names_list)
+ self.number_of_face_tracked_points = mp.python.solutions.face_mesh.FACEMESH_NUM_LANDMARKS_WITH_IRISES
+
+ self.number_of_tracked_points_total = self.number_of_body_tracked_points + \
+ self.number_of_left_hand_tracked_points + \
+ self.number_of_right_hand_tracked_points + \
+ self.number_of_face_tracked_points
+
+ def detect_skeleton_in_image(self, raw_frame_payload: FramePayload,
+ annotated_image: np.ndarray = None) -> Mediapipe2dDataPayload:
+
+ mediapipe_results = self._holistic_tracker.process(
+ raw_frame_payload.image) # <-this is where the magic happens, i.e. where the raw image is processed by a convolutional neural network to provide an estimate of joint position in pixel coordinates. Please don't forget that this is insane and should not be possible lol
+
+ if annotated_image is None:
+ annotated_image = raw_frame_payload.image.copy()
+
+ annotated_image = self._annotate_image(annotated_image, mediapipe_results)
+
+ mediapipe_single_frame_npy_data = self._list_of_mediapipe_results_to_npy_arrays([mediapipe_results])
+ return Mediapipe2dDataPayload(raw_frame_payload=raw_frame_payload,
+ mediapipe_results=mediapipe_results,
+ annotated_image=annotated_image,
+ pixel_data_numpy_arrays=mediapipe_single_frame_npy_data)
def process_session_folder(self,
save_annotated_videos: bool = True):
@@ -194,45 +232,35 @@ class MediaPipeSkeletonDetector:
mediapipe_results_list: List,
image_width: int = 1,
image_height: int = 1
- ) -> Mediapipe2dSingleCameraNpyArrays:
-
- body_names_list = self._mediapipe_tracked_point_names_dict["body"]
- right_hand_names_list = self._mediapipe_tracked_point_names_dict["right_hand"]
- left_hand_names_list = self._mediapipe_tracked_point_names_dict["left_hand"]
- face_names_list = self._mediapipe_tracked_point_names_dict["face"]
+ ) -> Mediapipe2dNumpyArrays:
# apparently `mediapipe_results.pose_landmarks.landmark` returns something iterable Β―\_(ツ)_/Β―
mediapipe_pose_landmark_iterator = mp.python.solutions.pose.PoseLandmark
mediapipe_hand_landmark_iterator = mp.python.solutions.hands.HandLandmark
- # TODO - build a better iterator and list of `face_marker_names` that will only pull out the face_counters & iris edges (mp.python.solutions.face_mesh_connections.FACEMESH_CONTOURS, FACE_MESH_IRISES)
number_of_frames = len(mediapipe_results_list)
-
- number_of_body_trackedPoints = len(body_names_list)
- number_of_right_hand_trackedPoints = len(right_hand_names_list)
- number_of_left_hand_trackedPoints = len(left_hand_names_list)
- number_of_face_trackedPoints = mp.python.solutions.face_mesh.FACEMESH_NUM_LANDMARKS_WITH_IRISES
number_of_spatial_dimensions = 2 # this will be 2d XY pixel data
body2d_frameNumber_trackedPointNumber_XY = np.zeros(
- (number_of_frames, number_of_body_trackedPoints, number_of_spatial_dimensions))
+ (number_of_frames, self.number_of_body_tracked_points, number_of_spatial_dimensions))
body2d_frameNumber_trackedPointNumber_XY[:] = np.nan
- body2d_frameNumber_trackedPointNumber_confidence = np.zeros((number_of_frames, number_of_body_trackedPoints))
+ body2d_frameNumber_trackedPointNumber_confidence = np.zeros(
+ (number_of_frames, self.number_of_body_tracked_points))
body2d_frameNumber_trackedPointNumber_confidence[:] = np.nan # only body markers get a 'confidence' value
rightHand2d_frameNumber_trackedPointNumber_XY = np.zeros((number_of_frames,
- number_of_right_hand_trackedPoints,
+ self.number_of_right_hand_tracked_points,
number_of_spatial_dimensions))
rightHand2d_frameNumber_trackedPointNumber_XY[:] = np.nan
leftHand2d_frameNumber_trackedPointNumber_XY = np.zeros((number_of_frames,
- number_of_left_hand_trackedPoints,
+ self.number_of_left_hand_tracked_points,
number_of_spatial_dimensions))
leftHand2d_frameNumber_trackedPointNumber_XY[:] = np.nan
face2d_frameNumber_trackedPointNumber_XY = np.zeros((number_of_frames,
- number_of_face_trackedPoints,
+ self.number_of_face_tracked_points,
number_of_spatial_dimensions))
face2d_frameNumber_trackedPointNumber_XY[:] = np.nan
@@ -312,7 +340,7 @@ class MediaPipeSkeletonDetector:
all_tracked_points_visible_on_this_frame_list.append(all_points_visible)
- return Mediapipe2dSingleCameraNpyArrays(
+ return Mediapipe2dNumpyArrays(
body2d_frameNumber_trackedPointNumber_XY=body2d_frameNumber_trackedPointNumber_XY,
rightHand2d_frameNumber_trackedPointNumber_XY=rightHand2d_frameNumber_trackedPointNumber_XY,
leftHand2d_frameNumber_trackedPointNumber_XY=leftHand2d_frameNumber_trackedPointNumber_XY,
diff --git a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
index 446ab473..cdf765bc 100644
--- a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
+++ b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
@@ -157,6 +157,7 @@ class SessionPipelineOrchestrator:
self.anipose_camera_calibration_object = CalibrationPipelineOrchestrator().load_most_recent_calibration()
dictionary_of_charuco_frame_payloads_on_this_multi_frame = {}
+ dictionary_of_mediapipe_payloads_on_this_multi_frame = {}
with self._open_cv_camera_manager.start_capture_session_all_cams() as connected_cameras_dict:
@@ -173,11 +174,19 @@ class SessionPipelineOrchestrator:
if detect_charuco:
if len(dictionary_of_charuco_frame_payloads_on_this_multi_frame) > 0:
- charuco3d_multi_frame_payload = self._reconstruct_3d_charuco(dictionary_of_charuco_frame_payloads_on_this_multi_frame)
+ charuco3d_multi_frame_payload = self._reconstruct_3d_charuco(
+ dictionary_of_charuco_frame_payloads_on_this_multi_frame)
if show_visualizer_gui and charuco3d_multi_frame_payload is not None:
self._visualizer_gui.update_charuco_3d_dottos(charuco3d_multi_frame_payload)
dictionary_of_charuco_frame_payloads_on_this_multi_frame = {}
+ if detect_mediapipe:
+ if len(dictionary_of_mediapipe_payloads_on_this_multi_frame) > 0:
+ mediapipe3d_multi_frame_payload = self._reconstruct_3d_mediapipe(
+ dictionary_of_mediapipe_payloads_on_this_multi_frame)
+ if show_visualizer_gui and mediapipe3d_multi_frame_payload is not None:
+ self._visualizer_gui.update_medaiapipe3d_skel(mediapipe3d_multi_frame_payload)
+ dictionary_of_mediapipe_payloads_on_this_multi_frame = {}
timestamp_manager.log_new_timestamp_for_main_loop_ns(time.perf_counter_ns())
@@ -226,6 +235,17 @@ class SessionPipelineOrchestrator:
dictionary_of_charuco_frame_payloads_on_this_multi_frame[
this_webcam_id] = this_charuco_frame_payload
+ if detect_mediapipe:
+ # detect mediapipe skeleton
+ this_mediapipe_frame_payload = self._mediapipe_skeleton_detector.detect_skeleton_in_image(
+ this_cam_latest_frame,
+ annotated_image=image_to_display)
+ image_to_display = this_mediapipe_frame_payload.annotated_image
+
+ if reconstruct_3d:
+ dictionary_of_mediapipe_payloads_on_this_multi_frame[
+ this_webcam_id] = this_mediapipe_frame_payload
+
if show_camera_views_in_windows:
should_continue = show_cam_window(
this_webcam_id, image_to_display, timestamp_manager
@@ -290,18 +310,17 @@ class SessionPipelineOrchestrator:
min_cameras_to_reconstruct = 2
if sum(is_there_any_charuco_data) >= min_cameras_to_reconstruct: # if at least 2 cameras have any data, then we can try to make some 3d dottos
- charuco2d_data_per_cam_dict = self.format_charuco2d_data(dictionary_of_charuco_payloads_on_this_multiframe)
+ charuco2d_data_per_cam_dict = self._format_charuco2d_data(dictionary_of_charuco_payloads_on_this_multiframe)
charuco_3d_data_payload = self._triangulate_2d_data(charuco2d_data_per_cam_dict,
self._charuco_board_detector.number_of_charuco_corners)
x = charuco_3d_data_payload.data3d_trackedPointNum_xyz[:, 0]
return charuco_3d_data_payload
- def format_charuco2d_data(self, dictionary_of_charuco_payloads_on_this_multiframe) -> Dict:
+ def _format_charuco2d_data(self, dictionary_of_charuco_payloads_on_this_multiframe) -> Dict:
number_of_tracked_points = self._charuco_board_detector.number_of_charuco_corners
-
base_charuco_data_npy_with_nans_for_missing_data_xy = np.zeros((number_of_tracked_points, 2))
base_charuco_data_npy_with_nans_for_missing_data_xy[:] = np.nan
@@ -320,6 +339,25 @@ class SessionPipelineOrchestrator:
return charuco2d_data_per_cam_dict
+ def _reconstruct_3d_mediapipe(self, dictionary_of_mediapipe_payloads_on_this_multi_frame) -> \
+ Union[None, Data3dMultiFramePayload]:
+
+ return
+
+ is_there_any_mediapipe_data = []
+ for this_cam_medipipe_data in dictionary_of_mediapipe_payloads_on_this_multi_frame.values():
+ is_there_any_mediapipe_data.append(
+ this_cam_medipipe_data.pixel_data_numpy_arrays.all_data2d_nFrames_nTrackedPts_XY is not None)
+
+ min_cameras_to_reconstruct = 2
+ if sum(is_there_any_mediapipe_data) >= min_cameras_to_reconstruct: # if at least 2 cameras have any data, then we can try to make some 3d dottos
+ mediapipe2d_data_per_cam_dict = self._format_charuco2d_data(
+ dictionary_of_mediapipe_payloads_on_this_multi_frame)
+ mediapipe_3d_data_payload = self._triangulate_2d_data(mediapipe2d_data_per_cam_dict,
+ self._mediapipe_skeleton_detector.number_of_tracked_points_total)
+
+ return mediapipe_3d_data_payload
+
def _triangulate_2d_data(self, data2d_per_cam_dict: Dict,
number_of_tracked_points: int) -> Data3dMultiFramePayload:
diff --git a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
index d7fee482..62fb0381 100644
--- a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
+++ b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
@@ -201,6 +201,15 @@ class QTVisualizerAndGui:
self.opengl_3d_plot_dock.addWidget(self.opengl_3d_plot_widget)
self._main_dock_area.addDock(self.opengl_3d_plot_dock, 'bottom', self._camera_views_dock)
+ def initialize_charuco_dottos(self, number_of_charuco_corners: int):
+ dummy_charuco_points = np.zeros((number_of_charuco_corners, 3))
+ self._charuco_scatter_item = gl.GLScatterPlotItem(
+ pos=dummy_charuco_points,
+ color=(1, 1, 0, 1),
+ size=20
+ )
+ self.opengl_3d_plot_widget.addItem(self._charuco_scatter_item)
+
def update_charuco_3d_dottos(self, charuco_frame_payload: Data3dMultiFramePayload):
# if not self._camera_set:
# mean_xyz = np.nanmean(charuco_frame_payload.data3d_trackedPointNum_xyz,axis=0)
@@ -211,14 +220,9 @@ class QTVisualizerAndGui:
pos=charuco_frame_payload.data3d_trackedPointNum_xyz
)
- def initialize_charuco_dottos(self, number_of_charuco_corners: int):
- dummy_charuco_points = np.zeros((number_of_charuco_corners, 3))
- self._charuco_scatter_item = gl.GLScatterPlotItem(
- pos=dummy_charuco_points,
- color=(1, 1, 0, 1),
- size=20
- )
- self.opengl_3d_plot_widget.addItem(self._charuco_scatter_item)
+
+ def update_medaiapipe3d_skel(self, mediapipe3d_multi_frame_payload):
+ pass
if __name__ == "__main__":