Free Motion Capture for Everyone πβ¨
TypeScript
something of mediapipe is making it in, but the geometry is borked
Jondd52678cd8f5
diff --git a/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py b/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.pyindex dcc773b1..acd103a0 100644--- a/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py+++ b/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py@@ -279,10 +279,8 @@ class QtGlLaserSkeletonVisualizer():if self.current_frame_number % 10 == 0:print(f'frame number: {self.current_frame_number}')- self.skeleton_scatter_item.setData(- pos=self.mediapipe_fr_mar_xyz[self.current_frame_number, :, :]- )- self.update_skeleton_lines()++ self.update_skeleton()if self.session_data is not None:self.update_head_axis_lines()self.update_eye_axis_lines()@@ -290,7 +288,13 @@ class QtGlLaserSkeletonVisualizer():self.update_gaze_lasers()self.update_gaze_laser_tails()- def update_skeleton_lines(self):+ def update_skeleton(self):+ # skel dottos+ self.skeleton_scatter_item.setData(+ pos=self.mediapipe_fr_mar_xyz[self.current_frame_number, :, :]+ )++ #skel linesfor this_skeleton_line_number, this_connection in enumerate(self.mediapipe_body_connections):self.skeleton_connections_list[this_skeleton_line_number].setData(pos=self.mediapipe_fr_mar_xyz[self.current_frame_number, this_connection, :]diff --git a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.pyindex d16a0cc9..7283dcda 100644--- a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py+++ b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py@@ -28,13 +28,22 @@ class Mediapipe2dNumpyArrays:body2d_frameNumber_trackedPointNumber_confidence: np.ndarray = None+ @property+ def has_data(self):+ return not np.isnan(self.body2d_frameNumber_trackedPointNumber_XY).all()+@propertydef all_data2d_nFrames_nTrackedPts_XY(self):"""dimensions will be [number_of_frames , number_of_markers, XY]"""- return np.hstack([self.body2d_frameNumber_trackedPointNumber_XY,- self.rightHand2d_frameNumber_trackedPointNumber_XY,- self.leftHand2d_frameNumber_trackedPointNumber_XY,- self.face2d_frameNumber_trackedPointNumber_XY])++ if self.body2d_frameNumber_trackedPointNumber_XY is None:+ # if there's no body data, there's no hand or face data either+ return++ return np.squeeze(np.hstack([self.body2d_frameNumber_trackedPointNumber_XY,+ self.rightHand2d_frameNumber_trackedPointNumber_XY,+ self.leftHand2d_frameNumber_trackedPointNumber_XY,+ self.face2d_frameNumber_trackedPointNumber_XY]))@dataclassdiff --git a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py b/src/pipelines/session_pipeline/session_pipeline_orchestrator.pyindex cdf765bc..d20a0226 100644--- a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py+++ b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py@@ -185,7 +185,7 @@ class SessionPipelineOrchestrator:mediapipe3d_multi_frame_payload = self._reconstruct_3d_mediapipe(dictionary_of_mediapipe_payloads_on_this_multi_frame)if show_visualizer_gui and mediapipe3d_multi_frame_payload is not None:- self._visualizer_gui.update_medaiapipe3d_skel(mediapipe3d_multi_frame_payload)+ self._visualizer_gui.update_medaiapipe3d_skeleton(mediapipe3d_multi_frame_payload)dictionary_of_mediapipe_payloads_on_this_multi_frame = {}timestamp_manager.log_new_timestamp_for_main_loop_ns(time.perf_counter_ns())@@ -341,18 +341,17 @@ class SessionPipelineOrchestrator:def _reconstruct_3d_mediapipe(self, dictionary_of_mediapipe_payloads_on_this_multi_frame) -> \Union[None, Data3dMultiFramePayload]:-- return-is_there_any_mediapipe_data = []for this_cam_medipipe_data in dictionary_of_mediapipe_payloads_on_this_multi_frame.values():is_there_any_mediapipe_data.append(- this_cam_medipipe_data.pixel_data_numpy_arrays.all_data2d_nFrames_nTrackedPts_XY is not None)+ this_cam_medipipe_data.pixel_data_numpy_arrays.has_data)min_cameras_to_reconstruct = 2if sum(is_there_any_mediapipe_data) >= min_cameras_to_reconstruct: # if at least 2 cameras have any data, then we can try to make some 3d dottos- mediapipe2d_data_per_cam_dict = self._format_charuco2d_data(- dictionary_of_mediapipe_payloads_on_this_multi_frame)+ mediapipe2d_data_per_cam_dict = {}+ for this_webcam_id, this_mediapipe_payload in dictionary_of_mediapipe_payloads_on_this_multi_frame.items():+ mediapipe2d_data_per_cam_dict[this_webcam_id] = this_mediapipe_payload.pixel_data_numpy_arrays.all_data2d_nFrames_nTrackedPts_XY+mediapipe_3d_data_payload = self._triangulate_2d_data(mediapipe2d_data_per_cam_dict,self._mediapipe_skeleton_detector.number_of_tracked_points_total)diff --git a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.pyindex 62fb0381..769c32d9 100644--- a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py+++ b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py@@ -9,6 +9,7 @@ import pyqtgraph.opengl as glfrom pyqtgraph.dockarea.Dock import Dockfrom pyqtgraph.dockarea.DockArea import DockAreafrom pyqtgraph.Qt import QtWidgets+from mediapipe.python.solutions import holistic as mp_holisticfrom src.core_processor.timestamp_manager.timestamp_manager import TimestampManagerfrom src.pipelines.session_pipeline.data_classes.data_3d_single_frame_payload import Data3dMultiFramePayload@@ -19,6 +20,8 @@ logger = logging.getLogger(__name__)class QTVisualizerAndGui:def __init__(self):# https://pyqtgraph.readthedocs.io/en/latest/config_options.html+ self._skeleton_connections_list = None+ self._mediapipe_skeleton_initialized = Falseself.pyqtgraph_app = pg.mkQApp('freemocap! :D')pg.setConfigOptions(imageAxisOrder='row-major')@@ -59,6 +62,7 @@ class QTVisualizerAndGui:self._setup_camera_views_dock()self._setup_3d_viewport()logger.info('launching QT Visualizer and GUI window')+self._main_window_widget.show()def update_camera_view_image(self, webcam_id, image_to_display):@@ -72,7 +76,6 @@ class QTVisualizerAndGui:raise ecamera_image_item_widget.setImage(cv2.cvtColor(image_to_display, cv2.COLOR_BGR2RGB))-def _setup_main_window(self, window_width: int = 1000, window_height: int = 1000):"""This is the main main window for the GUI.@@ -166,13 +169,12 @@ class QTVisualizerAndGui:# self.opengl_3d_plot_widget.opts['azimuth'] = 0# self.opengl_3d_plot_widget.opts['elevation'] = 0- self.opengl_grid_item = gl.GLGridItem()- self.opengl_3d_plot_widget.addItem(self.opengl_grid_item)+ self.create_grid_planes()# create XYZ axes- x_axis_line_array = np.array([[0, 0, 0], [1, 0, 0]])- y_axis_line_array = np.array([[0, 0, 0], [0, 1, 0]])- z_axis_line_array = np.array([[0, 0, 0], [0, 0, 1]])+ x_axis_line_array = np.array([[0, 0, 0], [100, 0, 0]])+ y_axis_line_array = np.array([[0, 0, 0], [0, 100, 0]])+ z_axis_line_array = np.array([[0, 0, 0], [0, 0, 100]])self.origin_x_axis_gl_lineplot_item = pg.opengl.GLLinePlotItem(pos=x_axis_line_array,color=(1, 0, 0, 1),@@ -201,6 +203,29 @@ class QTVisualizerAndGui:self.opengl_3d_plot_dock.addWidget(self.opengl_3d_plot_widget)self._main_dock_area.addDock(self.opengl_3d_plot_dock, 'bottom', self._camera_views_dock)+ def create_grid_planes(self):+ grid_scale = 2e3+ # create the background grids+ grid_plane_x = gl.GLGridItem()+ grid_plane_x.setSize(grid_scale, grid_scale, grid_scale)+ grid_plane_x.setSpacing(grid_scale / 10, grid_scale / 10, grid_scale / 10)+ grid_plane_x.rotate(90, 0, 1, 0)+ grid_plane_x.translate(-grid_scale, 0, 0)+ self.opengl_3d_plot_widget.addItem(grid_plane_x)++ grid_plane_y = gl.GLGridItem()+ grid_plane_y.setSize(grid_scale, grid_scale, grid_scale)+ grid_plane_y.setSpacing(grid_scale / 10, grid_scale / 10, grid_scale / 10)+ grid_plane_y.rotate(90, 1, 0, 0)+ grid_plane_y.translate(0, -grid_scale, 0)+ self.opengl_3d_plot_widget.addItem(grid_plane_y)++ grid_plane_z = gl.GLGridItem()+ grid_plane_z.setSize(grid_scale, grid_scale, grid_scale)+ grid_plane_z.setSpacing(grid_scale / 10, grid_scale / 10, grid_scale / 10)+ grid_plane_z.translate(0, 0, -grid_scale)+ self.opengl_3d_plot_widget.addItem(grid_plane_z)+def initialize_charuco_dottos(self, number_of_charuco_corners: int):dummy_charuco_points = np.zeros((number_of_charuco_corners, 3))self._charuco_scatter_item = gl.GLScatterPlotItem(@@ -220,9 +245,45 @@ class QTVisualizerAndGui:pos=charuco_frame_payload.data3d_trackedPointNum_xyz)+ def get_mediapipe_connections(self):+ self.mediapipe_body_connections = [this_connection for this_connection in mp_holistic.POSE_CONNECTIONS]+ self.mediapipe_hand_connections = [this_connection for this_connection in mp_holistic.HAND_CONNECTIONS]+ self.mediapipe_face_connections = [this_connection for this_connection in mp_holistic.FACEMESH_TESSELATION]++ def initialize_skel_dottos(self, mediapipe_trackedPoint_xyz: np.ndarray):+ self.skeleton_scatter_item = gl.GLScatterPlotItem(+ pos=mediapipe_trackedPoint_xyz, color=(0, 1, 1, 1), size=10,+ pxMode=False+ )+ self.opengl_3d_plot_widget.addItem(self.skeleton_scatter_item)++ def initialize_skel_lines(self, mediapipe_trackedPoint_xyz: np.ndarray):+ self._skeleton_connections_list = []+ for this_connection in self.mediapipe_body_connections:+ this_skel_line = gl.GLLinePlotItem(+ pos=mediapipe_trackedPoint_xyz[this_connection, :])+ self._skeleton_connections_list.append(this_skel_line)+ self.opengl_3d_plot_widget.addItem(this_skel_line)++ def update_medaiapipe3d_skeleton(self, mediapipe3d_multi_frame_payload):+ mediapipe3d_trackedPoint_xyz = mediapipe3d_multi_frame_payload.data3d_trackedPointNum_xyz+ if not self._mediapipe_skeleton_initialized:+ self.get_mediapipe_connections()+ self.initialize_skel_dottos(mediapipe3d_trackedPoint_xyz)+ self.initialize_skel_lines(mediapipe3d_trackedPoint_xyz)+ self._mediapipe_skeleton_initialized = True++ # skel dottos+ self.skeleton_scatter_item.setData(+ pos=mediapipe3d_trackedPoint_xyz+ )++ #skel lines+ for this_skeleton_line_number, this_connection in enumerate(self.mediapipe_body_connections):+ self._skeleton_connections_list[this_skeleton_line_number].setData(+ pos=mediapipe3d_trackedPoint_xyz[this_connection, :]+ )- def update_medaiapipe3d_skel(self, mediapipe3d_multi_frame_payload):- passif __name__ == "__main__":