Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

something of mediapipe is making it in, but the geometry is borked

Jondd52678cd8f5
diff --git a/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py b/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py
index dcc773b1..acd103a0 100644
--- a/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py
+++ b/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py
@@ -279,10 +279,8 @@ class QtGlLaserSkeletonVisualizer():
if self.current_frame_number % 10 == 0:
print(f'frame number: {self.current_frame_number}')
- self.skeleton_scatter_item.setData(
- pos=self.mediapipe_fr_mar_xyz[self.current_frame_number, :, :]
- )
- self.update_skeleton_lines()
+
+ self.update_skeleton()
if self.session_data is not None:
self.update_head_axis_lines()
self.update_eye_axis_lines()
@@ -290,7 +288,13 @@ class QtGlLaserSkeletonVisualizer():
self.update_gaze_lasers()
self.update_gaze_laser_tails()
- def update_skeleton_lines(self):
+ def update_skeleton(self):
+ # skel dottos
+ self.skeleton_scatter_item.setData(
+ pos=self.mediapipe_fr_mar_xyz[self.current_frame_number, :, :]
+ )
+
+ #skel lines
for this_skeleton_line_number, this_connection in enumerate(self.mediapipe_body_connections):
self.skeleton_connections_list[this_skeleton_line_number].setData(
pos=self.mediapipe_fr_mar_xyz[self.current_frame_number, this_connection, :]
diff --git a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
index d16a0cc9..7283dcda 100644
--- a/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
+++ b/src/core_processor/mediapipe_skeleton_detector/mediapipe_skeleton_detector.py
@@ -28,13 +28,22 @@ class Mediapipe2dNumpyArrays:
body2d_frameNumber_trackedPointNumber_confidence: np.ndarray = None
+ @property
+ def has_data(self):
+ return not np.isnan(self.body2d_frameNumber_trackedPointNumber_XY).all()
+
@property
def all_data2d_nFrames_nTrackedPts_XY(self):
"""dimensions will be [number_of_frames , number_of_markers, XY]"""
- return np.hstack([self.body2d_frameNumber_trackedPointNumber_XY,
- self.rightHand2d_frameNumber_trackedPointNumber_XY,
- self.leftHand2d_frameNumber_trackedPointNumber_XY,
- self.face2d_frameNumber_trackedPointNumber_XY])
+
+ if self.body2d_frameNumber_trackedPointNumber_XY is None:
+ # if there's no body data, there's no hand or face data either
+ return
+
+ return np.squeeze(np.hstack([self.body2d_frameNumber_trackedPointNumber_XY,
+ self.rightHand2d_frameNumber_trackedPointNumber_XY,
+ self.leftHand2d_frameNumber_trackedPointNumber_XY,
+ self.face2d_frameNumber_trackedPointNumber_XY]))
@dataclass
diff --git a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
index cdf765bc..d20a0226 100644
--- a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
+++ b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py
@@ -185,7 +185,7 @@ class SessionPipelineOrchestrator:
mediapipe3d_multi_frame_payload = self._reconstruct_3d_mediapipe(
dictionary_of_mediapipe_payloads_on_this_multi_frame)
if show_visualizer_gui and mediapipe3d_multi_frame_payload is not None:
- self._visualizer_gui.update_medaiapipe3d_skel(mediapipe3d_multi_frame_payload)
+ self._visualizer_gui.update_medaiapipe3d_skeleton(mediapipe3d_multi_frame_payload)
dictionary_of_mediapipe_payloads_on_this_multi_frame = {}
timestamp_manager.log_new_timestamp_for_main_loop_ns(time.perf_counter_ns())
@@ -341,18 +341,17 @@ class SessionPipelineOrchestrator:
def _reconstruct_3d_mediapipe(self, dictionary_of_mediapipe_payloads_on_this_multi_frame) -> \
Union[None, Data3dMultiFramePayload]:
-
- return
-
is_there_any_mediapipe_data = []
for this_cam_medipipe_data in dictionary_of_mediapipe_payloads_on_this_multi_frame.values():
is_there_any_mediapipe_data.append(
- this_cam_medipipe_data.pixel_data_numpy_arrays.all_data2d_nFrames_nTrackedPts_XY is not None)
+ this_cam_medipipe_data.pixel_data_numpy_arrays.has_data)
min_cameras_to_reconstruct = 2
if sum(is_there_any_mediapipe_data) >= min_cameras_to_reconstruct: # if at least 2 cameras have any data, then we can try to make some 3d dottos
- mediapipe2d_data_per_cam_dict = self._format_charuco2d_data(
- dictionary_of_mediapipe_payloads_on_this_multi_frame)
+ mediapipe2d_data_per_cam_dict = {}
+ for this_webcam_id, this_mediapipe_payload in dictionary_of_mediapipe_payloads_on_this_multi_frame.items():
+ mediapipe2d_data_per_cam_dict[this_webcam_id] = this_mediapipe_payload.pixel_data_numpy_arrays.all_data2d_nFrames_nTrackedPts_XY
+
mediapipe_3d_data_payload = self._triangulate_2d_data(mediapipe2d_data_per_cam_dict,
self._mediapipe_skeleton_detector.number_of_tracked_points_total)
diff --git a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
index 62fb0381..769c32d9 100644
--- a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
+++ b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
@@ -9,6 +9,7 @@ import pyqtgraph.opengl as gl
from pyqtgraph.dockarea.Dock import Dock
from pyqtgraph.dockarea.DockArea import DockArea
from pyqtgraph.Qt import QtWidgets
+from mediapipe.python.solutions import holistic as mp_holistic
from src.core_processor.timestamp_manager.timestamp_manager import TimestampManager
from src.pipelines.session_pipeline.data_classes.data_3d_single_frame_payload import Data3dMultiFramePayload
@@ -19,6 +20,8 @@ logger = logging.getLogger(__name__)
class QTVisualizerAndGui:
def __init__(self):
# https://pyqtgraph.readthedocs.io/en/latest/config_options.html
+ self._skeleton_connections_list = None
+ self._mediapipe_skeleton_initialized = False
self.pyqtgraph_app = pg.mkQApp('freemocap! :D')
pg.setConfigOptions(imageAxisOrder='row-major')
@@ -59,6 +62,7 @@ class QTVisualizerAndGui:
self._setup_camera_views_dock()
self._setup_3d_viewport()
logger.info('launching QT Visualizer and GUI window')
+
self._main_window_widget.show()
def update_camera_view_image(self, webcam_id, image_to_display):
@@ -72,7 +76,6 @@ class QTVisualizerAndGui:
raise e
camera_image_item_widget.setImage(cv2.cvtColor(image_to_display, cv2.COLOR_BGR2RGB))
-
def _setup_main_window(self, window_width: int = 1000, window_height: int = 1000):
"""
This is the main main window for the GUI.
@@ -166,13 +169,12 @@ class QTVisualizerAndGui:
# self.opengl_3d_plot_widget.opts['azimuth'] = 0
# self.opengl_3d_plot_widget.opts['elevation'] = 0
- self.opengl_grid_item = gl.GLGridItem()
- self.opengl_3d_plot_widget.addItem(self.opengl_grid_item)
+ self.create_grid_planes()
# create XYZ axes
- x_axis_line_array = np.array([[0, 0, 0], [1, 0, 0]])
- y_axis_line_array = np.array([[0, 0, 0], [0, 1, 0]])
- z_axis_line_array = np.array([[0, 0, 0], [0, 0, 1]])
+ x_axis_line_array = np.array([[0, 0, 0], [100, 0, 0]])
+ y_axis_line_array = np.array([[0, 0, 0], [0, 100, 0]])
+ z_axis_line_array = np.array([[0, 0, 0], [0, 0, 100]])
self.origin_x_axis_gl_lineplot_item = pg.opengl.GLLinePlotItem(pos=x_axis_line_array,
color=(1, 0, 0, 1),
@@ -201,6 +203,29 @@ class QTVisualizerAndGui:
self.opengl_3d_plot_dock.addWidget(self.opengl_3d_plot_widget)
self._main_dock_area.addDock(self.opengl_3d_plot_dock, 'bottom', self._camera_views_dock)
+ def create_grid_planes(self):
+ grid_scale = 2e3
+ # create the background grids
+ grid_plane_x = gl.GLGridItem()
+ grid_plane_x.setSize(grid_scale, grid_scale, grid_scale)
+ grid_plane_x.setSpacing(grid_scale / 10, grid_scale / 10, grid_scale / 10)
+ grid_plane_x.rotate(90, 0, 1, 0)
+ grid_plane_x.translate(-grid_scale, 0, 0)
+ self.opengl_3d_plot_widget.addItem(grid_plane_x)
+
+ grid_plane_y = gl.GLGridItem()
+ grid_plane_y.setSize(grid_scale, grid_scale, grid_scale)
+ grid_plane_y.setSpacing(grid_scale / 10, grid_scale / 10, grid_scale / 10)
+ grid_plane_y.rotate(90, 1, 0, 0)
+ grid_plane_y.translate(0, -grid_scale, 0)
+ self.opengl_3d_plot_widget.addItem(grid_plane_y)
+
+ grid_plane_z = gl.GLGridItem()
+ grid_plane_z.setSize(grid_scale, grid_scale, grid_scale)
+ grid_plane_z.setSpacing(grid_scale / 10, grid_scale / 10, grid_scale / 10)
+ grid_plane_z.translate(0, 0, -grid_scale)
+ self.opengl_3d_plot_widget.addItem(grid_plane_z)
+
def initialize_charuco_dottos(self, number_of_charuco_corners: int):
dummy_charuco_points = np.zeros((number_of_charuco_corners, 3))
self._charuco_scatter_item = gl.GLScatterPlotItem(
@@ -220,9 +245,45 @@ class QTVisualizerAndGui:
pos=charuco_frame_payload.data3d_trackedPointNum_xyz
)
+ def get_mediapipe_connections(self):
+ self.mediapipe_body_connections = [this_connection for this_connection in mp_holistic.POSE_CONNECTIONS]
+ self.mediapipe_hand_connections = [this_connection for this_connection in mp_holistic.HAND_CONNECTIONS]
+ self.mediapipe_face_connections = [this_connection for this_connection in mp_holistic.FACEMESH_TESSELATION]
+
+ def initialize_skel_dottos(self, mediapipe_trackedPoint_xyz: np.ndarray):
+ self.skeleton_scatter_item = gl.GLScatterPlotItem(
+ pos=mediapipe_trackedPoint_xyz, color=(0, 1, 1, 1), size=10,
+ pxMode=False
+ )
+ self.opengl_3d_plot_widget.addItem(self.skeleton_scatter_item)
+
+ def initialize_skel_lines(self, mediapipe_trackedPoint_xyz: np.ndarray):
+ self._skeleton_connections_list = []
+ for this_connection in self.mediapipe_body_connections:
+ this_skel_line = gl.GLLinePlotItem(
+ pos=mediapipe_trackedPoint_xyz[this_connection, :])
+ self._skeleton_connections_list.append(this_skel_line)
+ self.opengl_3d_plot_widget.addItem(this_skel_line)
+
+ def update_medaiapipe3d_skeleton(self, mediapipe3d_multi_frame_payload):
+ mediapipe3d_trackedPoint_xyz = mediapipe3d_multi_frame_payload.data3d_trackedPointNum_xyz
+ if not self._mediapipe_skeleton_initialized:
+ self.get_mediapipe_connections()
+ self.initialize_skel_dottos(mediapipe3d_trackedPoint_xyz)
+ self.initialize_skel_lines(mediapipe3d_trackedPoint_xyz)
+ self._mediapipe_skeleton_initialized = True
+
+ # skel dottos
+ self.skeleton_scatter_item.setData(
+ pos=mediapipe3d_trackedPoint_xyz
+ )
+
+ #skel lines
+ for this_skeleton_line_number, this_connection in enumerate(self.mediapipe_body_connections):
+ self._skeleton_connections_list[this_skeleton_line_number].setData(
+ pos=mediapipe3d_trackedPoint_xyz[this_connection, :]
+ )
- def update_medaiapipe3d_skel(self, mediapipe3d_multi_frame_payload):
- pass
if __name__ == "__main__":