Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

qt_gl - plotting skeleton with head axes (not working, but moving in the right direction)

Jonb4ba55a07615
diff --git a/jon_scratch/pupil_calibration_pipeline/data_classes/freemocap_session_data_class.py b/jon_scratch/pupil_calibration_pipeline/data_classes/freemocap_session_data_class.py
index b8da2fca..ecfd0c62 100644
--- a/jon_scratch/pupil_calibration_pipeline/data_classes/freemocap_session_data_class.py
+++ b/jon_scratch/pupil_calibration_pipeline/data_classes/freemocap_session_data_class.py
@@ -4,14 +4,18 @@ from typing import List
import numpy as np
from jon_scratch.pupil_calibration_pipeline.data_classes.pupil_dataclass_and_handler import PupilData
+from jon_scratch.pupil_calibration_pipeline.data_classes.rotation_data_class import RotationDataClass
@dataclass
class FreemocapSessionDataClass:
session_id: str = None
+ timestamps: np.ndarray = None
mediapipe_skel_fr_mar_dim: np.ndarray = None
right_eye_data: PupilData = None
left_eye_data: PupilData = None
- head_rotation_matricies: List = None
- timestamps: np.ndarray = None
+ head_rotation_data: RotationDataClass = None
+ right_eye_socket_rotation_matricies: List = None
+ left_eye_socket_rotation_matricies: List = None
+
diff --git a/jon_scratch/pupil_calibration_pipeline/data_classes/rotation_data_class.py b/jon_scratch/pupil_calibration_pipeline/data_classes/rotation_data_class.py
new file mode 100644
index 00000000..16030066
--- /dev/null
+++ b/jon_scratch/pupil_calibration_pipeline/data_classes/rotation_data_class.py
@@ -0,0 +1,10 @@
+from dataclasses import dataclass
+from typing import List
+
+import numpy as np
+
+
+@dataclass
+class RotationDataClass:
+ rotation_matricies: List[np.ndarray]
+ local_origin_fr_xyz: np.ndarray = np.array([0, 0, 0])
\ No newline at end of file
diff --git a/jon_scratch/pupil_calibration_pipeline/head_rotation_calculator.py b/jon_scratch/pupil_calibration_pipeline/head_rotation_calculator.py
index f6088fcc..dba76ecb 100644
--- a/jon_scratch/pupil_calibration_pipeline/head_rotation_calculator.py
+++ b/jon_scratch/pupil_calibration_pipeline/head_rotation_calculator.py
@@ -7,6 +7,8 @@ import matplotlib.pyplot as plt
import matplotlib
import keyboard
+from jon_scratch.pupil_calibration_pipeline.data_classes.rotation_data_class import RotationDataClass
+
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
@@ -21,7 +23,7 @@ def normalize_length(point_xyz):
return point_xyz / np.linalg.norm(point_xyz, axis=1)[:, None]
-class HeadRotationCalculator:
+class RotationMatrixCalculator:
mediapipe_skeleton_fr_mar_xyz: np.ndarray
def __init__(self, mediapipe_skeleton_fr_mar_xyz):
@@ -51,19 +53,19 @@ class HeadRotationCalculator:
left_ear_xyz = self.mediapipe_skeleton_fr_mar_xyz[:, left_ear_id, :]
head_center_xyz = (right_ear_xyz + left_ear_xyz) / 2
- head_rotation_matricies = self.define_orthonormal_basis_vectors(center_point_fr_xyz=head_center_xyz,
+ head_rotation_data= self.define_orthonormal_basis_vectors(center_point_fr_xyz=head_center_xyz,
x_direction_fr_xyz=nose_xyz,
y_direction_fr_xyz=left_ear_xyz)
if debug:
- self.show_head_rotation_debug_plot(head_rotation_matricies)
+ self.show_head_rotation_debug_plot(head_rotation_data.rotation_matricies)
- return head_rotation_matricies
+ return head_rotation_data
def define_orthonormal_basis_vectors(self,
center_point_fr_xyz: np.ndarray = None,
x_direction_fr_xyz: np.ndarray = None,
y_direction_fr_xyz: np.ndarray = None,
- z_direction_fr_xyz: np.ndarray = None) -> List[np.ndarray]:
+ z_direction_fr_xyz: np.ndarray = None) -> RotationDataClass:
"""
create orthonormal basis vectors based on a center point and at least TWO of the following = a point defining X direction, a point defining Y direction, and a point defining Z direction
"""
@@ -124,9 +126,10 @@ class HeadRotationCalculator:
rotation_matricies = [this_rotation_martix.T for this_rotation_martix in rotation_matricies]
- return rotation_matricies
+ return RotationDataClass(rotation_matricies=rotation_matricies,
+ local_origin_fr_xyz=center_point_fr_xyz)
- def create_unrotated_gaze_laser(self, debug, eye: str) -> np.ndarray:
+ def calculate_eye_rotation_matricies(self, debug, eye: str) -> List[np.ndarray]:
nose_index = 0
if eye == 'left':
inner_eye_index = 1
@@ -145,31 +148,25 @@ class HeadRotationCalculator:
eye_mid_fr_xyz = self.mediapipe_skeleton_fr_mar_xyz[:, eye_mid_index, :]
- eye_socket_rotation_matricies = self.define_orthonormal_basis_vectors(center_point_fr_xyz=eye_mid_fr_xyz,
- x_direction_fr_xyz=x_direction_fr_xyz,
- y_direction_fr_xyz=y_direction_fr_xyz)
-
- initial_eye_laser_unit_vector = np.array([0, 0, 1])
- eye_laser_unit_vectors_fr_xyz = [this_eye_socket_rotation_matrix.T @ initial_eye_laser_unit_vector for this_eye_socket_rotation_matrix in eye_socket_rotation_matricies]
-
- eye_laser_unit_vectors_fr_xyz *= 1000 #make it a meter long
- eye_laser_unit_vectors_fr_xyz += eye_mid_fr_xyz #situation gaze lasers onto eye_mid_fr_xyz
+ eye_socket_rotation_data = self.define_orthonormal_basis_vectors(center_point_fr_xyz=eye_mid_fr_xyz,
+ x_direction_fr_xyz=x_direction_fr_xyz,
+ y_direction_fr_xyz=y_direction_fr_xyz)
- return eye_laser_unit_vectors_fr_xyz # this is a unit vector pointing outward from eyeball center (as if eyes were paralyzed in center of their orbits)
+ return eye_socket_rotation_data
- def show_head_rotation_debug_plot(self, head_rotation_matricies):
+ def show_head_rotation_debug_plot(self, rotation_matricies):
- x_hat_xyz = np.asarray([this_rot_mat[0, :] for this_rot_mat in head_rotation_matricies])
- y_hat_xyz = np.asarray([this_rot_mat[1, :] for this_rot_mat in head_rotation_matricies])
- z_hat_xyz = np.asarray([this_rot_mat[2, :] for this_rot_mat in head_rotation_matricies])
+ x_hat_xyz = np.asarray([this_rot_mat[0, :] for this_rot_mat in rotation_matricies])
+ y_hat_xyz = np.asarray([this_rot_mat[1, :] for this_rot_mat in rotation_matricies])
+ z_hat_xyz = np.asarray([this_rot_mat[2, :] for this_rot_mat in rotation_matricies])
test_point_xyz_og = np.array((1, 1, 1))
test_point_xyz_rot = np.empty(z_hat_xyz.shape)
logger.warning('We shouldnt have to transpose the rotation matix here, need to fix this upstream')
- for frame_number in range(len(head_rotation_matricies)):
+ for frame_number in range(len(rotation_matricies)):
# see https://peps.python.org/pep-0465/ for explanation of the `@` operator (which is a "matrix multiplication" operator)
- test_point_xyz_rot[frame_number, :] = head_rotation_matricies[frame_number].T @ test_point_xyz_og
+ test_point_xyz_rot[frame_number, :] = rotation_matricies[frame_number].T @ test_point_xyz_og
plt.close('all')
diff --git a/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_calibration_pipeline_orchestrator.py b/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_calibration_pipeline_orchestrator.py
index f9c88dad..3ec93ebb 100644
--- a/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_calibration_pipeline_orchestrator.py
+++ b/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_calibration_pipeline_orchestrator.py
@@ -4,7 +4,7 @@ from typing import Union
import numpy as np
from rich import inspect
from jon_scratch.pupil_calibration_pipeline.data_classes.freemocap_session_data_class import FreemocapSessionDataClass
-from jon_scratch.pupil_calibration_pipeline.head_rotation_calculator import HeadRotationCalculator
+from jon_scratch.pupil_calibration_pipeline.head_rotation_calculator import RotationMatrixCalculator
from jon_scratch.pupil_calibration_pipeline.pupil_freemocap_synchronizer import PupilFreemocapSynchronizer
from jon_scratch.pupil_calibration_pipeline.qt_gl_laser_skeleton_visualizer import QtGlLaserSkeletonVisualizer
from jon_scratch.pupil_calibration_pipeline.session_data_loader import SessionDataLoader
@@ -18,11 +18,11 @@ class PupilFreemocapCalibrationPipelineOrchestrator:
session_data_loader: SessionDataLoader = None
raw_session_data = FreemocapSessionDataClass()
synchronized_session_data: FreemocapSessionDataClass = None
- laser_skeleton_visualizer = QtGlLaserSkeletonVisualizer()
vor_frame_start: int = None,
vor_frame_end: int = None
- def __init__(self, session_path: Union[Path, str], debug: bool = False, vor_frame_start: int = None, vor_frame_end: int = None):
+ def __init__(self, session_path: Union[Path, str], debug: bool = False, vor_frame_start: int = None,
+ vor_frame_end: int = None):
logger.info(f'initializing pupil-freemocap calibration pipeline orchestrator for session: {session_path}')
self.session_path = session_path
self.session_id = self.session_path.stem
@@ -49,19 +49,6 @@ class PupilFreemocapCalibrationPipelineOrchestrator:
logger.info(
f'self.raw_session_data.mediapipe_skel_fr_mar_dim.shape: {self.raw_session_data.mediapipe_skel_fr_mar_dim.shape}')
- ####
- # Calculate Head Rotation matrix for each frame (gaze data will be rotated by head_rot, then calibrated_offset_rot)
- ####
- head_rotation_calculator = HeadRotationCalculator(self.raw_session_data.mediapipe_skel_fr_mar_dim)
- head_rotation_matricies = head_rotation_calculator.calculate_head_rotation_matricies(debug=True)
- self.unrotated_right_gaze_laser_fr_xyz = head_rotation_calculator.create_unrotated_gaze_laser(debug=True, eye='right')
- self.unrotated_left_gaze_laser_fr_xyz = head_rotation_calculator.create_unrotated_gaze_laser(debug=True, eye='left')
-
- self.raw_session_data.head_rotation_matricies = head_rotation_matricies
- self.save_head_rotation_matricies()
- logger.info(
- f'len(self.raw_session_data.head_rotation_matricies): {len(self.raw_session_data.head_rotation_matricies)}')
-
####
# load pupil data
####
@@ -72,21 +59,55 @@ class PupilFreemocapCalibrationPipelineOrchestrator:
####
# Synchronize pupil data with freemocap data - results in self.synchronized_session_data (each stream has exactly the same number of frames)
####
- self.synchronized_session_data = PupilFreemocapSynchronizer(self.raw_session_data).synchronize(debug=False, vor_frame_start=self.vor_frame_start, vor_frame_end=self.vor_frame_end)
- logger.info('synchronization complete - I should add a test to make sure everything has the same number of frames')
+ self.synchronized_session_data = PupilFreemocapSynchronizer(self.raw_session_data).synchronize(debug=False,
+ vor_frame_start=self.vor_frame_start,
+ vor_frame_end=self.vor_frame_end)
+ logger.info(
+ 'synchronization complete - I should add a test to make sure everything has the same number of frames')
+
+ ####
+ # Calculate Head Rotation matrix for each frame (gaze data will be rotated by head_rot, then calibrated_offset_rot)
+ ####
+ rotation_matrix_calculator = RotationMatrixCalculator(self.raw_session_data.mediapipe_skel_fr_mar_dim)
+
+ self.synchronized_session_data.head_rotation_data = rotation_matrix_calculator.calculate_head_rotation_matricies(
+ debug=False)
+
+ self.synchronized_session_data.right_eye_socket_rotation_data= rotation_matrix_calculator.calculate_eye_rotation_matricies(
+ debug=False,
+ eye='right')
+
+ self.synchronized_session_data.left_eye_socket_rotation_data = rotation_matrix_calculator.calculate_eye_rotation_matricies(
+ debug=False,
+ eye='left')
+
+ logger.info(
+ f'len(self.synchronized_session_data.head_rotation_data.head_rotation_matricies): {len(self.synchronized_session_data.head_rotation_data.rotation_matricies)}')
####
# Perform Vestibular-Ocular-Reflex based calibration (see methods from (Matthis et al, 2018 and 2022) for deetos)
####
- vor_calibrator = VorCalibrator(self.synchronized_session_data, vor_start_frame=self.vor_frame_start, vor_end_frame=self.vor_frame_end)
- fixation_point_fr_xyz = self.synchronized_session_data.mediapipe_skel_fr_mar_dim[self.vor_frame_start:self.vor_frame_end, :, :3]
+ vor_calibrator = VorCalibrator(self.synchronized_session_data, vor_start_frame=self.vor_frame_start,
+ vor_end_frame=self.vor_frame_end)
+ right_index_fingertip_idx = 41 # pretty sure this is right?
+ fixation_point_fr_xyz = self.synchronized_session_data.mediapipe_skel_fr_mar_dim[
+ self.vor_frame_start:self.vor_frame_end, right_index_fingertip_idx, :3]
vor_calibrator.calibrate(fixation_point_fr_xyz)
+ ####
+ # Play laser skeleton animation (as both a cool thing and a debug tool)
+ ####
+
+ qt_gl_laser_skeleton = QtGlLaserSkeletonVisualizer(session_data=self.synchronized_session_data,
+ move_data_to_origin=True,
+ start_frame=self.vor_frame_start,
+ end_frame=self.vor_frame_end)
+ qt_gl_laser_skeleton.start_animation()
+
############################################
############################################
### helper methods
############################################
- ###########################################
def save_head_rotation_matricies(self):
save_path = self.session_path / 'DataArrays' / 'mediaPipeSkel_3d_head_rotation_matricies_fr_row_col.npy'
logger.info(f'saving head rotation matricies to: {save_path}')
diff --git a/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.py b/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.py
index db6f445c..b4bef7a6 100644
--- a/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.py
+++ b/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.py
@@ -110,8 +110,6 @@ class PupilFreemocapSynchronizer:
mediapipe_skel_fr_mar_dim=self.raw_session_data.mediapipe_skel_fr_mar_dim[
freemocap_start_frame:freemocap_end_frame,
:, :],
- head_rotation_matricies=self.raw_session_data.head_rotation_matricies[
- freemocap_start_frame:freemocap_end_frame],
right_eye_data=synchronized_right_eye_data,
left_eye_data=synchronized_left_eye_data)
return synchronized_session_data
diff --git a/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py b/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py
index d35065d0..4c304e93 100644
--- a/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py
+++ b/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py
@@ -1,6 +1,7 @@
# adapted from - https://gist.github.com/markjay4k/da2f55e28514be7160a7c5fbf95bd243
import logging
from pathlib import Path
+from typing import Union, List
from pyqtgraph.Qt import QtCore, QtGui
import pyqtgraph.opengl as gl
@@ -10,6 +11,7 @@ import sys
from mediapipe.python.solutions import holistic as mp_holistic
from jon_scratch.pupil_calibration_pipeline.data_classes.freemocap_session_data_class import FreemocapSessionDataClass
+from jon_scratch.pupil_calibration_pipeline.data_classes.rotation_data_class import RotationDataClass
from jon_scratch.pupil_calibration_pipeline.session_data_loader import SessionDataLoader
logger = logging.getLogger(__name__)
@@ -18,17 +20,33 @@ logger = logging.getLogger(__name__)
class QtGlLaserSkeletonVisualizer():
session_path: Path = None
- def __init__(self, session_data: FreemocapSessionDataClass = None, mediapipe_skel_fr_mar_xyz: np.ndarray = None):
+ def __init__(self,
+ session_data: FreemocapSessionDataClass = None,
+ mediapipe_skel_fr_mar_xyz: np.ndarray = None,
+ start_frame: int = None,
+ end_frame: int = None,
+ move_data_to_origin: bool = True):
+
if session_data is not None:
+ self.session_data = session_data
self.mediapipe_fr_mar_xyz = session_data.mediapipe_skel_fr_mar_dim
elif mediapipe_skel_fr_mar_xyz is not None:
self.mediapipe_fr_mar_xyz = mediapipe_skel_fr_mar_xyz
else:
raise ValueError("Must provide either session data or mediapipe skeleton data")
- self.traces = dict()
- self.start_frame_number = 1000
- self.end_frame_number = 2000
+ if move_data_to_origin:
+ self.move_data_to_origin()
+
+ if start_frame is None:
+ self.start_frame_number = 0
+ else:
+ self.start_frame_number = start_frame
+ if end_frame is None:
+ self.end_frame_number = self.mediapipe_fr_mar_xyz.shape[0]
+ else:
+ self.end_frame_number = end_frame
+
self.current_frame_number = self.start_frame_number
def initialize_display_window(self):
@@ -37,7 +55,7 @@ class QtGlLaserSkeletonVisualizer():
self.get_mediapipe_connections()
self.initialize_skel_dottos()
self.initialize_skel_lines()
- # self.initialize_gaze_laser()
+ self.initialize_head_axes()
def create_app_window(self):
self.app = pg.mkQApp("Laser Skeleton")
@@ -90,13 +108,81 @@ class QtGlLaserSkeletonVisualizer():
self.skeleton_connections_list.append(this_skel_line)
self.gl_view_widget.addItem(this_skel_line)
- def initialize_gaze_laser(self):
+ def initialize_head_axes(self):
+ self.head_axes_scale = 1e3
+
+ first_frame_head_center_xyz = self.session_data.head_rotation_data.local_origin_fr_xyz[0, :]
+
+ self.head_axes_x_vector_line_item = self.create_axis_vector_line_item(0,
+ self.session_data.head_rotation_data,
+ dimension='x',
+ scale=self.head_axes_scale,
+ color=(1, 0, 0, 1),
+ width=3)
+ self.head_axes_y_vector_line_item = self.create_axis_vector_line_item(0,
+ self.session_data.head_rotation_data,
+ dimension='y',
+ scale=self.head_axes_scale,
+ color=(0, 1, 0, 1),
+ width=3)
+ self.head_axes_z_vector_line_item = self.create_axis_vector_line_item(0,
+ self.session_data.head_rotation_data,
+ dimension='z',
+ scale=self.head_axes_scale,
+ color=(0, 0, 1, 1),
+ width=3)
+
+ self.gl_view_widget.addItem(self.head_axes_x_vector_line_item)
+ self.gl_view_widget.addItem(self.head_axes_y_vector_line_item)
+ self.gl_view_widget.addItem(self.head_axes_z_vector_line_item)
+
+ def create_axis_vector_line_item(self,
+ frame_number: int,
+ rotation_data: RotationDataClass,
+ dimension: Union[str, int] = None,
+ scale: Union[int, float] = 1,
+ color=(1, 1, 1, 1),
+ width=3,
+ ) -> np.ndarray:
+
+ this_frame_axis_vector_xyz = self.unit_vector_from_rotation_matrix(frame_number,
+ rotation_data,
+ dimension=dimension,
+ scale=scale)
+ this_frame_axis_vector_line_item = gl.GLLinePlotItem(pos=this_frame_axis_vector_xyz,
+ color=color,
+ width=width)
+ self.gl_view_widget.addItem(this_frame_axis_vector_line_item)
+ return this_frame_axis_vector_line_item
+
+ def unit_vector_from_rotation_matrix(self,
+ frame_number: int,
+ rotation_data: RotationDataClass,
+ dimension: Union[str, int] = None,
+ scale: Union[int, float] = 1,
+ ) -> np.ndarray:
+
+ try:
+ if dimension == 'x':
+ dimension = 0
+ elif dimension == 'y':
+ dimension = 1
+ elif dimension == 'z':
+ dimension = 2
- this_gaze_laser = np.vstack(
- (self.right_eye_xyz[self.current_frame_number, :], self.right_gaze_xyz[self.current_frame_number, :]))
- self.right_gaze_line_item = gl.GLLinePlotItem(pos=this_gaze_laser,
- color=(1, 0, 0, 1), )
- self.gl_view_widget.addItem(self.right_gaze_line_item)
+ if dimension > rotation_data.rotation_matricies[0].shape[1]:
+ raise ValueError(
+ 'dimension must be `x`, `y`, `z`, or an integer less than the number of dimensions in the rotation matrix')
+ except:
+ raise ValueError('something weird about the `dimension` argument')
+
+ this_rot_mat = rotation_data.rotation_matricies[frame_number]
+ this_axis_vector_endpoint_xyz = (this_rot_mat[dimension, :] * scale) + rotation_data.local_origin_fr_xyz[frame_number,:]
+ this_axis_vector_origin_xyz = rotation_data.local_origin_fr_xyz[frame_number,:]
+
+ this_axis_vector_line_xyz = np.array([this_axis_vector_origin_xyz, this_axis_vector_endpoint_xyz])
+
+ return this_axis_vector_line_xyz
def update(self):
self.update_frame_number()
@@ -107,6 +193,7 @@ class QtGlLaserSkeletonVisualizer():
pos=self.mediapipe_fr_mar_xyz[self.current_frame_number, :, :]
)
self.update_skeleton_lines()
+ self.update_head_axis_lines()
# this_gaze_laser = np.vstack(
# (self.right_eye_xyz[self.current_frame_number, :], self.right_gaze_xyz[self.current_frame_number, :]))
# self.right_gaze_line_item.setData(pos=this_gaze_laser)
@@ -117,12 +204,33 @@ class QtGlLaserSkeletonVisualizer():
pos=self.mediapipe_fr_mar_xyz[self.current_frame_number, this_connection, :]
)
+ def update_head_axis_lines(self):
+ # X
+ this_frame_head_axis_x_vector_xyz = self.unit_vector_from_rotation_matrix(self.current_frame_number,
+ self.session_data.head_rotation_data,
+ dimension='x',
+ scale=self.head_axes_scale)
+ self.head_axes_x_vector_line_item.setData(pos=this_frame_head_axis_x_vector_xyz)
+
+ # Y
+ this_frame_head_axis_y_vector_xyz = self.unit_vector_from_rotation_matrix(self.current_frame_number,
+ self.session_data.head_rotation_data,
+ dimension='y',
+ scale=self.head_axes_scale)
+ self.head_axes_y_vector_line_item.setData(pos=this_frame_head_axis_y_vector_xyz)
+
+ # Z
+ this_frame_head_axis_z_vector_xyz = self.unit_vector_from_rotation_matrix(self.current_frame_number,
+ self.session_data.head_rotation_data,
+ dimension='z',
+ scale=self.head_axes_scale)
+ self.head_axes_z_vector_line_item.setData(pos=this_frame_head_axis_z_vector_xyz)
+
def update_frame_number(self):
self.current_frame_number += 1
if self.current_frame_number >= self.end_frame_number:
self.current_frame_number = self.start_frame_number
-
def start(self):
print('starting animation')
pg.exec()
@@ -135,6 +243,21 @@ class QtGlLaserSkeletonVisualizer():
timer.start(33)
self.start()
+ def move_data_to_origin(self):
+ mean_position_xyz = np.nanmedian(np.nanmedian(self.mediapipe_fr_mar_xyz, axis=0), axis=0)
+
+ self.mediapipe_fr_mar_xyz[:, :, 0] -= mean_position_xyz[0]
+ self.mediapipe_fr_mar_xyz[:, :, 1] -= mean_position_xyz[1]
+ self.mediapipe_fr_mar_xyz[:, :, 2] -= mean_position_xyz[2]
+
+ self.session_data.head_rotation_data.local_origin_fr_xyz[:,0] -= mean_position_xyz[0]
+ self.session_data.head_rotation_data.local_origin_fr_xyz[:,1] -= mean_position_xyz[1]
+ self.session_data.head_rotation_data.local_origin_fr_xyz[:,2] -= mean_position_xyz[2]
+
+ def create_head_axes_vector_line_item(self, param, head_rotation_matricies, dimension, scale):
+ pass
+
+
if __name__ == '__main__':
# session_id = 'sesh_2022-02-15_11_54_28_pupil_maybe'
@@ -145,5 +268,5 @@ if __name__ == '__main__':
session_data_loader = SessionDataLoader(session_path)
mediapipe_skel_fr_mar_xyz_in = session_data_loader.load_mediapipe_data(move_to_origin=True)
print(f'mediapipe_skel_fr_mar_xyz.shape: {mediapipe_skel_fr_mar_xyz_in.shape}')
- qt_gl_laser_skeleton = QtGlLaserSkeletonVisualizer(mediapipe_skel_fr_mar_xyz = mediapipe_skel_fr_mar_xyz_in,)
+ qt_gl_laser_skeleton = QtGlLaserSkeletonVisualizer(mediapipe_skel_fr_mar_xyz=mediapipe_skel_fr_mar_xyz_in, )
qt_gl_laser_skeleton.start_animation()
diff --git a/jon_scratch/pupil_calibration_pipeline/vor_calibrator.py b/jon_scratch/pupil_calibration_pipeline/vor_calibrator.py
index 1554a4f8..d812d41b 100644
--- a/jon_scratch/pupil_calibration_pipeline/vor_calibrator.py
+++ b/jon_scratch/pupil_calibration_pipeline/vor_calibrator.py
@@ -27,17 +27,16 @@ class VorCalibrator:
if not fixation_point_fr_xyz.shape[1] == 3:
raise Exception("fixation_point_fr_xyz must be a numpy array with 3 columns")
- if not fixation_point_fr_xyz.shape[0] == self.skel_fr_mar_dim.shape[0]:
+ if not fixation_point_fr_xyz.shape[0] == self.mediapipe_skel_fr_mar_dim.shape[0]:
raise Exception("fixation_point_xyz must have precisely the same number of rows as the skeleton/gaze data")
- self.create_unrotated_gaze_vector()
self.calculate_optimal_rotational_offset()
self.create_skeleton_gaze_lasers()
return self.session_data
def clip_out_vor_frames(self, vor_start_frame, vor_end_frame):
- self.skel_fr_mar_dim = self.session_data.skel_fr_mar_dim[vor_start_frame:vor_end_frame,:,:]
- self.head_rot_matricies = self.session_data.head_rotation_matricies[vor_start_frame:vor_end_frame]
+ self.mediapipe_skel_fr_mar_dim = self.session_data.mediapipe_skel_fr_mar_dim[vor_start_frame:vor_end_frame,:,:]
+ self.head_rotation_matricies = self.session_data.head_rotation_data.rotation_matricies[vor_start_frame:vor_end_frame]
self.right_eye_theta = self.session_data.right_eye_data.theta[vor_start_frame:vor_end_frame]
self.right_eye_phi = self.session_data.right_eye_data.phi[vor_start_frame:vor_end_frame]
@@ -45,6 +44,7 @@ class VorCalibrator:
self.left_eye_phi = self.session_data.left_eye_data.phi[vor_start_frame:vor_end_frame]
self.left_eye_theta = self.session_data.left_eye_data.theta[vor_start_frame:vor_end_frame]
+
def calculate_optimal_rotational_offset(self):
pass
diff --git a/jon_scratch/tensorflow_gpu_tester.py b/jon_scratch/tensorflow_gpu_tester.py
new file mode 100644
index 00000000..583ba6e8
--- /dev/null
+++ b/jon_scratch/tensorflow_gpu_tester.py
@@ -0,0 +1,2 @@
+import tensorflow as tf
+print("Num GPUs Available: ", len(tf.config.list_physical_devices('GPU')))
diff --git a/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py b/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py
index 69971e30..e00d61ec 100644
--- a/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py
+++ b/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py
@@ -4,13 +4,16 @@ import numpy as np
import mediapipe as mp
-@dataclass
class MediaPipeSkeletonDetection:
- model_complexity: int = 1 # can be 0,1, or 2 - higher numbers are more accurate but heavier computationally
- min_detection_confidence: float = .5
- min_tracking_confidence: float = .5
- def __post_init__(self):
+ def __init__(self,
+ model_complexity: int = 1,
+ min_detection_confidence: float = .5,
+ min_tracking_confidence: float = .5,
+ ):
+ self.model_complexity = model_complexity # can be 0,1, or 2 - higher numbers are more accurate but heavier computationally
+ self.min_detection_confidence = min_detection_confidence
+ self.min_tracking_confidence = min_tracking_confidence
self._mp_drawing = mp.solutions.drawing_utils
self._mp_drawing_styles = mp.solutions.drawing_styles
self._mp_holistic = mp.solutions.holistic
@@ -28,7 +31,6 @@ class MediaPipeSkeletonDetection:
if annotate_image:
return self._annotate_image(raw_image, mediapipe_results)
-
def _annotate_image(self, image, mediapipe_results):
self._mp_drawing.draw_landmarks(image=image,
landmark_list=mediapipe_results.face_landmarks,
diff --git a/src/session_pipeline/session_pipeline_orchestrator.py b/src/session_pipeline/session_pipeline_orchestrator.py
index ccdfdc0e..1e908eeb 100644
--- a/src/session_pipeline/session_pipeline_orchestrator.py
+++ b/src/session_pipeline/session_pipeline_orchestrator.py
@@ -16,6 +16,9 @@ from src.qt_visualizer_and_gui.qt_visualizer_and_gui import QTVisualizerAndGui
logger = logging.getLogger(__name__)
+import os
+
+
def get_canonical_time_str():
return time.strftime("%m-%d-%Y-%H_%M_%S")