Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

creates QT/GL plots, need to verify that unrotated gazers are goodo

Jondc20dccb1055
diff --git a/jon_scratch/pupil_calibration_pipeline/data_classes/pupil_dataclass_and_handler.py b/jon_scratch/pupil_calibration_pipeline/data_classes/pupil_dataclass_and_handler.py
index 7a114769..d2aec820 100644
--- a/jon_scratch/pupil_calibration_pipeline/data_classes/pupil_dataclass_and_handler.py
+++ b/jon_scratch/pupil_calibration_pipeline/data_classes/pupil_dataclass_and_handler.py
@@ -66,13 +66,19 @@ class PupilDataHandler:
pupil_recording_info_json[
'start_time_system_s']
- def get_eye_data(self, this_eye_d: int):
+ def get_eye_data(self, eye_str: str):
"""
get data for the right eye
pull out data according to `eye_d` (right_eye == 0, left_eye==1) and tracking method (because pupil interleaves 2d and 3d data)
"""
- this_eye_logical_indicies = np.logical_and(self.pupil_data.eye_d == this_eye_d,
+ if eye_str == 'right':
+ eye_d = 0
+
+ if eye_str == 'left':
+ eye_d = 1
+
+ this_eye_logical_indicies = np.logical_and(self.pupil_data.eye_d == eye_d,
self.pupil_data.method != '2d c++')
return PupilData(
timestamps=self.pupil_data.timestamps[this_eye_logical_indicies],
diff --git a/jon_scratch/pupil_calibration_pipeline/head_rotation_calculator.py b/jon_scratch/pupil_calibration_pipeline/head_rotation_calculator.py
index 18a1b4eb..f6088fcc 100644
--- a/jon_scratch/pupil_calibration_pipeline/head_rotation_calculator.py
+++ b/jon_scratch/pupil_calibration_pipeline/head_rotation_calculator.py
@@ -1,4 +1,5 @@
import logging
+from typing import List
import numpy as np
@@ -50,46 +51,125 @@ class HeadRotationCalculator:
left_ear_xyz = self.mediapipe_skeleton_fr_mar_xyz[:, left_ear_id, :]
head_center_xyz = (right_ear_xyz + left_ear_xyz) / 2
- # zero everything out, aka pin it to the origin, aka "subtract the position of the thing you want to be the origin from everything else", aka "do the thing that makes the X,Y,Z coordinates of the thing that should be (0,0,0) equal to (0,0,0)"
- nose_xyz_z = pin_point_new_origin(nose_xyz, head_center_xyz)
- left_ear_xyz_z = pin_point_new_origin(left_ear_xyz, head_center_xyz)
-
- # normalize the length of the vectors (so that each one is a unit vector)
- z_norm_nose_xyz = normalize_length(nose_xyz_z)
- z_norm_left_ear_xyz = normalize_length(left_ear_xyz_z)
-
- # define x_hat as the zero'd and normalized nose vector (i.e. the +X axis of the head starts at the head center and goes out the person's nose)
- x_hat_xyz = z_norm_nose_xyz
-
- # define z_hat as x_hat crossed with the normalized left ear vector
- # create `z_hat` a vector perpendicular to `x_hat` and the vector from the nose to the left ear(aka. the pseudo y_hat)
- z_hat_xyz = np.cross(x_hat_xyz, z_norm_left_ear_xyz)
-
- # define y_hat as z_hat cross with x_hat (this ensures that they are all orthogonal, where as the original vector to the left ear probably wasn't)
- y_hat_xyz = np.cross(x_hat_xyz, z_hat_xyz)
- # not sure if this needs normalizing, but it couldn't hurt
- y_hat_xyz = y_hat_xyz / np.linalg.norm(y_hat_xyz, axis=1)[:, None]
-
- self.head_rotation_matricies = [np.dstack((this_x_hat_xyz, this_y_hat_xyz, this_z_hat_xyz))
- for this_x_hat_xyz, this_y_hat_xyz, this_z_hat_xyz in
- zip(x_hat_xyz, y_hat_xyz, z_hat_xyz)]
+ head_rotation_matricies = self.define_orthonormal_basis_vectors(center_point_fr_xyz=head_center_xyz,
+ x_direction_fr_xyz=nose_xyz,
+ y_direction_fr_xyz=left_ear_xyz)
+ if debug:
+ self.show_head_rotation_debug_plot(head_rotation_matricies)
- self.x_hat_xyz = x_hat_xyz
- self.y_hat_xyz = y_hat_xyz
- self.z_hat_xyz = z_hat_xyz
+ return head_rotation_matricies
- if debug:
- self.show_debug_plot()
+ def define_orthonormal_basis_vectors(self,
+ center_point_fr_xyz: np.ndarray = None,
+ x_direction_fr_xyz: np.ndarray = None,
+ y_direction_fr_xyz: np.ndarray = None,
+ z_direction_fr_xyz: np.ndarray = None) -> List[np.ndarray]:
+ """
+ create orthonormal basis vectors based on a center point and at least TWO of the following = a point defining X direction, a point defining Y direction, and a point defining Z direction
+ """
- return self.head_rotation_matricies
+ x_unknown = False
+ y_unknown = False
+ z_unknown = False
+
+ if x_direction_fr_xyz is None:
+ x_unknown = True
+ x_direction_fr_xyz = np.empty(center_point_fr_xyz.shape)
+ x_direction_fr_xyz[:] = np.nan
+ if y_direction_fr_xyz is None:
+ y_unknown = True
+ y_direction_fr_xyz = np.empty(center_point_fr_xyz.shape)
+ y_direction_fr_xyz[:] = np.nan
+ if z_direction_fr_xyz is None:
+ z_unknown = True
+ z_direction_fr_xyz = np.empty(center_point_fr_xyz.shape)
+ z_direction_fr_xyz[:] = np.nan
+
+ if sum([x_unknown, y_unknown, z_unknown]) > 1:
+ raise ValueError(
+ "Must specify at least two of the following: x_direction_fr_xyz, y_direction_fr_xyz, z_direction_fr_xyz")
+
+ if center_point_fr_xyz is not None: # if center point is specified, then use it to zero out the x,y,z directions vectors
+ zero_x_direction_xyz = pin_point_new_origin(center_point_fr_xyz, x_direction_fr_xyz)
+ zero_y_direction_xyz = pin_point_new_origin(center_point_fr_xyz, y_direction_fr_xyz)
+ zero_z_direction_xyz = pin_point_new_origin(center_point_fr_xyz, z_direction_fr_xyz)
+ else:
+ zero_x_direction_xyz = x_direction_fr_xyz
+ zero_y_direction_xyz = y_direction_fr_xyz
+ zero_z_direction_xyz = z_direction_fr_xyz
+
+ x_hat_xyz = None
+ y_hat_xyz = None
+ z_hat_xyz = None
+
+ if not x_unknown:
+ x_hat_xyz = normalize_length(zero_x_direction_xyz)
+ if not y_unknown: # we know x and y directions and need to find Z by X cross Y
+ z_hat_xyz = normalize_length(np.cross(x_hat_xyz, zero_y_direction_xyz))
+ y_hat_xyz = normalize_length(np.cross(z_hat_xyz, x_hat_xyz))
+
+ else: # we know x and z directions and need to find Y by Z cross X
+ y_hat_xyz = normalize_length(np.cross(zero_y_direction_xyz, x_hat_xyz))
+ z_hat_xyz = normalize_length(np.cross(x_hat_xyz, y_hat_xyz))
+
+ elif not y_unknown: # we know y and z directions and need to find X by Y cross Z
+ y_hat_xyz = normalize_length(zero_y_direction_xyz)
+ x_hat_xyz = normalize_length(np.cross(y_hat_xyz, zero_z_direction_xyz))
+ z_hat_xyz = normalize_length(np.cross(x_hat_xyz, y_hat_xyz))
+
+ # create 3x3 cosine rotation matricies by stacking X_hat, Y_hat, and Z_hat
+ rotation_matricies = [np.vstack((this_x_hat_xyz, this_y_hat_xyz, this_z_hat_xyz))
+ for this_x_hat_xyz, this_y_hat_xyz, this_z_hat_xyz in
+ zip(x_hat_xyz, y_hat_xyz, z_hat_xyz)]
+
+ rotation_matricies = [this_rotation_martix.T for this_rotation_martix in rotation_matricies]
+
+ return rotation_matricies
+
+ def create_unrotated_gaze_laser(self, debug, eye: str) -> np.ndarray:
+ nose_index = 0
+ if eye == 'left':
+ inner_eye_index = 1
+ eye_mid_index = 2
+ outer_eye_index = 3
+ x_direction_fr_xyz = self.mediapipe_skeleton_fr_mar_xyz[:, inner_eye_index,
+ :] # this will make Z+ point forward
+ if eye == 'right':
+ inner_eye_index = 4
+ eye_mid_index = 5
+ outer_eye_index = 6
+ x_direction_fr_xyz = self.mediapipe_skeleton_fr_mar_xyz[:, outer_eye_index,
+ :] # this will make Z+ point forward
+
+ y_direction_fr_xyz = self.mediapipe_skeleton_fr_mar_xyz[:, nose_index, :]
+
+ eye_mid_fr_xyz = self.mediapipe_skeleton_fr_mar_xyz[:, eye_mid_index, :]
+
+ eye_socket_rotation_matricies = self.define_orthonormal_basis_vectors(center_point_fr_xyz=eye_mid_fr_xyz,
+ x_direction_fr_xyz=x_direction_fr_xyz,
+ y_direction_fr_xyz=y_direction_fr_xyz)
+
+ initial_eye_laser_unit_vector = np.array([0, 0, 1])
+ eye_laser_unit_vectors_fr_xyz = [this_eye_socket_rotation_matrix.T @ initial_eye_laser_unit_vector for this_eye_socket_rotation_matrix in eye_socket_rotation_matricies]
+
+ eye_laser_unit_vectors_fr_xyz *= 1000 #make it a meter long
+ eye_laser_unit_vectors_fr_xyz += eye_mid_fr_xyz #situation gaze lasers onto eye_mid_fr_xyz
+
+ return eye_laser_unit_vectors_fr_xyz # this is a unit vector pointing outward from eyeball center (as if eyes were paralyzed in center of their orbits)
+
+ def show_head_rotation_debug_plot(self, head_rotation_matricies):
+
+ x_hat_xyz = np.asarray([this_rot_mat[0, :] for this_rot_mat in head_rotation_matricies])
+ y_hat_xyz = np.asarray([this_rot_mat[1, :] for this_rot_mat in head_rotation_matricies])
+ z_hat_xyz = np.asarray([this_rot_mat[2, :] for this_rot_mat in head_rotation_matricies])
- def show_debug_plot(self):
test_point_xyz_og = np.array((1, 1, 1))
- test_point_xyz_rot = np.empty(self.z_hat_xyz.shape)
+ test_point_xyz_rot = np.empty(z_hat_xyz.shape)
+ logger.warning('We shouldnt have to transpose the rotation matix here, need to fix this upstream')
- for frame_number in range(len(self.head_rotation_matricies)):
+ for frame_number in range(len(head_rotation_matricies)):
# see https://peps.python.org/pep-0465/ for explanation of the `@` operator (which is a "matrix multiplication" operator)
- test_point_xyz_rot[frame_number, :] = self.head_rotation_matricies[frame_number] @ test_point_xyz_og
+ test_point_xyz_rot[frame_number, :] = head_rotation_matricies[frame_number].T @ test_point_xyz_og
plt.close('all')
@@ -99,33 +179,33 @@ class HeadRotationCalculator:
ax_range = 2
- for frame_number in range(1200, self.x_hat_xyz.shape[0]):
+ for frame_number in range(1200, x_hat_xyz.shape[0]):
ax.clear()
# plot head orthonormal basis vectors
- ax.plot(self.x_hat_xyz[frame_number - 30:frame_number, 0],
- self.x_hat_xyz[frame_number - 30:frame_number, 1],
- self.x_hat_xyz[frame_number - 30:frame_number, 2], 'r-')
+ ax.plot(x_hat_xyz[frame_number - 30:frame_number, 0],
+ x_hat_xyz[frame_number - 30:frame_number, 1],
+ x_hat_xyz[frame_number - 30:frame_number, 2], 'r-')
- ax.plot(self.y_hat_xyz[frame_number - 30:frame_number, 0],
- self.y_hat_xyz[frame_number - 30:frame_number, 1],
- self.y_hat_xyz[frame_number - 30:frame_number, 2], 'g-')
+ ax.plot(y_hat_xyz[frame_number - 30:frame_number, 0],
+ y_hat_xyz[frame_number - 30:frame_number, 1],
+ y_hat_xyz[frame_number - 30:frame_number, 2], 'g-')
- ax.plot(self.z_hat_xyz[frame_number - 30:frame_number, 0],
- self.z_hat_xyz[frame_number - 30:frame_number, 1],
- self.z_hat_xyz[frame_number - 30:frame_number, 2], 'b-')
+ ax.plot(z_hat_xyz[frame_number - 30:frame_number, 0],
+ z_hat_xyz[frame_number - 30:frame_number, 1],
+ z_hat_xyz[frame_number - 30:frame_number, 2], 'b-')
- ax.plot([0, self.x_hat_xyz[frame_number, 0]],
- [0, self.x_hat_xyz[frame_number, 1]],
- [0, self.x_hat_xyz[frame_number, 2]], 'r-o', label='x_hat')
+ ax.plot([0, x_hat_xyz[frame_number, 0]],
+ [0, x_hat_xyz[frame_number, 1]],
+ [0, x_hat_xyz[frame_number, 2]], 'r-o', label='x_hat')
- ax.plot([0, self.y_hat_xyz[frame_number, 0]],
- [0, self.y_hat_xyz[frame_number, 1]],
- [0, self.y_hat_xyz[frame_number, 2]], 'g-o', label='y_hat')
+ ax.plot([0, y_hat_xyz[frame_number, 0]],
+ [0, y_hat_xyz[frame_number, 1]],
+ [0, y_hat_xyz[frame_number, 2]], 'g-o', label='y_hat')
- ax.plot([0, self.z_hat_xyz[frame_number, 0]],
- [0, self.z_hat_xyz[frame_number, 1]],
- [0, self.z_hat_xyz[frame_number, 2]], 'b-o', label='z_hat')
+ ax.plot([0, z_hat_xyz[frame_number, 0]],
+ [0, z_hat_xyz[frame_number, 1]],
+ [0, z_hat_xyz[frame_number, 2]], 'b-o', label='z_hat')
# plot test point rotated with head rotation matricies
@@ -149,4 +229,3 @@ class HeadRotationCalculator:
break # finishing the loop
plt.close(fig)
-
diff --git a/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_calibration_pipeline_orchestrator.py b/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_calibration_pipeline_orchestrator.py
index d8c238f0..f9c88dad 100644
--- a/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_calibration_pipeline_orchestrator.py
+++ b/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_calibration_pipeline_orchestrator.py
@@ -18,7 +18,6 @@ class PupilFreemocapCalibrationPipelineOrchestrator:
session_data_loader: SessionDataLoader = None
raw_session_data = FreemocapSessionDataClass()
synchronized_session_data: FreemocapSessionDataClass = None
- vor_calibrator = VorCalibrator()
laser_skeleton_visualizer = QtGlLaserSkeletonVisualizer()
vor_frame_start: int = None,
vor_frame_end: int = None
@@ -39,9 +38,9 @@ class PupilFreemocapCalibrationPipelineOrchestrator:
def run(self):
logger.info(f'loading session data from {self.session_data_loader.session_path}')
- #########
+ ####
# load raw freemocap data
- #########
+ ####
self.raw_session_data.timestamps = self.session_data_loader.load_freemocap_unix_timestamps()
logger.info(
f'self.raw_session_data.freemocap_timestamps.shape: {self.raw_session_data.timestamps.shape}')
@@ -50,29 +49,44 @@ class PupilFreemocapCalibrationPipelineOrchestrator:
logger.info(
f'self.raw_session_data.mediapipe_skel_fr_mar_dim.shape: {self.raw_session_data.mediapipe_skel_fr_mar_dim.shape}')
- #########
+ ####
# Calculate Head Rotation matrix for each frame (gaze data will be rotated by head_rot, then calibrated_offset_rot)
- #########
+ ####
head_rotation_calculator = HeadRotationCalculator(self.raw_session_data.mediapipe_skel_fr_mar_dim)
head_rotation_matricies = head_rotation_calculator.calculate_head_rotation_matricies(debug=True)
+ self.unrotated_right_gaze_laser_fr_xyz = head_rotation_calculator.create_unrotated_gaze_laser(debug=True, eye='right')
+ self.unrotated_left_gaze_laser_fr_xyz = head_rotation_calculator.create_unrotated_gaze_laser(debug=True, eye='left')
+
self.raw_session_data.head_rotation_matricies = head_rotation_matricies
self.save_head_rotation_matricies()
logger.info(
f'len(self.raw_session_data.head_rotation_matricies): {len(self.raw_session_data.head_rotation_matricies)}')
- #########
+ ####
# load pupil data
- #########
+ ####
pupil_data_handler = self.session_data_loader.load_pupil_data()
- self.raw_session_data.right_eye_data = pupil_data_handler.get_eye_data(0)
- self.raw_session_data.left_eye_data = pupil_data_handler.get_eye_data(1)
+ self.raw_session_data.right_eye_data = pupil_data_handler.get_eye_data('right')
+ self.raw_session_data.left_eye_data = pupil_data_handler.get_eye_data('left')
- #########
+ ####
# Synchronize pupil data with freemocap data - results in self.synchronized_session_data (each stream has exactly the same number of frames)
- #########
- self.synchronized_session_data = PupilFreemocapSynchronizer(self.raw_session_data).synchronize(debug=True, vor_frame_start=self.vor_frame_start, vor_frame_end=self.vor_frame_end)
- print(self.synchronized_session_data)
-
+ ####
+ self.synchronized_session_data = PupilFreemocapSynchronizer(self.raw_session_data).synchronize(debug=False, vor_frame_start=self.vor_frame_start, vor_frame_end=self.vor_frame_end)
+ logger.info('synchronization complete - I should add a test to make sure everything has the same number of frames')
+
+ ####
+ # Perform Vestibular-Ocular-Reflex based calibration (see methods from (Matthis et al, 2018 and 2022) for deetos)
+ ####
+ vor_calibrator = VorCalibrator(self.synchronized_session_data, vor_start_frame=self.vor_frame_start, vor_end_frame=self.vor_frame_end)
+ fixation_point_fr_xyz = self.synchronized_session_data.mediapipe_skel_fr_mar_dim[self.vor_frame_start:self.vor_frame_end, :, :3]
+ vor_calibrator.calibrate(fixation_point_fr_xyz)
+
+ ############################################
+ ############################################
+ ### helper methods
+ ############################################
+ ###########################################
def save_head_rotation_matricies(self):
save_path = self.session_path / 'DataArrays' / 'mediaPipeSkel_3d_head_rotation_matricies_fr_row_col.npy'
logger.info(f'saving head rotation matricies to: {save_path}')
diff --git a/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py b/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py
index b4dab671..d35065d0 100644
--- a/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py
+++ b/jon_scratch/pupil_calibration_pipeline/qt_gl_laser_skeleton_visualizer.py
@@ -9,13 +9,23 @@ import numpy as np
import sys
from mediapipe.python.solutions import holistic as mp_holistic
+from jon_scratch.pupil_calibration_pipeline.data_classes.freemocap_session_data_class import FreemocapSessionDataClass
+from jon_scratch.pupil_calibration_pipeline.session_data_loader import SessionDataLoader
+
logger = logging.getLogger(__name__)
+
class QtGlLaserSkeletonVisualizer():
session_path: Path = None
- def __init__(self):
- self.mediapipe_fr_mar_xyz = None
+ def __init__(self, session_data: FreemocapSessionDataClass = None, mediapipe_skel_fr_mar_xyz: np.ndarray = None):
+ if session_data is not None:
+ self.mediapipe_fr_mar_xyz = session_data.mediapipe_skel_fr_mar_dim
+ elif mediapipe_skel_fr_mar_xyz is not None:
+ self.mediapipe_fr_mar_xyz = mediapipe_skel_fr_mar_xyz
+ else:
+ raise ValueError("Must provide either session data or mediapipe skeleton data")
+
self.traces = dict()
self.start_frame_number = 1000
self.end_frame_number = 2000
@@ -27,7 +37,7 @@ class QtGlLaserSkeletonVisualizer():
self.get_mediapipe_connections()
self.initialize_skel_dottos()
self.initialize_skel_lines()
- self.initialize_gaze_laser()
+ # self.initialize_gaze_laser()
def create_app_window(self):
self.app = pg.mkQApp("Laser Skeleton")
@@ -90,14 +100,16 @@ class QtGlLaserSkeletonVisualizer():
def update(self):
self.update_frame_number()
+ if self.current_frame_number % 10 == 0:
+ print(f'frame number: {self.current_frame_number}')
self.skeleton_scatter_item.setData(
pos=self.mediapipe_fr_mar_xyz[self.current_frame_number, :, :]
)
self.update_skeleton_lines()
- this_gaze_laser = np.vstack(
- (self.right_eye_xyz[self.current_frame_number, :], self.right_gaze_xyz[self.current_frame_number, :]))
- self.right_gaze_line_item.setData(pos=this_gaze_laser)
+ # this_gaze_laser = np.vstack(
+ # (self.right_eye_xyz[self.current_frame_number, :], self.right_gaze_xyz[self.current_frame_number, :]))
+ # self.right_gaze_line_item.setData(pos=this_gaze_laser)
def update_skeleton_lines(self):
for this_skeleton_line_number, this_connection in enumerate(self.mediapipe_body_connections):
@@ -109,15 +121,15 @@ class QtGlLaserSkeletonVisualizer():
self.current_frame_number += 1
if self.current_frame_number >= self.end_frame_number:
self.current_frame_number = self.start_frame_number
- if self.current_frame_number % 10 == 0:
- print(f'frame number: {self.current_frame_number}')
+
def start(self):
- print('starting')
+ print('starting animation')
pg.exec()
def start_animation(self):
- print('starting animation')
+ print('initializing display window')
+ self.initialize_display_window()
timer = QtCore.QTimer()
timer.timeout.connect(self.update)
timer.start(33)
@@ -125,8 +137,13 @@ class QtGlLaserSkeletonVisualizer():
if __name__ == '__main__':
+ # session_id = 'sesh_2022-02-15_11_54_28_pupil_maybe'
+ session_id = 'sesh_2022-05-03_13_43_00_JSM_treadmill_day2_t0'
data_path = Path('C:/Users/jonma/Dropbox/FreeMoCapProject/FreeMocap_Data/')
- session_path = data_path / 'sesh_2022-02-15_11_54_28_pupil_maybe'
+ session_path = data_path / session_id
- qt_gl_laser_skeleton = QtGlLaserSkeletonVisualizer(session_path)
+ session_data_loader = SessionDataLoader(session_path)
+ mediapipe_skel_fr_mar_xyz_in = session_data_loader.load_mediapipe_data(move_to_origin=True)
+ print(f'mediapipe_skel_fr_mar_xyz.shape: {mediapipe_skel_fr_mar_xyz_in.shape}')
+ qt_gl_laser_skeleton = QtGlLaserSkeletonVisualizer(mediapipe_skel_fr_mar_xyz = mediapipe_skel_fr_mar_xyz_in,)
qt_gl_laser_skeleton.start_animation()
diff --git a/jon_scratch/pupil_calibration_pipeline/session_data_loader.py b/jon_scratch/pupil_calibration_pipeline/session_data_loader.py
index f3ba396d..e6f44094 100644
--- a/jon_scratch/pupil_calibration_pipeline/session_data_loader.py
+++ b/jon_scratch/pupil_calibration_pipeline/session_data_loader.py
@@ -21,6 +21,7 @@ class SessionDataLoader:
def session_path(self):
return self._session_path
+
def load_freemocap_unix_timestamps(self):
"""
load/calculate the unix timestamps for the freemocap data (as mean of each camera's timestamp on each frame)
@@ -57,10 +58,6 @@ class SessionDataLoader:
logger.info(f'loading mediapipe data from {mediapipe_data_path}')
mediapipe_fr_mar_xyz = np.load(str(mediapipe_data_path))
- # logging.warning(
- # 'SLOPPY BUG FIX - Removing first frame of mediapipe_data due to annoying \'off-by-one\' error in the timestamp logger')
- # mediapipe_fr_mar_xyz = np.delete(mediapipe_fr_mar_xyz, 0, axis=0)
-
if move_to_origin:
mean_position_xyz = np.nanmedian(np.nanmedian(mediapipe_fr_mar_xyz, axis=0), axis=0)
mediapipe_fr_mar_xyz[:, :, 0] -= mean_position_xyz[0]
@@ -99,3 +96,12 @@ class SessionDataLoader:
return pupil_data_handler
+if __name__ == '__main__':
+ data_path = Path('C:/Users/jonma/Dropbox/FreeMoCapProject/FreeMocap_Data/')
+ session_path = data_path / 'sesh_2022-02-15_11_54_28_pupil_maybe'
+
+ session_data_loader = SessionDataLoader(session_path)
+ pupil_data = session_data_loader.load_pupil_data()
+ print('pupil data loaded, but I don\'t have an easy way to print out the shape of things yet, lol')
+ mediapipe_skel_fr_mar_xyz = session_data_loader.load_mediapipe_data()
+ print(f'mediapipe_skel_fr_mar_xyz.shape: {mediapipe_skel_fr_mar_xyz.shape}')
diff --git a/jon_scratch/pupil_calibration_pipeline/vor_calibrator.py b/jon_scratch/pupil_calibration_pipeline/vor_calibrator.py
index 54a29a37..1554a4f8 100644
--- a/jon_scratch/pupil_calibration_pipeline/vor_calibrator.py
+++ b/jon_scratch/pupil_calibration_pipeline/vor_calibrator.py
@@ -1,2 +1,52 @@
+import logging
+
+import numpy as np
+from matplotlib import pyplot as plt
+import matplotlib
+
+from jon_scratch.pupil_calibration_pipeline.data_classes.freemocap_session_data_class import FreemocapSessionDataClass
+
+matplotlib.use('qt5agg')
+logger = logging.getLogger(__name__)
+
+
class VorCalibrator:
- pass
\ No newline at end of file
+ """
+ Calibrate synchronized eye data from a Pupil Labs eye tracker with FreeMoCap skeleton
+ see methods sections of (Matthis, Yates, and Hayhoe, Current Biology, 2018) and (Matthis, Muller, Bonnen, Hayehoe, PLoS Computation Biology 2022) for details
+ """
+
+ def __init__(self, session_data: FreemocapSessionDataClass, vor_start_frame: int = None, vor_end_frame: int = None):
+ self.session_data = session_data
+ if vor_start_frame is not None:
+ if vor_end_frame is None:
+ raise ValueError("Must specify vor_end_frame if vor_start_frame is specified")
+ self.clip_out_vor_frames(vor_start_frame, vor_end_frame)
+
+ def calibrate(self, fixation_point_fr_xyz: np.ndarray) -> FreemocapSessionDataClass:
+
+ if not fixation_point_fr_xyz.shape[1] == 3:
+ raise Exception("fixation_point_fr_xyz must be a numpy array with 3 columns")
+ if not fixation_point_fr_xyz.shape[0] == self.skel_fr_mar_dim.shape[0]:
+ raise Exception("fixation_point_xyz must have precisely the same number of rows as the skeleton/gaze data")
+
+ self.create_unrotated_gaze_vector()
+ self.calculate_optimal_rotational_offset()
+ self.create_skeleton_gaze_lasers()
+ return self.session_data
+
+ def clip_out_vor_frames(self, vor_start_frame, vor_end_frame):
+ self.skel_fr_mar_dim = self.session_data.skel_fr_mar_dim[vor_start_frame:vor_end_frame,:,:]
+ self.head_rot_matricies = self.session_data.head_rotation_matricies[vor_start_frame:vor_end_frame]
+
+ self.right_eye_theta = self.session_data.right_eye_data.theta[vor_start_frame:vor_end_frame]
+ self.right_eye_phi = self.session_data.right_eye_data.phi[vor_start_frame:vor_end_frame]
+
+ self.left_eye_phi = self.session_data.left_eye_data.phi[vor_start_frame:vor_end_frame]
+ self.left_eye_theta = self.session_data.left_eye_data.theta[vor_start_frame:vor_end_frame]
+
+ def calculate_optimal_rotational_offset(self):
+ pass
+
+ def create_skeleton_gaze_lasers(self):
+ pass