Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

pupil/freemocap sync appears to work

Jonc8be6fe172ab
diff --git a/jon_scratch/pupil_calibration_pipeline/data_classes/freemocap_session_data_class.py b/jon_scratch/pupil_calibration_pipeline/data_classes/freemocap_session_data_class.py
index 893dd4a0..b8da2fca 100644
--- a/jon_scratch/pupil_calibration_pipeline/data_classes/freemocap_session_data_class.py
+++ b/jon_scratch/pupil_calibration_pipeline/data_classes/freemocap_session_data_class.py
@@ -9,7 +9,7 @@ from jon_scratch.pupil_calibration_pipeline.data_classes.pupil_dataclass_and_han
@dataclass
class FreemocapSessionDataClass:
session_id: str = None
- mediapipe_fr_mar_dim: np.ndarray = None
+ mediapipe_skel_fr_mar_dim: np.ndarray = None
right_eye_data: PupilData = None
left_eye_data: PupilData = None
head_rotation_matricies: List = None
diff --git a/jon_scratch/pupil_calibration_pipeline/data_classes/pupil_dataclass_and_handler.py b/jon_scratch/pupil_calibration_pipeline/data_classes/pupil_dataclass_and_handler.py
index 7cd433a8..7a114769 100644
--- a/jon_scratch/pupil_calibration_pipeline/data_classes/pupil_dataclass_and_handler.py
+++ b/jon_scratch/pupil_calibration_pipeline/data_classes/pupil_dataclass_and_handler.py
@@ -45,9 +45,12 @@ class PupilDataHandler:
# resituate pupil data so that the pupil normal direction is situated on the origin (instead of the arbitrary eye-camera-based eyeball sphere center)
# ... I think... I need to check this. Their docs are a bit unclear about the origin of the pupil normal direction.
- pupil_center_normal_x = np.array(pupil_dataframe['sphere_center_x']) - np.array(pupil_dataframe['circle_3d_normal_x'])
- pupil_center_normal_y = np.array(pupil_dataframe['sphere_center_y']) - np.array(pupil_dataframe['circle_3d_normal_y'])
- pupil_center_normal_z = np.array(pupil_dataframe['sphere_center_z']) - np.array(pupil_dataframe['circle_3d_normal_z'])
+ # pupil_center_normal_x = np.array(pupil_dataframe['sphere_center_x']) - np.array(pupil_dataframe['circle_3d_normal_x'])
+ # pupil_center_normal_y = np.array(pupil_dataframe['sphere_center_y']) - np.array(pupil_dataframe['circle_3d_normal_y'])
+ # pupil_center_normal_z = np.array(pupil_dataframe['sphere_center_z']) - np.array(pupil_dataframe['circle_3d_normal_z'])
+ pupil_center_normal_x = np.array(pupil_dataframe['circle_3d_normal_x'])
+ pupil_center_normal_y = np.array(pupil_dataframe['circle_3d_normal_y'])
+ pupil_center_normal_z = np.array(pupil_dataframe['circle_3d_normal_z'])
self.pupil_data = PupilData(timestamps=np.array(pupil_dataframe['pupil_timestamp']),
theta=np.array(pupil_dataframe['theta']),
@@ -59,9 +62,9 @@ class PupilDataHandler:
method=pupil_dataframe['method'])
def convert_to_unix_timestamps(self, pupil_recording_info_json: dict):
- unix_time_offset = pupil_recording_info_json['start_time_synced_s'] + pupil_recording_info_json[
- 'start_time_system_s']
- self.pupil_data.timestamps = self.pupil_data.timestamps - unix_time_offset
+ self.pupil_data.timestamps = self.pupil_data.timestamps - pupil_recording_info_json['start_time_synced_s'] + \
+ pupil_recording_info_json[
+ 'start_time_system_s']
def get_eye_data(self, this_eye_d: int):
"""
@@ -69,7 +72,8 @@ class PupilDataHandler:
pull out data according to `eye_d` (right_eye == 0, left_eye==1) and tracking method (because pupil interleaves 2d and 3d data)
"""
- this_eye_logical_indicies = np.logical_and(self.pupil_data.eye_d == this_eye_d, self.pupil_data.method != '2d c++')
+ this_eye_logical_indicies = np.logical_and(self.pupil_data.eye_d == this_eye_d,
+ self.pupil_data.method != '2d c++')
return PupilData(
timestamps=self.pupil_data.timestamps[this_eye_logical_indicies],
theta=self.pupil_data.theta[this_eye_logical_indicies],
diff --git a/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.py b/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.py
index 64343cae..db6f445c 100644
--- a/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.py
+++ b/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.py
@@ -21,7 +21,7 @@ class PupilFreemocapSynchronizer:
self.synchronized_session_data: FreemocapSessionDataClass = None
def synchronise(self, debug: bool = False):
- self.synchronize(debug=debug) # lol
+ self.synchronize(debug=debug) # lol brits
def synchronize(self, debug: bool = False, vor_frame_start: int = None, vor_frame_end: int = None):
"""
@@ -81,7 +81,9 @@ class PupilFreemocapSynchronizer:
self.resample_eye_data()
self.normalize_eye_data()
- self.synchronized_timestamps = self.raw_session_data.timestamps[freemocap_start_frame:freemocap_end_frame]
+ self.synchronized_timestamps = self.raw_session_data.timestamps[freemocap_start_frame:freemocap_end_frame+1]
+
+ self.synchronized_timestamps = self.synchronized_timestamps - self.synchronized_timestamps[0]
assert (self.synchronized_timestamps.shape[0] == self.right_eye_theta.shape[0])
assert (self.synchronized_timestamps.shape[0] == self.left_eye_theta.shape[0])
@@ -104,40 +106,50 @@ class PupilFreemocapSynchronizer:
pupil_center_normal_z=self.left_eye_pupil_center_normal_z,
eye_d=1)
- self.synchronized_freemocap_data = FreemocapSessionDataClass(timestamps=self.synchronized_timestamps,
- mediapipe_fr_mar_dim=self.raw_session_data.mediapipe_fr_mar_dim[
+ synchronized_session_data = FreemocapSessionDataClass(timestamps=self.synchronized_timestamps,
+ mediapipe_skel_fr_mar_dim=self.raw_session_data.mediapipe_skel_fr_mar_dim[
freemocap_start_frame:freemocap_end_frame,
:, :],
head_rotation_matricies=self.raw_session_data.head_rotation_matricies[
freemocap_start_frame:freemocap_end_frame],
right_eye_data=synchronized_right_eye_data,
left_eye_data=synchronized_left_eye_data)
-
+ return synchronized_session_data
def clip_eye_data(self):
self.right_eye_timestamps_clipped = self.raw_session_data.right_eye_data.timestamps[self.right_eye_start_frame:self.right_eye_end_frame]
+
self.right_eye_pupil_center_normal_x_clipped = self.raw_session_data.right_eye_data.pupil_center_normal_x[
self.right_eye_start_frame:self.right_eye_end_frame]
+
self.right_eye_pupil_center_normal_y_clipped = self.raw_session_data.right_eye_data.pupil_center_normal_y[
self.right_eye_start_frame:self.right_eye_end_frame]
+
self.right_eye_pupil_center_normal_z_clipped = self.raw_session_data.right_eye_data.pupil_center_normal_z[
self.right_eye_start_frame:self.right_eye_end_frame]
+
self.right_eye_theta_clipped = self.raw_session_data.right_eye_data.theta[
self.right_eye_start_frame:self.right_eye_end_frame]
+
self.right_eye_phi_clipped = self.raw_session_data.right_eye_data.phi[
self.right_eye_start_frame:self.right_eye_end_frame]
self.left_eye_timestamps_clipped = self.raw_session_data.left_eye_data.timestamps[
self.left_eye_start_frame:self.left_eye_end_frame]
+
self.left_eye_pupil_center_normal_x_clipped = self.raw_session_data.left_eye_data.pupil_center_normal_x[
self.left_eye_start_frame:self.left_eye_end_frame]
- self.left_eye_pupil_center_normal_y_clipped = self.raw_session_data.left_eye_data.pupil_center_normal_x[
+
+ self.left_eye_pupil_center_normal_y_clipped = self.raw_session_data.left_eye_data.pupil_center_normal_y[
self.left_eye_start_frame:self.left_eye_end_frame]
- self.left_eye_pupil_center_normal_z_clipped = self.raw_session_data.left_eye_data.pupil_center_normal_x[
+
+ self.left_eye_pupil_center_normal_z_clipped = self.raw_session_data.left_eye_data.pupil_center_normal_z[
self.left_eye_start_frame:self.left_eye_end_frame]
+
self.left_eye_theta_clipped = self.raw_session_data.left_eye_data.theta[
self.left_eye_start_frame:self.left_eye_end_frame]
+
self.left_eye_phi_clipped = self.raw_session_data.left_eye_data.phi[
self.left_eye_start_frame:self.left_eye_end_frame]
@@ -178,9 +190,49 @@ class PupilFreemocapSynchronizer:
self.left_eye_pupil_center_normal_z = self.left_eye_pupil_center_normal_z / np.linalg.norm(self.left_eye_pupil_center_normal_z)
def show_debug_plots(self, vor_frame_start, vor_frame_end):
- vor_frame_start = 0
- vor_frame_end = self.synchronized_timestamps.shape[0]
+
+ ###########################
+ # Plot Raw Data
+ ###########################
+ fig = plt.figure(num=653412, figsize=(10, 20))
+ fig.suptitle('Raw data')
+ ax1 = fig.add_subplot(411)
+ ax1.plot(self.raw_session_data.right_eye_data.timestamps, self.raw_session_data.right_eye_data.pupil_center_normal_x, '.-', label='right_eye_pupil_center_normal_x')
+ ax1.plot(self.raw_session_data.right_eye_data.timestamps, self.raw_session_data.right_eye_data.pupil_center_normal_y, '.-', label='right_eye_pupil_center_normal_y')
+ ax1.plot(self.raw_session_data.right_eye_data.timestamps, self.raw_session_data.right_eye_data.pupil_center_normal_z, '.-', label='right_eye_pupil_center_normal_z')
+ ax1.legend(loc='upper left')
+
+ ax2 = fig.add_subplot(412)
+ ax2.plot(self.raw_session_data.right_eye_data.timestamps,
+ self.raw_session_data.right_eye_data.theta, '.-',
+ label='right_eye_theta')
+ ax2.plot(self.raw_session_data.right_eye_data.timestamps,
+ self.raw_session_data.right_eye_data.phi, '.-',
+ label='right_eye_phi')
+ ax2.legend(loc='upper left')
+
+ ax3 = fig.add_subplot(413)
+ ax3.plot(self.raw_session_data.timestamps, self.left_eye_pupil_center_normal_x, '.-', label='left_eye_pupil_center_normal_x')
+ ax3.plot(self.raw_session_data.timestamps, self.left_eye_pupil_center_normal_y, '.-', label='left_eye_pupil_center_normal_y')
+ ax3.plot(self.raw_session_data.timestamps, self.left_eye_pupil_center_normal_z, '.-', label='left_eye_pupil_center_normal_z')
+ ax3.legend(loc='upper left')
+
+ ax4 = fig.add_subplot(414)
+ ax4.plot(self.raw_session_data.left_eye_data.timestamps,
+ self.raw_session_data.left_eye_data.theta, '.-',
+ label='left_eye_theta')
+ ax4.plot(self.raw_session_data.left_eye_data.timestamps,
+ self.raw_session_data.left_eye_data.phi, '.-',
+ label='left_eye_phi')
+ ax4.legend(loc='upper left')
+
+ ###########################
+ # Plot synchronized data
+ ###########################
+
+
fig = plt.figure(num=65341, figsize=(10, 20))
+ fig.suptitle('Synchronized data')
ax1 = fig.add_subplot(411)
ax1.plot(self.synchronized_timestamps[vor_frame_start:vor_frame_end], self.right_eye_pupil_center_normal_x[vor_frame_start:vor_frame_end], '.-', label='right_eye_pupil_center_normal_x')
ax1.plot(self.synchronized_timestamps[vor_frame_start:vor_frame_end], self.right_eye_pupil_center_normal_y[vor_frame_start:vor_frame_end], '.-', label='right_eye_pupil_center_normal_y')
diff --git a/jon_scratch/pupil_calibration_pipeline/session_data_loader.py b/jon_scratch/pupil_calibration_pipeline/session_data_loader.py
index 32a9afcb..f3ba396d 100644
--- a/jon_scratch/pupil_calibration_pipeline/session_data_loader.py
+++ b/jon_scratch/pupil_calibration_pipeline/session_data_loader.py
@@ -57,9 +57,9 @@ class SessionDataLoader:
logger.info(f'loading mediapipe data from {mediapipe_data_path}')
mediapipe_fr_mar_xyz = np.load(str(mediapipe_data_path))
- logging.warning(
- 'SLOPPY BUG FIX - Removing first frame of mediapipe_data due to annoying \'off-by-one\' error in the timestamp logger')
- mediapipe_fr_mar_xyz = np.delete(mediapipe_fr_mar_xyz, 0, axis=0)
+ # logging.warning(
+ # 'SLOPPY BUG FIX - Removing first frame of mediapipe_data due to annoying \'off-by-one\' error in the timestamp logger')
+ # mediapipe_fr_mar_xyz = np.delete(mediapipe_fr_mar_xyz, 0, axis=0)
if move_to_origin:
mean_position_xyz = np.nanmedian(np.nanmedian(mediapipe_fr_mar_xyz, axis=0), axis=0)