Free Motion Capture for Everyone πβ¨
TypeScript
pupil/freemocap sync appears to work
Jonc8be6fe172ab
diff --git a/jon_scratch/pupil_calibration_pipeline/data_classes/freemocap_session_data_class.py b/jon_scratch/pupil_calibration_pipeline/data_classes/freemocap_session_data_class.pyindex 893dd4a0..b8da2fca 100644--- a/jon_scratch/pupil_calibration_pipeline/data_classes/freemocap_session_data_class.py+++ b/jon_scratch/pupil_calibration_pipeline/data_classes/freemocap_session_data_class.py@@ -9,7 +9,7 @@ from jon_scratch.pupil_calibration_pipeline.data_classes.pupil_dataclass_and_han@dataclassclass FreemocapSessionDataClass:session_id: str = None- mediapipe_fr_mar_dim: np.ndarray = None+ mediapipe_skel_fr_mar_dim: np.ndarray = Noneright_eye_data: PupilData = Noneleft_eye_data: PupilData = Nonehead_rotation_matricies: List = Nonediff --git a/jon_scratch/pupil_calibration_pipeline/data_classes/pupil_dataclass_and_handler.py b/jon_scratch/pupil_calibration_pipeline/data_classes/pupil_dataclass_and_handler.pyindex 7cd433a8..7a114769 100644--- a/jon_scratch/pupil_calibration_pipeline/data_classes/pupil_dataclass_and_handler.py+++ b/jon_scratch/pupil_calibration_pipeline/data_classes/pupil_dataclass_and_handler.py@@ -45,9 +45,12 @@ class PupilDataHandler:# resituate pupil data so that the pupil normal direction is situated on the origin (instead of the arbitrary eye-camera-based eyeball sphere center)# ... I think... I need to check this. Their docs are a bit unclear about the origin of the pupil normal direction.- pupil_center_normal_x = np.array(pupil_dataframe['sphere_center_x']) - np.array(pupil_dataframe['circle_3d_normal_x'])- pupil_center_normal_y = np.array(pupil_dataframe['sphere_center_y']) - np.array(pupil_dataframe['circle_3d_normal_y'])- pupil_center_normal_z = np.array(pupil_dataframe['sphere_center_z']) - np.array(pupil_dataframe['circle_3d_normal_z'])+ # pupil_center_normal_x = np.array(pupil_dataframe['sphere_center_x']) - np.array(pupil_dataframe['circle_3d_normal_x'])+ # pupil_center_normal_y = np.array(pupil_dataframe['sphere_center_y']) - np.array(pupil_dataframe['circle_3d_normal_y'])+ # pupil_center_normal_z = np.array(pupil_dataframe['sphere_center_z']) - np.array(pupil_dataframe['circle_3d_normal_z'])+ pupil_center_normal_x = np.array(pupil_dataframe['circle_3d_normal_x'])+ pupil_center_normal_y = np.array(pupil_dataframe['circle_3d_normal_y'])+ pupil_center_normal_z = np.array(pupil_dataframe['circle_3d_normal_z'])self.pupil_data = PupilData(timestamps=np.array(pupil_dataframe['pupil_timestamp']),theta=np.array(pupil_dataframe['theta']),@@ -59,9 +62,9 @@ class PupilDataHandler:method=pupil_dataframe['method'])def convert_to_unix_timestamps(self, pupil_recording_info_json: dict):- unix_time_offset = pupil_recording_info_json['start_time_synced_s'] + pupil_recording_info_json[- 'start_time_system_s']- self.pupil_data.timestamps = self.pupil_data.timestamps - unix_time_offset+ self.pupil_data.timestamps = self.pupil_data.timestamps - pupil_recording_info_json['start_time_synced_s'] + \+ pupil_recording_info_json[+ 'start_time_system_s']def get_eye_data(self, this_eye_d: int):"""@@ -69,7 +72,8 @@ class PupilDataHandler:pull out data according to `eye_d` (right_eye == 0, left_eye==1) and tracking method (because pupil interleaves 2d and 3d data)"""- this_eye_logical_indicies = np.logical_and(self.pupil_data.eye_d == this_eye_d, self.pupil_data.method != '2d c++')+ this_eye_logical_indicies = np.logical_and(self.pupil_data.eye_d == this_eye_d,+ self.pupil_data.method != '2d c++')return PupilData(timestamps=self.pupil_data.timestamps[this_eye_logical_indicies],theta=self.pupil_data.theta[this_eye_logical_indicies],diff --git a/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.py b/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.pyindex 64343cae..db6f445c 100644--- a/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.py+++ b/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.py@@ -21,7 +21,7 @@ class PupilFreemocapSynchronizer:self.synchronized_session_data: FreemocapSessionDataClass = Nonedef synchronise(self, debug: bool = False):- self.synchronize(debug=debug) # lol+ self.synchronize(debug=debug) # lol britsdef synchronize(self, debug: bool = False, vor_frame_start: int = None, vor_frame_end: int = None):"""@@ -81,7 +81,9 @@ class PupilFreemocapSynchronizer:self.resample_eye_data()self.normalize_eye_data()- self.synchronized_timestamps = self.raw_session_data.timestamps[freemocap_start_frame:freemocap_end_frame]+ self.synchronized_timestamps = self.raw_session_data.timestamps[freemocap_start_frame:freemocap_end_frame+1]++ self.synchronized_timestamps = self.synchronized_timestamps - self.synchronized_timestamps[0]assert (self.synchronized_timestamps.shape[0] == self.right_eye_theta.shape[0])assert (self.synchronized_timestamps.shape[0] == self.left_eye_theta.shape[0])@@ -104,40 +106,50 @@ class PupilFreemocapSynchronizer:pupil_center_normal_z=self.left_eye_pupil_center_normal_z,eye_d=1)- self.synchronized_freemocap_data = FreemocapSessionDataClass(timestamps=self.synchronized_timestamps,- mediapipe_fr_mar_dim=self.raw_session_data.mediapipe_fr_mar_dim[+ synchronized_session_data = FreemocapSessionDataClass(timestamps=self.synchronized_timestamps,+ mediapipe_skel_fr_mar_dim=self.raw_session_data.mediapipe_skel_fr_mar_dim[freemocap_start_frame:freemocap_end_frame,:, :],head_rotation_matricies=self.raw_session_data.head_rotation_matricies[freemocap_start_frame:freemocap_end_frame],right_eye_data=synchronized_right_eye_data,left_eye_data=synchronized_left_eye_data)-+ return synchronized_session_datadef clip_eye_data(self):self.right_eye_timestamps_clipped = self.raw_session_data.right_eye_data.timestamps[self.right_eye_start_frame:self.right_eye_end_frame]+self.right_eye_pupil_center_normal_x_clipped = self.raw_session_data.right_eye_data.pupil_center_normal_x[self.right_eye_start_frame:self.right_eye_end_frame]+self.right_eye_pupil_center_normal_y_clipped = self.raw_session_data.right_eye_data.pupil_center_normal_y[self.right_eye_start_frame:self.right_eye_end_frame]+self.right_eye_pupil_center_normal_z_clipped = self.raw_session_data.right_eye_data.pupil_center_normal_z[self.right_eye_start_frame:self.right_eye_end_frame]+self.right_eye_theta_clipped = self.raw_session_data.right_eye_data.theta[self.right_eye_start_frame:self.right_eye_end_frame]+self.right_eye_phi_clipped = self.raw_session_data.right_eye_data.phi[self.right_eye_start_frame:self.right_eye_end_frame]self.left_eye_timestamps_clipped = self.raw_session_data.left_eye_data.timestamps[self.left_eye_start_frame:self.left_eye_end_frame]+self.left_eye_pupil_center_normal_x_clipped = self.raw_session_data.left_eye_data.pupil_center_normal_x[self.left_eye_start_frame:self.left_eye_end_frame]- self.left_eye_pupil_center_normal_y_clipped = self.raw_session_data.left_eye_data.pupil_center_normal_x[++ self.left_eye_pupil_center_normal_y_clipped = self.raw_session_data.left_eye_data.pupil_center_normal_y[self.left_eye_start_frame:self.left_eye_end_frame]- self.left_eye_pupil_center_normal_z_clipped = self.raw_session_data.left_eye_data.pupil_center_normal_x[++ self.left_eye_pupil_center_normal_z_clipped = self.raw_session_data.left_eye_data.pupil_center_normal_z[self.left_eye_start_frame:self.left_eye_end_frame]+self.left_eye_theta_clipped = self.raw_session_data.left_eye_data.theta[self.left_eye_start_frame:self.left_eye_end_frame]+self.left_eye_phi_clipped = self.raw_session_data.left_eye_data.phi[self.left_eye_start_frame:self.left_eye_end_frame]@@ -178,9 +190,49 @@ class PupilFreemocapSynchronizer:self.left_eye_pupil_center_normal_z = self.left_eye_pupil_center_normal_z / np.linalg.norm(self.left_eye_pupil_center_normal_z)def show_debug_plots(self, vor_frame_start, vor_frame_end):- vor_frame_start = 0- vor_frame_end = self.synchronized_timestamps.shape[0]++ ###########################+ # Plot Raw Data+ ###########################+ fig = plt.figure(num=653412, figsize=(10, 20))+ fig.suptitle('Raw data')+ ax1 = fig.add_subplot(411)+ ax1.plot(self.raw_session_data.right_eye_data.timestamps, self.raw_session_data.right_eye_data.pupil_center_normal_x, '.-', label='right_eye_pupil_center_normal_x')+ ax1.plot(self.raw_session_data.right_eye_data.timestamps, self.raw_session_data.right_eye_data.pupil_center_normal_y, '.-', label='right_eye_pupil_center_normal_y')+ ax1.plot(self.raw_session_data.right_eye_data.timestamps, self.raw_session_data.right_eye_data.pupil_center_normal_z, '.-', label='right_eye_pupil_center_normal_z')+ ax1.legend(loc='upper left')++ ax2 = fig.add_subplot(412)+ ax2.plot(self.raw_session_data.right_eye_data.timestamps,+ self.raw_session_data.right_eye_data.theta, '.-',+ label='right_eye_theta')+ ax2.plot(self.raw_session_data.right_eye_data.timestamps,+ self.raw_session_data.right_eye_data.phi, '.-',+ label='right_eye_phi')+ ax2.legend(loc='upper left')++ ax3 = fig.add_subplot(413)+ ax3.plot(self.raw_session_data.timestamps, self.left_eye_pupil_center_normal_x, '.-', label='left_eye_pupil_center_normal_x')+ ax3.plot(self.raw_session_data.timestamps, self.left_eye_pupil_center_normal_y, '.-', label='left_eye_pupil_center_normal_y')+ ax3.plot(self.raw_session_data.timestamps, self.left_eye_pupil_center_normal_z, '.-', label='left_eye_pupil_center_normal_z')+ ax3.legend(loc='upper left')++ ax4 = fig.add_subplot(414)+ ax4.plot(self.raw_session_data.left_eye_data.timestamps,+ self.raw_session_data.left_eye_data.theta, '.-',+ label='left_eye_theta')+ ax4.plot(self.raw_session_data.left_eye_data.timestamps,+ self.raw_session_data.left_eye_data.phi, '.-',+ label='left_eye_phi')+ ax4.legend(loc='upper left')++ ###########################+ # Plot synchronized data+ ###########################++fig = plt.figure(num=65341, figsize=(10, 20))+ fig.suptitle('Synchronized data')ax1 = fig.add_subplot(411)ax1.plot(self.synchronized_timestamps[vor_frame_start:vor_frame_end], self.right_eye_pupil_center_normal_x[vor_frame_start:vor_frame_end], '.-', label='right_eye_pupil_center_normal_x')ax1.plot(self.synchronized_timestamps[vor_frame_start:vor_frame_end], self.right_eye_pupil_center_normal_y[vor_frame_start:vor_frame_end], '.-', label='right_eye_pupil_center_normal_y')diff --git a/jon_scratch/pupil_calibration_pipeline/session_data_loader.py b/jon_scratch/pupil_calibration_pipeline/session_data_loader.pyindex 32a9afcb..f3ba396d 100644--- a/jon_scratch/pupil_calibration_pipeline/session_data_loader.py+++ b/jon_scratch/pupil_calibration_pipeline/session_data_loader.py@@ -57,9 +57,9 @@ class SessionDataLoader:logger.info(f'loading mediapipe data from {mediapipe_data_path}')mediapipe_fr_mar_xyz = np.load(str(mediapipe_data_path))- logging.warning(- 'SLOPPY BUG FIX - Removing first frame of mediapipe_data due to annoying \'off-by-one\' error in the timestamp logger')- mediapipe_fr_mar_xyz = np.delete(mediapipe_fr_mar_xyz, 0, axis=0)+ # logging.warning(+ # 'SLOPPY BUG FIX - Removing first frame of mediapipe_data due to annoying \'off-by-one\' error in the timestamp logger')+ # mediapipe_fr_mar_xyz = np.delete(mediapipe_fr_mar_xyz, 0, axis=0)if move_to_origin:mean_position_xyz = np.nanmedian(np.nanmedian(mediapipe_fr_mar_xyz, axis=0), axis=0)