Free Motion Capture for Everyone πβ¨
TypeScript
freemocap laser skeletons, baby :D
Jon78a40f6cdc8e
diff --git a/freemocap/play_skeleton_animation.py b/freemocap/play_skeleton_animation.pyindex 9e3aff1b..b31b36e8 100644--- a/freemocap/play_skeleton_animation.py+++ b/freemocap/play_skeleton_animation.py@@ -328,7 +328,7 @@ def PlaySkeletonAnimation(if useMediaPipe:try:- mediaPipe_skel_fr_mar_xyz = np.load(session.dataArrayPath / 'mediaPipeSkel_3d.npy')+ mediaPipe_skel_fr_mar_xyz = np.load(session.dataArrayPath / 'mediaPipeSkel_3d_smoothed.npy')mediaPipe_nCams_nFrames_nImgPts_XYC = np.load(session.dataArrayPath / 'mediaPipeData_2d.npy')except:print('No mediaPipe data found.')diff --git a/jon_scratch/pupil_calibration_pipeline/load_pupil_data_blender_script.py b/jon_scratch/pupil_calibration_pipeline/load_pupil_data_blender_script.pyindex be277391..777f6310 100644--- a/jon_scratch/pupil_calibration_pipeline/load_pupil_data_blender_script.py+++ b/jon_scratch/pupil_calibration_pipeline/load_pupil_data_blender_script.py@@ -1,69 +1,82 @@import bpyimport numpy as np+import sys+from pathlib import Pathprint("loading gaze data as empties")-path_to_right_eye_gaze_xyz_npy = r"C:\Users\jonma\Dropbox\FreeMoCapProject\FreeMocap_Data\sesh_2022-02-15_11_54_28_pupil_maybe\DataArrays\right_eye_gaze_fr_xyz.npy"-r_gaze_eye_in_head_xyz = np.load(path_to_right_eye_gaze_xyz_npy)--r_gaze_eye_in_head_x = r_gaze_eye_in_head_xyz[:,0]*.001-r_gaze_eye_in_head_y = r_gaze_eye_in_head_xyz[:,1]*.001-r_gaze_eye_in_head_z = r_gaze_eye_in_head_xyz[:,2]*.001--# #%%-# %matplotlib-# plt.plot(r_eye_norm_pos_x, '.')-#%%-print(f'loading right_eye_data...')-bpy.ops.object.empty_add(type='SPHERE')-this_empty = bpy.context.active_object-# this_empty.name = 'right_eye_gaze'--for frame_num in range(r_gaze_eye_in_head_xyz.shape[0]):- if frame_num % 100 == 0:- print(f'frame {frame_num}')-- this_empty.location = (r_gaze_eye_in_head_x[frame_num],- r_gaze_eye_in_head_y[frame_num],- r_gaze_eye_in_head_z[frame_num])- this_empty.scale = (0.01, 0.01, 0.01)-- this_empty.name = 'right_eye_gaze'+def load_gaze_data(gaze_fr_xyz:np.ndarray, eye_empty_name:str):- this_empty.keyframe_insert(data_path='location',frame=frame_num)+ this_eyeball_empty = bpy.data.objects[eye_empty_name]+ bpy.ops.object.empty_add(type='SPHERE')+ this_gaze_empty = bpy.context.active_object+ print(f'loading {eye_empty_name} gaze data...')+ for frame_num in range(r_gaze_eye_in_head_xyz.shape[0]):+ this_gaze_empty.location = (gaze_fr_xyz[frame_num,0],+ gaze_fr_xyz[frame_num,1],+ gaze_fr_xyz[frame_num,2])+ this_gaze_empty.scale = (0.01, 0.01, 0.01)++ this_gaze_empty.name = eye_empty_name + '_gaze_empty'++ this_gaze_empty.keyframe_insert(data_path='location',frame=frame_num)+ bpy.ops.object.armature_add(enter_editmode=False, align='WORLD', location=(0, 0, 0), scale=(1, 1, 1))+ this_gaze_armature = bpy.context.active_object+ this_gaze_armature.name = eye_empty_name + '_gaze_armature'+ this_gaze_bone_name = eye_empty_name + '_gaze_bone'+ this_gaze_bone = this_gaze_armature.data.bones[0]+ this_gaze_bone.name = this_gaze_bone_name++ # #create gaze bone+ # this_gaze_edit_bone= this_gaze_armature.data.edit_bones[0] # note this is using EDIT_bones, not regular bones b/c blender is a wild wacky place+ # this_gaze_edit_bone.tail = this_eyeball_empty.location+ # this_gaze_edit_bone.head = this_gaze_empty.location-path_to_left_eye_gaze_xyz_npy = r"C:\Users\jonma\Dropbox\FreeMoCapProject\FreeMocap_Data\sesh_2022-02-15_11_54_28_pupil_maybe\DataArrays\left_eye_gaze_fr_xyz.npy"-l_gaze_eye_in_head_xyz = np.load(path_to_left_eye_gaze_xyz_npy)--l_gaze_eye_in_head_x = l_gaze_eye_in_head_xyz[:,0]*.001-l_gaze_eye_in_head_y = l_gaze_eye_in_head_xyz[:,1]*.001-l_gaze_eye_in_head_z = l_gaze_eye_in_head_xyz[:,2]*.001--aprint(f'loading left_eye_data...')-bpy.ops.object.empty_add(type='SPHERE')-this_empty = bpy.context.active_object-# this_empty.name = 'right_eye_gaze'--for frame_num in range(l_gaze_eye_in_head_xyz.shape[0]):- if frame_num % 100 == 0:- print(f'frame {frame_num}')-- this_empty.location = (l_gaze_eye_in_head_x[frame_num],- l_gaze_eye_in_head_y[frame_num],- l_gaze_eye_in_head_z[frame_num])- this_empty.scale = (0.01, 0.01, 0.01)+ #track gaze bone to gaze empty+ this_gaze_pose_bone = this_gaze_armature.pose.bones[0]++ this_gaze_constraint = this_gaze_pose_bone.constraints.new('COPY_LOCATION')+ this_gaze_constraint.target = this_eyeball_empty+ this_gaze_constraint = this_gaze_pose_bone.constraints.new('DAMPED_TRACK')+ this_gaze_constraint.target = this_gaze_empty++try:+ ##% Session path+ # #get session path as command line argument+ argv = sys.argv+ argv = argv[argv.index("--") + 1:]+ session_path = argv[0]++ if len(argv)>1:+ good_clean_frame_number = int(argv[1])+ else:+ good_clean_frame_number = 0+except:+ print('we appear to be running from the Blender Scripting tab! Manually enter your `session_path` at line 23')+ session_path = Path(r'C:\Users\jonma\Dropbox\FreeMoCapProject\FreeMocap_Data\sesh_2022-05-07_17_15_05_pupil_wobble_juggle_0')+- this_empty.name = 'left_eye_gaze'-- this_empty.keyframe_insert(data_path='location',frame=frame_num)+data_arrays_path = session_path / 'DataArrays'+path_to_right_eye_gaze_xyz_npy = data_arrays_path / 'right_eye_gaze_fr_xyz.npy'+path_to_left_eye_gaze_xyz_npy = data_arrays_path / 'left_eye_gaze_fr_xyz.npy'+# right eye+print(f'loading right_eye_data...')+r_gaze_eye_in_head_xyz = np.load(path_to_right_eye_gaze_xyz_npy)+r_gaze_eye_in_head_xyz *= .001 #convert to meters+load_gaze_data(r_gaze_eye_in_head_xyz, 'right_eye')+# right eye+print(f'loading left_eye_data...')+l_gaze_eye_in_head_xyz = np.load(path_to_left_eye_gaze_xyz_npy)+l_gaze_eye_in_head_xyz *= .001 #convert to meters+load_gaze_data(l_gaze_eye_in_head_xyz, 'left_eye')diff --git a/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_calibration_pipeline_orchestrator.py b/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_calibration_pipeline_orchestrator.pyindex 9b2a828d..908c9439 100644--- a/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_calibration_pipeline_orchestrator.py+++ b/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_calibration_pipeline_orchestrator.py@@ -59,16 +59,18 @@ class PupilFreemocapCalibrationPipelineOrchestrator:##### Synchronize pupil data with freemocap data - results in synchronized_session_data (each stream has exactly the same number of frames)####- synchronized_session_data = PupilFreemocapSynchronizer(self.raw_session_data).synchronize(debug=False,- vor_frame_start=self.vor_frame_start,- vor_frame_end=self.vor_frame_end)+ synchronized_session_data = PupilFreemocapSynchronizer(self.raw_session_data).synchronize(+ vor_frame_start=self.vor_frame_start,+ vor_frame_end=self.vor_frame_end,+ debug=False, )+logger.info('synchronization complete - I should add a test to make sure everything has the same number of frames')##### Calculate Head Rotation matrix for each frame (gaze data will be rotated by head_rot, then calibrated_offset_rot)####- rotation_matrix_calculator = RotationMatrixCalculator(self.raw_session_data.mediapipe_skel_fr_mar_dim)+ rotation_matrix_calculator = RotationMatrixCalculator(synchronized_session_data.mediapipe_skel_fr_mar_dim)synchronized_session_data.head_rotation_data = rotation_matrix_calculator.calculate_head_rotation_matricies(debug=False)@@ -111,36 +113,42 @@ class PupilFreemocapCalibrationPipelineOrchestrator:copy.deepcopy(synchronized_session_data.head_rotation_data),fixation_point_fr_xyz)- #save that data+ # save that dataself.save_gaze_data(synchronized_session_data)##### Play laser skeleton animation (as both a cool thing and a debug tool)####qt_gl_laser_skeleton = QtGlLaserSkeletonVisualizer(session_data=synchronized_session_data,- move_data_to_origin=True,)- # start_frame=self.vor_frame_start,- # end_frame=self.vor_frame_end)+ move_data_to_origin=True, )+ # start_frame=self.vor_frame_start,+ # end_frame=self.vor_frame_end)qt_gl_laser_skeleton.start_animation()def save_gaze_data(self, synchronized_session_data):data_save_path = self.session_path / 'DataArrays'- #right eye+ # right eyesave_right_eye_data_path = data_save_path / 'right_eye_gaze_fr_xyz.npy'np.save(save_right_eye_data_path, synchronized_session_data.right_gaze_vector_endpoint_fr_xyz)- #left eye+ # left eyesave_left_eye_data_path = data_save_path / 'left_eye_gaze_fr_xyz.npy'np.save(save_left_eye_data_path, synchronized_session_data.left_gaze_vector_endpoint_fr_xyz)+if __name__ == '__main__':+ # session_id = 'sesh_2022-05-07_17_15_05_pupil_wobble_juggle_0'+ # vor_frame_start_in = 614+ # vor_frame_end_in = 1073+session_id = 'sesh_2022-02-15_11_54_28_pupil_maybe'+ vor_frame_start_in = 1200+ vor_frame_end_in = 1500+data_path = Path('C:/Users/jonma/Dropbox/FreeMoCapProject/FreeMocap_Data/')this_session_path = data_path / session_id- vor_frame_start_in = 1200- vor_frame_end_in = 1500pupil_freemocap_calibration_pipeline_orchestrator = PupilFreemocapCalibrationPipelineOrchestrator(this_session_path,vor_frame_start=vor_frame_start_in,diff --git a/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.py b/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.pyindex 7add08cd..42f19211 100644--- a/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.py+++ b/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.py@@ -71,6 +71,8 @@ class PupilFreemocapSynchronizer:self.left_eye_end_frame = left_eye_end_frame# rebase time onto freemocap's framerate (b/c it's slower than pupil) <- sloppy, assumes mocap slower than eye tracker, which is untrue for, say, GoPros+ self.synchronized_timestamps = self.raw_session_data.timestamps[freemocap_start_frame:freemocap_end_frame ]+logger.warning("SLOPPY ASSUMPTION: assuming freemocap framerate is always slower than eye tracker (true for webcams, not true for GoPros)")@@ -78,7 +80,6 @@ class PupilFreemocapSynchronizer:self.resample_eye_data()# self.normalize_eye_data()- self.synchronized_timestamps = self.raw_session_data.timestamps[freemocap_start_frame:freemocap_end_frame + 1]self.synchronized_timestamps = self.synchronized_timestamps - self.synchronized_timestamps[0]@@ -149,7 +150,7 @@ class PupilFreemocapSynchronizer:self.left_eye_start_frame:self.left_eye_end_frame]def resample_eye_data(self):- freemocap_timestamps = self.raw_session_data.timestamps+ freemocap_timestamps = self.synchronized_timestampsright_eye_timestamps = self.right_eye_timestamps_clippedleft_eye_timestamps = self.left_eye_timestamps_clipped@@ -219,11 +220,11 @@ class PupilFreemocapSynchronizer:ax2.legend(loc='upper left')ax3 = fig.add_subplot(413)- ax3.plot(self.raw_session_data.timestamps, self.left_eye_pupil_center_normal_x, '.-',+ ax3.plot(self.synchronized_timestamps, self.left_eye_pupil_center_normal_x, '.-',label='left_eye_pupil_center_normal_x')- ax3.plot(self.raw_session_data.timestamps, self.left_eye_pupil_center_normal_y, '.-',+ ax3.plot(self.synchronized_timestamps, self.left_eye_pupil_center_normal_y, '.-',label='left_eye_pupil_center_normal_y')- ax3.plot(self.raw_session_data.timestamps, self.left_eye_pupil_center_normal_z, '.-',+ ax3.plot(self.synchronized_timestamps, self.left_eye_pupil_center_normal_z, '.-',label='left_eye_pupil_center_normal_z')ax3.legend(loc='upper left')