Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

freemocap laser skeletons, baby :D

Jon78a40f6cdc8e
diff --git a/freemocap/play_skeleton_animation.py b/freemocap/play_skeleton_animation.py
index 9e3aff1b..b31b36e8 100644
--- a/freemocap/play_skeleton_animation.py
+++ b/freemocap/play_skeleton_animation.py
@@ -328,7 +328,7 @@ def PlaySkeletonAnimation(
if useMediaPipe:
try:
- mediaPipe_skel_fr_mar_xyz = np.load(session.dataArrayPath / 'mediaPipeSkel_3d.npy')
+ mediaPipe_skel_fr_mar_xyz = np.load(session.dataArrayPath / 'mediaPipeSkel_3d_smoothed.npy')
mediaPipe_nCams_nFrames_nImgPts_XYC = np.load(session.dataArrayPath / 'mediaPipeData_2d.npy')
except:
print('No mediaPipe data found.')
diff --git a/jon_scratch/pupil_calibration_pipeline/load_pupil_data_blender_script.py b/jon_scratch/pupil_calibration_pipeline/load_pupil_data_blender_script.py
index be277391..777f6310 100644
--- a/jon_scratch/pupil_calibration_pipeline/load_pupil_data_blender_script.py
+++ b/jon_scratch/pupil_calibration_pipeline/load_pupil_data_blender_script.py
@@ -1,69 +1,82 @@
import bpy
import numpy as np
+import sys
+from pathlib import Path
print("loading gaze data as empties")
-path_to_right_eye_gaze_xyz_npy = r"C:\Users\jonma\Dropbox\FreeMoCapProject\FreeMocap_Data\sesh_2022-02-15_11_54_28_pupil_maybe\DataArrays\right_eye_gaze_fr_xyz.npy"
-r_gaze_eye_in_head_xyz = np.load(path_to_right_eye_gaze_xyz_npy)
-
-r_gaze_eye_in_head_x = r_gaze_eye_in_head_xyz[:,0]*.001
-r_gaze_eye_in_head_y = r_gaze_eye_in_head_xyz[:,1]*.001
-r_gaze_eye_in_head_z = r_gaze_eye_in_head_xyz[:,2]*.001
-
-# #%%
-# %matplotlib
-# plt.plot(r_eye_norm_pos_x, '.')
-#%%
-print(f'loading right_eye_data...')
-bpy.ops.object.empty_add(type='SPHERE')
-this_empty = bpy.context.active_object
-# this_empty.name = 'right_eye_gaze'
-
-for frame_num in range(r_gaze_eye_in_head_xyz.shape[0]):
- if frame_num % 100 == 0:
- print(f'frame {frame_num}')
-
- this_empty.location = (r_gaze_eye_in_head_x[frame_num],
- r_gaze_eye_in_head_y[frame_num],
- r_gaze_eye_in_head_z[frame_num])
- this_empty.scale = (0.01, 0.01, 0.01)
-
- this_empty.name = 'right_eye_gaze'
+def load_gaze_data(gaze_fr_xyz:np.ndarray, eye_empty_name:str):
- this_empty.keyframe_insert(data_path='location',frame=frame_num)
+ this_eyeball_empty = bpy.data.objects[eye_empty_name]
+ bpy.ops.object.empty_add(type='SPHERE')
+ this_gaze_empty = bpy.context.active_object
+ print(f'loading {eye_empty_name} gaze data...')
+ for frame_num in range(r_gaze_eye_in_head_xyz.shape[0]):
+ this_gaze_empty.location = (gaze_fr_xyz[frame_num,0],
+ gaze_fr_xyz[frame_num,1],
+ gaze_fr_xyz[frame_num,2])
+ this_gaze_empty.scale = (0.01, 0.01, 0.01)
+
+ this_gaze_empty.name = eye_empty_name + '_gaze_empty'
+
+ this_gaze_empty.keyframe_insert(data_path='location',frame=frame_num)
+ bpy.ops.object.armature_add(enter_editmode=False, align='WORLD', location=(0, 0, 0), scale=(1, 1, 1))
+ this_gaze_armature = bpy.context.active_object
+ this_gaze_armature.name = eye_empty_name + '_gaze_armature'
+ this_gaze_bone_name = eye_empty_name + '_gaze_bone'
+ this_gaze_bone = this_gaze_armature.data.bones[0]
+ this_gaze_bone.name = this_gaze_bone_name
+
+ # #create gaze bone
+ # this_gaze_edit_bone= this_gaze_armature.data.edit_bones[0] # note this is using EDIT_bones, not regular bones b/c blender is a wild wacky place
+ # this_gaze_edit_bone.tail = this_eyeball_empty.location
+ # this_gaze_edit_bone.head = this_gaze_empty.location
-path_to_left_eye_gaze_xyz_npy = r"C:\Users\jonma\Dropbox\FreeMoCapProject\FreeMocap_Data\sesh_2022-02-15_11_54_28_pupil_maybe\DataArrays\left_eye_gaze_fr_xyz.npy"
-l_gaze_eye_in_head_xyz = np.load(path_to_left_eye_gaze_xyz_npy)
-
-l_gaze_eye_in_head_x = l_gaze_eye_in_head_xyz[:,0]*.001
-l_gaze_eye_in_head_y = l_gaze_eye_in_head_xyz[:,1]*.001
-l_gaze_eye_in_head_z = l_gaze_eye_in_head_xyz[:,2]*.001
-
-aprint(f'loading left_eye_data...')
-bpy.ops.object.empty_add(type='SPHERE')
-this_empty = bpy.context.active_object
-# this_empty.name = 'right_eye_gaze'
-
-for frame_num in range(l_gaze_eye_in_head_xyz.shape[0]):
- if frame_num % 100 == 0:
- print(f'frame {frame_num}')
-
- this_empty.location = (l_gaze_eye_in_head_x[frame_num],
- l_gaze_eye_in_head_y[frame_num],
- l_gaze_eye_in_head_z[frame_num])
- this_empty.scale = (0.01, 0.01, 0.01)
+ #track gaze bone to gaze empty
+ this_gaze_pose_bone = this_gaze_armature.pose.bones[0]
+
+ this_gaze_constraint = this_gaze_pose_bone.constraints.new('COPY_LOCATION')
+ this_gaze_constraint.target = this_eyeball_empty
+ this_gaze_constraint = this_gaze_pose_bone.constraints.new('DAMPED_TRACK')
+ this_gaze_constraint.target = this_gaze_empty
+
+try:
+ ##% Session path
+ # #get session path as command line argument
+ argv = sys.argv
+ argv = argv[argv.index("--") + 1:]
+ session_path = argv[0]
+
+ if len(argv)>1:
+ good_clean_frame_number = int(argv[1])
+ else:
+ good_clean_frame_number = 0
+except:
+ print('we appear to be running from the Blender Scripting tab! Manually enter your `session_path` at line 23')
+ session_path = Path(r'C:\Users\jonma\Dropbox\FreeMoCapProject\FreeMocap_Data\sesh_2022-05-07_17_15_05_pupil_wobble_juggle_0')
+
- this_empty.name = 'left_eye_gaze'
-
- this_empty.keyframe_insert(data_path='location',frame=frame_num)
+data_arrays_path = session_path / 'DataArrays'
+path_to_right_eye_gaze_xyz_npy = data_arrays_path / 'right_eye_gaze_fr_xyz.npy'
+path_to_left_eye_gaze_xyz_npy = data_arrays_path / 'left_eye_gaze_fr_xyz.npy'
+# right eye
+print(f'loading right_eye_data...')
+r_gaze_eye_in_head_xyz = np.load(path_to_right_eye_gaze_xyz_npy)
+r_gaze_eye_in_head_xyz *= .001 #convert to meters
+load_gaze_data(r_gaze_eye_in_head_xyz, 'right_eye')
+# right eye
+print(f'loading left_eye_data...')
+l_gaze_eye_in_head_xyz = np.load(path_to_left_eye_gaze_xyz_npy)
+l_gaze_eye_in_head_xyz *= .001 #convert to meters
+load_gaze_data(l_gaze_eye_in_head_xyz, 'left_eye')
diff --git a/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_calibration_pipeline_orchestrator.py b/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_calibration_pipeline_orchestrator.py
index 9b2a828d..908c9439 100644
--- a/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_calibration_pipeline_orchestrator.py
+++ b/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_calibration_pipeline_orchestrator.py
@@ -59,16 +59,18 @@ class PupilFreemocapCalibrationPipelineOrchestrator:
####
# Synchronize pupil data with freemocap data - results in synchronized_session_data (each stream has exactly the same number of frames)
####
- synchronized_session_data = PupilFreemocapSynchronizer(self.raw_session_data).synchronize(debug=False,
- vor_frame_start=self.vor_frame_start,
- vor_frame_end=self.vor_frame_end)
+ synchronized_session_data = PupilFreemocapSynchronizer(self.raw_session_data).synchronize(
+ vor_frame_start=self.vor_frame_start,
+ vor_frame_end=self.vor_frame_end,
+ debug=False, )
+
logger.info(
'synchronization complete - I should add a test to make sure everything has the same number of frames')
####
# Calculate Head Rotation matrix for each frame (gaze data will be rotated by head_rot, then calibrated_offset_rot)
####
- rotation_matrix_calculator = RotationMatrixCalculator(self.raw_session_data.mediapipe_skel_fr_mar_dim)
+ rotation_matrix_calculator = RotationMatrixCalculator(synchronized_session_data.mediapipe_skel_fr_mar_dim)
synchronized_session_data.head_rotation_data = rotation_matrix_calculator.calculate_head_rotation_matricies(
debug=False)
@@ -111,36 +113,42 @@ class PupilFreemocapCalibrationPipelineOrchestrator:
copy.deepcopy(synchronized_session_data.head_rotation_data),
fixation_point_fr_xyz)
- #save that data
+ # save that data
self.save_gaze_data(synchronized_session_data)
####
# Play laser skeleton animation (as both a cool thing and a debug tool)
####
qt_gl_laser_skeleton = QtGlLaserSkeletonVisualizer(session_data=synchronized_session_data,
- move_data_to_origin=True,)
- # start_frame=self.vor_frame_start,
- # end_frame=self.vor_frame_end)
+ move_data_to_origin=True, )
+ # start_frame=self.vor_frame_start,
+ # end_frame=self.vor_frame_end)
qt_gl_laser_skeleton.start_animation()
def save_gaze_data(self, synchronized_session_data):
data_save_path = self.session_path / 'DataArrays'
- #right eye
+ # right eye
save_right_eye_data_path = data_save_path / 'right_eye_gaze_fr_xyz.npy'
np.save(save_right_eye_data_path, synchronized_session_data.right_gaze_vector_endpoint_fr_xyz)
- #left eye
+ # left eye
save_left_eye_data_path = data_save_path / 'left_eye_gaze_fr_xyz.npy'
np.save(save_left_eye_data_path, synchronized_session_data.left_gaze_vector_endpoint_fr_xyz)
+
if __name__ == '__main__':
+ # session_id = 'sesh_2022-05-07_17_15_05_pupil_wobble_juggle_0'
+ # vor_frame_start_in = 614
+ # vor_frame_end_in = 1073
+
session_id = 'sesh_2022-02-15_11_54_28_pupil_maybe'
+ vor_frame_start_in = 1200
+ vor_frame_end_in = 1500
+
data_path = Path('C:/Users/jonma/Dropbox/FreeMoCapProject/FreeMocap_Data/')
this_session_path = data_path / session_id
- vor_frame_start_in = 1200
- vor_frame_end_in = 1500
pupil_freemocap_calibration_pipeline_orchestrator = PupilFreemocapCalibrationPipelineOrchestrator(this_session_path,
vor_frame_start=vor_frame_start_in,
diff --git a/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.py b/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.py
index 7add08cd..42f19211 100644
--- a/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.py
+++ b/jon_scratch/pupil_calibration_pipeline/pupil_freemocap_synchronizer.py
@@ -71,6 +71,8 @@ class PupilFreemocapSynchronizer:
self.left_eye_end_frame = left_eye_end_frame
# rebase time onto freemocap's framerate (b/c it's slower than pupil) <- sloppy, assumes mocap slower than eye tracker, which is untrue for, say, GoPros
+ self.synchronized_timestamps = self.raw_session_data.timestamps[freemocap_start_frame:freemocap_end_frame ]
+
logger.warning(
"SLOPPY ASSUMPTION: assuming freemocap framerate is always slower than eye tracker (true for webcams, not true for GoPros)")
@@ -78,7 +80,6 @@ class PupilFreemocapSynchronizer:
self.resample_eye_data()
# self.normalize_eye_data()
- self.synchronized_timestamps = self.raw_session_data.timestamps[freemocap_start_frame:freemocap_end_frame + 1]
self.synchronized_timestamps = self.synchronized_timestamps - self.synchronized_timestamps[0]
@@ -149,7 +150,7 @@ class PupilFreemocapSynchronizer:
self.left_eye_start_frame:self.left_eye_end_frame]
def resample_eye_data(self):
- freemocap_timestamps = self.raw_session_data.timestamps
+ freemocap_timestamps = self.synchronized_timestamps
right_eye_timestamps = self.right_eye_timestamps_clipped
left_eye_timestamps = self.left_eye_timestamps_clipped
@@ -219,11 +220,11 @@ class PupilFreemocapSynchronizer:
ax2.legend(loc='upper left')
ax3 = fig.add_subplot(413)
- ax3.plot(self.raw_session_data.timestamps, self.left_eye_pupil_center_normal_x, '.-',
+ ax3.plot(self.synchronized_timestamps, self.left_eye_pupil_center_normal_x, '.-',
label='left_eye_pupil_center_normal_x')
- ax3.plot(self.raw_session_data.timestamps, self.left_eye_pupil_center_normal_y, '.-',
+ ax3.plot(self.synchronized_timestamps, self.left_eye_pupil_center_normal_y, '.-',
label='left_eye_pupil_center_normal_y')
- ax3.plot(self.raw_session_data.timestamps, self.left_eye_pupil_center_normal_z, '.-',
+ ax3.plot(self.synchronized_timestamps, self.left_eye_pupil_center_normal_z, '.-',
label='left_eye_pupil_center_normal_z')
ax3.legend(loc='upper left')