Free Motion Capture for Everyone πβ¨
TypeScript
trying a couple plotting methods for debug visualization
Jon109b01e45f79
diff --git a/jon_scratch/calculate_head_rotation_matrix.ipynb b/jon_scratch/calculate_head_rotation_matrix.ipynbindex dd9900b0..a066aa25 100644--- a/jon_scratch/calculate_head_rotation_matrix.ipynb+++ b/jon_scratch/calculate_head_rotation_matrix.ipynb@@ -2,7 +2,11 @@"cells": [{"cell_type": "markdown",- "metadata": {},+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },"source": ["# Calculate head rotation matricies\n","\n",@@ -19,16 +23,46 @@},{"cell_type": "markdown",- "metadata": {},+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },"source": ["install requirements to Python kernel (I think a 'kernel' is roughly equivalent to a conda environment, i.e. telling the system which version of `python.exe` you want to use to evaluate this Python code )"]},{"cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],+ "execution_count": 1,+ "metadata": {+ "pycharm": {+ "name": "#%%\n"+ }+ },+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Requirement already satisfied: numpy in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (1.22.3)\n",+ "Requirement already satisfied: matplotlib in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (3.5.1)\n",+ "Requirement already satisfied: cycler>=0.10 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from matplotlib) (0.11.0)\n",+ "Requirement already satisfied: pyparsing>=2.2.1 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from matplotlib) (3.0.8)\n",+ "Requirement already satisfied: numpy>=1.17 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from matplotlib) (1.22.3)\n",+ "Requirement already satisfied: fonttools>=4.22.0 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from matplotlib) (4.33.2)\n",+ "Requirement already satisfied: pillow>=6.2.0 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from matplotlib) (9.1.0)\n",+ "Requirement already satisfied: packaging>=20.0 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from matplotlib) (21.3)\n",+ "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from matplotlib) (1.4.2)\n",+ "Requirement already satisfied: python-dateutil>=2.7 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from matplotlib) (2.8.2)\n",+ "Requirement already satisfied: six>=1.5 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n",+ "Requirement already satisfied: pyqt5 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (5.15.6)\n",+ "Requirement already satisfied: PyQt5-Qt5>=5.15.2 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from pyqt5) (5.15.2)\n",+ "Requirement already satisfied: PyQt5-sip<13,>=12.8 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from pyqt5) (12.10.1)\n",+ "Requirement already satisfied: keyboard in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (0.13.5)\n"+ ]+ }+ ],"source": ["#I should specify the versions to ensure future stability\n","!pip install numpy\n",@@ -39,8 +73,12 @@},{"cell_type": "code",- "execution_count": null,- "metadata": {},+ "execution_count": 2,+ "metadata": {+ "pycharm": {+ "name": "#%%\n"+ }+ },"outputs": [],"source": ["\n",@@ -53,8 +91,12 @@},{"cell_type": "code",- "execution_count": null,- "metadata": {},+ "execution_count": 3,+ "metadata": {+ "pycharm": {+ "name": "#%%\n"+ }+ },"outputs": [],"source": ["debug = True\n",@@ -68,8 +110,12 @@},{"cell_type": "code",- "execution_count": 25,- "metadata": {},+ "execution_count": 4,+ "metadata": {+ "pycharm": {+ "name": "#%%\n"+ }+ },"outputs": [],"source": ["\n",@@ -87,7 +133,11 @@},{"cell_type": "markdown",- "metadata": {},+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },"source": ["\n"," ### load in data\n",@@ -100,8 +150,12 @@},{"cell_type": "code",- "execution_count": null,- "metadata": {},+ "execution_count": 5,+ "metadata": {+ "pycharm": {+ "name": "#%%\n"+ }+ },"outputs": [],"source": ["mediapipe_skeleton_fr_mar_xyz = np.load(mediapipe_skeleton_data_path)"@@ -109,8 +163,12 @@},{"cell_type": "code",- "execution_count": null,- "metadata": {},+ "execution_count": 6,+ "metadata": {+ "pycharm": {+ "name": "#%%\n"+ }+ },"outputs": [],"source": ["nose_id= 0\n",@@ -126,8 +184,12 @@},{"cell_type": "code",- "execution_count": null,- "metadata": {},+ "execution_count": 7,+ "metadata": {+ "pycharm": {+ "name": "#%%\n"+ }+ },"outputs": [],"source": ["fig = plt.figure(num=123)\n",@@ -151,16 +213,41 @@},{"cell_type": "markdown",- "metadata": {},+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },"source": ["zero everything out, aka pin it to the origin, aka \"subtract the position of the thing you want to be the origin from everything else\", aka \"do the thing that makes the X,Y,Z coordinates of the thing that should be (0,0,0) equal to (0,0,0)\""]},{"cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],+ "execution_count": 8,+ "metadata": {+ "pycharm": {+ "name": "#%%\n"+ }+ },+ "outputs": [+ {+ "data": {+ "text/plain": [+ "array([[ nan, nan, nan],\n",+ " [ nan, nan, nan],\n",+ " [ nan, nan, nan],\n",+ " ...,\n",+ " [ 12.47374782, 69.0853272 , -22.37502357],\n",+ " [ 14.10629544, 60.84139366, -6.04679083],\n",+ " [ 25.76475466, 63.75538109, -21.2651289 ]])"+ ]+ },+ "execution_count": 8,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],"source": ["z_nose_xyz = nose_xyz - head_center_xyz\n","z_nose_x = z_nose_xyz[:,0]\n",@@ -177,15 +264,23 @@},{"cell_type": "markdown",- "metadata": {},+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },"source": ["Normalize so they are unit vectors"]},{"cell_type": "code",- "execution_count": null,- "metadata": {},+ "execution_count": 9,+ "metadata": {+ "pycharm": {+ "name": "#%%\n"+ }+ },"outputs": [],"source": ["z_norm_nose_xyz = z_nose_xyz / np.linalg.norm(z_nose_xyz, axis=1)[:,None]\n",@@ -201,16 +296,41 @@},{"cell_type": "markdown",- "metadata": {},+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },"source": ["define x_hat as the zero'd out nose vector (i.e. the +X axis of the head starts at the head center and goes out the person's nose)"]},{"cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],+ "execution_count": 10,+ "metadata": {+ "pycharm": {+ "name": "#%%\n"+ }+ },+ "outputs": [+ {+ "data": {+ "text/plain": [+ "array([[ nan, nan, nan],\n",+ " [ nan, nan, nan],\n",+ " [ nan, nan, nan],\n",+ " ...,\n",+ " [ 0.16929195, 0.93761632, -0.30367067],\n",+ " [ 0.22481111, 0.96962534, -0.09636731],\n",+ " [ 0.35795511, 0.8857668 , -0.29544087]])"+ ]+ },+ "execution_count": 10,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],"source": ["x_hat_xyz = z_norm_nose_xyz\n","x_hat_x = x_hat_xyz[:,0]\n",@@ -222,16 +342,41 @@},{"cell_type": "markdown",- "metadata": {},+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },"source": ["define z as x cross y"]},{"cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],+ "execution_count": 11,+ "metadata": {+ "pycharm": {+ "name": "#%%\n"+ }+ },+ "outputs": [+ {+ "data": {+ "text/plain": [+ "array([[ nan, nan, nan],\n",+ " [ nan, nan, nan],\n",+ " [ nan, nan, nan],\n",+ " ...,\n",+ " [ 0.07844496, -0.31995954, -0.9441781 ],\n",+ " [ 0.23993877, -0.15094018, -0.95898199],\n",+ " [ 0.15626882, -0.36877272, -0.91628966]])"+ ]+ },+ "execution_count": 11,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],"source": ["z_hat_xyz = np.cross(x_hat_xyz, z_norm_left_ear_xyz) #create `z_hat` a vector perpendicular to `x_hat` and the vector from the nose to the left ear(i.e. pseudo y_hat)\n","z_hat_xyz = z_hat_xyz / np.linalg.norm(z_hat_xyz, axis=1)[:,None] #not sure if this needs normalizing, but it couldn't hurt\n",@@ -244,16 +389,41 @@},{"cell_type": "markdown",- "metadata": {},+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },"source": ["define y_hat as z_hat cross with x_hat (this ensures that they are all orthogonal, where as the original vector to the left ear might not have been )"]},{"cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],+ "execution_count": 12,+ "metadata": {+ "pycharm": {+ "name": "#%%\n"+ }+ },+ "outputs": [+ {+ "data": {+ "text/plain": [+ "array([[ nan, nan, nan],\n",+ " [ nan, nan, nan],\n",+ " [ nan, nan, nan],\n",+ " ...,\n",+ " [ 0.98243912, -0.13602032, 0.12771785],\n",+ " [ 0.94439894, -0.19246755, 0.26658374],\n",+ " [ 0.9205695 , -0.28182237, 0.27042181]])"+ ]+ },+ "execution_count": 12,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],"source": ["y_hat_xyz = np.cross(z_hat_xyz, z_norm_nose_xyz)\n","y_hat_xyz = y_hat_xyz / np.linalg.norm(y_hat_xyz, axis=1)[:,None] #not sure if this needs normalizing, but it couldn't hurt\n",@@ -266,16 +436,35 @@},{"cell_type": "markdown",- "metadata": {},+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },"source": ["create head rotation matricies by stacking x_hat_xyz, y_hat_xyz, and z_hat_xyz"]},{"cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],+ "execution_count": 13,+ "metadata": {+ "pycharm": {+ "name": "#%%\n"+ }+ },+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "4916\n",+ "[[[nan nan nan]\n",+ " [nan nan nan]\n",+ " [nan nan nan]]]\n"+ ]+ }+ ],"source": ["head_rotation_matricies = [np.dstack((this_x_hat_xyz, this_y_hat_xyz, this_z_hat_xyz))\n"," for this_x_hat_xyz, this_y_hat_xyz, this_z_hat_xyz in zip(x_hat_xyz, y_hat_xyz, z_hat_xyz)]\n",@@ -286,16 +475,41 @@},{"cell_type": "markdown",- "metadata": {},+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },"source": ["make a point in head space and rotate it with the head rotation matricies, as a test of these strange mathemagics"]},{"cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],+ "execution_count": 14,+ "metadata": {+ "pycharm": {+ "name": "#%%\n"+ }+ },+ "outputs": [+ {+ "data": {+ "text/plain": [+ "array([[ nan, nan, nan],\n",+ " [ nan, nan, nan],\n",+ " [ nan, nan, nan],\n",+ " ...,\n",+ " [ 1.23017602, 0.48163646, -1.12013092],\n",+ " [ 1.40914882, 0.62621761, -0.78876556],\n",+ " [ 1.43479342, 0.23517171, -0.94130872]])"+ ]+ },+ "execution_count": 14,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],"source": ["test_point_xyz_og = np.array((1,1,1))\n","test_point_xyz_rot = np.empty(z_hat_xyz.shape)\n",@@ -310,15 +524,23 @@},{"cell_type": "markdown",- "metadata": {},+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },"source": ["Visualize results! Press ESC to quit"]},{"cell_type": "code",- "execution_count": 17,- "metadata": {},+ "execution_count": 15,+ "metadata": {+ "pycharm": {+ "name": "#%%\n"+ }+ },"outputs": [{"name": "stdout",@@ -401,8 +623,12 @@},{"cell_type": "code",- "execution_count": 24,- "metadata": {},+ "execution_count": 16,+ "metadata": {+ "pycharm": {+ "name": "#%%\n"+ }+ },"outputs": [{"data": {@@ -410,7 +636,7 @@"(4916, 3, 3)"]},- "execution_count": 24,+ "execution_count": 16,"metadata": {},"output_type": "execute_result"}@@ -422,8 +648,12 @@},{"cell_type": "code",- "execution_count": 28,- "metadata": {},+ "execution_count": 17,+ "metadata": {+ "pycharm": {+ "name": "#%%\n"+ }+ },"outputs": [],"source": ["save_head_rot_path = data_arrays_path / 'mediaPipeSkel_3d_head_rotation_matricies_fr_row_col.npy'\n",@@ -433,7 +663,11 @@{"cell_type": "code","execution_count": null,- "metadata": {},+ "metadata": {+ "pycharm": {+ "name": "#%%\n"+ }+ },"outputs": [],"source": []}diff --git a/jon_scratch/freemocap_plotly.py b/jon_scratch/freemocap_plotly.pynew file mode 100644index 00000000..7cf531e0--- /dev/null+++ b/jon_scratch/freemocap_plotly.py@@ -0,0 +1,112 @@+# Import data+import time+import numpy as np+import plotly.graph_objects as go+from skimage import io+from pathlib import Path+## load in freemocap data+session_id = 'sesh_2022-02-15_11_54_28_pupil_maybe'+freemocap_data_folder = Path('C:/Users/jonma/Dropbox/FreeMoCapProject/FreeMocap_Data')+session_folder_path = freemocap_data_folder / session_id+data_arrays_path = session_folder_path / 'DataArrays'+mediapipe_skeleton_data_path = data_arrays_path / 'mediaPipeSkel_3d_smoothed.npy'+mediapipe_skeleton_fr_mar_xyz = np.load(mediapipe_skeleton_data_path)++##+vol = io.imread("https://s3.amazonaws.com/assets.datacamp.com/blog_assets/attention-mri.tif")+volume = vol.T+r, c = volume[0].shape++# Define frames+number_of_frames = mediapipe_skeleton_fr_mar_xyz.shape[0]++fig = go.Figure(frames=[+ go.Frame(++ data=go.Scatter3d(+ x=mediapipe_skeleton_fr_mar_xyz[this_frame_number, :, 0],+ y=mediapipe_skeleton_fr_mar_xyz[this_frame_number, :, 1],+ z=mediapipe_skeleton_fr_mar_xyz[this_frame_number, :, 2],+ mode='markers',+ marker=dict(+ size=12,+ color='red', # set color to an array/list of desired values+ colorscale='Viridis', # choose a colorscale+ opacity=0.8+ )+ ),++ name=str(this_frame_number) # you need to name the frame for the animation to behave properly+ )+ for this_frame_number in range(number_of_frames)])++# Add data to be displayed before animation starts+fig.add_trace(go.Surface(+ z=6.7 * np.ones((r, c)),+ surfacecolor=np.flipud(volume[67]),+ colorscale='Gray',+ cmin=0, cmax=200,+ colorbar=dict(thickness=20, ticklen=4)+))+++def frame_args(duration):+ return {+ "frame": {"duration": duration},+ "mode": "immediate",+ "fromcurrent": True,+ "transition": {"duration": duration, "easing": "linear"},+ }+++sliders = [+ {+ "pad": {"b": 10, "t": 60},+ "len": 0.9,+ "x": 0.1,+ "y": 0,+ "steps": [+ {+ "args": [[f.name], frame_args(0)],+ "label": str(k),+ "method": "animate",+ }+ for k, f in enumerate(fig.frames)+ ],+ }+]++# Layout+fig.update_layout(+ title='Slices in volumetric data',+ width=600,+ height=600,+ scene=dict(+ zaxis=dict(range=[-2e3, 2e3], autorange=False),+ aspectratio=dict(x=1, y=1, z=1),+ ),+ updatemenus=[+ {+ "buttons": [+ {+ "args": [None, frame_args(0)],+ "label": "▶", # play symbol+ "method": "animate",+ },+ {+ "args": [[None], frame_args(0)],+ "label": "◼", # pause symbol+ "method": "animate",+ },+ ],+ "direction": "left",+ "pad": {"r": 10, "t": 70},+ "type": "buttons",+ "x": 0.1,+ "y": 0,+ }+ ],+ sliders=sliders+)++fig.show()diff --git a/jon_scratch/karls_pupil_freemocap_code.ipynb b/jon_scratch/karls_pupil_freemocap_code.ipynbnew file mode 100644index 00000000..2d05d69c--- /dev/null+++ b/jon_scratch/karls_pupil_freemocap_code.ipynb@@ -0,0 +1,1622 @@+{+ "cells": [+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "\n",+ "# this is Jon's attempt to put Karl's `alignPupilFMC.py` into an jupyter notebook "+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "### import stuff"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 1,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Requirement already satisfied: scipy in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (1.8.0)\n",+ "Requirement already satisfied: numpy<1.25.0,>=1.17.3 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from scipy) (1.22.3)\n",+ "Requirement already satisfied: pandas in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (1.4.2)\n",+ "Requirement already satisfied: pytz>=2020.1 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from pandas) (2022.1)\n",+ "Requirement already satisfied: python-dateutil>=2.8.1 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from pandas) (2.8.2)\n",+ "Requirement already satisfied: numpy>=1.18.5 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from pandas) (1.22.3)\n",+ "Requirement already satisfied: six>=1.5 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from python-dateutil>=2.8.1->pandas) (1.16.0)\n",+ "Requirement already satisfied: numpy in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (1.22.3)\n",+ "Requirement already satisfied: pyqt5 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (5.15.6)\n",+ "Requirement already satisfied: PyQt5-Qt5>=5.15.2 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from pyqt5) (5.15.2)\n",+ "Requirement already satisfied: PyQt5-sip<13,>=12.8 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from pyqt5) (12.10.1)\n"+ ]+ }+ ],+ "source": [+ "!pip install scipy\n",+ "!pip install pandas\n",+ "!pip install numpy\n",+ "!pip install pyqt5"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 2,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "\n",+ "import numpy as np\n",+ "import pandas as pd\n",+ "import json\n",+ "from pathlib import Path\n",+ "\n",+ "from scipy import optimize\n",+ "from scipy.spatial.transform import Rotation "+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 3,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "debug = True\n",+ "if debug:\n",+ " %matplotlib qt\n",+ " # %matplotlib \n",+ " import matplotlib.pyplot as plt\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "### get yr paths right"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 4,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "session_id = 'sesh_2022-02-15_11_54_28_pupil_maybe'\n",+ "freemocap_data_folder = Path('C:/Users/jonma/Dropbox/FreeMoCapProject/FreeMocap_Data')\n",+ "\n",+ "session_folder_path = freemocap_data_folder / session_id\n",+ "\n",+ "mediapipe_skeleton_data_path = session_folder_path / 'DataArrays'/ 'mediaPipeSkel_3d_smoothed.npy'\n",+ "mediapipe_head_rotations_path = session_folder_path / 'DataArrays'/ 'mediaPipeSkel_3d_head_rotation_matricies_fr_row_col.npy'\n",+ "\n",+ "pupil_data_path = session_folder_path / 'pupil_000'\n",+ "pupil_data_exports_path = pupil_data_path / 'exports' / '000'\n",+ "pupil_positions_path = pupil_data_exports_path / 'pupil_positions.csv'\n",+ "pupil_recording_info_path = pupil_data_path / 'info.player.json'\n",+ "\n",+ "freemocap_unix_timestamp_path = session_folder_path /'unix_synced_timestamps.csv'\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ " ### load in data\n",+ "\n",+ "- full body 3d kinematic data - `mediapipe_skeleton_frame_mar_xyz`\n",+ " - type: numpy array\n",+ " - dimensions: [frame_number, tracked_point_number, dimension (xyz)]\n",+ "- pupil data from `pupil_positions.csv` in the **pupil exports** folder\n",+ " - definitions in `pupil_gaze_positions_info.txt`\n",+ "- pupil start time from `info.player.json` for sync\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 5,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "mediapipe_skeleton_fr_mar_xyz = np.load(mediapipe_skeleton_data_path)\n",+ "mediapipe_head_rotation_matricies_fr_row_col = np.load(mediapipe_head_rotations_path)\n",+ "pupil_dataframe = pd.read_csv(pupil_positions_path)\n",+ "pupil_recording_info_json = json.load(open(pupil_recording_info_path))\n",+ "freemocap_unix_timestamps_dataframe = pd.read_csv(freemocap_unix_timestamp_path)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "### pull out relevant pupil data"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "data for both eyes"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 6,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "\n",+ "timestamps = np.array(pupil_dataframe['pupil_timestamp'])\n",+ "eye_theta = np.array(pupil_dataframe['theta'])\n",+ "eye_phi = np.array(pupil_dataframe['phi'])\n",+ "sphere_center_x = np.array(pupil_dataframe['sphere_center_x'])\n",+ "sphere_center_y = np.array(pupil_dataframe['sphere_center_y'])\n",+ "sphere_center_z = np.array(pupil_dataframe['sphere_center_z'])\n",+ "pupil_center_normal_x = np.array(pupil_dataframe['circle_3d_normal_x'])\n",+ "pupil_center_normal_y = np.array(pupil_dataframe['circle_3d_normal_y'])\n",+ "pupil_center_normal_z = np.array(pupil_dataframe['circle_3d_normal_z'])\n",+ "\n",+ "eye_d = pupil_dataframe['eye_id']\n",+ "method = pupil_dataframe['method']\n",+ "# tracking_method_name = 'pye3d 0.3.0 real-time'\n",+ "\n",+ "\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "get data for the right eye\n",+ "\n",+ "pull out data according to `eye_d` (right_eye == 0) and tracking method (because pupil interleaves 2d and 3d data)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 7,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "right_eye_d = 0 \n",+ "\n",+ "#timestamps\n",+ "r_eye_timestamps_og = timestamps[np.logical_and(eye_d==right_eye_d, method!='2d c++')]\n",+ "\n",+ "r_eye_theta = eye_theta[np.logical_and(eye_d==right_eye_d, method!='2d c++')]\n",+ "r_eye_phi = eye_phi[np.logical_and(eye_d==right_eye_d, method!='2d c++')]\n",+ "\n",+ "r_eye_sphere_center_x = sphere_center_x[np.logical_and(eye_d==right_eye_d, method!='2d c++')]\n",+ "r_eye_sphere_center_y = sphere_center_y[np.logical_and(eye_d==right_eye_d, method!='2d c++')]\n",+ "r_eye_sphere_center_z = sphere_center_z[np.logical_and(eye_d==right_eye_d, method!='2d c++')]\n",+ "\n",+ "r_eye_pupil_center_normal_x = pupil_center_normal_x[np.logical_and(eye_d==right_eye_d, method!='2d c++')]\n",+ "r_eye_pupil_center_normal_y = pupil_center_normal_y[np.logical_and(eye_d==right_eye_d, method!='2d c++')]\n",+ "r_eye_pupil_center_normal_z = pupil_center_normal_z[np.logical_and(eye_d==right_eye_d, method!='2d c++')]\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "Get data for the left eye\n",+ "\n",+ "pull out data according to `eye_d` (left eye == 1) and tracking method (because pupil interleaves 2d and 3d data)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 8,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "left_eye_d = 1 \n",+ "\n",+ "#timestamps\n",+ "l_eye_timestamps_og = timestamps[np.logical_and(eye_d==left_eye_d, method!='2d c++')]\n",+ "\n",+ "l_eye_theta = eye_theta[np.logical_and(eye_d==left_eye_d, method!='2d c++')]\n",+ "l_eye_phi = eye_phi[np.logical_and(eye_d==left_eye_d, method!='2d c++')]\n",+ "\n",+ "l_eye_sphere_center_x = sphere_center_x[np.logical_and(eye_d==left_eye_d, method!='2d c++')]\n",+ "l_eye_sphere_center_y = sphere_center_y[np.logical_and(eye_d==left_eye_d, method!='2d c++')]\n",+ "l_eye_sphere_center_z = sphere_center_z[np.logical_and(eye_d==left_eye_d, method!='2d c++')]\n",+ "\n",+ "l_eye_pupil_center_normal_x = pupil_center_normal_x[np.logical_and(eye_d==left_eye_d, method!='2d c++')]\n",+ "l_eye_pupil_center_normal_y = pupil_center_normal_y[np.logical_and(eye_d==left_eye_d, method!='2d c++')]\n",+ "l_eye_pupil_center_normal_z = pupil_center_normal_z[np.logical_and(eye_d==left_eye_d, method!='2d c++')]\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 9,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "if debug:\n",+ " fig = plt.figure(num=2134,figsize=(10,10))\n",+ " \n",+ " ax1 = fig.add_subplot(321)\n",+ " ax1.plot(r_eye_timestamps_og, r_eye_theta, '.-', label='r eye theta')\n",+ " ax1.plot(r_eye_timestamps_og, r_eye_phi, '.-', label='r eye phi')\n",+ "\n",+ " ax1.legend(loc='upper right')\n",+ " \n",+ " ax3 = fig.add_subplot(323)\n",+ " ax3.plot(r_eye_timestamps_og, r_eye_pupil_center_normal_x, '.-', label='r_eye_pupil_center_normal_x')\n",+ " ax3.plot(r_eye_timestamps_og, r_eye_pupil_center_normal_y, '.-', label='r_eye_pupil_center_normal_y')\n",+ " ax3.plot(r_eye_timestamps_og, r_eye_pupil_center_normal_z, '.-', label='r_eye_pupil_center_normal_z')\n",+ " ax3.legend(loc='upper right')\n",+ "\n",+ " ax5 = fig.add_subplot(325)\n",+ " ax5.plot(r_eye_timestamps_og, r_eye_sphere_center_x, '.-', label='r_eye_sphere_center_x')\n",+ " ax5.plot(r_eye_timestamps_og, r_eye_sphere_center_y, '.-', label='r_eye_sphere_center_y')\n",+ " ax5.plot(r_eye_timestamps_og, r_eye_sphere_center_z, '.-', label='r_eye_sphere_center_z')\n",+ " ax5.legend(loc='upper right')\n",+ "\n",+ " \n",+ " ax2 = fig.add_subplot(322)\n",+ " ax2.plot(l_eye_timestamps_og, l_eye_theta, '.-', label='l eye theta')\n",+ " ax2.plot(l_eye_timestamps_og, l_eye_phi, '.-', label='l eye phi')\n",+ " ax2.legend(loc='upper right')\n",+ " \n",+ " ax4 = fig.add_subplot(324)\n",+ " ax4.plot(l_eye_timestamps_og, l_eye_pupil_center_normal_x, '.-', label='l_eye_pupil_center_normal_x')\n",+ " ax4.plot(l_eye_timestamps_og, l_eye_pupil_center_normal_y, '.-', label='l_eye_pupil_center_normal_y')\n",+ " ax4.plot(l_eye_timestamps_og, l_eye_pupil_center_normal_z, '.-', label='l_eye_pupil_center_normal_z')\n",+ " ax4.legend(loc='upper right')\n",+ "\n",+ " ax6 = fig.add_subplot(326)\n",+ " ax6.plot(l_eye_timestamps_og, l_eye_sphere_center_x, '.-', label='l_eye_sphere_center_x')\n",+ " ax6.plot(l_eye_timestamps_og, l_eye_sphere_center_y, '.-', label='l_eye_sphere_center_y')\n",+ " ax6.plot(l_eye_timestamps_og, l_eye_sphere_center_z, '.-', label='l_eye_sphere_center_z')\n",+ "\n",+ " plt.pause(.1)\n",+ " plt.show()\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "### create XYZ gaze array\n",+ "\n",+ "\n",+ "subtract eyeball center from pupil center to re-base the gaze vector to the origin "+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "skipping 'remove nans' step for now, but i might need to add it later? "+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "### synchronize pupil and freemocap timestamps "+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "#### convert pupil timestamps into unix time"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 10,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "r_eye_timestamps = r_eye_timestamps_og - pupil_recording_info_json['start_time_synced_s'] +pupil_recording_info_json['start_time_system_s'] \n",+ "l_eye_timestamps = l_eye_timestamps_og - pupil_recording_info_json['start_time_synced_s'] +pupil_recording_info_json['start_time_system_s'] \n",+ "\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "calculate freemocap unix timestamps (as mean of each camera's timestamp on each frame)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 11,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "\n",+ "each_cam_timestamps_list = []\n",+ "\n",+ "camera_names = [column_name for column_name in list(freemocap_unix_timestamps_dataframe.columns) if 'Cam' in column_name]\n",+ "\n",+ "for this_camera_name in camera_names:\n",+ " each_cam_timestamps_list.append(np.array(freemocap_unix_timestamps_dataframe[this_camera_name]))\n",+ " \n",+ "cameras_timestamp_array = np.stack(each_cam_timestamps_list,axis=1)\n",+ "cameras_timestamp_array[cameras_timestamp_array==-1]=np.nan #replace -1 with nan\n",+ "\n",+ "freemocap_timestamps = np.nanmean(cameras_timestamp_array,axis=1)\n",+ "\n",+ "freemocap_frame_durations = np.diff(freemocap_timestamps)\n",+ "freemocap_frames_per_second = np.nanmean(freemocap_frame_durations**-1)\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 12,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "if debug:\n",+ "\n",+ " timestamp_diff_on_each_frame_rel_cam0 = np.diff(cameras_timestamp_array, axis=1)\n",+ " frame_duration = np.diff(freemocap_timestamps)\n",+ "\n",+ " fig = plt.figure(234899234)\n",+ " ax1 = fig.add_subplot(411)\n",+ " ax1.plot(cameras_timestamp_array, '.-',label='each cam')\n",+ " ax1.plot(freemocap_timestamps, 'r-',label='freemocap_timestamps')\n",+ " ax1.set_title('cameras_timestamps')\n",+ " ax1.set_xlabel('frame number')\n",+ " ax1.set_ylabel('timestamp')\n",+ " ax1.legend(loc='upper left')\n",+ " \n",+ " ax2 = fig.add_subplot(412)\n",+ " for this_cam_num in range(timestamp_diff_on_each_frame_rel_cam0.shape[1]):\n",+ " ax2.hist(timestamp_diff_on_each_frame_rel_cam0[:,this_cam_num], label=f'camera {this_cam_num}', alpha=0.5)\n",+ " ax2.set_xlim(-.1,.1)\n",+ " ax2.set_xlabel('timestamp diff on each frame (s)')\n",+ " ax2.legend()\n",+ "\n",+ " ax3 = fig.add_subplot(413)\n",+ " ax3.plot(freemocap_frame_durations, '.-',label='frame duration')\n",+ " ax3.plot([0, len(freemocap_frame_durations)],\n",+ " [np.nanmean(freemocap_frame_durations), np.nanmean(freemocap_frame_durations)],\n",+ " 'r-',\n",+ " label=f'mean frame duration (ms):{str(np.nanmean(freemocap_frame_durations)*1000)[:5]}',\n",+ " )\n",+ " ax3.legend(loc='upper left')\n",+ " ax3.set_xlabel('frame number')\n",+ " ax3.set_ylabel('frame duration (s)')\n",+ "\n",+ " ax4 = fig.add_subplot(414)\n",+ " ax4.hist(freemocap_frame_durations**-1, label='frame duration')\n",+ " ax4.legend(loc='upper left')\n",+ " ax4.set_xlabel('frame rate (frames per second')\n",+ " ax4.set_xlim(0,50)\n",+ " \n",+ " plt.pause(.1)\n",+ " plt.show()"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 13,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "freemocap_timestamps[-1]: 1644944570.8511524\n",+ "r_eye_timestamps[-1]: 1644944575.0656984\n"+ ]+ },+ {+ "data": {+ "text/plain": [+ "1644944570.8511524"+ ]+ },+ "execution_count": 13,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "print(f'freemocap_timestamps[-1]: {freemocap_timestamps[-1]}')\n",+ "print(f'r_eye_timestamps[-1]: {r_eye_timestamps[-1]}')\n",+ "\n",+ "np.min((freemocap_timestamps[-1],r_eye_timestamps[-1]))"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "#### align freemocap and pupil timestamps and clip the starts and ends of the various data traces so that everything covers the same timespacn"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 14,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "\n",+ "#find start and end frames shared by all datastreams\n",+ "start_time_unix = np.max((freemocap_timestamps[0],r_eye_timestamps[0], l_eye_timestamps[0]))\n",+ "end_time_unix = np.min((freemocap_timestamps[-1],r_eye_timestamps[-1], l_eye_timestamps[-1]))\n",+ "\n",+ "#freemocap\n",+ "if any(freemocap_timestamps>=start_time_unix):\n",+ " freemocap_start = np.where(freemocap_timestamps>=start_time_unix)[0][0]\n",+ "else:\n",+ " freemocap_start = 0\n",+ "\n",+ "if any(freemocap_timestamps<=end_time_unix):\n",+ " freemocap_end = np.where(freemocap_timestamps<=end_time_unix)[0][-1]\n",+ "else:\n",+ " freemocap_end = len(freemocap_timestamps)\n",+ "\n",+ "#right eye\n",+ "if any(r_eye_timestamps>=start_time_unix):\n",+ " r_eye_start = np.where(r_eye_timestamps>=start_time_unix)[0][0]\n",+ "else:\n",+ " r_eye_start = 0\n",+ " \n",+ "if any(r_eye_timestamps<=end_time_unix):\n",+ " r_eye_end = np.where(r_eye_timestamps<=end_time_unix)[0][-1]\n",+ "else:\n",+ " r_eye_end = len(r_eye_timestamps)\n",+ "\n",+ "#left eye\n",+ "if any(l_eye_timestamps>=start_time_unix):\n",+ " l_eye_start = np.where(l_eye_timestamps>=start_time_unix)[0][0]\n",+ "else:\n",+ " l_eye_start = 0\n",+ " \n",+ "if any(l_eye_timestamps<=end_time_unix):\n",+ " l_eye_end = np.where(l_eye_timestamps<=end_time_unix)[0][-1]\n",+ "else:\n",+ " l_eye_end = len(l_eye_timestamps)\n",+ "\n",+ "#rebase time onto freemocap's framerate (b/c it's slower than pupil) <- sloppy, assumes mocap slower than eye tracker, which is untrue for say GoPros\n",+ "new_timestamps = np.arange(start_time_unix,end_time_unix,1/freemocap_frames_per_second) \n",+ "\n",+ "\n",+ "freemocap_timestamps = freemocap_timestamps[freemocap_start:freemocap_end]\n",+ "r_eye_timestamps = r_eye_timestamps[r_eye_start:r_eye_end]\n",+ "l_eye_timestamps = l_eye_timestamps[l_eye_start:l_eye_end]\n",+ "\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "clip out portions of each eye's data stream that correspond to the shared time period\n",+ "\n",+ "(I appear to be skiping the freemocap data, which might cause fail on other recordings?)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 15,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "r_eye_sphere_center_x = r_eye_sphere_center_x[r_eye_start:r_eye_end]\n",+ "r_eye_sphere_center_y = r_eye_sphere_center_y[r_eye_start:r_eye_end]\n",+ "r_eye_sphere_center_z = r_eye_sphere_center_z[r_eye_start:r_eye_end]\n",+ "\n",+ "r_eye_pupil_center_normal_x = r_eye_pupil_center_normal_x[r_eye_start:r_eye_end]\n",+ "r_eye_pupil_center_normal_y = r_eye_pupil_center_normal_y[r_eye_start:r_eye_end]\n",+ "r_eye_pupil_center_normal_z = r_eye_pupil_center_normal_z[r_eye_start:r_eye_end]"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 16,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "l_eye_sphere_center_x = l_eye_sphere_center_x[l_eye_start:l_eye_end]\n",+ "l_eye_sphere_center_y = l_eye_sphere_center_y[l_eye_start:l_eye_end]\n",+ "l_eye_sphere_center_z = l_eye_sphere_center_z[l_eye_start:l_eye_end]\n",+ "\n",+ "l_eye_pupil_center_normal_x = l_eye_pupil_center_normal_x[l_eye_start:l_eye_end]\n",+ "l_eye_pupil_center_normal_y = l_eye_pupil_center_normal_y[l_eye_start:l_eye_end]\n",+ "l_eye_pupil_center_normal_z = l_eye_pupil_center_normal_z[l_eye_start:l_eye_end]"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "create r_eye_gaze_xyz : the direction the eye is pointing *in head centered coordinates*"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 17,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "r_eye_gaze_xyz_og.shape: (17737, 3)\n"+ ]+ }+ ],+ "source": [+ "r_eye_gaze_x = r_eye_pupil_center_normal_x\n",+ "r_eye_gaze_y = r_eye_pupil_center_normal_y\n",+ "r_eye_gaze_z = r_eye_pupil_center_normal_z\n",+ "\n",+ "r_eye_gaze_xyz_og = np.vstack((r_eye_gaze_x, r_eye_gaze_y, r_eye_gaze_z)).T\n",+ "\n",+ "print(f'r_eye_gaze_xyz_og.shape: {r_eye_gaze_xyz_og.shape}')"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 18,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "l_eye_gaze_xyz_og.shape: (16324, 3)\n"+ ]+ }+ ],+ "source": [+ "l_eye_gaze_x = l_eye_pupil_center_normal_x\n",+ "l_eye_gaze_y = l_eye_pupil_center_normal_y\n",+ "l_eye_gaze_z = l_eye_pupil_center_normal_z\n",+ "\n",+ "l_eye_gaze_xyz_og = np.vstack((l_eye_gaze_x, l_eye_gaze_y, l_eye_gaze_z)).T\n",+ "\n",+ "print(f'l_eye_gaze_xyz_og.shape: {l_eye_gaze_xyz_og.shape}')"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "#### resample gaze and skeleton data so that each has the same number of frames\n",+ "\n",+ "resample pupil data onto freemocap timestamps"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 19,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "\n",+ "r_eye_gaze_xyz = np.zeros((len(freemocap_timestamps),3))\n",+ "l_eye_gaze_xyz = np.zeros((len(freemocap_timestamps),3))\n",+ "\n",+ "for this_dimension in range(3): #loop over the X Y and Z simensions and resample to new timestamps\n",+ " r_eye_gaze_xyz[:,this_dimension] = np.interp(freemocap_timestamps,\n",+ " r_eye_timestamps,\n",+ " r_eye_gaze_xyz_og[:,this_dimension])\n",+ " l_eye_gaze_xyz[:,this_dimension] = np.interp(freemocap_timestamps,\n",+ " l_eye_timestamps,\n",+ " l_eye_gaze_xyz_og[:,this_dimension])\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 20,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "r_eye_gaze_xyz = r_eye_gaze_xyz/np.linalg.norm(r_eye_gaze_xyz,axis=1,ord=2)[:,None]\n",+ "l_eye_gaze_xyz = l_eye_gaze_xyz/np.linalg.norm(l_eye_gaze_xyz,axis=1,ord=2)[:,None]\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "# snip off the first frame of the freemocap data becuase there is a -> BUG <- such that there are one fewer timestamp than there are recorded frames. \n",+ "\n",+ "V V V V V V"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 21,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "if r_eye_gaze_xyz.shape[0] != mediapipe_skeleton_fr_mar_xyz.shape[0]:\n",+ " mediapipe_skeleton_fr_mar_xyz = np.delete(mediapipe_skeleton_fr_mar_xyz, 0, axis=0)\n",+ " mediapipe_head_rotation_matricies_fr_row_col = np.delete(mediapipe_head_rotation_matricies_fr_row_col, 0, axis=0)\n",+ "# DOESN'T BOTHER ME AT ALL HAHA"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "These should all have the exact same number of frame"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 22,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "r_eye_gaze_xyz.shape: (4915, 3)\n",+ "l_eye_gaze_xyz.shape: (4915, 3)\n",+ "mediapipe_skeleton_fr_mar_xyz.shape: (4915, 543, 3)\n",+ "mediapipe_head_rotation_matricies_fr_row_col.shape: (4915, 3, 3)\n"+ ]+ }+ ],+ "source": [+ "print(f'r_eye_gaze_xyz.shape: {r_eye_gaze_xyz.shape}')\n",+ "print(f'l_eye_gaze_xyz.shape: {l_eye_gaze_xyz.shape}')\n",+ "print(f'mediapipe_skeleton_fr_mar_xyz.shape: {mediapipe_skeleton_fr_mar_xyz.shape}')\n",+ "print(f'mediapipe_head_rotation_matricies_fr_row_col.shape: {mediapipe_head_rotation_matricies_fr_row_col.shape}')"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 23,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"+ ]+ }+ ],+ "source": [+ "if debug:\n",+ " fig = plt.figure(num=65341,figsize=(10,20))\n",+ " ax1 = fig.add_subplot(211)\n",+ " ax1.plot(freemocap_timestamps, r_eye_gaze_xyz[:,0], '.-', label='r_eye_gaze_x')\n",+ " ax1.plot(freemocap_timestamps, r_eye_gaze_xyz[:,1], '.-', label='r_eye_gaze_y')\n",+ " ax1.plot(freemocap_timestamps, r_eye_gaze_xyz[:,2], '.-', label='r_eye_gaze_z')\n",+ " ax1.legend(loc='upper left')\n",+ " \n",+ " ax2 = fig.add_subplot(212)\n",+ " ax2.plot(freemocap_timestamps, l_eye_gaze_xyz[:,0], '.-', label='_eye_gaze_x')\n",+ " ax2.plot(freemocap_timestamps, l_eye_gaze_xyz[:,1], '.-', label='_eye_gaze_y')\n",+ " ax2.plot(freemocap_timestamps, l_eye_gaze_xyz[:,2], '.-', label='_eye_gaze_z')\n",+ " ax2.legend(loc='upper left')\n",+ " plt.show()\n",+ "\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "normalize gaze vector to unit length"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "save out `npy` of gaze_xyz data"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "Smooth a bit with savitsky golay filter"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 24,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "from scipy.signal import savgol_filter\n",+ "\n",+ "smoothWinLength = 5\n",+ "smoothOrder = 3\n",+ "for dim in range(r_eye_gaze_xyz.shape[1]):\n",+ " r_eye_gaze_xyz[:,dim] = savgol_filter(r_eye_gaze_xyz[:,dim], smoothWinLength, smoothOrder)\n",+ " l_eye_gaze_xyz[:,dim] = savgol_filter(l_eye_gaze_xyz[:,dim], smoothWinLength, smoothOrder)\n",+ "\n",+ "\n",+ "# #reorient gaze (I think because it comes in in image coordiates? not really sure )\n",+ "# r_eye_gaze_xyz[:,1] = -r_eye_gaze_xyz[:,1]\n",+ "# r_eye_gaze_xyz[:,(0,2)] = r_eye_gaze_xyz[:,(2,0)]\n",+ "\n",+ "# l_eye_gaze_xyz[:,1] = -l_eye_gaze_xyz[:,1]\n",+ "# l_eye_gaze_xyz[:,(0,2)] = l_eye_gaze_xyz[:,(2,0)]"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 25,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "r_gaze_xyz_npy_save_path = pupil_data_exports_path / 'right_eye_gaze_xyz_synchronized_w_freemocap_uncalibrated.npy'\n",+ "np.save(r_gaze_xyz_npy_save_path, r_eye_gaze_xyz)\n",+ "\n",+ "l_gaze_xyz_npy_save_path = pupil_data_exports_path / 'left_eye_gaze_xyz_synchronized_w_freemocap_uncalibrated.npy'\n",+ "np.save(l_gaze_xyz_npy_save_path, l_eye_gaze_xyz)\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "# VOR Calibration"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "# prep data for VOR calibration\n",+ "\n",+ "we'll be finding a rotational offset such that gaze_xyz rotated-by-the offset-then-rotated-by-head-orientation will align with the fixation point\n",+ "\n",+ " so we want to put the fixation point (in this case the right index finger in a eyeball-centered reference frame (i.e. where `r_eye_xyz` is at `(0,0,0)`)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 26,+ "metadata": {+ "collapsed": false,+ "pycharm": {+ "name": "#%%\n"+ }+ },+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "head_rotation_martices_fr_row_col.shape: (4915, 3, 3)\n",+ "r_eye_gaze_xyz.shape: (4915, 3)\n",+ "fixation_point_xyz.shape: (4915, 3)\n"+ ]+ }+ ],+ "source": [+ "vor_frame_start =1000\n",+ "vor_frame_end = 1400\n",+ "\n",+ "right_index_fingertip_idx = 41 # pretty sure this is right?\n",+ "r_eye_idx = 5\n",+ "\n",+ "fixation_point_xyz = np.squeeze(mediapipe_skeleton_fr_mar_xyz[:, right_index_fingertip_idx, :])\n",+ "r_eyeball_center_xyz = np.squeeze(mediapipe_skeleton_fr_mar_xyz[:, r_eye_idx, :])\n",+ "#make sure everyone is the right size and shape\n",+ "print(f'head_rotation_martices_fr_row_col.shape: {mediapipe_head_rotation_matricies_fr_row_col.shape}')\n",+ "print(f'r_eye_gaze_xyz.shape: {r_eye_gaze_xyz.shape}')\n",+ "print(f'fixation_point_xyz.shape: {fixation_point_xyz.shape}')\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 27,+ "metadata": {+ "collapsed": false,+ "pycharm": {+ "name": "#%%\n"+ }+ },+ "outputs": [+ {+ "data": {+ "text/plain": [+ "array([[-140.99623459, 371.93667349, -908.18373013],\n",+ " [-162.83809499, 344.26373174, -912.30887986],\n",+ " [-165.73504142, 346.97592809, -911.18040645],\n",+ " ...,\n",+ " [-104.44826484, -31.40865991, -522.0636855 ],\n",+ " [-101.70694275, -47.25579649, -518.36201151],\n",+ " [-102.06426831, -55.07294944, -524.55510472]])"+ ]+ },+ "execution_count": 27,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "\n",+ "fixation_point_during_vor_xyz = mediapipe_skeleton_fr_mar_xyz[vor_frame_start:vor_frame_end, right_index_fingertip_idx, :]\n",+ "r_eyeball_center_during_vor_xyz = mediapipe_skeleton_fr_mar_xyz[vor_frame_start:vor_frame_end, r_eye_idx, :]\n",+ "\n",+ "fixation_point_in_r_eyeball_coordinates_fr_xyz = r_eyeball_center_during_vor_xyz - fixation_point_during_vor_xyz #Subtract the-thing-that-will-be-at-origin from the-other-things to put other-things in ## thing-that-will-be-at-origin-centered-reference-frame\n",+ "fixation_point_distance_xyz = np.squeeze(np.linalg.norm(fixation_point_in_r_eyeball_coordinates_fr_xyz,axis=1,ord=2)[:,None])\n",+ "\n",+ "\n",+ "\n",+ "r_gaze_during_vor_fr_xyz = r_eye_gaze_xyz[vor_frame_start:vor_frame_end]\n",+ "r_gaze_during_vor_fr_xyz[:,0] = r_gaze_during_vor_fr_xyz[:,0] * fixation_point_distance_xyz #scale gaze vector to reach fixation point\n",+ "r_gaze_during_vor_fr_xyz[:,1] = r_gaze_during_vor_fr_xyz[:,1] * fixation_point_distance_xyz #scale gaze vector to reach fixation point\n",+ "r_gaze_during_vor_fr_xyz[:,2] = r_gaze_during_vor_fr_xyz[:,2] * fixation_point_distance_xyz #scale gaze vector to reach fixation point\n",+ "r_gaze_during_vor_fr_xyz"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 28,+ "metadata": {+ "collapsed": false,+ "pycharm": {+ "name": "#%%\n"+ }+ },+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "head_rotation_matrices_during_vor_fr_row_col.shape: (400, 3, 3)\n",+ "r_gaze_during_vor_xyz.shape: (400, 3)\n",+ "fixation_point_in_r_eyeball_coordinates_xyz.shape: (400, 3)\n"+ ]+ }+ ],+ "source": [+ "\n",+ "head_rotation_matrices_during_vor_fr_row_col = mediapipe_head_rotation_matricies_fr_row_col[vor_frame_start:vor_frame_end, :, :]\n",+ "\n",+ "#make sure everyone is the right size and shape\n",+ "print(f'head_rotation_matrices_during_vor_fr_row_col.shape: {head_rotation_matrices_during_vor_fr_row_col.shape}')\n",+ "print(f'r_gaze_during_vor_xyz.shape: {r_gaze_during_vor_fr_xyz.shape}')\n",+ "print(f'fixation_point_in_r_eyeball_coordinates_xyz.shape: {fixation_point_in_r_eyeball_coordinates_fr_xyz.shape}')\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "collapsed": false,+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "put mediapipe skel into eyeball centered coordinates (for debugging)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 29,+ "metadata": {+ "collapsed": false,+ "pycharm": {+ "name": "#%%\n"+ }+ },+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "skel_during_vor_fr_mar_dim.shape: (400, 543, 3)\n"+ ]+ }+ ],+ "source": [+ "skel_during_vor_fr_mar_dim = mediapipe_skeleton_fr_mar_xyz[vor_frame_start:vor_frame_end,:,:]\n",+ "\n",+ "for this_marker_number in range(skel_during_vor_fr_mar_dim.shape[1]):\n",+ " skel_during_vor_fr_mar_dim[:,this_marker_number,:] = r_eyeball_center_during_vor_xyz - skel_during_vor_fr_mar_dim[:,this_marker_number,:]\n",+ "\n",+ "\n",+ "print(f'skel_during_vor_fr_mar_dim.shape: {skel_during_vor_fr_mar_dim.shape}')"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "collapsed": false,+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "# define error functions and run optimization\n",+ "\n",+ "according to lines 58:68 of - [https://github.com/trentwirth/ARGP_Matlab_LaserSkeleton/blob/main/Toolbox/vorPupilAlignErrFun_eyeCam.m](https://github.com/trentwirth/ARGP_Matlab_LaserSkeleton/blob/main/Toolbox/vorPupilAlignErrFun_eyeCam.m)\n",+ "```\n",+ "\n",+ "%%\n",+ "%%%%%%% This part's important - Rotate gaze vector by [this guess at the proper alignment matrix], prior to resituating the origin on on the eyeball\n",+ "%%%%\n",+ "\n",+ "\n",+ "for rr = 1:length(gazeXYZ)\n",+ " \n",+ " thisET_frame_unrot = camRotMatGuess * [gazeXYZ(rr,1); gazeXYZ(rr,2); gazeXYZ(rr,3)];\n",+ " thisETframe = headRotMat_row_col_fr(:,:,rr) * thisET_frame_unrot;\n",+ " \n",+ " headOrVec(rr,:) = headRotMat_row_col_fr(:,:,rr) * [2e3; 0;0];\n",+ " \n",+ " gazeXYZ(rr,:) = thisETframe;\n",+ " \n",+ "end\n",+ "\n",+ "```\n",+ "\n",+ "we want to rotate by rotation_guess THEN by head rotation \n",+ "\n",+ "(which makes sense, since we're using the optimizer to find the rotational offset between the eye camera reference frame and the eye-in-head reference frame)\n",+ "\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {+ "collapsed": false,+ "pycharm": {+ "name": "#%% md\n"+ }+ },+ "source": [+ "### optimization error animation function \n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 30,+ "metadata": {+ "collapsed": false,+ "pycharm": {+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "plt.close('all')\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 32,+ "metadata": {+ "collapsed": false,+ "pycharm": {+ "name": "#%%\n"+ }+ },+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ " Iteration Total nfev Cost Cost reduction Step norm Optimality \n",+ " 0 1 1.9160e+11 2.28e+11 \n",+ " 1 2 2.9245e+10 1.62e+11 1.00e+00 6.24e+10 \n",+ " 2 4 1.0953e+10 1.83e+10 5.00e-01 1.18e+10 \n",+ " 3 6 9.8908e+09 1.06e+09 2.50e-01 3.05e+09 \n",+ " 4 8 9.7599e+09 1.31e+08 6.25e-02 4.77e+08 \n",+ " 5 9 9.7498e+09 1.01e+07 6.25e-02 3.66e+08 \n",+ " 6 10 9.7457e+09 4.13e+06 1.56e-02 1.41e+08 \n",+ " 7 12 9.7451e+09 5.70e+05 3.91e-03 1.34e+08 \n",+ " 8 13 9.7447e+09 4.71e+05 3.91e-03 9.35e+07 \n",+ " 9 14 9.7438e+09 8.17e+05 7.81e-03 8.51e+07 \n",+ " 10 15 9.7437e+09 1.83e+05 1.56e-02 3.72e+08 \n",+ " 11 16 9.7423e+09 1.35e+06 3.91e-03 1.78e+08 \n",+ " 12 17 9.7417e+09 5.74e+05 7.81e-03 2.08e+08 \n",+ " 13 18 9.7411e+09 6.42e+05 7.81e-03 1.87e+08 \n",+ " 14 19 9.7404e+09 6.81e+05 7.81e-03 2.02e+08 \n",+ " 15 20 9.7397e+09 7.35e+05 7.81e-03 1.93e+08 \n",+ " 16 21 9.7389e+09 7.89e+05 7.81e-03 1.96e+08 \n",+ " 17 22 9.7380e+09 8.46e+05 7.81e-03 1.97e+08 \n",+ " 18 23 9.7371e+09 9.07e+05 7.81e-03 1.89e+08 \n",+ " 19 24 9.7362e+09 9.72e+05 7.81e-03 2.02e+08 \n",+ " 20 25 9.7351e+09 1.04e+06 7.81e-03 1.80e+08 \n",+ " 21 26 9.7340e+09 1.11e+06 7.81e-03 2.06e+08 \n",+ " 22 27 9.7328e+09 1.19e+06 7.81e-03 1.69e+08 \n",+ " 23 28 9.7316e+09 1.26e+06 7.81e-03 2.09e+08 \n",+ " 24 29 9.7302e+09 1.35e+06 7.81e-03 1.66e+08 \n",+ " 25 30 9.7288e+09 1.43e+06 7.81e-03 2.12e+08 \n",+ " 26 31 9.7273e+09 1.52e+06 7.81e-03 1.71e+08 \n",+ " 27 32 9.7248e+09 2.43e+06 1.56e-02 3.03e+08 \n",+ " 28 33 9.7234e+09 1.45e+06 1.56e-02 4.48e+08 \n",+ " 29 34 9.7217e+09 1.66e+06 3.91e-03 2.56e+08 \n",+ " 30 35 9.7199e+09 1.82e+06 7.81e-03 2.25e+08 \n",+ " 31 36 9.7180e+09 1.93e+06 7.81e-03 1.87e+08 \n",+ " 32 37 9.7142e+09 3.82e+06 1.56e-02 2.73e+08 \n",+ " 33 38 9.7128e+09 1.31e+06 3.13e-02 8.53e+08 \n",+ " 34 39 9.7072e+09 5.66e+06 7.81e-03 4.38e+08 \n",+ " 35 40 9.7039e+09 3.25e+06 1.56e-02 3.53e+08 \n",+ " 36 41 9.7008e+09 3.15e+06 1.56e-02 3.79e+08 \n",+ " 37 42 9.6974e+09 3.37e+06 1.56e-02 3.64e+08 \n",+ " 38 43 9.6939e+09 3.55e+06 1.56e-02 3.64e+08 \n",+ " 39 44 9.6901e+09 3.74e+06 1.56e-02 3.68e+08 \n",+ " 40 45 9.6862e+09 3.93e+06 1.56e-02 3.49e+08 \n",+ " 41 46 9.6821e+09 4.13e+06 1.56e-02 3.71e+08 \n",+ " 42 47 9.6777e+09 4.34e+06 1.56e-02 3.32e+08 \n",+ " 43 48 9.6732e+09 4.55e+06 1.56e-02 3.73e+08 \n",+ " 44 49 9.6684e+09 4.76e+06 1.56e-02 3.17e+08 \n",+ " 45 50 9.6634e+09 4.98e+06 1.56e-02 3.74e+08 \n",+ " 46 51 9.6582e+09 5.20e+06 1.56e-02 3.25e+08 \n",+ " 47 52 9.6528e+09 5.42e+06 1.56e-02 3.75e+08 \n",+ " 48 53 9.6471e+09 5.65e+06 1.56e-02 3.32e+08 \n",+ " 49 54 9.6413e+09 5.88e+06 1.56e-02 3.75e+08 \n",+ " 50 55 9.6320e+09 9.31e+06 3.13e-02 6.40e+08 \n",+ " 51 56 9.6270e+09 4.91e+06 3.13e-02 7.83e+08 \n",+ " 52 57 9.6207e+09 6.37e+06 7.81e-03 4.66e+08 \n",+ " 53 58 9.6141e+09 6.61e+06 1.56e-02 3.56e+08 \n",+ " 54 59 9.6072e+09 6.88e+06 1.56e-02 3.78e+08 \n",+ " 55 60 9.5943e+09 1.29e+07 3.13e-02 4.62e+08 \n",+ " 56 62 9.5872e+09 7.08e+06 1.56e-02 3.91e+08 \n",+ " 57 63 9.5798e+09 7.45e+06 1.56e-02 3.77e+08 \n",+ " 58 64 9.5652e+09 1.46e+07 3.12e-02 4.43e+08 \n",+ " 59 65 9.5596e+09 5.55e+06 6.25e-02 1.63e+09 \n",+ " 60 66 9.5390e+09 2.06e+07 1.56e-02 8.00e+08 \n",+ " 61 67 9.5284e+09 1.06e+07 3.12e-02 6.33e+08 \n",+ " 62 68 9.5180e+09 1.05e+07 3.13e-02 7.53e+08 \n",+ " 63 69 9.5071e+09 1.09e+07 3.12e-02 6.33e+08 \n",+ " 64 70 9.4959e+09 1.12e+07 3.12e-02 7.38e+08 \n",+ " 65 71 9.4844e+09 1.15e+07 3.12e-02 6.21e+08 \n",+ " 66 72 9.4726e+09 1.18e+07 3.12e-02 7.25e+08 \n",+ " 67 73 9.4604e+09 1.21e+07 3.12e-02 6.07e+08 \n",+ " 68 74 9.4479e+09 1.25e+07 3.12e-02 7.13e+08 \n",+ " 69 75 9.4352e+09 1.28e+07 3.12e-02 5.94e+08 \n",+ " 70 76 9.4221e+09 1.31e+07 3.12e-02 7.00e+08 \n",+ " 71 77 9.4086e+09 1.34e+07 3.12e-02 5.79e+08 \n",+ " 72 78 9.3949e+09 1.37e+07 3.12e-02 6.87e+08 \n",+ " 73 79 9.3808e+09 1.41e+07 3.12e-02 5.64e+08 \n",+ " 74 80 9.3664e+09 1.44e+07 3.12e-02 6.74e+08 \n",+ " 75 81 9.3517e+09 1.47e+07 3.12e-02 5.49e+08 \n",+ " 76 82 9.3367e+09 1.50e+07 3.12e-02 6.61e+08 \n",+ " 77 83 9.3213e+09 1.54e+07 3.12e-02 5.37e+08 \n",+ " 78 84 9.3056e+09 1.57e+07 3.12e-02 6.48e+08 \n",+ " 79 85 9.2896e+09 1.60e+07 3.13e-02 5.61e+08 \n",+ " 80 86 9.2733e+09 1.63e+07 3.13e-02 6.35e+08 \n",+ " 81 87 9.2567e+09 1.66e+07 3.12e-02 5.84e+08 \n",+ " 82 88 9.2397e+09 1.70e+07 3.12e-02 6.22e+08 \n",+ " 83 89 9.2224e+09 1.73e+07 3.12e-02 6.07e+08 \n",+ " 84 90 9.2048e+09 1.76e+07 3.12e-02 6.09e+08 \n",+ " 85 91 9.1869e+09 1.79e+07 3.12e-02 6.30e+08 \n",+ " 86 92 9.1687e+09 1.82e+07 3.12e-02 5.97e+08 \n",+ " 87 93 9.1502e+09 1.85e+07 3.12e-02 6.52e+08 \n",+ " 88 94 9.1313e+09 1.89e+07 3.12e-02 5.85e+08 \n",+ " 89 95 9.1121e+09 1.92e+07 3.12e-02 6.73e+08 \n",+ " 90 96 9.0926e+09 1.95e+07 3.12e-02 5.73e+08 \n",+ " 91 97 9.0728e+09 1.98e+07 3.12e-02 6.93e+08 \n",+ " 92 98 9.0527e+09 2.01e+07 3.12e-02 5.61e+08 \n",+ " 93 99 9.0323e+09 2.04e+07 3.12e-02 7.13e+08 \n",+ " 94 100 9.0116e+09 2.07e+07 3.12e-02 5.49e+08 \n",+ " 95 101 8.9906e+09 2.10e+07 3.12e-02 7.32e+08 \n",+ " 96 102 8.9693e+09 2.13e+07 3.12e-02 5.38e+08 \n",+ " 97 103 8.9476e+09 2.16e+07 3.12e-02 7.50e+08 \n",+ " 98 104 8.9257e+09 2.19e+07 3.12e-02 5.48e+08 \n",+ " 99 105 8.8951e+09 3.06e+07 6.25e-02 1.22e+09 \n",+ " 100 106 8.8807e+09 1.44e+07 6.25e-02 1.84e+09 \n",+ " 101 107 8.8554e+09 2.53e+07 1.56e-02 1.03e+09 \n",+ " 102 108 8.8321e+09 2.33e+07 3.12e-02 7.97e+08 \n",+ " 103 109 8.8089e+09 2.33e+07 3.12e-02 5.74e+08 \n",+ " 104 110 8.7740e+09 3.49e+07 6.25e-02 1.12e+09 \n",+ " 105 111 8.7594e+09 1.46e+07 6.25e-02 1.86e+09 \n",+ " 106 112 8.7334e+09 2.61e+07 1.56e-02 1.05e+09 \n",+ " 107 113 8.7090e+09 2.43e+07 3.12e-02 8.38e+08 \n",+ " 108 114 8.6847e+09 2.43e+07 3.12e-02 5.97e+08 \n",+ " 109 115 8.6468e+09 3.79e+07 6.25e-02 1.10e+09 \n",+ " 110 116 8.6318e+09 1.50e+07 6.25e-02 1.87e+09 \n",+ " 111 117 8.6053e+09 2.65e+07 1.56e-02 1.06e+09 \n",+ " 112 118 8.5803e+09 2.50e+07 3.12e-02 8.70e+08 \n",+ " 113 119 8.5554e+09 2.49e+07 3.12e-02 6.10e+08 \n",+ " 114 120 8.5163e+09 3.91e+07 6.25e-02 1.13e+09 \n",+ " 115 121 8.5017e+09 1.47e+07 6.25e-02 1.88e+09 \n",+ " 116 122 8.4748e+09 2.69e+07 1.56e-02 1.07e+09 \n",+ " 117 123 8.4498e+09 2.50e+07 3.12e-02 8.87e+08 \n",+ " 118 124 8.4251e+09 2.47e+07 3.12e-02 6.03e+08 \n",+ " 119 125 8.3879e+09 3.72e+07 6.25e-02 1.16e+09 \n",+ " 120 126 8.3750e+09 1.29e+07 6.25e-02 1.90e+09 \n",+ " 121 127 8.3479e+09 2.71e+07 1.56e-02 1.09e+09 \n",+ " 122 128 8.3237e+09 2.42e+07 3.12e-02 8.83e+08 \n",+ " 123 129 8.3004e+09 2.33e+07 3.12e-02 5.65e+08 \n",+ " 124 130 8.2699e+09 3.05e+07 6.25e-02 1.20e+09 \n",+ " 125 131 8.2591e+09 1.08e+07 6.25e-02 1.90e+09 \n",+ " 126 132 8.2322e+09 2.69e+07 1.56e-02 1.08e+09 \n",+ " 127 133 8.2102e+09 2.20e+07 3.13e-02 8.55e+08 \n",+ " 128 134 8.1900e+09 2.02e+07 3.12e-02 6.22e+08 \n",+ " 129 135 8.1708e+09 1.91e+07 3.12e-02 8.58e+08 \n",+ " 130 136 8.1527e+09 1.81e+07 3.12e-02 6.77e+08 \n",+ " 131 137 8.1356e+09 1.71e+07 3.12e-02 8.46e+08 \n",+ " 132 138 8.1195e+09 1.61e+07 3.13e-02 7.16e+08 \n",+ " 133 139 8.1044e+09 1.51e+07 3.12e-02 8.28e+08 \n",+ " 134 140 8.0902e+09 1.41e+07 3.12e-02 7.49e+08 \n",+ " 135 141 8.0771e+09 1.31e+07 3.12e-02 8.04e+08 \n",+ " 136 142 8.0650e+09 1.21e+07 3.12e-02 7.79e+08 \n",+ " 137 143 8.0538e+09 1.12e+07 3.12e-02 7.77e+08 \n",+ " 138 144 8.0436e+09 1.02e+07 3.12e-02 8.05e+08 \n",+ " 139 145 8.0343e+09 9.32e+06 3.12e-02 7.47e+08 \n",+ " 140 146 8.0258e+09 8.47e+06 3.12e-02 8.26e+08 \n",+ " 141 147 8.0181e+09 7.65e+06 3.12e-02 7.15e+08 \n",+ " 142 148 8.0119e+09 6.22e+06 7.81e-03 5.58e+08 \n",+ " 143 149 8.0056e+09 6.36e+06 1.56e-02 3.10e+08 \n",+ " 144 150 7.9994e+09 6.17e+06 1.56e-02 4.51e+08 \n",+ " 145 151 7.9916e+09 7.82e+06 3.13e-02 6.71e+08 \n",+ " 146 152 7.9909e+09 6.36e+05 3.13e-02 8.63e+08 \n",+ " 147 153 7.9840e+09 6.91e+06 7.81e-03 5.02e+08 \n",+ " 148 154 7.9793e+09 4.77e+06 1.56e-02 2.93e+08 \n",+ " 149 155 7.9758e+09 3.42e+06 1.56e-02 3.91e+08 \n",+ " 150 156 7.9731e+09 2.80e+06 1.56e-02 3.92e+08 \n",+ " 151 157 7.9706e+09 2.45e+06 1.56e-02 3.73e+08 \n",+ " 152 158 7.9685e+09 2.11e+06 1.56e-02 4.09e+08 \n",+ " 153 159 7.9667e+09 1.83e+06 1.56e-02 3.51e+08 \n",+ " 154 160 7.9651e+09 1.54e+06 3.91e-03 2.71e+08 \n",+ " 155 161 7.9636e+09 1.49e+06 7.81e-03 1.47e+08 \n",+ " 156 162 7.9622e+09 1.39e+06 7.81e-03 2.16e+08 \n",+ " 157 163 7.9610e+09 1.24e+06 7.81e-03 1.50e+08 \n",+ " 158 164 7.9599e+09 1.05e+06 7.81e-03 2.02e+08 \n",+ " 159 165 7.9591e+09 8.70e+05 7.81e-03 1.82e+08 \n",+ " 160 166 7.9583e+09 7.25e+05 7.81e-03 1.92e+08 \n",+ " 161 167 7.9577e+09 6.13e+05 7.81e-03 1.98e+08 \n",+ " 162 168 7.9572e+09 5.19e+05 7.81e-03 1.79e+08 \n",+ " 163 169 7.9568e+09 4.40e+05 7.81e-03 2.07e+08 \n",+ " 164 170 7.9564e+09 3.84e+05 1.95e-03 1.10e+08 \n",+ " 165 171 7.9560e+09 3.58e+05 3.91e-03 1.12e+08 \n",+ " 166 172 7.9557e+09 3.28e+05 3.91e-03 7.26e+07 \n",+ " 167 173 7.9554e+09 2.83e+05 3.91e-03 1.02e+08 \n",+ " 168 174 7.9552e+09 2.33e+05 3.91e-03 8.75e+07 \n",+ " 169 175 7.9550e+09 1.90e+05 3.91e-03 9.62e+07 \n",+ " 170 176 7.9548e+09 1.58e+05 3.91e-03 9.83e+07 \n",+ " 171 177 7.9547e+09 1.32e+05 3.91e-03 8.96e+07 \n",+ " 172 178 7.9546e+09 1.11e+05 3.91e-03 1.03e+08 \n",+ " 173 179 7.9545e+09 9.60e+04 9.77e-04 5.48e+07 \n",+ " 174 180 7.9544e+09 8.96e+04 1.95e-03 5.59e+07 \n",+ " 175 181 7.9543e+09 8.16e+04 1.95e-03 3.61e+07 \n",+ " 176 182 7.9543e+09 7.00e+04 1.95e-03 5.05e+07 \n",+ " 177 183 7.9542e+09 5.71e+04 1.95e-03 4.39e+07 \n",+ " 178 184 7.9542e+09 4.62e+04 1.95e-03 4.77e+07 \n",+ " 179 185 7.9541e+09 3.82e+04 1.95e-03 4.93e+07 \n",+ " 180 186 7.9541e+09 3.19e+04 1.95e-03 4.43e+07 \n",+ " 181 187 7.9541e+09 2.66e+04 1.95e-03 5.17e+07 \n",+ " 182 188 7.9540e+09 2.39e+04 4.88e-04 2.74e+07 \n",+ " 183 189 7.9540e+09 2.17e+04 9.77e-04 2.75e+07 \n",+ " 184 190 7.9540e+09 1.95e+04 9.77e-04 1.86e+07 \n",+ " 185 191 7.9540e+09 1.65e+04 9.77e-04 2.50e+07 \n",+ " 186 192 7.9540e+09 1.33e+04 9.77e-04 2.25e+07 \n",+ " 187 193 7.9540e+09 1.08e+04 9.77e-04 2.36e+07 \n",+ " 188 194 7.9539e+09 8.98e+03 9.77e-04 2.49e+07 \n",+ " 189 195 7.9539e+09 7.48e+03 9.77e-04 2.18e+07 \n",+ " 190 196 7.9539e+09 6.24e+03 9.77e-04 2.60e+07 \n",+ " 191 197 7.9539e+09 5.97e+03 2.44e-04 1.37e+07 \n",+ " 192 198 7.9539e+09 5.16e+03 4.88e-04 1.35e+07 \n",+ " 193 199 7.9539e+09 4.56e+03 4.88e-04 9.69e+06 \n",+ " 194 200 7.9539e+09 3.79e+03 4.88e-04 1.24e+07 \n",+ " 195 201 7.9539e+09 3.06e+03 4.88e-04 1.16e+07 \n",+ " 196 202 7.9539e+09 2.51e+03 4.88e-04 1.16e+07 \n",+ " 197 203 7.9539e+09 2.08e+03 4.88e-04 1.26e+07 \n",+ " 198 204 7.9539e+09 1.73e+03 4.88e-04 1.09e+07 \n",+ " 199 205 7.9539e+09 1.50e+03 1.22e-04 8.24e+06 \n",+ " 200 206 7.9539e+09 1.39e+03 2.44e-04 4.66e+06 \n",+ " 201 207 7.9539e+09 1.26e+03 2.44e-04 6.51e+06 \n",+ " 202 208 7.9539e+09 1.07e+03 2.44e-04 4.97e+06 \n",+ " 203 209 7.9539e+09 8.65e+02 2.44e-04 6.10e+06 \n",+ " 204 210 7.9539e+09 6.96e+02 2.44e-04 5.96e+06 \n",+ " 205 211 7.9539e+09 5.75e+02 2.44e-04 5.70e+06 \n",+ " 206 212 7.9539e+09 4.78e+02 2.44e-04 6.36e+06 \n",+ " 207 213 7.9539e+09 3.98e+02 2.44e-04 5.56e+06 \n",+ " 208 214 7.9539e+09 3.73e+02 6.10e-05 4.01e+06 \n",+ " 209 215 7.9539e+09 3.27e+02 1.22e-04 2.31e+06 \n",+ " 210 216 7.9539e+09 2.90e+02 1.22e-04 3.21e+06 \n",+ " 211 217 7.9539e+09 2.42e+02 1.22e-04 2.63e+06 \n",+ " 212 218 7.9539e+09 1.95e+02 1.22e-04 3.01e+06 \n",+ " 213 219 7.9539e+09 1.59e+02 1.22e-04 3.04e+06 \n",+ " 214 220 7.9539e+09 1.32e+02 1.22e-04 2.80e+06 \n",+ " 215 221 7.9539e+09 1.10e+02 1.22e-04 3.21e+06 \n",+ " 216 222 7.9539e+09 9.37e+01 3.05e-05 1.70e+06 \n",+ " 217 223 7.9539e+09 8.77e+01 6.10e-05 1.74e+06 \n",+ " 218 224 7.9539e+09 7.95e+01 6.10e-05 1.12e+06 \n",+ "`ftol` termination condition is satisfied.\n",+ "Function evaluations 224, initial cost 1.9160e+11, final cost 7.9539e+09, first-order optimality 1.12e+06.\n"+ ]+ }+ ],+ "source": [+ "\n",+ "if debug:\n",+ " def plot_optimization_error(error,gaze_xyz, gaze_rotated_by_guess_then_head_rotation_xyz, fixation_point_in_eye_coordinates_xyz, skel_during_vor_fr_mar_dim):\n",+ " figure_number=13451\n",+ "\n",+ " if not plt.fignum_exists(figure_number):\n",+ " fig = plt.figure(figure_number)\n",+ " ax = fig.add_subplot(111, projection='3d')\n",+ " else:\n",+ " fig = plt.gcf()\n",+ " ax = plt.gca()\n",+ "\n",+ "\n",+ " fig.suptitle(f'error: {error}')\n",+ " ax_range = 1e3\n",+ " ax.clear() \n",+ "\n",+ " ax.plot(0,0,0, 'mo',label='origin')\n",+ "\n",+ " ax.plot(gaze_xyz[:,0],\n",+ " gaze_xyz[:,1], \n",+ " gaze_xyz[:,2], 'k-o',label='original gaze xyz')\n",+ "\n",+ " gaze_rot_xyz =np.array(gaze_rotated_by_guess_then_head_rotation_xyz)\n",+ " ax.plot(gaze_rot_xyz[:,0],\n",+ " gaze_rot_xyz[:,1], \n",+ " gaze_rot_xyz[:,2], 'r-o',label='gaze_rotated_by_guess_then_head_rotation_xyz')\n",+ "\n",+ " ax.plot(fixation_point_in_eye_coordinates_xyz[:,0],\n",+ " fixation_point_in_eye_coordinates_xyz[:,1], \n",+ " fixation_point_in_eye_coordinates_xyz[:,2], 'b-o',label='fixation_point_in_eye_coordinates_xyz')\n",+ "\n",+ " ax.set_xlim([-ax_range, ax_range])\n",+ " ax.set_ylim([-ax_range, ax_range])\n",+ " ax.set_zlim([-ax_range, ax_range])\n",+ " ax.legend()\n",+ " plt.pause(.01)\n",+ "\n",+ "\n",+ "\n",+ "def get_error_between_two_rotation_matricies(euler_angle_guess,\n",+ " gaze_xyz,\n",+ " fixation_point_in_eye_coordinates_xyz,\n",+ " head_rotation_maticies_fr_row_col,\n",+ " skel_during_vor_fr_mar_dim):\n",+ " #convert euler angles to rotation matrix\n",+ " rotation_matrix_guess = Rotation.from_euler('XYZ',euler_angle_guess).as_matrix()\n",+ "\n",+ " #rotate gaze by rotation guess\n",+ " gaze_rotated_by_guess = [rotation_matrix_guess @ gaze_xyz[this_frame_number,:] for this_frame_number in range(gaze_xyz.shape[0])]\n",+ "\n",+ " #...then rotate THAT by head_rotation matrix\n",+ " gaze_rotated_by_guess_then_head_rotation_xyz =[head_rotation_maticies_fr_row_col[this_frame_number,:,:] @ gaze_rotated_by_guess[this_frame_number]\n",+ " for this_frame_number in range(gaze_xyz.shape[0])]\n",+ "\n",+ " # mean_fixation_point_xyz = np.mean(fixation_point_in_eye_coordinates_xyz, axis=0)\n",+ " #define error as difference between the fixation point and that rotated gaze estimate (these are both normalized, I think)\n",+ " error_per_frame = gaze_rotated_by_guess_then_head_rotation_xyz - fixation_point_in_eye_coordinates_xyz\n",+ " error = np.nanmean(np.nansum(error_per_frame**2)/error_per_frame.shape[0])\n",+ " \n",+ " plot_optimization_error(error,gaze_xyz, gaze_rotated_by_guess_then_head_rotation_xyz, fixation_point_in_eye_coordinates_xyz, skel_during_vor_fr_mar_dim)\n",+ "\n",+ " return error\n",+ "\n",+ "\n",+ "def get_optimal_rotation_matrix_to_align_gaze_with_target(gaze_xyz,\n",+ " fixation_point_in_eye_coordinates_xyz,\n",+ " head_rotation_matricies_fr_row_col,\n",+ " skel_during_vor_fr_mar_dim):\n",+ " euler_angles = optimize.least_squares(get_error_between_two_rotation_matricies,\n",+ " [0,0,0],\n",+ " args=(gaze_xyz,fixation_point_in_eye_coordinates_xyz, head_rotation_matricies_fr_row_col, skel_during_vor_fr_mar_dim),\n",+ " gtol=1e-10,\n",+ " verbose=2).x\n",+ " return Rotation.from_euler('XYZ',euler_angles).as_matrix()\n",+ "\n",+ "\n",+ "######################################\n",+ "\n",+ "\n",+ "rotation_matrix_to_align_pupil_data_to_eye_in_head_coordinates = get_optimal_rotation_matrix_to_align_gaze_with_target(r_gaze_during_vor_fr_xyz,\n",+ " fixation_point_in_r_eyeball_coordinates_fr_xyz,\n",+ " head_rotation_matrices_during_vor_fr_row_col,\n",+ " skel_during_vor_fr_mar_dim)\n",+ "\n",+ "r_eye_gaze_xyz_calibrated = [rotation_matrix_to_align_pupil_data_to_eye_in_head_coordinates @ r_eye_gaze_xyz[this_frame_number,:] for this_frame_number in range(r_eye_gaze_xyz.shape[0])]\n",+ "\n",+ "\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 33,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "gaze_xyz_npy_save_path = pupil_data_exports_path / 'right_eye_gaze_xyz.npy'\n",+ "np.save(gaze_xyz_npy_save_path, r_eye_gaze_xyz_calibrated)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": [+ "import plotly.offline as py\n",+ "import plotly.express as px \n",+ "import plotly.graph_objects as go\n",+ "\n",+ "fig = go.Figure()\n",+ "\n",+ "fig.add_trace(go.Scatter(x=np.arange(r_eye_gaze_xyz.shape[0]), y=r_eye_gaze_xyz[:,0], mode='lines+markers'))\n",+ "fig.add_trace(go.Scatter(x=np.arange(r_eye_gaze_xyz.shape[0]), y=r_eye_gaze_xyz[:,1], mode='lines+markers'))\n",+ "fig.add_trace(go.Scatter(x=np.arange(r_eye_gaze_xyz.shape[0]), y=r_eye_gaze_xyz[:,2], mode='lines+markers'))\n",+ "fig.show()"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {+ "pycharm": {+ "is_executing": true,+ "name": "#%%\n"+ }+ },+ "outputs": [],+ "source": []+ }+ ],+ "metadata": {+ "interpreter": {+ "hash": "4ef2e72e5b895021b427c707c9f05b4880b5a5a00e886b84de197a42e54c7f32"+ },+ "kernelspec": {+ "display_name": "Python 3.9.12 ('humon-env')",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.9.12"+ },+ "orig_nbformat": 4+ },+ "nbformat": 4,+ "nbformat_minor": 2+}diff --git a/jon_scratch/matplotlib_laser_skeleton.py b/jon_scratch/matplotlib_laser_skeleton.pynew file mode 100644index 00000000..27d59552--- /dev/null+++ b/jon_scratch/matplotlib_laser_skeleton.py@@ -0,0 +1,1039 @@+"""+============+ββββββ ββ βββββ ββ ββ βββββββ ββ ββ βββββββ ββ βββββββ ββββββββ ββββββ βββ ββ βββββ βββ ββ ββ βββ βββ βββββ ββββββββ ββ ββββββ βββ ββ+ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββββ ββ ββ ββ ββββ ββ ββ ββββ ββββ ββ ββ ββ ββ ββ ββ ββββ ββ+ββββββ ββ βββββββ ββββ βββββββ βββββ βββββ ββ βββββ ββ ββ ββ ββ ββ ββ βββββββ ββ ββ ββ ββ ββ ββββ ββ βββββββ ββ ββ ββ ββ ββ ββ ββ+ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ+ββ βββββββ ββ ββ ββ βββββββ ββ ββ βββββββ βββββββ βββββββ ββ ββββββ ββ ββββ ββ ββ ββ ββββ ββ ββ ββ ββ ββ ββ ββ ββββββ ββ ββββ+============+Font - ANSI Regular - https://patorjk.com/software/taag/#p=display&f=ANSI%20Regular&t=Play%20Skeleton%20Animation+============+Originally based on tutorial from - https://matplotlib.org/2.1.2/gallery/animation/simple_3danim.html++Create a matplotlib animation from a FreeMoCap recording session.+"""+++################################################################################################################+################################################################################################################+###+### ββ βββ βββ ββββββ ββββββ ββββββ ββββββββ+### ββ ββββ ββββ ββ ββ ββ ββ ββ ββ ββ+### ββ ββ ββββ ββ ββββββ ββ ββ ββββββ ββ+### ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ+### ββ ββ ββ ββ ββββββ ββ ββ ββ+###+################################################################################################################+################################################################################################################+++import numpy as np++import matplotlib as mpl+import matplotlib.pyplot as plt+import mpl_toolkits.mplot3d.axes3d as p3+import matplotlib.animation as animation+from matplotlib.widgets import Slider+import cv2+from scipy.signal import savgol_filter+import moviepy.editor as mp++import os+import copy+from pathlib import Path++#RICH CONSOLE STUFF+from rich import pretty+pretty.install() #makes all print statement output pretty+from rich import inspect+from rich.console import Console+console = Console()+from rich.traceback import install as rich_traceback_install+from rich.markdown import Markdown++#colors from Taylor Davis branding -+humon_dark = np.array([37, 67, 66])/255+humon_green = np.array([53, 93, 95])/255+humon_blue = np.array([14, 90, 253])/255+humon_red = np.array([217, 61, 67])/255+++def PlaySkeletonAnimation(+ session=None,+ vidType=1,+ startFrame=0,+ azimuth=-90,+ elevation=-70,+ useOpenPose=True,+ useMediaPipe=False,+ useDLC=False,+ recordVid = True,+ showAnimation =True,+ ):+++++ ###+ ###+ ### ββ ββ ββββββ ββββββ βββββ ββββββββ βββββββ βββββββ ββ ββββββ ββ ββ ββββββ βββββββ+ ### ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ+ ### ββ ββ ββββββ ββ ββ βββββββ ββ βββββ βββββ ββ ββ βββ ββ ββ ββββββ βββββ+ ### ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ+ ### ββββββ ββ ββββββ ββ ββ ββ βββββββ ββ ββ ββββββ ββββββ ββ ββ βββββββ+ ###+ def update_figure(frameNum):+ """+ Called by matplotlib animator for each frame.+ """++ if useOpenPose:+ op_skel_dottos = matplotlib_artist_objs['op_skel_dottos']+ op_skel_trajectories = figure_data['openPose_skel_trajectories|mar|fr_dim']++ if useMediaPipe:+ mp_skel_dottos = matplotlib_artist_objs['mp_skel_dottos']+ mp_skel_trajectories = figure_data['mediaPipe_skel_trajectories|mar|fr_dim']++ charuco_dottos = matplotlib_artist_objs['charuco_dottos']+ charuco_trajectories = figure_data['charuco_trajectories|mar|fr_dim']++ if useDLC:+ dlc_dottos = matplotlib_artist_objs['dlc_dottos']+ dlc_trajectories = figure_data['dlc_trajectories|mar|fr_dim']++ #function to update the lines for each 3d body segment+ def update_3d_skel_segments(key, data_fr_mar_xyz):+ """+ updates the Artist of each body segment with the current frame data.+ """+ split = key.split('_')++ if split[0] == 'op':+ dict_of_segments_idxs = dict_of_openPoseSegmentIdx_dicts[key]+ if split[0] == 'mp':+ dict_of_segments_idxs = dict_of_mediaPipeSegmentIdx_dicts[key]+ elif split[0] == 'dlc':+ dict_of_segments_idxs = dict_of_dlcSegmentIdx_dicts[key]++ try:+ dict_of_artists = matplotlib_artist_objs[key]+ except:+ print('KeyError:', key)+++ for thisItem in dict_of_segments_idxs.items():+ segName = thisItem[0]+ segArtist = dict_of_artists[segName]+ segArtist.set_data((data_fr_mar_xyz[frameNum, dict_of_segments_idxs[segName], 0], data_fr_mar_xyz[frameNum, dict_of_segments_idxs[segName], 1]))+ segArtist.set_3d_properties(data_fr_mar_xyz[frameNum, dict_of_segments_idxs[segName], 2])++ if useOpenPose:+ update_3d_skel_segments('op_body', openPose_skel_fr_mar_xyz)+ update_3d_skel_segments('op_rHand', openPose_skel_fr_mar_xyz)+ update_3d_skel_segments('op_lHand', openPose_skel_fr_mar_xyz)+ update_3d_skel_segments('op_face', openPose_skel_fr_mar_xyz)++ if useMediaPipe:+ update_3d_skel_segments('mp_body', mediaPipe_skel_fr_mar_xyz)+ update_3d_skel_segments('mp_rHand', mediaPipe_skel_fr_mar_xyz)+ update_3d_skel_segments('mp_lHand', mediaPipe_skel_fr_mar_xyz)++ if useDLC:+ update_3d_skel_segments('dlc_pinkBall', dlc_fr_mar_xyz)+ update_3d_skel_segments('dlc_redBall', dlc_fr_mar_xyz)+ update_3d_skel_segments('dlc_greenBall', dlc_fr_mar_xyz)+ update_3d_skel_segments('dlc_wobbleBoard', dlc_fr_mar_xyz)+ update_3d_skel_segments('dlc_wobbleWheel', dlc_fr_mar_xyz)++ for bb in range(3):+ thisTailArtist = matplotlib_artist_objs['ball_tails'][bb]++ thisTailArtist.set_data((dlc_trajectories[bb][frameNum-ballTailLen:frameNum+1, 0],+ dlc_trajectories[bb][frameNum-ballTailLen:frameNum+1, 1]))++ thisTailArtist.set_3d_properties(dlc_trajectories[bb][frameNum-ballTailLen:frameNum+1, 2])+ #dlc data+ marNum = -1+ for thisDotto, thisTraj in zip(dlc_dottos,dlc_trajectories):+ marNum+=1+ # NOTE: there is no .set_data() for 3 dim data...+ thisDotto.set_data(thisTraj[ frameNum, 0:2])+ thisDotto.set_3d_properties(thisTraj[ frameNum, 2])++ #Plots the dots!+ #openpose data+ if useOpenPose:+ marNum = -1+ for thisSkelDotto, thisTraj in zip(op_skel_dottos,op_skel_trajectories):+ marNum+=1+ # NOTE: there is no .set_data() for 3 dim data...+ thisSkelDotto.set_data(thisTraj[ frameNum, 0:2])+ thisSkelDotto.set_3d_properties(thisTraj[ frameNum, 2])++ #mediapipe data+ if useMediaPipe:+ marNum = -1+ for thisSkelDotto, thisTraj in zip(mp_skel_dottos,mp_skel_trajectories):+ marNum+=1+ # NOTE: there is no .set_data() for 3 dim data...+ thisSkelDotto.set_data(thisTraj[ frameNum, 0:2])+ thisSkelDotto.set_3d_properties(thisTraj[ frameNum, 2])++++ #charuco data+ for thisDotto, thisTraj in zip(charuco_dottos,charuco_trajectories):++ if frameNum<charuco_trajectories[0].shape[0]:+ thisDotto.set_data(thisTraj[ frameNum, 0:2])+ thisDotto.set_3d_properties(thisTraj[ frameNum, 2])+ else:+ thisDotto.set_color('none')+++++ # function to update the lines for each body segment+ def update_2d_skel_segments(vidNum,key):+ """+ updates the Artist of each body segment with the current frame data.+ """+ split = key.split('_')+ if split[0] == 'op':+ dict_of_segments_idxs = dict_of_openPoseSegmentIdx_dicts[key]+ dict_of_artists = thisVidOpenPoseArtist_dict[key]+ elif split[0] == 'mp':+ dict_of_segments_idxs = dict_of_mediaPipeSegmentIdx_dicts[key]+ dict_of_artists = thisVidMediaPipeArtist_dict[key]+++ for thisItem in dict_of_segments_idxs.items():+ segName = thisItem[0]+ segArtist = dict_of_artists[segName]++ if split[0] == 'op':+ xData = openPose_nCams_nFrames_nImgPts_XYC[vidNum, frameNum, dict_of_segments_idxs[segName],0]+ yData = openPose_nCams_nFrames_nImgPts_XYC[vidNum, frameNum, dict_of_segments_idxs[segName],1]+ elif split[0] == 'mp':+ xData = mediaPipe_nCams_nFrames_nImgPts_XYC[vidNum, frameNum, dict_of_segments_idxs[segName],0]+ yData = mediaPipe_nCams_nFrames_nImgPts_XYC[vidNum, frameNum, dict_of_segments_idxs[segName],1]++ xDataMasked = np.ma.masked_array(xData, mask=(xData==0))+ yDataMasked = np.ma.masked_array(yData, mask=(xData==0))+ segArtist.set_data(xDataMasked,yDataMasked)+++ #update video frames+ for vidNum, thisVidArtist in enumerate(vidAristList):+ vidCapObjList[vidNum].set(cv2.CAP_PROP_POS_FRAMES, frameNum)+ success, image = vidCapObjList[vidNum].read()++ if success:+ thisVidArtist.set_array(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))+ else:+ vidCapObjList[vidNum].set(cv2.CAP_PROP_POS_FRAMES, 0)+ success, image = vidCapObjList[vidNum].read()++ if useOpenPose:+ thisVidOpenPoseArtist_dict = list_of_vidOpenPoseArtistdicts[vidNum]+ update_2d_skel_segments(vidNum,'op_body')+ update_2d_skel_segments(vidNum,'op_rHand')+ update_2d_skel_segments(vidNum,'op_lHand')+ update_2d_skel_segments(vidNum,'op_face')++ if useMediaPipe:+ thisVidMediaPipeArtist_dict = list_of_vidMediaPipeArtistdicts[vidNum]+ update_2d_skel_segments(vidNum,'mp_body')+ update_2d_skel_segments(vidNum,'mp_rHand')+ update_2d_skel_segments(vidNum,'mp_lHand')++ if useDLC: #JSM NOTE - this one won't work, we'll need to fix it the next time we get DLC data+ thisVidDLCArtist[0].set_data( dlcData_nCams_nFrames_nImgPts_XYC[vidNum,frameNum,:,0],+ dlcData_nCams_nFrames_nImgPts_XYC[vidNum,frameNum,:,1],)+++ #shift timeseries xlims along with the frameNum+ xTimeSeriesAx.set_xlim([(frameNum/fps)-timeRange, (frameNum/fps)+timeRange])+ yTimeSeriesAx.set_xlim([(frameNum/fps)-timeRange, (frameNum/fps)+timeRange])+ for thisArtistKey in xCurrTimeArtists:++ xCurrTimeArtists[thisArtistKey][0].set_xdata(frameNum/fps)+ yCurrTimeArtists[thisArtistKey][0].set_xdata(frameNum/fps)+ if not thisArtistKey == 'blackLine':+ xCurrTimeArtists[thisArtistKey][0].set_ydata(xCurrTimeYdata[thisArtistKey][frameNum] )+ yCurrTimeArtists[thisArtistKey][0].set_ydata(yCurrTimeYdata[thisArtistKey][frameNum] )++ yCurrTimeArtists['frameNumber'].set_x(frameNum/fps+3/fps)+ yCurrTimeArtists['frameNumber'].set_text('Frame# '+ str(frameNum))+ # if recordVid:+ # thisFramePath = str(animationFramePath) + '/'+ str(session.sessionID) + "_frame_" + str(frameNum).zfill(6) + ".png"+ # plt.savefig(thisFramePath)+ # # fig.suptitle("nSessionID: {}, Frame: {} of {}".format(session.sessionID, frameNum, numFrames), fontsize=10)+ # # animSlider.set_val(val=frameNum)++ ####+ #### ββββββ ββββββ βββββββ βββββ ββββββββ βββββββ βββββββ ββ ββββββ ββ ββ ββββββ βββββββ+ #### ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ+ #### ββ ββββββ βββββ βββββββ ββ βββββ βββββ ββ ββ βββ ββ ββ ββββββ βββββ+ #### ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ+ #### ββββββ ββ ββ βββββββ ββ ββ ββ βββββββ ββ ββ ββββββ ββββββ ββ ββ βββββββ+ ####+++ fig = plt.figure(dpi=200)+ plt.ion()+++ ###+ ### ββββββ ββββββ βββββββ βββββ ββββββββ βββββββ ββββββ ββββββ βββββ ββ ββ ββ βββββββ+ ### ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ+ ### ββ ββββββ βββββ βββββββ ββ βββββ βββββ βββββ ββ ββ βββββββ βββ ββ βββββββ+ ### ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ+ ### ββββββ ββ ββ βββββββ ββ ββ ββ βββββββ ββββββ ββββββ ββ ββ ββ ββ ββ βββββββ+ ###+ ###++ # ax3d = p3.Axes3D(fig)+ ax3d = fig.add_subplot(projection='3d')+ ax3d.set_position([-.065, .28, .7, .7]) # [left, bottom, width, height])+ ax3d.set_xticklabels([])+ ax3d.set_yticklabels([])+ ax3d.set_zticklabels([])++ # ax3d.set_xlabel('X')+ # ax3d.set_ylabel('Y')+ # ax3d.set_zlabel('Z')+ # ax3d.set_axis_off()+++ ax3d.tick_params(length=0) # WHY DOESNT THIS WORK? I HATE THOSE TICK MARKS >:(++ smoothData=True+ smoothWinLength = 5+ smoothOrder = 3++ if useOpenPose:+ try:+ openPose_skel_fr_mar_xyz = np.load(session.dataArrayPath / 'openPoseSkel_3d.npy')+ openPose_nCams_nFrames_nImgPts_XYC = np.load(session.dataArrayPath / 'openPoseData_2d.npy')+ except:+ print('No openPose data found.')++ if smoothData:+ for dim in range(openPose_skel_fr_mar_xyz.shape[2]):+ for mm in range(openPose_skel_fr_mar_xyz.shape[1]):+ openPose_skel_fr_mar_xyz[:,mm,dim] = savgol_filter(openPose_skel_fr_mar_xyz[:,mm,dim], smoothWinLength, smoothOrder)++ if useMediaPipe:+ try:+ mediaPipe_skel_fr_mar_xyz = np.load(session.dataArrayPath / 'mediaPipeSkel_3d.npy')+ mediaPipe_nCams_nFrames_nImgPts_XYC = np.load(session.dataArrayPath / 'mediaPipeData_2d.npy')+ except:+ print('No mediaPipe data found.')++ if smoothData:+ for dim in range(mediaPipe_skel_fr_mar_xyz.shape[2]):+ for mm in range(mediaPipe_skel_fr_mar_xyz.shape[1]):+ mediaPipe_skel_fr_mar_xyz[:,mm,dim] = savgol_filter(mediaPipe_skel_fr_mar_xyz[:,mm,dim], smoothWinLength, smoothOrder)++ try:+ charuco_fr_mar_xyz = np.load(session.dataArrayPath / 'charuco_3d_points.npy')+ except:+ console.warning('No charuco data found')+++++ figure_data = dict()+ if useOpenPose:+ figure_data['openPose_skel_trajectories|mar|fr_dim'] = [openPose_skel_fr_mar_xyz[:,markerNum,:] for markerNum in range(openPose_skel_fr_mar_xyz.shape[1])]+ figure_data['openPose_skel_fr_mar_xyz'] = openPose_skel_fr_mar_xyz++ dict_of_openPoseSegmentIdx_dicts, dict_of_op_skel_lineColor = formatOpenPoseStickIndices() #these will help us draw body and hands stick figures+++ if useMediaPipe:+ mediaPipe_trajectories = [mediaPipe_skel_fr_mar_xyz[:,markerNum,:] for markerNum in range(mediaPipe_skel_fr_mar_xyz.shape[1])]+ figure_data['mediaPipe_skel_trajectories|mar|fr_dim'] = [mediaPipe_skel_fr_mar_xyz[:,markerNum,:] for markerNum in range(mediaPipe_skel_fr_mar_xyz.shape[1])]+ figure_data['mediaPipe_skel_fr_mar_xyz'] = mediaPipe_skel_fr_mar_xyz++ dict_of_mediaPipeSegmentIdx_dicts, dict_of_mp_skel_lineColor = formatMediaPipeStickIndices() #these will help us draw body and hands stick figures+++ charuco_trajectories = [charuco_fr_mar_xyz[:,markerNum,:] for markerNum in range(charuco_fr_mar_xyz.shape[1])]+ figure_data['charuco_trajectories|mar|fr_dim'] = charuco_trajectories+ figure_data['charuco_fr_mar_xyz'] = charuco_fr_mar_xyz+++ if useDLC:+ dlc_fr_mar_xyz = np.load(session.dataArrayPath / "deepLabCut_3d.npy")+ dlcData_nCams_nFrames_nImgPts_XYC = np.load(session.dataArrayPath / "deepLabCutData_2d.npy")++ for mm in range(dlc_fr_mar_xyz.shape[1]):+ for dim in range(dlc_fr_mar_xyz.shape[2]):+ dlc_fr_mar_xyz[:,mm,dim] = savgol_filter(dlc_fr_mar_xyz[:,mm,dim], 11, 3)++ ballTailLen = 6+ if ballTailLen > startFrame:+ startFrame = ballTailLen+1 #gotta do this, or the tail will index negative frames (without requiring annoying checks later)++ dlc_trajectories = [dlc_fr_mar_xyz[:,markerNum,:] for markerNum in range(dlc_fr_mar_xyz.shape[1])]+ figure_data['dlc_trajectories|mar|fr_dim'] = dlc_trajectories+ figure_data['dlc_fr_mar_xyz'] = dlc_fr_mar_xyz+++++ def build_3d_segment_artist_dict(data_fr_mar_xyz,+ dict_of_list_of_segment_idxs,+ segColor = 'k',+ lineWidth = 1,+ lineStyle = '-',+ markerType = None,+ marSize = 12,+ markerEdgeColor = 'k',):+ """+ Builds a dictionary of line artists for each 3D body segment.+ """+ segNames = list(dict_of_list_of_segment_idxs)++++ dict_of_artist_objects = dict()+ for segNum, segName in enumerate(segNames):++ #determine color of segment, based on class of 'segColor' input+ if isinstance(segColor, str):+ thisRGBA = segColor+ elif isinstance(segColor, np.ndarray):+ thisRGBA = segColor+ elif isinstance(segColor, dict):+ thisRGBA = segColor[segName]+ elif isinstance(segColor, list):+ try:+ thisRGBA = segColor[segNum]+ except:+ print('Not enough colors provided, using Black instead')+ thisRGBA = 'k'+ else:+ thisRGBA = 'k'++ if isinstance(segName, str):+ idxsOG = dict_of_list_of_segment_idxs[segName]+ else:+ idxsOG++ if isinstance(idxsOG, int) or isinstance(idxsOG, float):+ idxs = [idxsOG]+ elif isinstance(idxsOG, dict):+ idxs = idxsOG[0]+ else:+ idxs = idxsOG.copy()+++ dict_of_artist_objects[segName] = ax3d.plot(+ data_fr_mar_xyz[startFrame, idxs ,0],+ data_fr_mar_xyz[startFrame, idxs ,1],+ data_fr_mar_xyz[startFrame, idxs ,2],+ linestyle=lineStyle,+ linewidth=lineWidth,+ markerSize = marSize,+ marker = markerType,+ color = thisRGBA,+ markeredgecolor = markerEdgeColor,+ )[0]+ return dict_of_artist_objects++++ matplotlib_artist_objs = dict()++ if useOpenPose:+ matplotlib_artist_objs['op_body'] = build_3d_segment_artist_dict(openPose_skel_fr_mar_xyz, dict_of_openPoseSegmentIdx_dicts['op_body'], segColor = dict_of_op_skel_lineColor)+ matplotlib_artist_objs['op_rHand'] = build_3d_segment_artist_dict(openPose_skel_fr_mar_xyz, dict_of_openPoseSegmentIdx_dicts['op_rHand'], segColor=np.append(humon_red, 1), markerType='.', markerEdgeColor = humon_red, lineWidth=1, marSize = 2)+ matplotlib_artist_objs['op_lHand'] = build_3d_segment_artist_dict(openPose_skel_fr_mar_xyz, dict_of_openPoseSegmentIdx_dicts['op_lHand'], segColor=np.append(humon_blue, 1), markerType='.', markerEdgeColor = humon_blue, lineWidth=1, marSize = 2)+ matplotlib_artist_objs['op_face'] = build_3d_segment_artist_dict(openPose_skel_fr_mar_xyz, dict_of_openPoseSegmentIdx_dicts['op_face'], segColor='k', lineWidth=.5)++ if useMediaPipe:+ matplotlib_artist_objs['mp_body'] = build_3d_segment_artist_dict(mediaPipe_skel_fr_mar_xyz, dict_of_mediaPipeSegmentIdx_dicts['mp_body'], segColor = dict_of_mp_skel_lineColor)+ matplotlib_artist_objs['mp_rHand'] = build_3d_segment_artist_dict(mediaPipe_skel_fr_mar_xyz, dict_of_mediaPipeSegmentIdx_dicts['mp_rHand'], segColor=np.append(humon_red, 1), markerType='.', markerEdgeColor = humon_red, lineWidth=1, marSize = 2)+ matplotlib_artist_objs['mp_lHand'] = build_3d_segment_artist_dict(mediaPipe_skel_fr_mar_xyz, dict_of_mediaPipeSegmentIdx_dicts['mp_lHand'], segColor=np.append(humon_blue, 1), markerType='.', markerEdgeColor = humon_blue, lineWidth=1, marSize = 2)++ if useDLC:+ dict_of_dlcSegmentIdx_dicts = dict()+ dict_of_dlcSegmentIdx_dicts['pinkBall'] = {'pinkBall': [0]}+ dict_of_dlcSegmentIdx_dicts['greenBall'] = {'greenBall': [1]}+ dict_of_dlcSegmentIdx_dicts['redBall'] = {'redBall': [2]}+ dict_of_dlcSegmentIdx_dicts['wobbleBoard'] = {'wobbleboard':[3,4,5,6,3,7,5,4,7,6]}+ dict_of_dlcSegmentIdx_dicts['wobbleWheel'] = {'wobbleWheel':[8,9]}+++ ballColor = ['darkviolet', 'forestgreen', 'xkcd:goldenrod']+ matplotlib_artist_objs['pinkBall'] = build_3d_segment_artist_dict(dlc_fr_mar_xyz, dict_of_dlcSegmentIdx_dicts['pinkBall'], segColor=ballColor[0], markerType = 'o', marSize = 6, markerEdgeColor = 'indigo')+ matplotlib_artist_objs['redBall'] = build_3d_segment_artist_dict(dlc_fr_mar_xyz, dict_of_dlcSegmentIdx_dicts['redBall'], segColor=ballColor[2], markerType = 'o', marSize = 6, markerEdgeColor ='orangered')+ matplotlib_artist_objs['greenBall'] = build_3d_segment_artist_dict(dlc_fr_mar_xyz, dict_of_dlcSegmentIdx_dicts['greenBall'], segColor=ballColor[1], markerType = 'o', marSize = 6, markerEdgeColor ='darkgreen')++ matplotlib_artist_objs['wobbleBoard'] = build_3d_segment_artist_dict(dlc_fr_mar_xyz, dict_of_dlcSegmentIdx_dicts['wobbleBoard'], segColor='k', lineWidth=1)+ matplotlib_artist_objs['wobbleWheel'] = build_3d_segment_artist_dict(dlc_fr_mar_xyz, dict_of_dlcSegmentIdx_dicts['wobbleWheel'], segColor='k', lineWidth=1)++ if useOpenPose:+ op_skel_dottos = []+ for mm in range(67): #getcher dottos off my face!+ thisTraj = openPose_skel_fr_mar_xyz[:, mm, :]+ if mm==15:+ col = 'r'+ markerSize = 2+ elif mm == 16:+ col = 'b'+ markerSize = 2+ else:+ col = 'k'+ markerSize = 1++ thisDotto =ax3d.plot(thisTraj[0, 0:1], thisTraj[1, 0:1], thisTraj[2, 0:1][0],'.', markersize=markerSize, color = col)+ op_skel_dottos.append(thisDotto[0])++ matplotlib_artist_objs['op_skel_dottos'] = op_skel_dottos++ if useMediaPipe:+ matplotlib_artist_objs['mp_skel_dottos'] = [ax3d.plot(thisTraj[0, 0:1], thisTraj[1, 0:1], thisTraj[2, 0:1],'m.', markersize=1)[0] for thisTraj in mediaPipe_trajectories]++ matplotlib_artist_objs['charuco_dottos'] = [ax3d.plot(thisTraj[0, 0:1], thisTraj[1, 0:1], thisTraj[2, 0:1],'m.', markersize=1)[0] for thisTraj in charuco_trajectories]+++ if useDLC:+ matplotlib_artist_objs['dlc_dottos'] = [ax3d.plot(thisTraj[0, 0:1], thisTraj[1, 0:1], thisTraj[2, 0:1],'b.', markersize=1)[0] for thisTraj in dlc_trajectories]++ matplotlib_artist_objs['ball_tails'] = [ax3d.plot(+ dlc_trajectories[bb][startFrame-ballTailLen:startFrame, 0],+ dlc_trajectories[bb][startFrame-ballTailLen:startFrame, 1],+ dlc_trajectories[bb][startFrame-ballTailLen:startFrame, 2],+ '-', color=ballColor[bb])[0] for bb in range(3)]+++ if useOpenPose:+ numFrames = openPose_skel_fr_mar_xyz.shape[0]+ mx = np.nanmean(openPose_skel_fr_mar_xyz[int(numFrames/2),:,0])+ my = np.nanmean(openPose_skel_fr_mar_xyz[int(numFrames/2),:,1])+ mz = np.nanmean(openPose_skel_fr_mar_xyz[int(numFrames/2),:,2])++ if useMediaPipe:+ numFrames = mediaPipe_skel_fr_mar_xyz.shape[0]+ mx = np.nanmean(mediaPipe_skel_fr_mar_xyz[int(numFrames/2),:,0])+ my = np.nanmean(mediaPipe_skel_fr_mar_xyz[int(numFrames/2),:,1])+ mz = np.nanmean(mediaPipe_skel_fr_mar_xyz[int(numFrames/2),:,2])+++ if np.isnan(mx) or np.isnan(my) or np.isnan(mz):+ mx = 0+ my = 0+ mz = 0+ # groundX = np.arange(mx-100,mx+100)+ # groundZ = np.arange(my-100,my+100)+ # groundXX, groundZZ = np.meshgrid(groundX, groundZ)+ # groundYY = np.zeros_like(groundXX)+ # groundMesh = ax3d.plot_surface(groundXX, groundZZ, groundYY, color='k', alpha=.5)++ axRange = 1000#session.board.square_length * 10++ # Setting the axes properties+ ax3d.set_xlim3d([mx-axRange, mx+axRange])+ ax3d.set_ylim3d([my-axRange, my+axRange])+ ax3d.set_zlim3d([mz-axRange, mz+axRange])++ tit1 = fig.text(.5, .96, "The FreeMoCap Project", fontsize=15, horizontalalignment='center')+ tit2 = fig.text(.5, .92, "Session: {}".format(session.sessionID), fontsize=8, horizontalalignment='center')++++ ax3d.view_init(azim=azimuth, elev=elevation)++++ ###+ ###+ ### βββ βββ βββββ ββ ββ βββββββ ββ ββ ββ ββββββ βββββββ ββββββ βββββ ββ ββ βββββββ βββββββ+ ### ββββ ββββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ+ ### ββ ββββ ββ βββββββ βββββ βββββ ββ ββ ββ ββ ββ βββββ ββ ββ βββββββ βββ βββββ βββββββ+ ### ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ+ ### ββ ββ ββ ββ ββ ββ βββββββ ββββ ββ ββββββ βββββββ ββββββ ββ ββ ββ ββ βββββββ βββββββ+ ###+ ###+++ syncedVidPathListAll = list(sorted(session.syncedVidPath.glob('*.mp4')))+ numVids = len(syncedVidPathListAll)++ syncedVidPathList = syncedVidPathListAll.copy()++ vidAxesList = []+ vidAristList = []+ vidCapObjList = []++ list_of_vidOpenPoseArtistdicts = []+ list_of_vidMediaPipeArtistdicts = []++ if useDLC: vidDLCArtistList = []++ def build_2d_segment_artist_dict(vidNum, data_nCams_nFrames_nImgPts_XYC, dict_of_list_of_segment_idxs, segColor = 'k', lineWidth = 1, lineStyle = '-'):+ """+ Builds a dictionary of line artists for each body 2d segment.+ """+ segNames = list(dict_of_list_of_segment_idxs)++ dict_of_artist_objects = dict()+ for segNum, segName in enumerate(segNames):++ #determine color of segment, based on class of 'segColor' input+ if isinstance(segColor, str):+ thisRGBA = segColor+ elif isinstance(segColor, np.ndarray):+ thisRGBA = segColor+ elif isinstance(segColor, dict):+ thisRGBA = segColor[segName].copy()+ thisRGBA[-1] = .75+ elif isinstance(segColor, list):+ try:+ thisRGBA = segColor[segNum]+ except:+ print('Not enough colors provided, using Black instead')+ thisRGBA = 'k'+ else:+ thisRGBA = 'k'++ xData = data_nCams_nFrames_nImgPts_XYC[vidNum, startFrame, dict_of_list_of_segment_idxs[segName],0]+ yData = data_nCams_nFrames_nImgPts_XYC[vidNum, startFrame, dict_of_list_of_segment_idxs[segName],1]++ # #make NaN's invisible (i thought they already would be but???!!!!)+ # thisRGBAall = np.tile(thisRGBA,(len(xData),1))+ # thisRGBAall[np.isnan(xData),3] = 0++ xDataMasked = np.ma.masked_where(xData, np.isnan(xData))+ yDataMasked = np.ma.masked_where(yData, np.isnan(yData))++ dict_of_artist_objects[segName] = thisVidAxis.plot(+ xDataMasked,+ yDataMasked,+ linestyle=lineStyle,+ linewidth=lineWidth,+ color = thisRGBA,+ )[0]+ return dict_of_artist_objects++ # vidImageGrid = ImageGrid(fig, [.45, 0.05, .54, .9], nrows_ncols = (2,2), axes_pad = 0.1) # https://matplotlib.org/stable/gallery/axes_grid1/simple_axesgrid.html+ if numVids < 4:+ numRows = numVids+ numCols = 1+ else:+ numRows = int(np.ceil(numVids/2))+ numCols = 2+++ vidAxGridSpec = fig.add_gridspec(numRows, numCols, left=.55, bottom=.025, right=.9, top=.9, wspace=.1, hspace=.1)+++ for thisVidNum, thisVidPath in enumerate(syncedVidPathList):+ #make subplot for figure (and set position)++ if numVids < 4:+ thisVidAxis = fig.add_subplot(vidAxGridSpec[thisVidNum])+ elif numVids %2 > 0: #if odd number videos plot the first vid across 2 spots+ if thisVidNum==0:+ thisVidAxis = fig.add_subplot(vidAxGridSpec[0,:])+ else:+ thisVidAxis = fig.add_subplot(vidAxGridSpec[thisVidNum+1])+ else:+ thisVidAxis = fig.add_subplot(vidAxGridSpec[thisVidNum])+++ thisVidAxis.set_axis_off()++ vidAxesList.append(thisVidAxis)++ #create video capture object+ thisVidCap = cv2.VideoCapture(str(thisVidPath))+++ #create artist object for each video+ success, image = thisVidCap.read()++ assert success==True, "{} - failed to load an image".format(thisVidPath.stem) #make sure we have a frame++ vidAristList.append(thisVidAxis.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)))+ vidCapObjList.append(thisVidCap)++++ if useOpenPose:+ vidOpenPoseArtist_dict = dict()+ vidOpenPoseArtist_dict['op_body'] = build_2d_segment_artist_dict(thisVidNum, openPose_nCams_nFrames_nImgPts_XYC, dict_of_openPoseSegmentIdx_dicts['op_body'], segColor = dict_of_op_skel_lineColor)+ vidOpenPoseArtist_dict['op_rHand'] = build_2d_segment_artist_dict(thisVidNum, openPose_nCams_nFrames_nImgPts_XYC, dict_of_openPoseSegmentIdx_dicts['op_rHand'], segColor=np.append(humon_red, .75), lineWidth=.5)+ vidOpenPoseArtist_dict['op_lHand'] = build_2d_segment_artist_dict(thisVidNum, openPose_nCams_nFrames_nImgPts_XYC, dict_of_openPoseSegmentIdx_dicts['op_lHand'], segColor=np.append(humon_blue, .75), lineWidth=.5)+ vidOpenPoseArtist_dict['op_face'] = build_2d_segment_artist_dict(thisVidNum, openPose_nCams_nFrames_nImgPts_XYC, dict_of_openPoseSegmentIdx_dicts['op_face'], segColor = np.array([1.,1.,1.,1]), lineWidth=.25)+ list_of_vidOpenPoseArtistdicts.append(vidOpenPoseArtist_dict)++ if useMediaPipe:+ vidMediaPipeArtist_dict = dict()+ vidMediaPipeArtist_dict['mp_body'] = build_2d_segment_artist_dict(thisVidNum, mediaPipe_nCams_nFrames_nImgPts_XYC, dict_of_mediaPipeSegmentIdx_dicts['mp_body'], segColor = 'g')+ vidMediaPipeArtist_dict['mp_rHand'] = build_2d_segment_artist_dict(thisVidNum, mediaPipe_nCams_nFrames_nImgPts_XYC, dict_of_mediaPipeSegmentIdx_dicts['mp_rHand'], segColor=np.append(humon_red, .75), lineWidth=.5)+ vidMediaPipeArtist_dict['mp_lHand'] = build_2d_segment_artist_dict(thisVidNum, mediaPipe_nCams_nFrames_nImgPts_XYC, dict_of_mediaPipeSegmentIdx_dicts['mp_lHand'], segColor=np.append(humon_blue, .75), lineWidth=.5)+ list_of_vidMediaPipeArtistdicts.append(vidMediaPipeArtist_dict)+++ if useDLC:++ vidDLCArtistList.append(thisVidAxis.plot(+ dlcData_nCams_nFrames_nImgPts_XYC[thisVidNum,startFrame,:,0],+ dlcData_nCams_nFrames_nImgPts_XYC[thisVidNum,startFrame,:,1],+ linestyle='none',+ marker = '.',+ markersize = 1,+ color='w',+ markerfacecolor='none' ))++ ###+ ###+ ### βββ βββ βββββ ββ ββ βββββββ ββ ββ βββ ββ βββββββ ββββββ ββ ββββββ ββββββββ βββββ ββ ββ βββββββ βββββββ+ ### ββββ ββββ ββ ββ ββ ββ ββ ββ ββ ββββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ+ ### ββ ββββ ββ βββββββ βββββ βββββ ββ ββ ββ ββ ββ βββββ ββββββ ββ ββ ββ ββ βββββββ βββ βββββ βββββββ+ ### ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ+ ### ββ ββ ββ ββ ββ ββ βββββββ βββββββ ββ ββ ββββ βββββββ ββ βββββββ ββββββ ββ ββ ββ ββ ββ βββββββ βββββββ+ ###+ ###+++ fps=30 #NOTE - This should be saved in the session Class some how+ time = np.arange(0, numFrames)/fps+ if useMediaPipe:+ rHandIdx = 20+ lHandIdx = 19+ rHandX = mediaPipe_skel_fr_mar_xyz[:,rHandIdx,0]/1000 #convert mm to meters+ rHandY = -mediaPipe_skel_fr_mar_xyz[:,rHandIdx,1]/1000+ lHandX = mediaPipe_skel_fr_mar_xyz[:,lHandIdx,0]/1000+ lHandY = -mediaPipe_skel_fr_mar_xyz[:,lHandIdx,1]/1000++ if useOpenPose:+ rHandIdx = 4+ lHandIdx = 7+ rHandX = openPose_skel_fr_mar_xyz[:,rHandIdx,0]/1000 #convert mm to meters+ rHandY = -openPose_skel_fr_mar_xyz[:,rHandIdx,1]/1000+ lHandX = openPose_skel_fr_mar_xyz[:,lHandIdx,0]/1000+ lHandY = -openPose_skel_fr_mar_xyz[:,lHandIdx,1]/1000+++ linePlotWidth = .45+ linePlotHeight = .14++ yTimeSeriesAx = fig.add_subplot(position=[.07, .225, linePlotWidth, linePlotHeight])+ yTimeSeriesAx.set_title('Hand Position vs Time', fontsize = 7, pad=2)++ yTimeSeriesAx.plot(time, rHandY, color=humon_red, linewidth=.75, label='Right Hand')+ yTimeSeriesAx.plot(time, lHandY, color=humon_blue, linewidth=.75, label='Left Hand')++ timeRange = 3+ ylimRange = .6++ yCurrTimeArtists = dict()+ yCurrTimeYdata = dict()+ yCurrTimeArtists['blackLine'] = yTimeSeriesAx.plot([startFrame/fps, startFrame/fps], [-ylimRange, ylimRange*2], color='k', linewidth=1)++ yCurrTimeArtists['rHandDot'] = yTimeSeriesAx.plot([startFrame/fps], [rHandY[startFrame]], markeredgecolor=humon_red, markerfacecolor = 'k',marker='o', markersize=3)+ yCurrTimeYdata['rHandDot'] = rHandY+ yCurrTimeArtists['lHandDot'] = yTimeSeriesAx.plot([startFrame/fps], [lHandY[startFrame]], markeredgecolor=humon_blue, markerfacecolor = 'k', marker='o', markersize=3)+ yCurrTimeYdata['lHandDot'] = lHandY+++ yCurrTimeArtists['frameNumber'] = yTimeSeriesAx.text(startFrame/fps+3/fps, ylimRange*.7, "Frame# " + str(startFrame), fontsize=6)++ yTimeSeriesAx.tick_params(labelsize=6, direction='in', width=.5)+ yTimeSeriesAx.tick_params( pad=2)+++ yTimeSeriesAx.set_ylabel('Vertical (m)', fontsize=7, labelpad=3)+ yTimeSeriesAx.set_ylim([-ylimRange, ylimRange])+ yTimeSeriesAx.set_xlim([(startFrame/fps)-timeRange, (startFrame/fps)+timeRange])++++ for axis in ['top','bottom','left','right']:+ yTimeSeriesAx.spines[axis].set_linewidth(0.5)++++ yTimeSeriesAx.legend(loc='upper left', fontsize=6)++++ xTimeSeriesAx = fig.add_subplot(position=[.07, 0.05, linePlotWidth, linePlotHeight])++ xTimeSeriesAx.plot(time, rHandX, color=humon_red, linewidth=.75, label='Right Hand')+ xTimeSeriesAx.plot(time, lHandX, color=humon_blue, linewidth=.75, label='Left Hand')++++ xCurrTimeArtists = dict()+ xCurrTimeYdata = dict()+ xCurrTimeArtists['blackLine'] = xTimeSeriesAx.plot([startFrame/fps, startFrame/fps], [-ylimRange, ylimRange], color='k', linewidth=1)+ xCurrTimeArtists['rHandDot'] = xTimeSeriesAx.plot([startFrame/fps], [rHandX[startFrame]], markeredgecolor=humon_red, markerfacecolor = 'k',marker='o', markersize=3)+ xCurrTimeYdata['rHandDot'] = rHandX+ xCurrTimeArtists['lHandDot'] = xTimeSeriesAx.plot([startFrame/fps], [lHandX[startFrame]], markeredgecolor=humon_blue, markerfacecolor = 'k', marker='o', markersize=3)+ xCurrTimeYdata['lHandDot'] = lHandX++++ xTimeSeriesAx.tick_params(labelsize=6, direction='in', width=.5)+ xTimeSeriesAx.tick_params( pad=2)++ xTimeSeriesAx.set_ylabel('Left/Right (m)', fontsize=7, labelpad=0)+ xTimeSeriesAx.set_xlabel('Time(sec)', fontsize=8, labelpad=0)++ xTimeSeriesAx.set_ylim([-ylimRange, ylimRange])++ xTimeSeriesAx.set_xlim([(startFrame/fps)-timeRange, (startFrame/fps)+timeRange])+++ for axis in ['top','bottom','left','right']:+ xTimeSeriesAx.spines[axis].set_linewidth(0.5)++++ # #slider+ # axControls = fig.add_subplot(2,1,2)+ # axControls.set_position([0.25, 0.01, 0.6, 0.05])+ # animSlider = Slider(+ # ax=axControls,+ # label="FrameNum",+ # valmin=10,+ # valmax=numFrames,+ # valinit=10,+ # orientation="horizontal"+ # )++ # thisVidAxis.text(-565,1440, 'Music - artist: Neon Exdeath, song: Meowmaline, album: Jewel Tones',color=humon_green, fontsize=6)+ thisVidAxis.text(0,thisVidCap.get(cv2.CAP_PROP_FRAME_HEIGHT)*1.06, 'github.com/jonmatthis/freemocap || freemocap.org',color=humon_green, fontsize=6)++ logoAx = fig.add_subplot(position=[.85, .85, .15, .15])+ logoIm = cv2.imread(r'logo\fmc-logo-black-border-white-bkgd.png') #JSM NOTE - THis doesn't work, but SOMETIMES IT DOES?!+ if logoIm is not None: logoAx.imshow(cv2.cvtColor(logoIm, cv2.COLOR_BGR2RGB))+ logoAx.axis('off')++ # if recordVid:+ # animationFramePath = session.sessionPath / "animationFrames"+ # animationFramePath.mkdir(parents=True, exist_ok=True)++ # Creating the Animation object+ line_animation = animation.FuncAnimation(fig, update_figure, range(startFrame,numFrames), fargs=(),+ interval=1, blit=False)++++ if recordVid:+ # vidSavePath = '{}_animVid.mp4'.format(str(session.sessionPath / session.sessionID))+ gifSavePath = '{}_animVid.gif'.format(str(session.sessionPath / session.sessionID)) #NOTE - saving as a gif first then converting to MP4, because saving directly to MP4 requires users to install ffmpeg (independently of the pip install)+ with console.status('Saving animation for {}'.format(session.sessionID)):+ # Writer = animation.writers['ffmpeg']+ Writer = animation.writers['pillow']+ writer = Writer(fps=fps, metadata=dict(artist='FreeMoCap'))#, bitrate=1800)+ line_animation.save(gifSavePath, writer = writer)+ # Writer = animation.FFMpegWriter(fps=30, metadata=dict(artist='FreeMoCap', comment=session.sessionID), bitrate=1800)+ # vidSavePath = '{}_outVid.mp4'.format(str(session.sessionPath / session.sessionID))+ # Writer.saving(fig = fig, outfile=vidSavePath, dpi=150)++ gif_filepath = mp.VideoFileClip(gifSavePath)+ gif_filepath.write_videofile(gifSavePath.replace('.gif','.mp4'))+ os.remove(gifSavePath)+++ try:+ if showAnimation:+ with console.status('Playing Skeleton animation! Close the `matplotlib` window to continue...'):+ plt.pause(0.1)+ plt.draw()+ except:+ pass+++++++ ###+ ###+ ### βββββββ ββββββ ββββββ βββ βββ βββββ ββββββββ ββββββ ββββββ βββββββ βββ ββ ββββββ ββββββ βββββββ βββββββ βββββββ ββ ββ βββββββ ββ+ ### ββ ββ ββ ββ ββ ββββ ββββ ββ ββ ββ ββ ββ ββ ββ ββ ββββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ+ ### βββββ ββ ββ ββββββ ββ ββββ ββ βββββββ ββ ββ ββ ββββββ βββββ ββ ββ ββ ββββββ ββ ββ βββββββ βββββ βββββββ βββββ βββββ ββ+ ### ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ+ ### ββ ββββββ ββ ββ ββ ββ ββ ββ ββ ββββββ ββ βββββββ ββ ββββ ββ ββββββ βββββββ βββββββ βββββββ ββ ββ βββββββ βββββββ+ ###+ ###+def formatOpenPoseStickIndices():+ """+ generate dictionary of arrays, each containing the 'connect-the-dots' order to draw a given body segment++ returns:+ openPoseBodySegmentIds= a dictionary of arrays containing indices of individual body segments (Note, a lot of markerless mocap comp sci types like to say 'pose' instead of 'body'. They also use 'pose' to refer to camera 6 DoF position sometimes. Comp sci is frustrating like that lol)+ openPoseHandIds = a dictionary of arrays containing indices of individual hand segments, along with offset to know where to start in the 'skel_fr_mar_xyz.shape[1]' part of the array+ dict_of_op_skel_lineColor = a dictionary of arrays, each containing the color (RGBA) to use for a given body segment+ """+ dict_of_openPoseSegmentIdx_dicts = dict()++ #make body dictionary+ openPoseBodySegmentIds = dict()+ openPoseBodySegmentIds['op_head'] = [17, 15, 0, 1,0, 16, 18, ]+ openPoseBodySegmentIds['op_spine'] = [1,8,5,1, 2, 12, 8, 9, 5, 1, 2, 8]+ openPoseBodySegmentIds['op_rArm'] = [1, 2, 3, 4, ]+ openPoseBodySegmentIds['op_lArm'] = [1, 5, 6, 7, ]+ openPoseBodySegmentIds['op_rLeg'] = [8, 9, 10, 11, 22, 23, 11, 24, ]+ openPoseBodySegmentIds['op_lLeg'] = [8,12, 13, 14, 19, 20, 14, 21,]+ dict_of_openPoseSegmentIdx_dicts['op_body'] = openPoseBodySegmentIds+++ #make colors dictionary+ openPoseBodyColor = np.array([50, 89, 0])/255+ openPoseRightColor = np.array([230, 50, 0])/255+ openPoseLeftColor = np.array([0, 50, 230])/255++ dict_of_op_skel_lineColor = dict()+ dict_of_op_skel_lineColor['op_head'] = np.append(openPoseBodyColor, 0.5)+ dict_of_op_skel_lineColor['op_spine'] = np.append(openPoseBodyColor, 1)+ dict_of_op_skel_lineColor['op_rArm'] = np.append(openPoseRightColor, 1)+ dict_of_op_skel_lineColor['op_lArm'] = np.append(openPoseLeftColor, 1)+ dict_of_op_skel_lineColor['op_rLeg'] = np.append(openPoseRightColor, 1)+ dict_of_op_skel_lineColor['op_lLeg'] = np.append(openPoseLeftColor, 1)+++ # Make some handy maps ;D+ openPoseHandIds = dict()+ rHandIDstart = 25+ lHandIDstart = rHandIDstart + 21++ openPoseHandIds['op_thumb'] = np.array([0, 1, 2, 3, 4, ])+ openPoseHandIds['op_index'] = np.array([0, 5, 6, 7, 8, ])+ openPoseHandIds['op_bird']= np.array([0, 9, 10, 11, 12, ])+ openPoseHandIds['op_ring']= np.array([0, 13, 14, 15, 16, ])+ openPoseHandIds['op_pinky'] = np.array([0, 17, 18, 19, 20, ])+++ rHand_dict = copy.deepcopy(openPoseHandIds.copy()) #copy.deepcopy() is necessary to make sure the dicts are independent of each other+ lHand_dict = copy.deepcopy(rHand_dict)++ for key in rHand_dict:+ rHand_dict[key] += rHandIDstart+ lHand_dict[key] += lHandIDstart++ dict_of_openPoseSegmentIdx_dicts['op_rHand'] = rHand_dict+ dict_of_openPoseSegmentIdx_dicts['op_lHand'] = lHand_dict+++ #how to face --> :D <--+ openPoseFaceIDs = dict()+ faceIDStart = 67+ #define face parts+ openPoseFaceIDs['op_jaw'] = np.arange(0,16) + faceIDStart+ openPoseFaceIDs['op_rBrow'] = np.arange(17,21) + faceIDStart+ openPoseFaceIDs['op_lBrow'] = np.arange(22,26) + faceIDStart+ openPoseFaceIDs['op_noseRidge'] = np.arange(27,30) + faceIDStart+ openPoseFaceIDs['op_noseBot'] = np.arange(31,35) + faceIDStart+ openPoseFaceIDs['op_rEye'] = np.concatenate((np.arange(36,41), [36])) + faceIDStart+ openPoseFaceIDs['op_lEye'] = np.concatenate((np.arange(42,47), [42])) + faceIDStart+ openPoseFaceIDs['op_upperLip'] = np.concatenate((np.arange(48,54), np.flip(np.arange(60, 64)), [48])) + faceIDStart+ openPoseFaceIDs['op_lowerLip'] = np.concatenate(([60], np.arange(64,67), np.arange(54, 59), [48], [60])) + faceIDStart+ openPoseFaceIDs['op_rPupil'] = np.array([68]) + faceIDStart+ openPoseFaceIDs['op_lPupil'] = np.array([69]) + faceIDStart #nice++ dict_of_openPoseSegmentIdx_dicts['op_face'] = openPoseFaceIDs++ return dict_of_openPoseSegmentIdx_dicts, dict_of_op_skel_lineColor++++###+### βββββββ ββββββ ββββββ βββ βββ βββββ ββββββββ βββ βββ βββββββ ββββββ ββ βββββ ββββββ ββ ββββββ βββββββ βββββββ ββ ββ βββββββ ββ+### ββ ββ ββ ββ ββ ββββ ββββ ββ ββ ββ ββββ ββββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ+### βββββ ββ ββ ββββββ ββ ββββ ββ βββββββ ββ ββ ββββ ββ βββββ ββ ββ ββ βββββββ ββββββ ββ ββββββ βββββ βββββββ βββββ βββββ ββ+### ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ+### ββ ββββββ ββ ββ ββ ββ ββ ββ ββ ββ ββ βββββββ ββββββ ββ ββ ββ ββ ββ ββ βββββββ βββββββ ββ ββ βββββββ βββββββ+###+###+def formatMediaPipeStickIndices():+ """+ generate dictionary of arrays, each containing the 'connect-the-dots' order to draw a given body segment++ returns:+ mediaPipeBodySegmentIds= a dictionary of arrays containing indices of individual body segments (Note, a lot of markerless mocap comp sci types like to say 'pose' instead of 'body'. They also use 'pose' to refer to camera 6 DoF position sometimes. Comp sci is frustrating like that lol)+ mediaPipeHandIds = a dictionary of arrays containing indices of individual hand segments, along with offset to know where to start in the 'skel_fr_mar_xyz.shape[1]' part of the array+ dict_of_mp_skel_lineColor = a dictionary of arrays, each containing the color (RGBA) to use for a given body segment+ """+ dict_of_mediaPipeSegmentIdx_dicts = dict()++ #make body dictionary+ mediaPipeBodySegmentIds = dict()+ mediaPipeBodySegmentIds['mp_head'] = [8, 6, 5, 4, 0, 10, 9, 0, 1, 2, 3, 7 ]+ mediaPipeBodySegmentIds['mp_tors'] = [12, 11, 24, 23, 12]+ mediaPipeBodySegmentIds['mp_rArm'] = [12, 14, 16, 18, 20, 16, 22 ]+ mediaPipeBodySegmentIds['mp_lArm'] = [11, 13, 15, 17, 19, 15, 21]+ mediaPipeBodySegmentIds['mp_rLeg'] = [24, 26, 28, 30, 32, 28, ]+ mediaPipeBodySegmentIds['mp_lLeg'] = [23, 25, 27, 29, 31, 27, ]+ dict_of_mediaPipeSegmentIdx_dicts['mp_body'] = mediaPipeBodySegmentIds++ #make colors dictionary+ mediaPipeBodyColor = np.array([125,106,0])/255+ mediaPipeRightColor = np.array([230, 0, 169])/255+ mediaPipeLeftColor = np.array([0, 176, 176])/255+ dict_of_mp_skel_lineColor = dict()++ dict_of_mp_skel_lineColor['mp_head'] = np.append(mediaPipeBodyColor, .5)+ dict_of_mp_skel_lineColor['mp_tors'] = np.append(mediaPipeBodyColor, 1)+ dict_of_mp_skel_lineColor['mp_rArm'] = np.append(mediaPipeRightColor, 1)+ dict_of_mp_skel_lineColor['mp_lArm'] = np.append(mediaPipeLeftColor, 1)+ dict_of_mp_skel_lineColor['mp_rLeg'] = np.append(mediaPipeRightColor, 1)+ dict_of_mp_skel_lineColor['mp_lLeg'] = np.append(mediaPipeLeftColor, 1)+++ # Make some handy maps ;D+ mediaPipeHandIds = dict()+ rHandIDstart = 33+ lHandIDstart = rHandIDstart + 21++ mediaPipeHandIds['mp_thumb'] = np.array([0, 1, 2, 3, 4, ])+ mediaPipeHandIds['mp_index'] = np.array([0, 5, 6, 7, 8, ])+ mediaPipeHandIds['mp_bird']= np.array([0, 9, 10, 11, 12, ])+ mediaPipeHandIds['mp_ring']= np.array([0, 13, 14, 15, 16, ])+ mediaPipeHandIds['mp_pinky'] = np.array([0, 17, 18, 19, 20, ])+++ rHand_dict = copy.deepcopy(mediaPipeHandIds.copy()) #copy.deepcopy() is necessary to make sure the dicts are independent of each other+ lHand_dict = copy.deepcopy(rHand_dict)++ for key in rHand_dict:+ rHand_dict[key] += rHandIDstart+ lHand_dict[key] += lHandIDstart++ dict_of_mediaPipeSegmentIdx_dicts['mp_rHand'] = rHand_dict+ dict_of_mediaPipeSegmentIdx_dicts['mp_lHand'] = lHand_dict+++ # #how to face --> :D <--+ # mediaPipeFaceIDs = dict()+ # faceIDStart = 67+ # #define face parts+ # mediaPipeFaceIDs['mp_jaw'] = np.arange(0,16) + faceIDStart+ # mediaPipeFaceIDs['mp_rBrow'] = np.arange(17,21) + faceIDStart+ # mediaPipeFaceIDs['mp_lBrow'] = np.arange(22,26) + faceIDStart+ # mediaPipeFaceIDs['mp_noseRidge'] = np.arange(27,30) + faceIDStart+ # mediaPipeFaceIDs['mp_noseBot'] = np.arange(31,35) + faceIDStart+ # mediaPipeFaceIDs['mp_rEye'] = np.concatenate((np.arange(36,41), [36])) + faceIDStart+ # mediaPipeFaceIDs['mp_lEye'] = np.concatenate((np.arange(42,47), [42])) + faceIDStart+ # mediaPipeFaceIDs['mp_upperLip'] = np.concatenate((np.arange(48,54), np.flip(np.arange(60, 64)), [48])) + faceIDStart+ # mediaPipeFaceIDs['mp_lowerLip'] = np.concatenate(([60], np.arange(64,67), np.arange(54, 59), [48], [60])) + faceIDStart+ # mediaPipeFaceIDs['mp_rPupil'] = np.array([68]) + faceIDStart+ # mediaPipeFaceIDs['mp_lPupil'] = np.array([69]) + faceIDStart #nice++ # dict_of_mediaPipeSegmentIdx_dicts['mp_face'] = mediaPipeFaceIDs++ return dict_of_mediaPipeSegmentIdx_dicts, dict_of_mp_skel_lineColor+++if __name__ == '__main__':+ MatplotlibLaserSkeleton()diff --git a/jon_scratch/matplotlib_laser_skeleton/__init__.py b/jon_scratch/matplotlib_laser_skeleton/__init__.pynew file mode 100644index 00000000..e69de29bdiff --git a/jon_scratch/matplotlib_laser_skeleton/matplotlib_laser_skeleton.py b/jon_scratch/matplotlib_laser_skeleton/matplotlib_laser_skeleton.pynew file mode 100644index 00000000..715328c8--- /dev/null+++ b/jon_scratch/matplotlib_laser_skeleton/matplotlib_laser_skeleton.py@@ -0,0 +1,68 @@+from pathlib import Path++import keyboard+import matplotlib.pyplot as plt+import matplotlib+import numpy as np+from mediapipe.python.solutions import holistic as mp_holistic++matplotlib.use('qtagg')++mediapipe_body_pose_connections = [this_connection for this_connection in mp_holistic.POSE_CONNECTIONS]+++class MatplotlibLaserSkeleton:+ def __init__(self, file_path):+ self.fig = None+ self.ax3d = None+ self.mediapipe_all_fr_mar_xyz = None+ self.file_path = Path(file_path)+ self.load_mediapipe_data()++ def load_mediapipe_data(self):+ mediapipe_data_path = self.file_path / 'DataArrays' / 'mediaPipeSkel_3d.npy'+ self.mediapipe_all_fr_mar_xyz = np.load(str(mediapipe_data_path))++ def play_laser_skeleton_animation(self, start_frame: int = 0, end_frame: int = None):+ if end_frame is None:+ end_frame = self.mediapipe_all_fr_mar_xyz.shape[0]++ self.create_figure()++ for frame_number in range(start_frame, end_frame):+ plt.cla()+ self.plot_mediapipe_skeleton(frame_number)+ plt.show()+ plt.pause(0.001)+ if keyboard.is_pressed('esc'):+ print('You Pressed Escape!')+ plt.close('all')+ break # finishing the loop++ def create_figure(self):+ plt.ion()+ self.fig = plt.figure()+ self.ax3d = self.fig.add_subplot(111, projection='3d')+ self.set_axis_limits()++ def set_axis_limits(self):+ median_position = np.nanmedian(self.mediapipe_all_fr_mar_xyz, axis=0)+ median_position = np.nanmedian(median_position, axis=0)+ stddev_position = np.nanstd(median_position, axis=0)+ axis_range = 1000+ self.ax3d.set_xlim(median_position[0] - axis_range, median_position[0] + axis_range)+ self.ax3d.set_ylim(median_position[1] - axis_range, median_position[1] + axis_range)+ self.ax3d.set_zlim(median_position[2] - axis_range, median_position[2] + axis_range)++ def plot_mediapipe_skeleton(self, frame_number: int):+ print(f'Plotting frame {frame_number}')+ self.ax3d.plot(self.mediapipe_all_fr_mar_xyz[frame_number, :, 0],+ self.mediapipe_all_fr_mar_xyz[frame_number, :, 1],+ self.mediapipe_all_fr_mar_xyz[frame_number, :, 2],+ color='k', marker='o', linestyle='none', markersize=1)+++if __name__ == '__main__':+ session_path = Path('C:/Users/jonma/Dropbox/FreeMoCapProject/FreeMocap_Data/sesh_2022-02-15_11_54_28_pupil_maybe')+ matplotlib_laser_skeleton = MatplotlibLaserSkeleton(session_path)+ matplotlib_laser_skeleton.play_laser_skeleton_animation(start_frame=1000, end_frame=1200)