Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

nice start on a gl visualizer - vor calibration still doesn't workl

Jonded5d41188f8
diff --git a/jon_scratch/karls_pupil_freemocap_code.ipynb b/jon_scratch/karls_pupil_freemocap_code.ipynb
index 2d05d69c..a613c97b 100644
--- a/jon_scratch/karls_pupil_freemocap_code.ipynb
+++ b/jon_scratch/karls_pupil_freemocap_code.ipynb
@@ -25,32 +25,13 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Requirement already satisfied: scipy in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (1.8.0)\n",
- "Requirement already satisfied: numpy<1.25.0,>=1.17.3 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from scipy) (1.22.3)\n",
- "Requirement already satisfied: pandas in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (1.4.2)\n",
- "Requirement already satisfied: pytz>=2020.1 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from pandas) (2022.1)\n",
- "Requirement already satisfied: python-dateutil>=2.8.1 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from pandas) (2.8.2)\n",
- "Requirement already satisfied: numpy>=1.18.5 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from pandas) (1.22.3)\n",
- "Requirement already satisfied: six>=1.5 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from python-dateutil>=2.8.1->pandas) (1.16.0)\n",
- "Requirement already satisfied: numpy in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (1.22.3)\n",
- "Requirement already satisfied: pyqt5 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (5.15.6)\n",
- "Requirement already satisfied: PyQt5-Qt5>=5.15.2 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from pyqt5) (5.15.2)\n",
- "Requirement already satisfied: PyQt5-sip<13,>=12.8 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from pyqt5) (12.10.1)\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"!pip install scipy\n",
"!pip install pandas\n",
@@ -60,10 +41,9 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -76,15 +56,15 @@
"from pathlib import Path\n",
"\n",
"from scipy import optimize\n",
- "from scipy.spatial.transform import Rotation "
+ "from scipy.spatial.transform import Rotation\n",
+ "\n"
]
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -110,10 +90,9 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -155,10 +134,9 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -195,10 +173,9 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -237,10 +214,9 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -278,10 +254,9 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -306,10 +281,9 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -406,10 +380,9 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -433,10 +406,9 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -461,10 +433,9 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -514,33 +485,13 @@
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "freemocap_timestamps[-1]: 1644944570.8511524\n",
- "r_eye_timestamps[-1]: 1644944575.0656984\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "1644944570.8511524"
- ]
- },
- "execution_count": 13,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
"print(f'freemocap_timestamps[-1]: {freemocap_timestamps[-1]}')\n",
"print(f'r_eye_timestamps[-1]: {r_eye_timestamps[-1]}')\n",
@@ -561,10 +512,9 @@
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -633,10 +583,9 @@
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -653,10 +602,9 @@
},
{
"cell_type": "code",
- "execution_count": 16,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -684,22 +632,13 @@
},
{
"cell_type": "code",
- "execution_count": 17,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "r_eye_gaze_xyz_og.shape: (17737, 3)\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"r_eye_gaze_x = r_eye_pupil_center_normal_x\n",
"r_eye_gaze_y = r_eye_pupil_center_normal_y\n",
@@ -712,22 +651,13 @@
},
{
"cell_type": "code",
- "execution_count": 18,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "l_eye_gaze_xyz_og.shape: (16324, 3)\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"l_eye_gaze_x = l_eye_pupil_center_normal_x\n",
"l_eye_gaze_y = l_eye_pupil_center_normal_y\n",
@@ -753,10 +683,9 @@
},
{
"cell_type": "code",
- "execution_count": 19,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -777,10 +706,9 @@
},
{
"cell_type": "code",
- "execution_count": 20,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -805,10 +733,9 @@
},
{
"cell_type": "code",
- "execution_count": 21,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -833,25 +760,13 @@
},
{
"cell_type": "code",
- "execution_count": 22,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "r_eye_gaze_xyz.shape: (4915, 3)\n",
- "l_eye_gaze_xyz.shape: (4915, 3)\n",
- "mediapipe_skeleton_fr_mar_xyz.shape: (4915, 543, 3)\n",
- "mediapipe_head_rotation_matricies_fr_row_col.shape: (4915, 3, 3)\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"print(f'r_eye_gaze_xyz.shape: {r_eye_gaze_xyz.shape}')\n",
"print(f'l_eye_gaze_xyz.shape: {l_eye_gaze_xyz.shape}')\n",
@@ -861,22 +776,13 @@
},
{
"cell_type": "code",
- "execution_count": 23,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"if debug:\n",
" fig = plt.figure(num=65341,figsize=(10,20))\n",
@@ -930,10 +836,9 @@
},
{
"cell_type": "code",
- "execution_count": 24,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -958,10 +863,9 @@
},
{
"cell_type": "code",
- "execution_count": 25,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -1002,24 +906,14 @@
},
{
"cell_type": "code",
- "execution_count": 26,
+ "execution_count": null,
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "head_rotation_martices_fr_row_col.shape: (4915, 3, 3)\n",
- "r_eye_gaze_xyz.shape: (4915, 3)\n",
- "fixation_point_xyz.shape: (4915, 3)\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"vor_frame_start =1000\n",
"vor_frame_end = 1400\n",
@@ -1037,31 +931,14 @@
},
{
"cell_type": "code",
- "execution_count": 27,
+ "execution_count": null,
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "array([[-140.99623459, 371.93667349, -908.18373013],\n",
- " [-162.83809499, 344.26373174, -912.30887986],\n",
- " [-165.73504142, 346.97592809, -911.18040645],\n",
- " ...,\n",
- " [-104.44826484, -31.40865991, -522.0636855 ],\n",
- " [-101.70694275, -47.25579649, -518.36201151],\n",
- " [-102.06426831, -55.07294944, -524.55510472]])"
- ]
- },
- "execution_count": 27,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
"\n",
"fixation_point_during_vor_xyz = mediapipe_skeleton_fr_mar_xyz[vor_frame_start:vor_frame_end, right_index_fingertip_idx, :]\n",
@@ -1081,24 +958,14 @@
},
{
"cell_type": "code",
- "execution_count": 28,
+ "execution_count": null,
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "head_rotation_matrices_during_vor_fr_row_col.shape: (400, 3, 3)\n",
- "r_gaze_during_vor_xyz.shape: (400, 3)\n",
- "fixation_point_in_r_eyeball_coordinates_xyz.shape: (400, 3)\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"\n",
"head_rotation_matrices_during_vor_fr_row_col = mediapipe_head_rotation_matricies_fr_row_col[vor_frame_start:vor_frame_end, :, :]\n",
@@ -1123,22 +990,14 @@
},
{
"cell_type": "code",
- "execution_count": 29,
+ "execution_count": null,
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "skel_during_vor_fr_mar_dim.shape: (400, 543, 3)\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"skel_during_vor_fr_mar_dim = mediapipe_skeleton_fr_mar_xyz[vor_frame_start:vor_frame_end,:,:]\n",
"\n",
@@ -1201,7 +1060,7 @@
},
{
"cell_type": "code",
- "execution_count": 30,
+ "execution_count": null,
"metadata": {
"collapsed": false,
"pycharm": {
@@ -1215,247 +1074,18 @@
},
{
"cell_type": "code",
- "execution_count": 32,
+ "execution_count": null,
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- " Iteration Total nfev Cost Cost reduction Step norm Optimality \n",
- " 0 1 1.9160e+11 2.28e+11 \n",
- " 1 2 2.9245e+10 1.62e+11 1.00e+00 6.24e+10 \n",
- " 2 4 1.0953e+10 1.83e+10 5.00e-01 1.18e+10 \n",
- " 3 6 9.8908e+09 1.06e+09 2.50e-01 3.05e+09 \n",
- " 4 8 9.7599e+09 1.31e+08 6.25e-02 4.77e+08 \n",
- " 5 9 9.7498e+09 1.01e+07 6.25e-02 3.66e+08 \n",
- " 6 10 9.7457e+09 4.13e+06 1.56e-02 1.41e+08 \n",
- " 7 12 9.7451e+09 5.70e+05 3.91e-03 1.34e+08 \n",
- " 8 13 9.7447e+09 4.71e+05 3.91e-03 9.35e+07 \n",
- " 9 14 9.7438e+09 8.17e+05 7.81e-03 8.51e+07 \n",
- " 10 15 9.7437e+09 1.83e+05 1.56e-02 3.72e+08 \n",
- " 11 16 9.7423e+09 1.35e+06 3.91e-03 1.78e+08 \n",
- " 12 17 9.7417e+09 5.74e+05 7.81e-03 2.08e+08 \n",
- " 13 18 9.7411e+09 6.42e+05 7.81e-03 1.87e+08 \n",
- " 14 19 9.7404e+09 6.81e+05 7.81e-03 2.02e+08 \n",
- " 15 20 9.7397e+09 7.35e+05 7.81e-03 1.93e+08 \n",
- " 16 21 9.7389e+09 7.89e+05 7.81e-03 1.96e+08 \n",
- " 17 22 9.7380e+09 8.46e+05 7.81e-03 1.97e+08 \n",
- " 18 23 9.7371e+09 9.07e+05 7.81e-03 1.89e+08 \n",
- " 19 24 9.7362e+09 9.72e+05 7.81e-03 2.02e+08 \n",
- " 20 25 9.7351e+09 1.04e+06 7.81e-03 1.80e+08 \n",
- " 21 26 9.7340e+09 1.11e+06 7.81e-03 2.06e+08 \n",
- " 22 27 9.7328e+09 1.19e+06 7.81e-03 1.69e+08 \n",
- " 23 28 9.7316e+09 1.26e+06 7.81e-03 2.09e+08 \n",
- " 24 29 9.7302e+09 1.35e+06 7.81e-03 1.66e+08 \n",
- " 25 30 9.7288e+09 1.43e+06 7.81e-03 2.12e+08 \n",
- " 26 31 9.7273e+09 1.52e+06 7.81e-03 1.71e+08 \n",
- " 27 32 9.7248e+09 2.43e+06 1.56e-02 3.03e+08 \n",
- " 28 33 9.7234e+09 1.45e+06 1.56e-02 4.48e+08 \n",
- " 29 34 9.7217e+09 1.66e+06 3.91e-03 2.56e+08 \n",
- " 30 35 9.7199e+09 1.82e+06 7.81e-03 2.25e+08 \n",
- " 31 36 9.7180e+09 1.93e+06 7.81e-03 1.87e+08 \n",
- " 32 37 9.7142e+09 3.82e+06 1.56e-02 2.73e+08 \n",
- " 33 38 9.7128e+09 1.31e+06 3.13e-02 8.53e+08 \n",
- " 34 39 9.7072e+09 5.66e+06 7.81e-03 4.38e+08 \n",
- " 35 40 9.7039e+09 3.25e+06 1.56e-02 3.53e+08 \n",
- " 36 41 9.7008e+09 3.15e+06 1.56e-02 3.79e+08 \n",
- " 37 42 9.6974e+09 3.37e+06 1.56e-02 3.64e+08 \n",
- " 38 43 9.6939e+09 3.55e+06 1.56e-02 3.64e+08 \n",
- " 39 44 9.6901e+09 3.74e+06 1.56e-02 3.68e+08 \n",
- " 40 45 9.6862e+09 3.93e+06 1.56e-02 3.49e+08 \n",
- " 41 46 9.6821e+09 4.13e+06 1.56e-02 3.71e+08 \n",
- " 42 47 9.6777e+09 4.34e+06 1.56e-02 3.32e+08 \n",
- " 43 48 9.6732e+09 4.55e+06 1.56e-02 3.73e+08 \n",
- " 44 49 9.6684e+09 4.76e+06 1.56e-02 3.17e+08 \n",
- " 45 50 9.6634e+09 4.98e+06 1.56e-02 3.74e+08 \n",
- " 46 51 9.6582e+09 5.20e+06 1.56e-02 3.25e+08 \n",
- " 47 52 9.6528e+09 5.42e+06 1.56e-02 3.75e+08 \n",
- " 48 53 9.6471e+09 5.65e+06 1.56e-02 3.32e+08 \n",
- " 49 54 9.6413e+09 5.88e+06 1.56e-02 3.75e+08 \n",
- " 50 55 9.6320e+09 9.31e+06 3.13e-02 6.40e+08 \n",
- " 51 56 9.6270e+09 4.91e+06 3.13e-02 7.83e+08 \n",
- " 52 57 9.6207e+09 6.37e+06 7.81e-03 4.66e+08 \n",
- " 53 58 9.6141e+09 6.61e+06 1.56e-02 3.56e+08 \n",
- " 54 59 9.6072e+09 6.88e+06 1.56e-02 3.78e+08 \n",
- " 55 60 9.5943e+09 1.29e+07 3.13e-02 4.62e+08 \n",
- " 56 62 9.5872e+09 7.08e+06 1.56e-02 3.91e+08 \n",
- " 57 63 9.5798e+09 7.45e+06 1.56e-02 3.77e+08 \n",
- " 58 64 9.5652e+09 1.46e+07 3.12e-02 4.43e+08 \n",
- " 59 65 9.5596e+09 5.55e+06 6.25e-02 1.63e+09 \n",
- " 60 66 9.5390e+09 2.06e+07 1.56e-02 8.00e+08 \n",
- " 61 67 9.5284e+09 1.06e+07 3.12e-02 6.33e+08 \n",
- " 62 68 9.5180e+09 1.05e+07 3.13e-02 7.53e+08 \n",
- " 63 69 9.5071e+09 1.09e+07 3.12e-02 6.33e+08 \n",
- " 64 70 9.4959e+09 1.12e+07 3.12e-02 7.38e+08 \n",
- " 65 71 9.4844e+09 1.15e+07 3.12e-02 6.21e+08 \n",
- " 66 72 9.4726e+09 1.18e+07 3.12e-02 7.25e+08 \n",
- " 67 73 9.4604e+09 1.21e+07 3.12e-02 6.07e+08 \n",
- " 68 74 9.4479e+09 1.25e+07 3.12e-02 7.13e+08 \n",
- " 69 75 9.4352e+09 1.28e+07 3.12e-02 5.94e+08 \n",
- " 70 76 9.4221e+09 1.31e+07 3.12e-02 7.00e+08 \n",
- " 71 77 9.4086e+09 1.34e+07 3.12e-02 5.79e+08 \n",
- " 72 78 9.3949e+09 1.37e+07 3.12e-02 6.87e+08 \n",
- " 73 79 9.3808e+09 1.41e+07 3.12e-02 5.64e+08 \n",
- " 74 80 9.3664e+09 1.44e+07 3.12e-02 6.74e+08 \n",
- " 75 81 9.3517e+09 1.47e+07 3.12e-02 5.49e+08 \n",
- " 76 82 9.3367e+09 1.50e+07 3.12e-02 6.61e+08 \n",
- " 77 83 9.3213e+09 1.54e+07 3.12e-02 5.37e+08 \n",
- " 78 84 9.3056e+09 1.57e+07 3.12e-02 6.48e+08 \n",
- " 79 85 9.2896e+09 1.60e+07 3.13e-02 5.61e+08 \n",
- " 80 86 9.2733e+09 1.63e+07 3.13e-02 6.35e+08 \n",
- " 81 87 9.2567e+09 1.66e+07 3.12e-02 5.84e+08 \n",
- " 82 88 9.2397e+09 1.70e+07 3.12e-02 6.22e+08 \n",
- " 83 89 9.2224e+09 1.73e+07 3.12e-02 6.07e+08 \n",
- " 84 90 9.2048e+09 1.76e+07 3.12e-02 6.09e+08 \n",
- " 85 91 9.1869e+09 1.79e+07 3.12e-02 6.30e+08 \n",
- " 86 92 9.1687e+09 1.82e+07 3.12e-02 5.97e+08 \n",
- " 87 93 9.1502e+09 1.85e+07 3.12e-02 6.52e+08 \n",
- " 88 94 9.1313e+09 1.89e+07 3.12e-02 5.85e+08 \n",
- " 89 95 9.1121e+09 1.92e+07 3.12e-02 6.73e+08 \n",
- " 90 96 9.0926e+09 1.95e+07 3.12e-02 5.73e+08 \n",
- " 91 97 9.0728e+09 1.98e+07 3.12e-02 6.93e+08 \n",
- " 92 98 9.0527e+09 2.01e+07 3.12e-02 5.61e+08 \n",
- " 93 99 9.0323e+09 2.04e+07 3.12e-02 7.13e+08 \n",
- " 94 100 9.0116e+09 2.07e+07 3.12e-02 5.49e+08 \n",
- " 95 101 8.9906e+09 2.10e+07 3.12e-02 7.32e+08 \n",
- " 96 102 8.9693e+09 2.13e+07 3.12e-02 5.38e+08 \n",
- " 97 103 8.9476e+09 2.16e+07 3.12e-02 7.50e+08 \n",
- " 98 104 8.9257e+09 2.19e+07 3.12e-02 5.48e+08 \n",
- " 99 105 8.8951e+09 3.06e+07 6.25e-02 1.22e+09 \n",
- " 100 106 8.8807e+09 1.44e+07 6.25e-02 1.84e+09 \n",
- " 101 107 8.8554e+09 2.53e+07 1.56e-02 1.03e+09 \n",
- " 102 108 8.8321e+09 2.33e+07 3.12e-02 7.97e+08 \n",
- " 103 109 8.8089e+09 2.33e+07 3.12e-02 5.74e+08 \n",
- " 104 110 8.7740e+09 3.49e+07 6.25e-02 1.12e+09 \n",
- " 105 111 8.7594e+09 1.46e+07 6.25e-02 1.86e+09 \n",
- " 106 112 8.7334e+09 2.61e+07 1.56e-02 1.05e+09 \n",
- " 107 113 8.7090e+09 2.43e+07 3.12e-02 8.38e+08 \n",
- " 108 114 8.6847e+09 2.43e+07 3.12e-02 5.97e+08 \n",
- " 109 115 8.6468e+09 3.79e+07 6.25e-02 1.10e+09 \n",
- " 110 116 8.6318e+09 1.50e+07 6.25e-02 1.87e+09 \n",
- " 111 117 8.6053e+09 2.65e+07 1.56e-02 1.06e+09 \n",
- " 112 118 8.5803e+09 2.50e+07 3.12e-02 8.70e+08 \n",
- " 113 119 8.5554e+09 2.49e+07 3.12e-02 6.10e+08 \n",
- " 114 120 8.5163e+09 3.91e+07 6.25e-02 1.13e+09 \n",
- " 115 121 8.5017e+09 1.47e+07 6.25e-02 1.88e+09 \n",
- " 116 122 8.4748e+09 2.69e+07 1.56e-02 1.07e+09 \n",
- " 117 123 8.4498e+09 2.50e+07 3.12e-02 8.87e+08 \n",
- " 118 124 8.4251e+09 2.47e+07 3.12e-02 6.03e+08 \n",
- " 119 125 8.3879e+09 3.72e+07 6.25e-02 1.16e+09 \n",
- " 120 126 8.3750e+09 1.29e+07 6.25e-02 1.90e+09 \n",
- " 121 127 8.3479e+09 2.71e+07 1.56e-02 1.09e+09 \n",
- " 122 128 8.3237e+09 2.42e+07 3.12e-02 8.83e+08 \n",
- " 123 129 8.3004e+09 2.33e+07 3.12e-02 5.65e+08 \n",
- " 124 130 8.2699e+09 3.05e+07 6.25e-02 1.20e+09 \n",
- " 125 131 8.2591e+09 1.08e+07 6.25e-02 1.90e+09 \n",
- " 126 132 8.2322e+09 2.69e+07 1.56e-02 1.08e+09 \n",
- " 127 133 8.2102e+09 2.20e+07 3.13e-02 8.55e+08 \n",
- " 128 134 8.1900e+09 2.02e+07 3.12e-02 6.22e+08 \n",
- " 129 135 8.1708e+09 1.91e+07 3.12e-02 8.58e+08 \n",
- " 130 136 8.1527e+09 1.81e+07 3.12e-02 6.77e+08 \n",
- " 131 137 8.1356e+09 1.71e+07 3.12e-02 8.46e+08 \n",
- " 132 138 8.1195e+09 1.61e+07 3.13e-02 7.16e+08 \n",
- " 133 139 8.1044e+09 1.51e+07 3.12e-02 8.28e+08 \n",
- " 134 140 8.0902e+09 1.41e+07 3.12e-02 7.49e+08 \n",
- " 135 141 8.0771e+09 1.31e+07 3.12e-02 8.04e+08 \n",
- " 136 142 8.0650e+09 1.21e+07 3.12e-02 7.79e+08 \n",
- " 137 143 8.0538e+09 1.12e+07 3.12e-02 7.77e+08 \n",
- " 138 144 8.0436e+09 1.02e+07 3.12e-02 8.05e+08 \n",
- " 139 145 8.0343e+09 9.32e+06 3.12e-02 7.47e+08 \n",
- " 140 146 8.0258e+09 8.47e+06 3.12e-02 8.26e+08 \n",
- " 141 147 8.0181e+09 7.65e+06 3.12e-02 7.15e+08 \n",
- " 142 148 8.0119e+09 6.22e+06 7.81e-03 5.58e+08 \n",
- " 143 149 8.0056e+09 6.36e+06 1.56e-02 3.10e+08 \n",
- " 144 150 7.9994e+09 6.17e+06 1.56e-02 4.51e+08 \n",
- " 145 151 7.9916e+09 7.82e+06 3.13e-02 6.71e+08 \n",
- " 146 152 7.9909e+09 6.36e+05 3.13e-02 8.63e+08 \n",
- " 147 153 7.9840e+09 6.91e+06 7.81e-03 5.02e+08 \n",
- " 148 154 7.9793e+09 4.77e+06 1.56e-02 2.93e+08 \n",
- " 149 155 7.9758e+09 3.42e+06 1.56e-02 3.91e+08 \n",
- " 150 156 7.9731e+09 2.80e+06 1.56e-02 3.92e+08 \n",
- " 151 157 7.9706e+09 2.45e+06 1.56e-02 3.73e+08 \n",
- " 152 158 7.9685e+09 2.11e+06 1.56e-02 4.09e+08 \n",
- " 153 159 7.9667e+09 1.83e+06 1.56e-02 3.51e+08 \n",
- " 154 160 7.9651e+09 1.54e+06 3.91e-03 2.71e+08 \n",
- " 155 161 7.9636e+09 1.49e+06 7.81e-03 1.47e+08 \n",
- " 156 162 7.9622e+09 1.39e+06 7.81e-03 2.16e+08 \n",
- " 157 163 7.9610e+09 1.24e+06 7.81e-03 1.50e+08 \n",
- " 158 164 7.9599e+09 1.05e+06 7.81e-03 2.02e+08 \n",
- " 159 165 7.9591e+09 8.70e+05 7.81e-03 1.82e+08 \n",
- " 160 166 7.9583e+09 7.25e+05 7.81e-03 1.92e+08 \n",
- " 161 167 7.9577e+09 6.13e+05 7.81e-03 1.98e+08 \n",
- " 162 168 7.9572e+09 5.19e+05 7.81e-03 1.79e+08 \n",
- " 163 169 7.9568e+09 4.40e+05 7.81e-03 2.07e+08 \n",
- " 164 170 7.9564e+09 3.84e+05 1.95e-03 1.10e+08 \n",
- " 165 171 7.9560e+09 3.58e+05 3.91e-03 1.12e+08 \n",
- " 166 172 7.9557e+09 3.28e+05 3.91e-03 7.26e+07 \n",
- " 167 173 7.9554e+09 2.83e+05 3.91e-03 1.02e+08 \n",
- " 168 174 7.9552e+09 2.33e+05 3.91e-03 8.75e+07 \n",
- " 169 175 7.9550e+09 1.90e+05 3.91e-03 9.62e+07 \n",
- " 170 176 7.9548e+09 1.58e+05 3.91e-03 9.83e+07 \n",
- " 171 177 7.9547e+09 1.32e+05 3.91e-03 8.96e+07 \n",
- " 172 178 7.9546e+09 1.11e+05 3.91e-03 1.03e+08 \n",
- " 173 179 7.9545e+09 9.60e+04 9.77e-04 5.48e+07 \n",
- " 174 180 7.9544e+09 8.96e+04 1.95e-03 5.59e+07 \n",
- " 175 181 7.9543e+09 8.16e+04 1.95e-03 3.61e+07 \n",
- " 176 182 7.9543e+09 7.00e+04 1.95e-03 5.05e+07 \n",
- " 177 183 7.9542e+09 5.71e+04 1.95e-03 4.39e+07 \n",
- " 178 184 7.9542e+09 4.62e+04 1.95e-03 4.77e+07 \n",
- " 179 185 7.9541e+09 3.82e+04 1.95e-03 4.93e+07 \n",
- " 180 186 7.9541e+09 3.19e+04 1.95e-03 4.43e+07 \n",
- " 181 187 7.9541e+09 2.66e+04 1.95e-03 5.17e+07 \n",
- " 182 188 7.9540e+09 2.39e+04 4.88e-04 2.74e+07 \n",
- " 183 189 7.9540e+09 2.17e+04 9.77e-04 2.75e+07 \n",
- " 184 190 7.9540e+09 1.95e+04 9.77e-04 1.86e+07 \n",
- " 185 191 7.9540e+09 1.65e+04 9.77e-04 2.50e+07 \n",
- " 186 192 7.9540e+09 1.33e+04 9.77e-04 2.25e+07 \n",
- " 187 193 7.9540e+09 1.08e+04 9.77e-04 2.36e+07 \n",
- " 188 194 7.9539e+09 8.98e+03 9.77e-04 2.49e+07 \n",
- " 189 195 7.9539e+09 7.48e+03 9.77e-04 2.18e+07 \n",
- " 190 196 7.9539e+09 6.24e+03 9.77e-04 2.60e+07 \n",
- " 191 197 7.9539e+09 5.97e+03 2.44e-04 1.37e+07 \n",
- " 192 198 7.9539e+09 5.16e+03 4.88e-04 1.35e+07 \n",
- " 193 199 7.9539e+09 4.56e+03 4.88e-04 9.69e+06 \n",
- " 194 200 7.9539e+09 3.79e+03 4.88e-04 1.24e+07 \n",
- " 195 201 7.9539e+09 3.06e+03 4.88e-04 1.16e+07 \n",
- " 196 202 7.9539e+09 2.51e+03 4.88e-04 1.16e+07 \n",
- " 197 203 7.9539e+09 2.08e+03 4.88e-04 1.26e+07 \n",
- " 198 204 7.9539e+09 1.73e+03 4.88e-04 1.09e+07 \n",
- " 199 205 7.9539e+09 1.50e+03 1.22e-04 8.24e+06 \n",
- " 200 206 7.9539e+09 1.39e+03 2.44e-04 4.66e+06 \n",
- " 201 207 7.9539e+09 1.26e+03 2.44e-04 6.51e+06 \n",
- " 202 208 7.9539e+09 1.07e+03 2.44e-04 4.97e+06 \n",
- " 203 209 7.9539e+09 8.65e+02 2.44e-04 6.10e+06 \n",
- " 204 210 7.9539e+09 6.96e+02 2.44e-04 5.96e+06 \n",
- " 205 211 7.9539e+09 5.75e+02 2.44e-04 5.70e+06 \n",
- " 206 212 7.9539e+09 4.78e+02 2.44e-04 6.36e+06 \n",
- " 207 213 7.9539e+09 3.98e+02 2.44e-04 5.56e+06 \n",
- " 208 214 7.9539e+09 3.73e+02 6.10e-05 4.01e+06 \n",
- " 209 215 7.9539e+09 3.27e+02 1.22e-04 2.31e+06 \n",
- " 210 216 7.9539e+09 2.90e+02 1.22e-04 3.21e+06 \n",
- " 211 217 7.9539e+09 2.42e+02 1.22e-04 2.63e+06 \n",
- " 212 218 7.9539e+09 1.95e+02 1.22e-04 3.01e+06 \n",
- " 213 219 7.9539e+09 1.59e+02 1.22e-04 3.04e+06 \n",
- " 214 220 7.9539e+09 1.32e+02 1.22e-04 2.80e+06 \n",
- " 215 221 7.9539e+09 1.10e+02 1.22e-04 3.21e+06 \n",
- " 216 222 7.9539e+09 9.37e+01 3.05e-05 1.70e+06 \n",
- " 217 223 7.9539e+09 8.77e+01 6.10e-05 1.74e+06 \n",
- " 218 224 7.9539e+09 7.95e+01 6.10e-05 1.12e+06 \n",
- "`ftol` termination condition is satisfied.\n",
- "Function evaluations 224, initial cost 1.9160e+11, final cost 7.9539e+09, first-order optimality 1.12e+06.\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"\n",
"if debug:\n",
- " def plot_optimization_error(error,gaze_xyz, gaze_rotated_by_guess_then_head_rotation_xyz, fixation_point_in_eye_coordinates_xyz, skel_during_vor_fr_mar_dim):\n",
+ " def plot_optimization_error(error,gaze_xyz, gaze_rotated_by_guess_then_head_rotation_xyz, mean_fixation_point_xyz, skel_during_vor_fr_mar_dim):\n",
" figure_number=13451\n",
"\n",
" if not plt.fignum_exists(figure_number):\n",
@@ -1481,9 +1111,9 @@
" gaze_rot_xyz[:,1], \n",
" gaze_rot_xyz[:,2], 'r-o',label='gaze_rotated_by_guess_then_head_rotation_xyz')\n",
"\n",
- " ax.plot(fixation_point_in_eye_coordinates_xyz[:,0],\n",
- " fixation_point_in_eye_coordinates_xyz[:,1], \n",
- " fixation_point_in_eye_coordinates_xyz[:,2], 'b-o',label='fixation_point_in_eye_coordinates_xyz')\n",
+ " ax.plot(mean_fixation_point_xyz[0],\n",
+ " mean_fixation_point_xyz[1], \n",
+ " mean_fixation_point_xyz[2], 'b-o',label='mean_fixation_point_xyz')\n",
"\n",
" ax.set_xlim([-ax_range, ax_range])\n",
" ax.set_ylim([-ax_range, ax_range])\n",
@@ -1508,12 +1138,12 @@
" gaze_rotated_by_guess_then_head_rotation_xyz =[head_rotation_maticies_fr_row_col[this_frame_number,:,:] @ gaze_rotated_by_guess[this_frame_number]\n",
" for this_frame_number in range(gaze_xyz.shape[0])]\n",
"\n",
- " # mean_fixation_point_xyz = np.mean(fixation_point_in_eye_coordinates_xyz, axis=0)\n",
+ " mean_fixation_point_xyz = np.mean(fixation_point_in_eye_coordinates_xyz, axis=0)\n",
" #define error as difference between the fixation point and that rotated gaze estimate (these are both normalized, I think)\n",
- " error_per_frame = gaze_rotated_by_guess_then_head_rotation_xyz - fixation_point_in_eye_coordinates_xyz\n",
+ " error_per_frame = gaze_rotated_by_guess_then_head_rotation_xyz - mean_fixation_point_xyz\n",
" error = np.nanmean(np.nansum(error_per_frame**2)/error_per_frame.shape[0])\n",
" \n",
- " plot_optimization_error(error,gaze_xyz, gaze_rotated_by_guess_then_head_rotation_xyz, fixation_point_in_eye_coordinates_xyz, skel_during_vor_fr_mar_dim)\n",
+ " plot_optimization_error(error,gaze_xyz, gaze_rotated_by_guess_then_head_rotation_xyz, mean_fixation_point_xyz, skel_during_vor_fr_mar_dim)\n",
"\n",
" return error\n",
"\n",
@@ -1545,10 +1175,9 @@
},
{
"cell_type": "code",
- "execution_count": 33,
+ "execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -1563,7 +1192,6 @@
"execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
@@ -1586,12 +1214,18 @@
"execution_count": null,
"metadata": {
"pycharm": {
- "is_executing": true,
"name": "#%%\n"
}
},
"outputs": [],
- "source": []
+ "source": [
+ "\n",
+ "data_path = Path('C:/Users/jonma/Dropbox/FreeMoCapProject/FreeMocap_Data/')\n",
+ "session_path = data_path / 'sesh_2022-02-15_11_54_28_pupil_maybe'\n",
+ "\n",
+ "qt_gl_laser_skeleton = QT_GL_LaserSkeleton(session_path)\n",
+ "qt_gl_laser_skeleton.start_animation()\n"
+ ]
}
],
"metadata": {
diff --git a/jon_scratch/pyqt_gl_laser_skeleton/__init__.py b/jon_scratch/pyqt_gl_laser_skeleton/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/jon_scratch/pyqt_gl_laser_skeleton/qt_gl_laser_skeleton.py b/jon_scratch/pyqt_gl_laser_skeleton/qt_gl_laser_skeleton.py
new file mode 100644
index 00000000..7a839d67
--- /dev/null
+++ b/jon_scratch/pyqt_gl_laser_skeleton/qt_gl_laser_skeleton.py
@@ -0,0 +1,157 @@
+# adapted from - https://gist.github.com/markjay4k/da2f55e28514be7160a7c5fbf95bd243
+from pathlib import Path
+
+from pyqtgraph.Qt import QtCore, QtGui
+import pyqtgraph.opengl as gl
+import pyqtgraph as pg
+import numpy as np
+import sys
+from mediapipe.python.solutions import holistic as mp_holistic
+
+
+class QT_GL_LaserSkeleton():
+ def __init__(self, session_path: Path):
+ self.mediapipe_fr_mar_xyz = None
+ self.session_path = Path(session_path)
+ self.traces = dict()
+ self.start_frame_number = 1000
+ self.end_frame_number = 2000
+ self.current_frame_number = self.start_frame_number
+
+ self.load_mediapipe_data()
+ self.load_pupil_data()
+
+ self.create_app_window()
+ self.create_grid_planes()
+ self.get_mediapipe_connections()
+ self.initialize_skel_dottos()
+ self.initialize_skel_lines()
+ self.initialize_gaze_laser()
+
+ def load_mediapipe_data(self):
+ mediapipe_data_path = self.session_path / 'DataArrays' / 'mediaPipeSkel_3d_smoothed.npy'
+ mediapipe_all_fr_mar_xyz = np.load(str(mediapipe_data_path))
+ self.mean_position_xyz = np.nanmedian(np.nanmedian(mediapipe_all_fr_mar_xyz, axis=0), axis=0)
+ mediapipe_all_fr_mar_xyz[:, :, 0] -= self.mean_position_xyz[0]
+ mediapipe_all_fr_mar_xyz[:, :, 1] -= self.mean_position_xyz[1]
+ mediapipe_all_fr_mar_xyz[:, :, 2] -= self.mean_position_xyz[2]
+
+ # remove first frame due to annoying 'off-by-one' error in the timestamp logger
+ mediapipe_all_fr_mar_xyz = np.delete(mediapipe_all_fr_mar_xyz, 0, axis=0)
+
+ self.mediapipe_fr_mar_xyz = mediapipe_all_fr_mar_xyz[:, :75, :]
+
+ def load_pupil_data(self):
+ pupil_data_path = self.session_path / 'pupil_000' / 'exports' / '000' / 'right_eye_gaze_xyz.npy'
+ right_gaze_xyz = np.load(str(pupil_data_path))
+
+ right_eye_d = 5
+ self.right_eye_xyz = np.squeeze(self.mediapipe_fr_mar_xyz[:, right_eye_d, :])
+
+ right_gaze_xyz[:, 0] += self.right_eye_xyz[:, 0]
+ right_gaze_xyz[:, 1] += self.right_eye_xyz[:, 1]
+ right_gaze_xyz[:, 2] += self.right_eye_xyz[:, 2]
+
+ self.right_gaze_xyz = right_gaze_xyz
+
+ def create_app_window(self):
+ self.app = pg.mkQApp("Laser Skeleton")
+ self.gl_view_widget = gl.GLViewWidget()
+ self.gl_view_widget.opts['distance'] = 2000
+ self.gl_view_widget.setWindowTitle('FreeMoCap - Laser Skeleton')
+ self.gl_view_widget.setGeometry(0, 110, 1920, 1080)
+ self.gl_view_widget.show()
+
+ def create_grid_planes(self):
+ grid_scale = 2e3
+ # create the background grids
+ grid_plane_x = gl.GLGridItem()
+ grid_plane_x.setSize(grid_scale, grid_scale, grid_scale)
+ grid_plane_x.setSpacing(grid_scale / 10, grid_scale / 10, grid_scale / 10)
+ grid_plane_x.rotate(90, 0, 1, 0)
+ grid_plane_x.translate(-grid_scale, 0, 0)
+ self.gl_view_widget.addItem(grid_plane_x)
+
+ grid_plane_y = gl.GLGridItem()
+ grid_plane_y.setSize(grid_scale, grid_scale, grid_scale)
+ grid_plane_y.setSpacing(grid_scale / 10, grid_scale / 10, grid_scale / 10)
+ grid_plane_y.rotate(90, 1, 0, 0)
+ grid_plane_y.translate(0, -grid_scale, 0)
+ self.gl_view_widget.addItem(grid_plane_y)
+
+ grid_plane_z = gl.GLGridItem()
+ grid_plane_z.setSize(grid_scale, grid_scale, grid_scale)
+ grid_plane_z.setSpacing(grid_scale / 10, grid_scale / 10, grid_scale / 10)
+ grid_plane_z.translate(0, 0, -grid_scale)
+ self.gl_view_widget.addItem(grid_plane_z)
+
+ def get_mediapipe_connections(self):
+ self.mediapipe_body_connections = [this_connection for this_connection in mp_holistic.POSE_CONNECTIONS]
+ self.mediapipe_hand_connections = [this_connection for this_connection in mp_holistic.HAND_CONNECTIONS]
+ self.mediapipe_face_connections = [this_connection for this_connection in mp_holistic.FACEMESH_TESSELATION]
+
+ def initialize_skel_dottos(self):
+ self.skeleton_scatter_item = gl.GLScatterPlotItem(
+ pos=self.mediapipe_fr_mar_xyz[self.current_frame_number, :, :], color=(0, 1, 1, 1), size=10,
+ pxMode=False
+ )
+ self.gl_view_widget.addItem(self.skeleton_scatter_item)
+
+ def initialize_skel_lines(self):
+ self.skeleton_connections_list = []
+ for this_connection in self.mediapipe_body_connections:
+ this_skel_line = gl.GLLinePlotItem(
+ pos=self.mediapipe_fr_mar_xyz[self.current_frame_number, this_connection, :])
+ self.skeleton_connections_list.append(this_skel_line)
+ self.gl_view_widget.addItem(this_skel_line)
+
+ def initialize_gaze_laser(self):
+
+ this_gaze_laser = np.vstack(
+ (self.right_eye_xyz[self.current_frame_number, :], self.right_gaze_xyz[self.current_frame_number, :]))
+ self.right_gaze_line_item = gl.GLLinePlotItem(pos=this_gaze_laser,
+ color=(1, 0, 0, 1), )
+ self.gl_view_widget.addItem(self.right_gaze_line_item)
+
+ def update(self):
+ self.update_frame_number()
+
+ self.skeleton_scatter_item.setData(
+ pos=self.mediapipe_fr_mar_xyz[self.current_frame_number, :, :]
+ )
+ self.update_skeleton_lines()
+ this_gaze_laser = np.vstack(
+ (self.right_eye_xyz[self.current_frame_number, :], self.right_gaze_xyz[self.current_frame_number, :]))
+ self.right_gaze_line_item.setData(pos=this_gaze_laser)
+
+ def update_skeleton_lines(self):
+ for this_skeleton_line_number, this_connection in enumerate(self.mediapipe_body_connections):
+ self.skeleton_connections_list[this_skeleton_line_number].setData(
+ pos=self.mediapipe_fr_mar_xyz[self.current_frame_number, this_connection, :]
+ )
+
+ def update_frame_number(self):
+ self.current_frame_number += 1
+ if self.current_frame_number >= self.end_frame_number:
+ self.current_frame_number = self.start_frame_number
+ if self.current_frame_number % 10 == 0:
+ print(f'frame number: {self.current_frame_number}')
+
+ def start(self):
+ print('starting')
+ pg.exec()
+
+ def start_animation(self):
+ print('starting animation')
+ timer = QtCore.QTimer()
+ timer.timeout.connect(self.update)
+ timer.start(33)
+ self.start()
+
+
+if __name__ == '__main__':
+ data_path = Path('C:/Users/jonma/Dropbox/FreeMoCapProject/FreeMocap_Data/')
+ session_path = data_path / 'sesh_2022-02-15_11_54_28_pupil_maybe'
+
+ qt_gl_laser_skeleton = QT_GL_LaserSkeleton(session_path)
+ qt_gl_laser_skeleton.start_animation()