Free Motion Capture for Everyone πβ¨
TypeScript
nice start on a gl visualizer - vor calibration still doesn't workl
Jonded5d41188f8
diff --git a/jon_scratch/karls_pupil_freemocap_code.ipynb b/jon_scratch/karls_pupil_freemocap_code.ipynbindex 2d05d69c..a613c97b 100644--- a/jon_scratch/karls_pupil_freemocap_code.ipynb+++ b/jon_scratch/karls_pupil_freemocap_code.ipynb@@ -25,32 +25,13 @@},{"cell_type": "code",- "execution_count": 1,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "Requirement already satisfied: scipy in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (1.8.0)\n",- "Requirement already satisfied: numpy<1.25.0,>=1.17.3 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from scipy) (1.22.3)\n",- "Requirement already satisfied: pandas in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (1.4.2)\n",- "Requirement already satisfied: pytz>=2020.1 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from pandas) (2022.1)\n",- "Requirement already satisfied: python-dateutil>=2.8.1 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from pandas) (2.8.2)\n",- "Requirement already satisfied: numpy>=1.18.5 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from pandas) (1.22.3)\n",- "Requirement already satisfied: six>=1.5 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from python-dateutil>=2.8.1->pandas) (1.16.0)\n",- "Requirement already satisfied: numpy in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (1.22.3)\n",- "Requirement already satisfied: pyqt5 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (5.15.6)\n",- "Requirement already satisfied: PyQt5-Qt5>=5.15.2 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from pyqt5) (5.15.2)\n",- "Requirement already satisfied: PyQt5-sip<13,>=12.8 in c:\\users\\jonma\\miniconda3\\envs\\humon-env\\lib\\site-packages (from pyqt5) (12.10.1)\n"- ]- }- ],+ "outputs": [],"source": ["!pip install scipy\n","!pip install pandas\n",@@ -60,10 +41,9 @@},{"cell_type": "code",- "execution_count": 2,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -76,15 +56,15 @@"from pathlib import Path\n","\n","from scipy import optimize\n",- "from scipy.spatial.transform import Rotation "+ "from scipy.spatial.transform import Rotation\n",+ "\n"]},{"cell_type": "code",- "execution_count": 3,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -110,10 +90,9 @@},{"cell_type": "code",- "execution_count": 4,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -155,10 +134,9 @@},{"cell_type": "code",- "execution_count": 5,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -195,10 +173,9 @@},{"cell_type": "code",- "execution_count": 6,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -237,10 +214,9 @@},{"cell_type": "code",- "execution_count": 7,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -278,10 +254,9 @@},{"cell_type": "code",- "execution_count": 8,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -306,10 +281,9 @@},{"cell_type": "code",- "execution_count": 9,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -406,10 +380,9 @@},{"cell_type": "code",- "execution_count": 10,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -433,10 +406,9 @@},{"cell_type": "code",- "execution_count": 11,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -461,10 +433,9 @@},{"cell_type": "code",- "execution_count": 12,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -514,33 +485,13 @@},{"cell_type": "code",- "execution_count": 13,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "freemocap_timestamps[-1]: 1644944570.8511524\n",- "r_eye_timestamps[-1]: 1644944575.0656984\n"- ]- },- {- "data": {- "text/plain": [- "1644944570.8511524"- ]- },- "execution_count": 13,- "metadata": {},- "output_type": "execute_result"- }- ],+ "outputs": [],"source": ["print(f'freemocap_timestamps[-1]: {freemocap_timestamps[-1]}')\n","print(f'r_eye_timestamps[-1]: {r_eye_timestamps[-1]}')\n",@@ -561,10 +512,9 @@},{"cell_type": "code",- "execution_count": 14,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -633,10 +583,9 @@},{"cell_type": "code",- "execution_count": 15,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -653,10 +602,9 @@},{"cell_type": "code",- "execution_count": 16,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -684,22 +632,13 @@},{"cell_type": "code",- "execution_count": 17,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "r_eye_gaze_xyz_og.shape: (17737, 3)\n"- ]- }- ],+ "outputs": [],"source": ["r_eye_gaze_x = r_eye_pupil_center_normal_x\n","r_eye_gaze_y = r_eye_pupil_center_normal_y\n",@@ -712,22 +651,13 @@},{"cell_type": "code",- "execution_count": 18,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "l_eye_gaze_xyz_og.shape: (16324, 3)\n"- ]- }- ],+ "outputs": [],"source": ["l_eye_gaze_x = l_eye_pupil_center_normal_x\n","l_eye_gaze_y = l_eye_pupil_center_normal_y\n",@@ -753,10 +683,9 @@},{"cell_type": "code",- "execution_count": 19,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -777,10 +706,9 @@},{"cell_type": "code",- "execution_count": 20,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -805,10 +733,9 @@},{"cell_type": "code",- "execution_count": 21,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -833,25 +760,13 @@},{"cell_type": "code",- "execution_count": 22,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "r_eye_gaze_xyz.shape: (4915, 3)\n",- "l_eye_gaze_xyz.shape: (4915, 3)\n",- "mediapipe_skeleton_fr_mar_xyz.shape: (4915, 543, 3)\n",- "mediapipe_head_rotation_matricies_fr_row_col.shape: (4915, 3, 3)\n"- ]- }- ],+ "outputs": [],"source": ["print(f'r_eye_gaze_xyz.shape: {r_eye_gaze_xyz.shape}')\n","print(f'l_eye_gaze_xyz.shape: {l_eye_gaze_xyz.shape}')\n",@@ -861,22 +776,13 @@},{"cell_type": "code",- "execution_count": 23,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},- "outputs": [- {- "name": "stderr",- "output_type": "stream",- "text": [- "No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n"- ]- }- ],+ "outputs": [],"source": ["if debug:\n"," fig = plt.figure(num=65341,figsize=(10,20))\n",@@ -930,10 +836,9 @@},{"cell_type": "code",- "execution_count": 24,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -958,10 +863,9 @@},{"cell_type": "code",- "execution_count": 25,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -1002,24 +906,14 @@},{"cell_type": "code",- "execution_count": 26,+ "execution_count": null,"metadata": {"collapsed": false,"pycharm": {"name": "#%%\n"}},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "head_rotation_martices_fr_row_col.shape: (4915, 3, 3)\n",- "r_eye_gaze_xyz.shape: (4915, 3)\n",- "fixation_point_xyz.shape: (4915, 3)\n"- ]- }- ],+ "outputs": [],"source": ["vor_frame_start =1000\n","vor_frame_end = 1400\n",@@ -1037,31 +931,14 @@},{"cell_type": "code",- "execution_count": 27,+ "execution_count": null,"metadata": {"collapsed": false,"pycharm": {"name": "#%%\n"}},- "outputs": [- {- "data": {- "text/plain": [- "array([[-140.99623459, 371.93667349, -908.18373013],\n",- " [-162.83809499, 344.26373174, -912.30887986],\n",- " [-165.73504142, 346.97592809, -911.18040645],\n",- " ...,\n",- " [-104.44826484, -31.40865991, -522.0636855 ],\n",- " [-101.70694275, -47.25579649, -518.36201151],\n",- " [-102.06426831, -55.07294944, -524.55510472]])"- ]- },- "execution_count": 27,- "metadata": {},- "output_type": "execute_result"- }- ],+ "outputs": [],"source": ["\n","fixation_point_during_vor_xyz = mediapipe_skeleton_fr_mar_xyz[vor_frame_start:vor_frame_end, right_index_fingertip_idx, :]\n",@@ -1081,24 +958,14 @@},{"cell_type": "code",- "execution_count": 28,+ "execution_count": null,"metadata": {"collapsed": false,"pycharm": {"name": "#%%\n"}},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "head_rotation_matrices_during_vor_fr_row_col.shape: (400, 3, 3)\n",- "r_gaze_during_vor_xyz.shape: (400, 3)\n",- "fixation_point_in_r_eyeball_coordinates_xyz.shape: (400, 3)\n"- ]- }- ],+ "outputs": [],"source": ["\n","head_rotation_matrices_during_vor_fr_row_col = mediapipe_head_rotation_matricies_fr_row_col[vor_frame_start:vor_frame_end, :, :]\n",@@ -1123,22 +990,14 @@},{"cell_type": "code",- "execution_count": 29,+ "execution_count": null,"metadata": {"collapsed": false,"pycharm": {"name": "#%%\n"}},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- "skel_during_vor_fr_mar_dim.shape: (400, 543, 3)\n"- ]- }- ],+ "outputs": [],"source": ["skel_during_vor_fr_mar_dim = mediapipe_skeleton_fr_mar_xyz[vor_frame_start:vor_frame_end,:,:]\n","\n",@@ -1201,7 +1060,7 @@},{"cell_type": "code",- "execution_count": 30,+ "execution_count": null,"metadata": {"collapsed": false,"pycharm": {@@ -1215,247 +1074,18 @@},{"cell_type": "code",- "execution_count": 32,+ "execution_count": null,"metadata": {"collapsed": false,"pycharm": {"name": "#%%\n"}},- "outputs": [- {- "name": "stdout",- "output_type": "stream",- "text": [- " Iteration Total nfev Cost Cost reduction Step norm Optimality \n",- " 0 1 1.9160e+11 2.28e+11 \n",- " 1 2 2.9245e+10 1.62e+11 1.00e+00 6.24e+10 \n",- " 2 4 1.0953e+10 1.83e+10 5.00e-01 1.18e+10 \n",- " 3 6 9.8908e+09 1.06e+09 2.50e-01 3.05e+09 \n",- " 4 8 9.7599e+09 1.31e+08 6.25e-02 4.77e+08 \n",- " 5 9 9.7498e+09 1.01e+07 6.25e-02 3.66e+08 \n",- " 6 10 9.7457e+09 4.13e+06 1.56e-02 1.41e+08 \n",- " 7 12 9.7451e+09 5.70e+05 3.91e-03 1.34e+08 \n",- " 8 13 9.7447e+09 4.71e+05 3.91e-03 9.35e+07 \n",- " 9 14 9.7438e+09 8.17e+05 7.81e-03 8.51e+07 \n",- " 10 15 9.7437e+09 1.83e+05 1.56e-02 3.72e+08 \n",- " 11 16 9.7423e+09 1.35e+06 3.91e-03 1.78e+08 \n",- " 12 17 9.7417e+09 5.74e+05 7.81e-03 2.08e+08 \n",- " 13 18 9.7411e+09 6.42e+05 7.81e-03 1.87e+08 \n",- " 14 19 9.7404e+09 6.81e+05 7.81e-03 2.02e+08 \n",- " 15 20 9.7397e+09 7.35e+05 7.81e-03 1.93e+08 \n",- " 16 21 9.7389e+09 7.89e+05 7.81e-03 1.96e+08 \n",- " 17 22 9.7380e+09 8.46e+05 7.81e-03 1.97e+08 \n",- " 18 23 9.7371e+09 9.07e+05 7.81e-03 1.89e+08 \n",- " 19 24 9.7362e+09 9.72e+05 7.81e-03 2.02e+08 \n",- " 20 25 9.7351e+09 1.04e+06 7.81e-03 1.80e+08 \n",- " 21 26 9.7340e+09 1.11e+06 7.81e-03 2.06e+08 \n",- " 22 27 9.7328e+09 1.19e+06 7.81e-03 1.69e+08 \n",- " 23 28 9.7316e+09 1.26e+06 7.81e-03 2.09e+08 \n",- " 24 29 9.7302e+09 1.35e+06 7.81e-03 1.66e+08 \n",- " 25 30 9.7288e+09 1.43e+06 7.81e-03 2.12e+08 \n",- " 26 31 9.7273e+09 1.52e+06 7.81e-03 1.71e+08 \n",- " 27 32 9.7248e+09 2.43e+06 1.56e-02 3.03e+08 \n",- " 28 33 9.7234e+09 1.45e+06 1.56e-02 4.48e+08 \n",- " 29 34 9.7217e+09 1.66e+06 3.91e-03 2.56e+08 \n",- " 30 35 9.7199e+09 1.82e+06 7.81e-03 2.25e+08 \n",- " 31 36 9.7180e+09 1.93e+06 7.81e-03 1.87e+08 \n",- " 32 37 9.7142e+09 3.82e+06 1.56e-02 2.73e+08 \n",- " 33 38 9.7128e+09 1.31e+06 3.13e-02 8.53e+08 \n",- " 34 39 9.7072e+09 5.66e+06 7.81e-03 4.38e+08 \n",- " 35 40 9.7039e+09 3.25e+06 1.56e-02 3.53e+08 \n",- " 36 41 9.7008e+09 3.15e+06 1.56e-02 3.79e+08 \n",- " 37 42 9.6974e+09 3.37e+06 1.56e-02 3.64e+08 \n",- " 38 43 9.6939e+09 3.55e+06 1.56e-02 3.64e+08 \n",- " 39 44 9.6901e+09 3.74e+06 1.56e-02 3.68e+08 \n",- " 40 45 9.6862e+09 3.93e+06 1.56e-02 3.49e+08 \n",- " 41 46 9.6821e+09 4.13e+06 1.56e-02 3.71e+08 \n",- " 42 47 9.6777e+09 4.34e+06 1.56e-02 3.32e+08 \n",- " 43 48 9.6732e+09 4.55e+06 1.56e-02 3.73e+08 \n",- " 44 49 9.6684e+09 4.76e+06 1.56e-02 3.17e+08 \n",- " 45 50 9.6634e+09 4.98e+06 1.56e-02 3.74e+08 \n",- " 46 51 9.6582e+09 5.20e+06 1.56e-02 3.25e+08 \n",- " 47 52 9.6528e+09 5.42e+06 1.56e-02 3.75e+08 \n",- " 48 53 9.6471e+09 5.65e+06 1.56e-02 3.32e+08 \n",- " 49 54 9.6413e+09 5.88e+06 1.56e-02 3.75e+08 \n",- " 50 55 9.6320e+09 9.31e+06 3.13e-02 6.40e+08 \n",- " 51 56 9.6270e+09 4.91e+06 3.13e-02 7.83e+08 \n",- " 52 57 9.6207e+09 6.37e+06 7.81e-03 4.66e+08 \n",- " 53 58 9.6141e+09 6.61e+06 1.56e-02 3.56e+08 \n",- " 54 59 9.6072e+09 6.88e+06 1.56e-02 3.78e+08 \n",- " 55 60 9.5943e+09 1.29e+07 3.13e-02 4.62e+08 \n",- " 56 62 9.5872e+09 7.08e+06 1.56e-02 3.91e+08 \n",- " 57 63 9.5798e+09 7.45e+06 1.56e-02 3.77e+08 \n",- " 58 64 9.5652e+09 1.46e+07 3.12e-02 4.43e+08 \n",- " 59 65 9.5596e+09 5.55e+06 6.25e-02 1.63e+09 \n",- " 60 66 9.5390e+09 2.06e+07 1.56e-02 8.00e+08 \n",- " 61 67 9.5284e+09 1.06e+07 3.12e-02 6.33e+08 \n",- " 62 68 9.5180e+09 1.05e+07 3.13e-02 7.53e+08 \n",- " 63 69 9.5071e+09 1.09e+07 3.12e-02 6.33e+08 \n",- " 64 70 9.4959e+09 1.12e+07 3.12e-02 7.38e+08 \n",- " 65 71 9.4844e+09 1.15e+07 3.12e-02 6.21e+08 \n",- " 66 72 9.4726e+09 1.18e+07 3.12e-02 7.25e+08 \n",- " 67 73 9.4604e+09 1.21e+07 3.12e-02 6.07e+08 \n",- " 68 74 9.4479e+09 1.25e+07 3.12e-02 7.13e+08 \n",- " 69 75 9.4352e+09 1.28e+07 3.12e-02 5.94e+08 \n",- " 70 76 9.4221e+09 1.31e+07 3.12e-02 7.00e+08 \n",- " 71 77 9.4086e+09 1.34e+07 3.12e-02 5.79e+08 \n",- " 72 78 9.3949e+09 1.37e+07 3.12e-02 6.87e+08 \n",- " 73 79 9.3808e+09 1.41e+07 3.12e-02 5.64e+08 \n",- " 74 80 9.3664e+09 1.44e+07 3.12e-02 6.74e+08 \n",- " 75 81 9.3517e+09 1.47e+07 3.12e-02 5.49e+08 \n",- " 76 82 9.3367e+09 1.50e+07 3.12e-02 6.61e+08 \n",- " 77 83 9.3213e+09 1.54e+07 3.12e-02 5.37e+08 \n",- " 78 84 9.3056e+09 1.57e+07 3.12e-02 6.48e+08 \n",- " 79 85 9.2896e+09 1.60e+07 3.13e-02 5.61e+08 \n",- " 80 86 9.2733e+09 1.63e+07 3.13e-02 6.35e+08 \n",- " 81 87 9.2567e+09 1.66e+07 3.12e-02 5.84e+08 \n",- " 82 88 9.2397e+09 1.70e+07 3.12e-02 6.22e+08 \n",- " 83 89 9.2224e+09 1.73e+07 3.12e-02 6.07e+08 \n",- " 84 90 9.2048e+09 1.76e+07 3.12e-02 6.09e+08 \n",- " 85 91 9.1869e+09 1.79e+07 3.12e-02 6.30e+08 \n",- " 86 92 9.1687e+09 1.82e+07 3.12e-02 5.97e+08 \n",- " 87 93 9.1502e+09 1.85e+07 3.12e-02 6.52e+08 \n",- " 88 94 9.1313e+09 1.89e+07 3.12e-02 5.85e+08 \n",- " 89 95 9.1121e+09 1.92e+07 3.12e-02 6.73e+08 \n",- " 90 96 9.0926e+09 1.95e+07 3.12e-02 5.73e+08 \n",- " 91 97 9.0728e+09 1.98e+07 3.12e-02 6.93e+08 \n",- " 92 98 9.0527e+09 2.01e+07 3.12e-02 5.61e+08 \n",- " 93 99 9.0323e+09 2.04e+07 3.12e-02 7.13e+08 \n",- " 94 100 9.0116e+09 2.07e+07 3.12e-02 5.49e+08 \n",- " 95 101 8.9906e+09 2.10e+07 3.12e-02 7.32e+08 \n",- " 96 102 8.9693e+09 2.13e+07 3.12e-02 5.38e+08 \n",- " 97 103 8.9476e+09 2.16e+07 3.12e-02 7.50e+08 \n",- " 98 104 8.9257e+09 2.19e+07 3.12e-02 5.48e+08 \n",- " 99 105 8.8951e+09 3.06e+07 6.25e-02 1.22e+09 \n",- " 100 106 8.8807e+09 1.44e+07 6.25e-02 1.84e+09 \n",- " 101 107 8.8554e+09 2.53e+07 1.56e-02 1.03e+09 \n",- " 102 108 8.8321e+09 2.33e+07 3.12e-02 7.97e+08 \n",- " 103 109 8.8089e+09 2.33e+07 3.12e-02 5.74e+08 \n",- " 104 110 8.7740e+09 3.49e+07 6.25e-02 1.12e+09 \n",- " 105 111 8.7594e+09 1.46e+07 6.25e-02 1.86e+09 \n",- " 106 112 8.7334e+09 2.61e+07 1.56e-02 1.05e+09 \n",- " 107 113 8.7090e+09 2.43e+07 3.12e-02 8.38e+08 \n",- " 108 114 8.6847e+09 2.43e+07 3.12e-02 5.97e+08 \n",- " 109 115 8.6468e+09 3.79e+07 6.25e-02 1.10e+09 \n",- " 110 116 8.6318e+09 1.50e+07 6.25e-02 1.87e+09 \n",- " 111 117 8.6053e+09 2.65e+07 1.56e-02 1.06e+09 \n",- " 112 118 8.5803e+09 2.50e+07 3.12e-02 8.70e+08 \n",- " 113 119 8.5554e+09 2.49e+07 3.12e-02 6.10e+08 \n",- " 114 120 8.5163e+09 3.91e+07 6.25e-02 1.13e+09 \n",- " 115 121 8.5017e+09 1.47e+07 6.25e-02 1.88e+09 \n",- " 116 122 8.4748e+09 2.69e+07 1.56e-02 1.07e+09 \n",- " 117 123 8.4498e+09 2.50e+07 3.12e-02 8.87e+08 \n",- " 118 124 8.4251e+09 2.47e+07 3.12e-02 6.03e+08 \n",- " 119 125 8.3879e+09 3.72e+07 6.25e-02 1.16e+09 \n",- " 120 126 8.3750e+09 1.29e+07 6.25e-02 1.90e+09 \n",- " 121 127 8.3479e+09 2.71e+07 1.56e-02 1.09e+09 \n",- " 122 128 8.3237e+09 2.42e+07 3.12e-02 8.83e+08 \n",- " 123 129 8.3004e+09 2.33e+07 3.12e-02 5.65e+08 \n",- " 124 130 8.2699e+09 3.05e+07 6.25e-02 1.20e+09 \n",- " 125 131 8.2591e+09 1.08e+07 6.25e-02 1.90e+09 \n",- " 126 132 8.2322e+09 2.69e+07 1.56e-02 1.08e+09 \n",- " 127 133 8.2102e+09 2.20e+07 3.13e-02 8.55e+08 \n",- " 128 134 8.1900e+09 2.02e+07 3.12e-02 6.22e+08 \n",- " 129 135 8.1708e+09 1.91e+07 3.12e-02 8.58e+08 \n",- " 130 136 8.1527e+09 1.81e+07 3.12e-02 6.77e+08 \n",- " 131 137 8.1356e+09 1.71e+07 3.12e-02 8.46e+08 \n",- " 132 138 8.1195e+09 1.61e+07 3.13e-02 7.16e+08 \n",- " 133 139 8.1044e+09 1.51e+07 3.12e-02 8.28e+08 \n",- " 134 140 8.0902e+09 1.41e+07 3.12e-02 7.49e+08 \n",- " 135 141 8.0771e+09 1.31e+07 3.12e-02 8.04e+08 \n",- " 136 142 8.0650e+09 1.21e+07 3.12e-02 7.79e+08 \n",- " 137 143 8.0538e+09 1.12e+07 3.12e-02 7.77e+08 \n",- " 138 144 8.0436e+09 1.02e+07 3.12e-02 8.05e+08 \n",- " 139 145 8.0343e+09 9.32e+06 3.12e-02 7.47e+08 \n",- " 140 146 8.0258e+09 8.47e+06 3.12e-02 8.26e+08 \n",- " 141 147 8.0181e+09 7.65e+06 3.12e-02 7.15e+08 \n",- " 142 148 8.0119e+09 6.22e+06 7.81e-03 5.58e+08 \n",- " 143 149 8.0056e+09 6.36e+06 1.56e-02 3.10e+08 \n",- " 144 150 7.9994e+09 6.17e+06 1.56e-02 4.51e+08 \n",- " 145 151 7.9916e+09 7.82e+06 3.13e-02 6.71e+08 \n",- " 146 152 7.9909e+09 6.36e+05 3.13e-02 8.63e+08 \n",- " 147 153 7.9840e+09 6.91e+06 7.81e-03 5.02e+08 \n",- " 148 154 7.9793e+09 4.77e+06 1.56e-02 2.93e+08 \n",- " 149 155 7.9758e+09 3.42e+06 1.56e-02 3.91e+08 \n",- " 150 156 7.9731e+09 2.80e+06 1.56e-02 3.92e+08 \n",- " 151 157 7.9706e+09 2.45e+06 1.56e-02 3.73e+08 \n",- " 152 158 7.9685e+09 2.11e+06 1.56e-02 4.09e+08 \n",- " 153 159 7.9667e+09 1.83e+06 1.56e-02 3.51e+08 \n",- " 154 160 7.9651e+09 1.54e+06 3.91e-03 2.71e+08 \n",- " 155 161 7.9636e+09 1.49e+06 7.81e-03 1.47e+08 \n",- " 156 162 7.9622e+09 1.39e+06 7.81e-03 2.16e+08 \n",- " 157 163 7.9610e+09 1.24e+06 7.81e-03 1.50e+08 \n",- " 158 164 7.9599e+09 1.05e+06 7.81e-03 2.02e+08 \n",- " 159 165 7.9591e+09 8.70e+05 7.81e-03 1.82e+08 \n",- " 160 166 7.9583e+09 7.25e+05 7.81e-03 1.92e+08 \n",- " 161 167 7.9577e+09 6.13e+05 7.81e-03 1.98e+08 \n",- " 162 168 7.9572e+09 5.19e+05 7.81e-03 1.79e+08 \n",- " 163 169 7.9568e+09 4.40e+05 7.81e-03 2.07e+08 \n",- " 164 170 7.9564e+09 3.84e+05 1.95e-03 1.10e+08 \n",- " 165 171 7.9560e+09 3.58e+05 3.91e-03 1.12e+08 \n",- " 166 172 7.9557e+09 3.28e+05 3.91e-03 7.26e+07 \n",- " 167 173 7.9554e+09 2.83e+05 3.91e-03 1.02e+08 \n",- " 168 174 7.9552e+09 2.33e+05 3.91e-03 8.75e+07 \n",- " 169 175 7.9550e+09 1.90e+05 3.91e-03 9.62e+07 \n",- " 170 176 7.9548e+09 1.58e+05 3.91e-03 9.83e+07 \n",- " 171 177 7.9547e+09 1.32e+05 3.91e-03 8.96e+07 \n",- " 172 178 7.9546e+09 1.11e+05 3.91e-03 1.03e+08 \n",- " 173 179 7.9545e+09 9.60e+04 9.77e-04 5.48e+07 \n",- " 174 180 7.9544e+09 8.96e+04 1.95e-03 5.59e+07 \n",- " 175 181 7.9543e+09 8.16e+04 1.95e-03 3.61e+07 \n",- " 176 182 7.9543e+09 7.00e+04 1.95e-03 5.05e+07 \n",- " 177 183 7.9542e+09 5.71e+04 1.95e-03 4.39e+07 \n",- " 178 184 7.9542e+09 4.62e+04 1.95e-03 4.77e+07 \n",- " 179 185 7.9541e+09 3.82e+04 1.95e-03 4.93e+07 \n",- " 180 186 7.9541e+09 3.19e+04 1.95e-03 4.43e+07 \n",- " 181 187 7.9541e+09 2.66e+04 1.95e-03 5.17e+07 \n",- " 182 188 7.9540e+09 2.39e+04 4.88e-04 2.74e+07 \n",- " 183 189 7.9540e+09 2.17e+04 9.77e-04 2.75e+07 \n",- " 184 190 7.9540e+09 1.95e+04 9.77e-04 1.86e+07 \n",- " 185 191 7.9540e+09 1.65e+04 9.77e-04 2.50e+07 \n",- " 186 192 7.9540e+09 1.33e+04 9.77e-04 2.25e+07 \n",- " 187 193 7.9540e+09 1.08e+04 9.77e-04 2.36e+07 \n",- " 188 194 7.9539e+09 8.98e+03 9.77e-04 2.49e+07 \n",- " 189 195 7.9539e+09 7.48e+03 9.77e-04 2.18e+07 \n",- " 190 196 7.9539e+09 6.24e+03 9.77e-04 2.60e+07 \n",- " 191 197 7.9539e+09 5.97e+03 2.44e-04 1.37e+07 \n",- " 192 198 7.9539e+09 5.16e+03 4.88e-04 1.35e+07 \n",- " 193 199 7.9539e+09 4.56e+03 4.88e-04 9.69e+06 \n",- " 194 200 7.9539e+09 3.79e+03 4.88e-04 1.24e+07 \n",- " 195 201 7.9539e+09 3.06e+03 4.88e-04 1.16e+07 \n",- " 196 202 7.9539e+09 2.51e+03 4.88e-04 1.16e+07 \n",- " 197 203 7.9539e+09 2.08e+03 4.88e-04 1.26e+07 \n",- " 198 204 7.9539e+09 1.73e+03 4.88e-04 1.09e+07 \n",- " 199 205 7.9539e+09 1.50e+03 1.22e-04 8.24e+06 \n",- " 200 206 7.9539e+09 1.39e+03 2.44e-04 4.66e+06 \n",- " 201 207 7.9539e+09 1.26e+03 2.44e-04 6.51e+06 \n",- " 202 208 7.9539e+09 1.07e+03 2.44e-04 4.97e+06 \n",- " 203 209 7.9539e+09 8.65e+02 2.44e-04 6.10e+06 \n",- " 204 210 7.9539e+09 6.96e+02 2.44e-04 5.96e+06 \n",- " 205 211 7.9539e+09 5.75e+02 2.44e-04 5.70e+06 \n",- " 206 212 7.9539e+09 4.78e+02 2.44e-04 6.36e+06 \n",- " 207 213 7.9539e+09 3.98e+02 2.44e-04 5.56e+06 \n",- " 208 214 7.9539e+09 3.73e+02 6.10e-05 4.01e+06 \n",- " 209 215 7.9539e+09 3.27e+02 1.22e-04 2.31e+06 \n",- " 210 216 7.9539e+09 2.90e+02 1.22e-04 3.21e+06 \n",- " 211 217 7.9539e+09 2.42e+02 1.22e-04 2.63e+06 \n",- " 212 218 7.9539e+09 1.95e+02 1.22e-04 3.01e+06 \n",- " 213 219 7.9539e+09 1.59e+02 1.22e-04 3.04e+06 \n",- " 214 220 7.9539e+09 1.32e+02 1.22e-04 2.80e+06 \n",- " 215 221 7.9539e+09 1.10e+02 1.22e-04 3.21e+06 \n",- " 216 222 7.9539e+09 9.37e+01 3.05e-05 1.70e+06 \n",- " 217 223 7.9539e+09 8.77e+01 6.10e-05 1.74e+06 \n",- " 218 224 7.9539e+09 7.95e+01 6.10e-05 1.12e+06 \n",- "`ftol` termination condition is satisfied.\n",- "Function evaluations 224, initial cost 1.9160e+11, final cost 7.9539e+09, first-order optimality 1.12e+06.\n"- ]- }- ],+ "outputs": [],"source": ["\n","if debug:\n",- " def plot_optimization_error(error,gaze_xyz, gaze_rotated_by_guess_then_head_rotation_xyz, fixation_point_in_eye_coordinates_xyz, skel_during_vor_fr_mar_dim):\n",+ " def plot_optimization_error(error,gaze_xyz, gaze_rotated_by_guess_then_head_rotation_xyz, mean_fixation_point_xyz, skel_during_vor_fr_mar_dim):\n"," figure_number=13451\n","\n"," if not plt.fignum_exists(figure_number):\n",@@ -1481,9 +1111,9 @@" gaze_rot_xyz[:,1], \n"," gaze_rot_xyz[:,2], 'r-o',label='gaze_rotated_by_guess_then_head_rotation_xyz')\n","\n",- " ax.plot(fixation_point_in_eye_coordinates_xyz[:,0],\n",- " fixation_point_in_eye_coordinates_xyz[:,1], \n",- " fixation_point_in_eye_coordinates_xyz[:,2], 'b-o',label='fixation_point_in_eye_coordinates_xyz')\n",+ " ax.plot(mean_fixation_point_xyz[0],\n",+ " mean_fixation_point_xyz[1], \n",+ " mean_fixation_point_xyz[2], 'b-o',label='mean_fixation_point_xyz')\n","\n"," ax.set_xlim([-ax_range, ax_range])\n"," ax.set_ylim([-ax_range, ax_range])\n",@@ -1508,12 +1138,12 @@" gaze_rotated_by_guess_then_head_rotation_xyz =[head_rotation_maticies_fr_row_col[this_frame_number,:,:] @ gaze_rotated_by_guess[this_frame_number]\n"," for this_frame_number in range(gaze_xyz.shape[0])]\n","\n",- " # mean_fixation_point_xyz = np.mean(fixation_point_in_eye_coordinates_xyz, axis=0)\n",+ " mean_fixation_point_xyz = np.mean(fixation_point_in_eye_coordinates_xyz, axis=0)\n"," #define error as difference between the fixation point and that rotated gaze estimate (these are both normalized, I think)\n",- " error_per_frame = gaze_rotated_by_guess_then_head_rotation_xyz - fixation_point_in_eye_coordinates_xyz\n",+ " error_per_frame = gaze_rotated_by_guess_then_head_rotation_xyz - mean_fixation_point_xyz\n"," error = np.nanmean(np.nansum(error_per_frame**2)/error_per_frame.shape[0])\n"," \n",- " plot_optimization_error(error,gaze_xyz, gaze_rotated_by_guess_then_head_rotation_xyz, fixation_point_in_eye_coordinates_xyz, skel_during_vor_fr_mar_dim)\n",+ " plot_optimization_error(error,gaze_xyz, gaze_rotated_by_guess_then_head_rotation_xyz, mean_fixation_point_xyz, skel_during_vor_fr_mar_dim)\n","\n"," return error\n","\n",@@ -1545,10 +1175,9 @@},{"cell_type": "code",- "execution_count": 33,+ "execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -1563,7 +1192,6 @@"execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},@@ -1586,12 +1214,18 @@"execution_count": null,"metadata": {"pycharm": {- "is_executing": true,"name": "#%%\n"}},"outputs": [],- "source": []+ "source": [+ "\n",+ "data_path = Path('C:/Users/jonma/Dropbox/FreeMoCapProject/FreeMocap_Data/')\n",+ "session_path = data_path / 'sesh_2022-02-15_11_54_28_pupil_maybe'\n",+ "\n",+ "qt_gl_laser_skeleton = QT_GL_LaserSkeleton(session_path)\n",+ "qt_gl_laser_skeleton.start_animation()\n"+ ]}],"metadata": {diff --git a/jon_scratch/pyqt_gl_laser_skeleton/__init__.py b/jon_scratch/pyqt_gl_laser_skeleton/__init__.pynew file mode 100644index 00000000..e69de29bdiff --git a/jon_scratch/pyqt_gl_laser_skeleton/qt_gl_laser_skeleton.py b/jon_scratch/pyqt_gl_laser_skeleton/qt_gl_laser_skeleton.pynew file mode 100644index 00000000..7a839d67--- /dev/null+++ b/jon_scratch/pyqt_gl_laser_skeleton/qt_gl_laser_skeleton.py@@ -0,0 +1,157 @@+# adapted from - https://gist.github.com/markjay4k/da2f55e28514be7160a7c5fbf95bd243+from pathlib import Path++from pyqtgraph.Qt import QtCore, QtGui+import pyqtgraph.opengl as gl+import pyqtgraph as pg+import numpy as np+import sys+from mediapipe.python.solutions import holistic as mp_holistic+++class QT_GL_LaserSkeleton():+ def __init__(self, session_path: Path):+ self.mediapipe_fr_mar_xyz = None+ self.session_path = Path(session_path)+ self.traces = dict()+ self.start_frame_number = 1000+ self.end_frame_number = 2000+ self.current_frame_number = self.start_frame_number++ self.load_mediapipe_data()+ self.load_pupil_data()++ self.create_app_window()+ self.create_grid_planes()+ self.get_mediapipe_connections()+ self.initialize_skel_dottos()+ self.initialize_skel_lines()+ self.initialize_gaze_laser()++ def load_mediapipe_data(self):+ mediapipe_data_path = self.session_path / 'DataArrays' / 'mediaPipeSkel_3d_smoothed.npy'+ mediapipe_all_fr_mar_xyz = np.load(str(mediapipe_data_path))+ self.mean_position_xyz = np.nanmedian(np.nanmedian(mediapipe_all_fr_mar_xyz, axis=0), axis=0)+ mediapipe_all_fr_mar_xyz[:, :, 0] -= self.mean_position_xyz[0]+ mediapipe_all_fr_mar_xyz[:, :, 1] -= self.mean_position_xyz[1]+ mediapipe_all_fr_mar_xyz[:, :, 2] -= self.mean_position_xyz[2]++ # remove first frame due to annoying 'off-by-one' error in the timestamp logger+ mediapipe_all_fr_mar_xyz = np.delete(mediapipe_all_fr_mar_xyz, 0, axis=0)++ self.mediapipe_fr_mar_xyz = mediapipe_all_fr_mar_xyz[:, :75, :]++ def load_pupil_data(self):+ pupil_data_path = self.session_path / 'pupil_000' / 'exports' / '000' / 'right_eye_gaze_xyz.npy'+ right_gaze_xyz = np.load(str(pupil_data_path))++ right_eye_d = 5+ self.right_eye_xyz = np.squeeze(self.mediapipe_fr_mar_xyz[:, right_eye_d, :])++ right_gaze_xyz[:, 0] += self.right_eye_xyz[:, 0]+ right_gaze_xyz[:, 1] += self.right_eye_xyz[:, 1]+ right_gaze_xyz[:, 2] += self.right_eye_xyz[:, 2]++ self.right_gaze_xyz = right_gaze_xyz++ def create_app_window(self):+ self.app = pg.mkQApp("Laser Skeleton")+ self.gl_view_widget = gl.GLViewWidget()+ self.gl_view_widget.opts['distance'] = 2000+ self.gl_view_widget.setWindowTitle('FreeMoCap - Laser Skeleton')+ self.gl_view_widget.setGeometry(0, 110, 1920, 1080)+ self.gl_view_widget.show()++ def create_grid_planes(self):+ grid_scale = 2e3+ # create the background grids+ grid_plane_x = gl.GLGridItem()+ grid_plane_x.setSize(grid_scale, grid_scale, grid_scale)+ grid_plane_x.setSpacing(grid_scale / 10, grid_scale / 10, grid_scale / 10)+ grid_plane_x.rotate(90, 0, 1, 0)+ grid_plane_x.translate(-grid_scale, 0, 0)+ self.gl_view_widget.addItem(grid_plane_x)++ grid_plane_y = gl.GLGridItem()+ grid_plane_y.setSize(grid_scale, grid_scale, grid_scale)+ grid_plane_y.setSpacing(grid_scale / 10, grid_scale / 10, grid_scale / 10)+ grid_plane_y.rotate(90, 1, 0, 0)+ grid_plane_y.translate(0, -grid_scale, 0)+ self.gl_view_widget.addItem(grid_plane_y)++ grid_plane_z = gl.GLGridItem()+ grid_plane_z.setSize(grid_scale, grid_scale, grid_scale)+ grid_plane_z.setSpacing(grid_scale / 10, grid_scale / 10, grid_scale / 10)+ grid_plane_z.translate(0, 0, -grid_scale)+ self.gl_view_widget.addItem(grid_plane_z)++ def get_mediapipe_connections(self):+ self.mediapipe_body_connections = [this_connection for this_connection in mp_holistic.POSE_CONNECTIONS]+ self.mediapipe_hand_connections = [this_connection for this_connection in mp_holistic.HAND_CONNECTIONS]+ self.mediapipe_face_connections = [this_connection for this_connection in mp_holistic.FACEMESH_TESSELATION]++ def initialize_skel_dottos(self):+ self.skeleton_scatter_item = gl.GLScatterPlotItem(+ pos=self.mediapipe_fr_mar_xyz[self.current_frame_number, :, :], color=(0, 1, 1, 1), size=10,+ pxMode=False+ )+ self.gl_view_widget.addItem(self.skeleton_scatter_item)++ def initialize_skel_lines(self):+ self.skeleton_connections_list = []+ for this_connection in self.mediapipe_body_connections:+ this_skel_line = gl.GLLinePlotItem(+ pos=self.mediapipe_fr_mar_xyz[self.current_frame_number, this_connection, :])+ self.skeleton_connections_list.append(this_skel_line)+ self.gl_view_widget.addItem(this_skel_line)++ def initialize_gaze_laser(self):++ this_gaze_laser = np.vstack(+ (self.right_eye_xyz[self.current_frame_number, :], self.right_gaze_xyz[self.current_frame_number, :]))+ self.right_gaze_line_item = gl.GLLinePlotItem(pos=this_gaze_laser,+ color=(1, 0, 0, 1), )+ self.gl_view_widget.addItem(self.right_gaze_line_item)++ def update(self):+ self.update_frame_number()++ self.skeleton_scatter_item.setData(+ pos=self.mediapipe_fr_mar_xyz[self.current_frame_number, :, :]+ )+ self.update_skeleton_lines()+ this_gaze_laser = np.vstack(+ (self.right_eye_xyz[self.current_frame_number, :], self.right_gaze_xyz[self.current_frame_number, :]))+ self.right_gaze_line_item.setData(pos=this_gaze_laser)++ def update_skeleton_lines(self):+ for this_skeleton_line_number, this_connection in enumerate(self.mediapipe_body_connections):+ self.skeleton_connections_list[this_skeleton_line_number].setData(+ pos=self.mediapipe_fr_mar_xyz[self.current_frame_number, this_connection, :]+ )++ def update_frame_number(self):+ self.current_frame_number += 1+ if self.current_frame_number >= self.end_frame_number:+ self.current_frame_number = self.start_frame_number+ if self.current_frame_number % 10 == 0:+ print(f'frame number: {self.current_frame_number}')++ def start(self):+ print('starting')+ pg.exec()++ def start_animation(self):+ print('starting animation')+ timer = QtCore.QTimer()+ timer.timeout.connect(self.update)+ timer.start(33)+ self.start()+++if __name__ == '__main__':+ data_path = Path('C:/Users/jonma/Dropbox/FreeMoCapProject/FreeMocap_Data/')+ session_path = data_path / 'sesh_2022-02-15_11_54_28_pupil_maybe'++ qt_gl_laser_skeleton = QT_GL_LaserSkeleton(session_path)+ qt_gl_laser_skeleton.start_animation()