Free Motion Capture for Everyone πβ¨
TypeScript
nice retinal centered video output :D
Jon9eb230f22b0c
diff --git a/jon_scratch/create_retina_centered_video_from_pupil_labs_data.ipynb b/jon_scratch/create_retina_centered_video_from_pupil_labs_data.ipynbnew file mode 100644index 00000000..0f92b8cf--- /dev/null+++ b/jon_scratch/create_retina_centered_video_from_pupil_labs_data.ipynb@@ -0,0 +1,1197 @@+{+ "cells": [+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "# a notebook to make a quick and dirty retina centered video from standard Pupil Player export video\n",+ "\n",+ "It'll make one retina centered video from a standard Pupil Payer export video\n",+ "\n",+ "and also a video of the gaze trace\n",+ "\n",+ "I'll put them together in Adobe Premier or, via this script. We'll see lol"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 1,+ "metadata": {},+ "outputs": [],+ "source": [+ "import cv2\n",+ "from pathlib import Path\n",+ "import matplotlib.pyplot as plt\n",+ "import pandas as pd\n",+ "import numpy as np\n",+ "from PIL import Image\n",+ "from scipy import signal"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 2,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "Using matplotlib backend: TkAgg\n"+ ]+ }+ ],+ "source": [+ "%matplotlib "+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "Getcher paths straight"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 4,+ "metadata": {},+ "outputs": [],+ "source": [+ "# session_id = 'sesh_2022-05-07_17_15_05_pupil_wobble_juggle_0'\n",+ "\n",+ "# freemocap_data_folder = Path('C:/Users/jonma/Dropbox/FreeMoCapProject/FreeMocap_Data')\n",+ "\n",+ "# session_folder_path = freemocap_data_folder / session_id\n",+ "\n",+ "# pupil_data_path = session_folder_path / 'pupil_000'\n",+ "# pupil_data_exports_path = pupil_data_path / 'exports' / '000'\n",+ "# pupil_export_video_path = pupil_data_exports_path / 'world.mp4'\n",+ "# pupil_world_video_path = pupil_data_path / 'world.mp4'\n",+ "\n",+ "pupil_data_path = Path(r'H:\\Other computers\\My Computer_MocapComputer\\Wirth_ARGP\\ARGP_Main\\data\\2022_04_29\\pupil_data')\n",+ "pupil_data_exports_path = pupil_data_path / 'exports' / 'Wirth_Pilot_ARGP_2022-04-09_export'\n",+ "pupil_export_video_path = pupil_data_exports_path / 'argp_pilot_May2022.mp4'\n",+ "\n",+ "pupil_world_video_path = pupil_data_path / 'world.mp4'\n",+ "\n",+ "\n",+ "gaze_positions_path = pupil_data_exports_path / 'gaze_positions.csv'\n",+ "\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "load stuff "+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 5,+ "metadata": {},+ "outputs": [],+ "source": [+ "#open cv video capture object\n",+ "pupil_export_video_cap_object = cv2.VideoCapture(str(pupil_export_video_path))\n",+ "\n",+ "#pupil labs - gaze data\n",+ "pupil_dataframe = pd.read_csv(gaze_positions_path)"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "get some info from the video cap"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 6,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "video_width : 1280\n",+ "video_height: 720\n",+ "video_framerate: 29\n"+ ]+ }+ ],+ "source": [+ "video_width = int(pupil_export_video_cap_object.get(cv2.CAP_PROP_FRAME_WIDTH))\n",+ "video_height = int(pupil_export_video_cap_object.get(cv2.CAP_PROP_FRAME_HEIGHT))\n",+ "video_framerate = int(pupil_export_video_cap_object.get(cv2.CAP_PROP_FPS))\n",+ "print(f'video_width : {video_width }')\n",+ "print(f'video_height: {video_height}')\n",+ "print(f'video_framerate: {video_framerate}')\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "gaze locations"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 7,+ "metadata": {},+ "outputs": [],+ "source": [+ "norm_pos_x = pupil_dataframe.norm_pos_x\n",+ "norm_pos_y = pupil_dataframe.norm_pos_y\n",+ "\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 8,+ "metadata": {},+ "outputs": [],+ "source": [+ "# fig = plt.Figure(123)\n",+ "# fig.plot(norm_pos_x, label = 'norm_pos_x')\n",+ "# fig.plot(norm_pos_y, label ='norm_pos_x')"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 9,+ "metadata": {},+ "outputs": [],+ "source": [+ "\n",+ "gaze_on_screen_x = norm_pos_x * video_width\n",+ "gaze_on_screen_y = norm_pos_y * video_height\n",+ "\n",+ "world_camera_frame_index_all = pupil_dataframe.world_index"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "zero-lag 4th order butterworth filter with a 7Hz cutoff"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 10,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "nans in gaze_on_screen_x: 0\n",+ "nans in gaze_on_screen_y: 0\n",+ "nans in gaze_on_screen_x_filtered: 0\n",+ "nans in gaze_on_screen_y_filtered: 0\n"+ ]+ }+ ],+ "source": [+ "\n",+ "\n",+ "order = 5\n",+ "cutoff=10**-1\n",+ "b, a = signal.butter(order, cutoff)\n",+ "gaze_on_screen_x_filtered = signal.filtfilt(b, a, gaze_on_screen_x)\n",+ "gaze_on_screen_y_filtered = signal.filtfilt(b, a, gaze_on_screen_y)\n",+ "\n",+ "print(f\"nans in gaze_on_screen_x: {np.isnan(gaze_on_screen_x).sum()}\")\n",+ "print(f\"nans in gaze_on_screen_y: {np.isnan(gaze_on_screen_y).sum()}\")\n",+ "print(f\"nans in gaze_on_screen_x_filtered: {np.isnan(gaze_on_screen_x_filtered).sum()}\")\n",+ "print(f\"nans in gaze_on_screen_y_filtered: {np.isnan(gaze_on_screen_y_filtered).sum()}\")\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 11,+ "metadata": {},+ "outputs": [+ {+ "data": {+ "text/plain": [+ "array([617.18147505, 617.38013879, 617.58062962, ..., 764.46021398,\n",+ " 790.05560885, 815.63926686])"+ ]+ },+ "execution_count": 11,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "gaze_on_screen_x_filtered"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 12,+ "metadata": {},+ "outputs": [+ {+ "data": {+ "text/plain": [+ "<matplotlib.legend.Legend at 0x1ba33e1af48>"+ ]+ },+ "execution_count": 12,+ "metadata": {},+ "output_type": "execute_result"+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "c:\\Users\\jonma\\miniconda3\\envs\\freemocap-env\\lib\\tkinter\\__init__.py:749: UserWarning: Creating legend with loc=\"best\" can be slow with large amounts of data.\n",+ " func(*args)\n",+ "c:\\Users\\jonma\\miniconda3\\envs\\freemocap-env\\lib\\tkinter\\__init__.py:1705: UserWarning: Creating legend with loc=\"best\" can be slow with large amounts of data.\n",+ " return self.func(*args)\n"+ ]+ }+ ],+ "source": [+ "\n",+ "plt.cla()\n",+ "fig2 = plt.Figure()\n",+ "ax = plt.gca()\n",+ "ax.plot(gaze_on_screen_x, label = 'gaze_on_screen_x_raw')\n",+ "ax.plot(gaze_on_screen_y, label = 'gaze_on_screen_y_raw')\n",+ "ax.plot(gaze_on_screen_x_filtered, label = 'gaze_on_screen_x_filtered')\n",+ "ax.plot(gaze_on_screen_y_filtered, label = 'gaze_on_screen_y_filtered')\n",+ "ax.set_ylim((0-100, video_width+100))\n",+ "ax.legend()\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "squash data down so each frame is the average of all data recorded on that frame"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 13,+ "metadata": {},+ "outputs": [],+ "source": [+ "number_of_frames = np.max(world_camera_frame_index_all)\n",+ "number_of_frames\n",+ "\n",+ "gaze_on_screen_x_downsampled = []\n",+ "gaze_on_screen_y_downsampled = []\n",+ "\n",+ "for this_world_frame_index in range(number_of_frames):\n",+ " this_frame_data = world_camera_frame_index_all == this_world_frame_index\n",+ " this_frame_x_data = gaze_on_screen_x_filtered[this_frame_data]\n",+ " this_frame_y_data = gaze_on_screen_y_filtered[this_frame_data]\n",+ " gaze_on_screen_x_downsampled.append(np.nanmedian(this_frame_x_data))\n",+ " gaze_on_screen_y_downsampled.append(np.nanmedian(this_frame_y_data))"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 14,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "nans in gaze_on_screen_x: 0\n",+ "nans in gaze_on_screen_y: 0\n",+ "nans in gaze_on_screen_x_filtered: 0\n",+ "nans in gaze_on_screen_y_filtered: 0\n",+ "nans in gaze_on_screen_x_downsampled: 0\n",+ "nans in gaze_on_screen_y_downsampled: 0\n"+ ]+ }+ ],+ "source": [+ "\n",+ "print(f\"nans in gaze_on_screen_x: {np.isnan(gaze_on_screen_x).sum()}\")\n",+ "print(f\"nans in gaze_on_screen_y: {np.isnan(gaze_on_screen_y).sum()}\")\n",+ "print(f\"nans in gaze_on_screen_x_filtered: {np.isnan(gaze_on_screen_x_filtered).sum()}\")\n",+ "print(f\"nans in gaze_on_screen_y_filtered: {np.isnan(gaze_on_screen_y_filtered).sum()}\")\n",+ "print(f\"nans in gaze_on_screen_x_downsampled: {np.isnan(gaze_on_screen_x_downsampled).sum()}\")\n",+ "print(f\"nans in gaze_on_screen_y_downsampled: {np.isnan(gaze_on_screen_y_downsampled).sum()}\")\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 15,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "nans in gaze_on_screen_x: 0\n",+ "nans in gaze_on_screen_y: 0\n",+ "nans in gaze_on_screen_x_filtered: 0\n",+ "nans in gaze_on_screen_y_filtered: 0\n"+ ]+ },+ {+ "data": {+ "text/plain": [+ "<matplotlib.legend.Legend at 0x1ba35d67c88>"+ ]+ },+ "execution_count": 15,+ "metadata": {},+ "output_type": "execute_result"+ },+ {+ "name": "stderr",+ "output_type": "stream",+ "text": [+ "c:\\Users\\jonma\\miniconda3\\envs\\freemocap-env\\lib\\tkinter\\__init__.py:749: UserWarning: Creating legend with loc=\"best\" can be slow with large amounts of data.\n",+ " func(*args)\n",+ "c:\\Users\\jonma\\miniconda3\\envs\\freemocap-env\\lib\\tkinter\\__init__.py:1705: UserWarning: Creating legend with loc=\"best\" can be slow with large amounts of data.\n",+ " return self.func(*args)\n"+ ]+ }+ ],+ "source": [+ "\n",+ "# order = 4\n",+ "# cutoff=10**-1\n",+ "# b, a = signal.butter(order, cutoff)\n",+ "# gaze_on_screen_x_filtered = signal.filtfilt(b, a, gaze_on_screen_x_downsampled)\n",+ "# gaze_on_screen_y_filtered = signal.filtfilt(b, a, gaze_on_screen_y_downsampled)\n",+ "\n",+ "print(f\"nans in gaze_on_screen_x: {np.isnan(gaze_on_screen_x).sum()}\")\n",+ "print(f\"nans in gaze_on_screen_y: {np.isnan(gaze_on_screen_y).sum()}\")\n",+ "print(f\"nans in gaze_on_screen_x_filtered: {np.isnan(gaze_on_screen_x_filtered).sum()}\")\n",+ "print(f\"nans in gaze_on_screen_y_filtered: {np.isnan(gaze_on_screen_y_filtered).sum()}\")\n",+ "\n",+ "plt.close('all')\n",+ "fig2 = plt.Figure()\n",+ "ax = plt.gca()\n",+ "# ax.plot(gaze_on_screen_x_downsampled, label = 'gaze_on_screen_x_downsampled')\n",+ "# ax.plot(gaze_on_screen_y_downsampled, label = 'gaze_on_screen_y_downsampled')\n",+ "ax.plot(gaze_on_screen_x_filtered, label = 'gaze_on_screen_x_filtered')\n",+ "ax.plot(gaze_on_screen_y_filtered, label = 'gaze_on_screen_y_filtered')\n",+ "ax.set_ylim((0-100, video_width+100))\n",+ "ax.legend()\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 16,+ "metadata": {},+ "outputs": [+ {+ "data": {+ "text/plain": [+ "<matplotlib.legend.Legend at 0x1ba35da7f08>"+ ]+ },+ "execution_count": 16,+ "metadata": {},+ "output_type": "execute_result"+ }+ ],+ "source": [+ "plt.close(\"all\")\n",+ "fig2 = plt.Figure()\n",+ "ax = plt.gca()\n",+ "ax.plot(gaze_on_screen_x_downsampled, '.-', label = 'gaze_on_screen_x_downsampled')\n",+ "ax.plot(gaze_on_screen_y_downsampled, '.-',label = 'gaze_on_screen_y_downsampled')\n",+ "ax.set_ylim((0-100, video_width+100))\n",+ "ax.legend()\n",+ "\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 17,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "nans in gaze_on_screen_x_downsampled: 0\n",+ "nans in gaze_on_screen_y_downsampled: 0\n",+ "nans in gaze_on_screen_x_downsampled_gapfilled: 0\n",+ "nans in gaze_on_screen_y_downsampled_gapfilled: 0\n"+ ]+ }+ ],+ "source": [+ "print(f\"nans in gaze_on_screen_x_downsampled: {np.isnan(gaze_on_screen_x_downsampled).sum()}\")\n",+ "print(f\"nans in gaze_on_screen_y_downsampled: {np.isnan(gaze_on_screen_y_downsampled).sum()}\")\n",+ "\n",+ "gaze_x_df = pd.DataFrame(gaze_on_screen_x_downsampled)\n",+ "gaze_x_df.interpolate(method = 'linear', inplace = True)\n",+ "gaze_on_screen_x_downsampled_gapfilled = gaze_x_df.to_numpy()\n",+ "\n",+ "gaze_y_df = pd.DataFrame(gaze_on_screen_y_downsampled)\n",+ "gaze_y_df.interpolate(method = 'linear', inplace = True)\n",+ "gaze_on_screen_y_downsampled_gapfilled = gaze_y_df.to_numpy()\n",+ "\n",+ "print(f\"nans in gaze_on_screen_x_downsampled_gapfilled: {np.isnan(gaze_on_screen_x_downsampled_gapfilled).sum()}\")\n",+ "print(f\"nans in gaze_on_screen_y_downsampled_gapfilled: {np.isnan(gaze_on_screen_y_downsampled_gapfilled).sum()}\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 18,+ "metadata": {},+ "outputs": [],+ "source": [+ "\n",+ "\n",+ "\n",+ "big_blank_image = np.ones((int(video_height)*2, int(video_width)*2, 3), dtype=np.uint8)\n",+ "\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 19,+ "metadata": {},+ "outputs": [],+ "source": [+ "# pupil_world_video_cap_object = cv2.VideoCapture(str(pupil_world_video_path))\n",+ "\n",+ "# frame_num = 0\n",+ "# success = True\n",+ "# while success: \n",+ "# success, image_raw = pupil_world_video_cap_object.read()\n",+ "# if not success:\n",+ "# break\n",+ "# frame_num += 1\n",+ "# print(f\"frame_num: {frame_num}\")"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "create circular mask\n",+ "\n",+ "from =- https://stackoverflow.com/questions/61516526/how-to-use-opencv-to-crop-circular-image"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 20,+ "metadata": {},+ "outputs": [],+ "source": [+ "#define circles\n",+ "radius = video_height #pixels\n",+ "x_center = video_width\n",+ "y_center = video_height\n",+ "\n",+ "# draw filled circles in white on black background as masks\n",+ "circular_mask = np.zeros_like(big_blank_image)\n",+ "circular_mask = cv2.circle(circular_mask, (x_center,y_center), radius, (255,255,255), -1)\n",+ "\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "video_framerate"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 21,+ "metadata": {},+ "outputs": [+ {+ "name": "stdout",+ "output_type": "stream",+ "text": [+ "frame_number: 0\n",+ "frame_number: 100\n",+ "frame_number: 200\n",+ "frame_number: 300\n",+ "frame_number: 400\n",+ "frame_number: 500\n",+ "frame_number: 600\n",+ "frame_number: 700\n",+ "frame_number: 800\n",+ "frame_number: 900\n",+ "frame_number: 1000\n",+ "frame_number: 1100\n",+ "frame_number: 1200\n",+ "frame_number: 1300\n",+ "frame_number: 1400\n",+ "frame_number: 1500\n",+ "frame_number: 1600\n",+ "frame_number: 1700\n",+ "frame_number: 1800\n",+ "frame_number: 1900\n",+ "frame_number: 2000\n",+ "frame_number: 2100\n",+ "frame_number: 2200\n",+ "frame_number: 2300\n",+ "frame_number: 2400\n",+ "frame_number: 2500\n",+ "frame_number: 2600\n",+ "frame_number: 2700\n",+ "frame_number: 2800\n",+ "frame_number: 2900\n",+ "frame_number: 3000\n",+ "frame_number: 3100\n",+ "frame_number: 3200\n",+ "frame_number: 3300\n",+ "frame_number: 3400\n",+ "frame_number: 3500\n",+ "frame_number: 3600\n",+ "frame_number: 3700\n",+ "frame_number: 3800\n",+ "frame_number: 3900\n",+ "frame_number: 4000\n",+ "frame_number: 4100\n",+ "frame_number: 4200\n",+ "frame_number: 4300\n",+ "frame_number: 4400\n",+ "frame_number: 4500\n",+ "frame_number: 4600\n",+ "frame_number: 4700\n",+ "frame_number: 4800\n",+ "frame_number: 4900\n",+ "frame_number: 5000\n",+ "frame_number: 5100\n",+ "frame_number: 5200\n",+ "frame_number: 5300\n",+ "frame_number: 5400\n",+ "frame_number: 5500\n",+ "frame_number: 5600\n",+ "frame_number: 5700\n",+ "frame_number: 5800\n",+ "frame_number: 5900\n",+ "frame_number: 6000\n",+ "frame_number: 6100\n",+ "frame_number: 6200\n",+ "frame_number: 6300\n",+ "frame_number: 6400\n",+ "frame_number: 6500\n",+ "frame_number: 6600\n",+ "frame_number: 6700\n",+ "frame_number: 6800\n",+ "frame_number: 6900\n",+ "frame_number: 7000\n",+ "frame_number: 7100\n",+ "frame_number: 7200\n",+ "frame_number: 7300\n",+ "frame_number: 7400\n",+ "frame_number: 7500\n",+ "frame_number: 7600\n",+ "frame_number: 7700\n",+ "frame_number: 7800\n",+ "frame_number: 7900\n",+ "frame_number: 8000\n",+ "frame_number: 8100\n",+ "frame_number: 8200\n",+ "frame_number: 8300\n",+ "frame_number: 8400\n",+ "frame_number: 8500\n",+ "frame_number: 8600\n",+ "frame_number: 8700\n",+ "frame_number: 8800\n",+ "frame_number: 8900\n",+ "frame_number: 9000\n",+ "frame_number: 9100\n",+ "frame_number: 9200\n",+ "frame_number: 9300\n",+ "frame_number: 9400\n",+ "frame_number: 9500\n",+ "frame_number: 9600\n",+ "frame_number: 9700\n",+ "frame_number: 9800\n",+ "frame_number: 9900\n",+ "frame_number: 10000\n",+ "frame_number: 10100\n",+ "frame_number: 10200\n",+ "frame_number: 10300\n",+ "frame_number: 10400\n",+ "frame_number: 10500\n",+ "frame_number: 10600\n",+ "frame_number: 10700\n",+ "frame_number: 10800\n",+ "frame_number: 10900\n",+ "frame_number: 11000\n",+ "frame_number: 11100\n",+ "frame_number: 11200\n",+ "frame_number: 11300\n",+ "frame_number: 11400\n",+ "frame_number: 11500\n",+ "frame_number: 11600\n",+ "frame_number: 11700\n",+ "frame_number: 11800\n",+ "frame_number: 11900\n",+ "frame_number: 12000\n",+ "frame_number: 12100\n",+ "frame_number: 12200\n",+ "frame_number: 12300\n",+ "frame_number: 12400\n",+ "frame_number: 12500\n",+ "frame_number: 12600\n",+ "frame_number: 12700\n",+ "frame_number: 12800\n",+ "frame_number: 12900\n",+ "frame_number: 13000\n",+ "frame_number: 13100\n",+ "frame_number: 13200\n",+ "frame_number: 13300\n",+ "frame_number: 13400\n",+ "frame_number: 13500\n",+ "frame_number: 13600\n",+ "frame_number: 13700\n",+ "frame_number: 13800\n",+ "frame_number: 13900\n",+ "frame_number: 14000\n",+ "frame_number: 14100\n",+ "frame_number: 14200\n",+ "frame_number: 14300\n",+ "frame_number: 14400\n",+ "frame_number: 14500\n",+ "frame_number: 14600\n",+ "frame_number: 14700\n",+ "frame_number: 14800\n",+ "frame_number: 14900\n",+ "frame_number: 15000\n",+ "frame_number: 15100\n",+ "frame_number: 15200\n",+ "frame_number: 15300\n",+ "frame_number: 15400\n",+ "frame_number: 15500\n",+ "frame_number: 15600\n",+ "frame_number: 15700\n",+ "frame_number: 15800\n",+ "frame_number: 15900\n",+ "frame_number: 16000\n",+ "frame_number: 16100\n",+ "frame_number: 16200\n",+ "frame_number: 16300\n",+ "frame_number: 16400\n",+ "frame_number: 16500\n",+ "frame_number: 16600\n",+ "frame_number: 16700\n",+ "frame_number: 16800\n",+ "frame_number: 16900\n",+ "frame_number: 17000\n",+ "frame_number: 17100\n",+ "frame_number: 17200\n",+ "frame_number: 17300\n",+ "frame_number: 17400\n",+ "frame_number: 17500\n",+ "frame_number: 17600\n",+ "frame_number: 17700\n",+ "frame_number: 17800\n",+ "frame_number: 17900\n",+ "frame_number: 18000\n",+ "frame_number: 18100\n",+ "frame_number: 18200\n",+ "frame_number: 18300\n",+ "frame_number: 18400\n",+ "frame_number: 18500\n",+ "frame_number: 18600\n",+ "frame_number: 18700\n",+ "frame_number: 18800\n",+ "frame_number: 18900\n",+ "frame_number: 19000\n",+ "frame_number: 19100\n",+ "frame_number: 19200\n",+ "frame_number: 19300\n",+ "frame_number: 19400\n",+ "frame_number: 19500\n",+ "frame_number: 19600\n",+ "frame_number: 19700\n",+ "frame_number: 19800\n",+ "frame_number: 19900\n",+ "frame_number: 20000\n",+ "frame_number: 20100\n",+ "frame_number: 20200\n",+ "frame_number: 20300\n",+ "frame_number: 20400\n",+ "frame_number: 20500\n",+ "frame_number: 20600\n",+ "frame_number: 20700\n",+ "frame_number: 20800\n",+ "frame_number: 20900\n",+ "frame_number: 21000\n",+ "frame_number: 21100\n",+ "frame_number: 21200\n",+ "frame_number: 21300\n",+ "frame_number: 21400\n",+ "frame_number: 21500\n",+ "frame_number: 21600\n",+ "frame_number: 21700\n",+ "frame_number: 21800\n",+ "frame_number: 21900\n",+ "frame_number: 22000\n",+ "frame_number: 22100\n",+ "frame_number: 22200\n",+ "frame_number: 22300\n",+ "frame_number: 22400\n",+ "frame_number: 22500\n",+ "frame_number: 22600\n",+ "frame_number: 22700\n",+ "frame_number: 22800\n",+ "frame_number: 22900\n",+ "frame_number: 23000\n",+ "frame_number: 23100\n",+ "frame_number: 23200\n",+ "frame_number: 23300\n",+ "frame_number: 23400\n",+ "frame_number: 23500\n",+ "frame_number: 23600\n",+ "frame_number: 23700\n",+ "frame_number: 23800\n",+ "frame_number: 23900\n",+ "frame_number: 24000\n",+ "frame_number: 24100\n",+ "frame_number: 24200\n",+ "frame_number: 24300\n",+ "frame_number: 24400\n",+ "frame_number: 24500\n",+ "frame_number: 24600\n",+ "frame_number: 24700\n",+ "frame_number: 24800\n",+ "frame_number: 24900\n",+ "frame_number: 25000\n",+ "frame_number: 25100\n",+ "frame_number: 25200\n",+ "frame_number: 25300\n",+ "frame_number: 25400\n",+ "frame_number: 25500\n",+ "frame_number: 25600\n",+ "frame_number: 25700\n",+ "frame_number: 25800\n",+ "frame_number: 25900\n",+ "frame_number: 26000\n",+ "frame_number: 26100\n",+ "frame_number: 26200\n",+ "frame_number: 26300\n",+ "frame_number: 26400\n",+ "frame_number: 26500\n",+ "frame_number: 26600\n",+ "frame_number: 26700\n",+ "frame_number: 26800\n",+ "frame_number: 26900\n",+ "frame_number: 27000\n",+ "frame_number: 27100\n",+ "frame_number: 27200\n",+ "frame_number: 27300\n",+ "frame_number: 27400\n",+ "frame_number: 27500\n",+ "frame_number: 27600\n",+ "frame_number: 27700\n",+ "frame_number: 27800\n",+ "frame_number: 27900\n",+ "frame_number: 28000\n",+ "frame_number: 28100\n",+ "frame_number: 28200\n",+ "frame_number: 28300\n",+ "frame_number: 28400\n",+ "frame_number: 28500\n",+ "frame_number: 28600\n",+ "frame_number: 28700\n",+ "frame_number: 28800\n",+ "frame_number: 28900\n",+ "frame_number: 29000\n",+ "frame_number: 29100\n",+ "frame_number: 29200\n",+ "frame_number: 29300\n",+ "frame_number: 29400\n",+ "frame_number: 29500\n",+ "frame_number: 29600\n",+ "frame_number: 29700\n",+ "frame_number: 29800\n",+ "frame_number: 29900\n",+ "frame_number: 30000\n",+ "frame_number: 30100\n",+ "frame_number: 30200\n",+ "frame_number: 30300\n",+ "frame_number: 30400\n",+ "frame_number: 30500\n",+ "frame_number: 30600\n",+ "frame_number: 30700\n",+ "frame_number: 30800\n",+ "frame_number: 30900\n",+ "frame_number: 31000\n",+ "frame_number: 31100\n",+ "frame_number: 31200\n",+ "frame_number: 31300\n",+ "frame_number: 31400\n",+ "frame_number: 31500\n",+ "frame_number: 31600\n",+ "frame_number: 31700\n",+ "frame_number: 31800\n",+ "frame_number: 31900\n",+ "frame_number: 32000\n",+ "frame_number: 32100\n",+ "frame_number: 32200\n",+ "frame_number: 32300\n",+ "frame_number: 32400\n",+ "frame_number: 32500\n",+ "frame_number: 32600\n",+ "frame_number: 32700\n",+ "frame_number: 32800\n",+ "frame_number: 32900\n",+ "frame_number: 33000\n",+ "frame_number: 33100\n",+ "frame_number: 33200\n",+ "frame_number: 33300\n",+ "frame_number: 33400\n",+ "frame_number: 33500\n",+ "frame_number: 33600\n",+ "frame_number: 33700\n",+ "frame_number: 33800\n",+ "frame_number: 33900\n",+ "frame_number: 34000\n",+ "frame_number: 34100\n",+ "frame_number: 34200\n",+ "frame_number: 34300\n",+ "frame_number: 34400\n",+ "frame_number: 34500\n",+ "frame_number: 34600\n",+ "frame_number: 34700\n",+ "frame_number: 34800\n",+ "frame_number: 34900\n",+ "frame_number: 35000\n",+ "frame_number: 35100\n",+ "frame_number: 35200\n",+ "frame_number: 35300\n",+ "frame_number: 35400\n",+ "frame_number: 35500\n",+ "frame_number: 35600\n",+ "frame_number: 35700\n",+ "frame_number: 35800\n",+ "frame_number: 35900\n",+ "frame_number: 36000\n",+ "frame_number: 36100\n",+ "frame_number: 36200\n",+ "frame_number: 36300\n",+ "frame_number: 36400\n",+ "frame_number: 36500\n",+ "frame_number: 36600\n",+ "frame_number: 36700\n",+ "frame_number: 36800\n",+ "frame_number: 36900\n",+ "frame_number: 37000\n",+ "frame_number: 37100\n",+ "frame_number: 37200\n",+ "frame_number: 37300\n",+ "frame_number: 37400\n",+ "frame_number: 37500\n",+ "frame_number: 37600\n",+ "frame_number: 37700\n",+ "frame_number: 37800\n",+ "frame_number: 37900\n",+ "frame_number: 38000\n",+ "frame_number: 38100\n",+ "frame_number: 38200\n",+ "frame_number: 38300\n",+ "frame_number: 38400\n",+ "frame_number: 38500\n",+ "frame_number: 38600\n",+ "frame_number: 38700\n",+ "frame_number: 38800\n",+ "frame_number: 38900\n",+ "frame_number: 39000\n",+ "frame_number: 39100\n",+ "frame_number: 39200\n",+ "frame_number: 39300\n",+ "frame_number: 39400\n",+ "frame_number: 39500\n",+ "frame_number: 39600\n",+ "frame_number: 39700\n",+ "frame_number: 39800\n",+ "frame_number: 39900\n",+ "frame_number: 40000\n",+ "frame_number: 40100\n",+ "frame_number: 40200\n",+ "frame_number: 40300\n",+ "frame_number: 40400\n",+ "frame_number: 40500\n",+ "frame_number: 40600\n",+ "frame_number: 40700\n",+ "frame_number: 40800\n",+ "frame_number: 40900\n",+ "frame_number: 41000\n",+ "frame_number: 41100\n",+ "frame_number: 41200\n",+ "frame_number: 41300\n",+ "frame_number: 41400\n",+ "frame_number: 41500\n",+ "frame_number: 41600\n",+ "frame_number: 41700\n",+ "frame_number: 41800\n",+ "frame_number: 41900\n",+ "frame_number: 42000\n",+ "frame_number: 42100\n",+ "frame_number: 42200\n",+ "frame_number: 42300\n",+ "frame_number: 42400\n",+ "frame_number: 42500\n",+ "frame_number: 42600\n",+ "frame_number: 42700\n",+ "frame_number: 42800\n",+ "frame_number: 42900\n",+ "frame_number: 43000\n",+ "frame_number: 43100\n",+ "frame_number: 43200\n",+ "frame_number: 43300\n",+ "frame_number: 43400\n",+ "frame_number: 43500\n",+ "frame_number: 43600\n",+ "frame_number: 43700\n",+ "frame_number: 43800\n",+ "frame_number: 43900\n",+ "frame_number: 44000\n",+ "frame_number: 44100\n",+ "frame_number: 44200\n",+ "frame_number: 44300\n",+ "frame_number: 44400\n",+ "frame_number: 44500\n",+ "frame_number: 44600\n",+ "frame_number: 44700\n",+ "frame_number: 44800\n",+ "frame_number: 44900\n",+ "frame_number: 45000\n",+ "frame_number: 45100\n",+ "frame_number: 45200\n",+ "frame_number: 45300\n",+ "frame_number: 45400\n",+ "frame_number: 45500\n",+ "frame_number: 45600\n",+ "frame_number: 45700\n",+ "frame_number: 45800\n",+ "frame_number: 45900\n",+ "frame_number: 46000\n",+ "frame_number: 46100\n",+ "frame_number: 46200\n",+ "frame_number: 46300\n",+ "frame_number: 46400\n",+ "frame_number: 46500\n",+ "frame_number: 46600\n",+ "frame_number: 46700\n",+ "frame_number: 46800\n",+ "frame_number: 46900\n",+ "frame_number: 47000\n",+ "frame_number: 47100\n",+ "frame_number: 47200\n",+ "frame_number: 47300\n",+ "frame_number: 47400\n",+ "frame_number: 47500\n",+ "frame_number: 47600\n",+ "frame_number: 47700\n",+ "frame_number: 47800\n",+ "frame_number: 47900\n",+ "frame_number: 48000\n",+ "frame_number: 48100\n",+ "frame_number: 48200\n",+ "frame_number: 48300\n",+ "frame_number: 48400\n",+ "frame_number: 48500\n",+ "frame_number: 48600\n",+ "frame_number: 48700\n",+ "frame_number: 48800\n",+ "frame_number: 48900\n",+ "frame_number: 49000\n",+ "frame_number: 49100\n",+ "frame_number: 49200\n",+ "frame_number: 49300\n",+ "frame_number: 49400\n",+ "frame_number: 49500\n",+ "frame_number: 49600\n",+ "frame_number: 49700\n",+ "frame_number: 49800\n",+ "frame_number: 49900\n",+ "frame_number: 50000\n",+ "frame_number: 50100\n",+ "frame_number: 50200\n",+ "frame_number: 50300\n",+ "frame_number: 50400\n",+ "frame_number: 50500\n",+ "frame_number: 50600\n",+ "frame_number: 50700\n",+ "frame_number: 50800\n",+ "frame_number: 50900\n",+ "frame_number: 51000\n",+ "frame_number: 51100\n",+ "frame_number: 51200\n",+ "frame_number: 51300\n",+ "frame_number: 51400\n",+ "frame_number: 51500\n",+ "frame_number: 51600\n",+ "frame_number: 51700\n",+ "frame_number: 51800\n",+ "frame_number: 51900\n",+ "frame_number: 52000\n",+ "frame_number: 52100\n",+ "frame_number: 52200\n",+ "frame_number: 52300\n",+ "frame_number: 52400\n",+ "frame_number: 52500\n",+ "frame_number: 52600\n",+ "frame_number: 52700\n",+ "frame_number: 52800\n",+ "frame_number: 52900\n",+ "frame_number: 53000\n",+ "frame_number: 53100\n",+ "frame_number: 53200\n",+ "frame_number: 53300\n",+ "frame_number: 53400\n"+ ]+ }+ ],+ "source": [+ "pupil_world_video_cap_object = cv2.VideoCapture(str(pupil_world_video_path))\n",+ "\n",+ "this_retinal_aligned_image = big_blank_image.copy()\n",+ "this_retinal_aligned_image[:, video_width-video_height:(video_width-video_height)+2*video_height,:]\n",+ "\n",+ "retinal_video_save_path = pupil_data_exports_path / 'retinal_aligned_video.mp4'\n",+ "retinal_video_writer = cv2.VideoWriter(str(retinal_video_save_path),\n",+ " cv2.VideoWriter_fourcc(*\"MP4V\"),\n",+ " video_framerate, \n",+ " (this_retinal_aligned_image.shape[0], this_retinal_aligned_image.shape[0]))\n",+ "\n",+ "success = True\n",+ "for frame_number in range(gaze_on_screen_x_downsampled_gapfilled.shape[0]):\n",+ " success, image= pupil_world_video_cap_object.read()\n",+ " if not success:\n",+ " print('failed to read frame')\n",+ " break\n",+ "\n",+ " # image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",+ " this_retinal_aligned_image = big_blank_image.copy()\n",+ "\n",+ " #pull out gaze data to align image to retinal coordinates\n",+ " gx = int(gaze_on_screen_x_downsampled_gapfilled[frame_number])\n",+ " gy = int(gaze_on_screen_y_downsampled_gapfilled[frame_number])\n",+ "\n",+ " bottom_edge_row = int(video_height)*2-(int(video_height)-gy)\n",+ " top_edge_row = int(video_height) -(int(video_height)-gy)\n",+ " left_col = (int(video_width)-gx)\n",+ " right_col = int(video_width)+(int(video_width)-gx)\n",+ "\n",+ " cols = np.arange(top_edge_row,bottom_edge_row)\n",+ "\n",+ " #if things are going off screen, use whatever gaze data worked most recently\n",+ " if top_edge_row < 0:\n",+ " gy = previous_gy\n",+ " top_edge_row = int(video_height) -(int(video_height)-gy)\n",+ " bottom_edge_row = int(video_height)*2-(int(video_height)-gy)\n",+ "\n",+ " if bottom_edge_row > int(video_height)*2:\n",+ " gy = previous_gy\n",+ " top_edge_row = int(video_height) -(int(video_height)-gy)\n",+ " bottom_edge_row = int(video_height)*2-(int(video_height)-gy)\n",+ " \n",+ " if left_col < 0:\n",+ " gx = previous_gx\n",+ " right_col = int(video_width)+(int(video_width)-gx)\n",+ " left_col = (int(video_width)-gx)\n",+ " \n",+ " if right_col > int(video_width)*2:\n",+ " gx = previous_gx\n",+ " right_col = int(video_width)+(int(video_width)-gx)\n",+ " left_col = (int(video_width)-gx)\n",+ "\n",+ " previous_gx = gx \n",+ " previous_gy = gy\n",+ " \n",+ " this_retinal_aligned_image[top_edge_row:bottom_edge_row, left_col:right_col,:] = image\n",+ "\n",+ " this_retinal_aligned_image[circular_mask==0] = 0\n",+ "\n",+ "\n",+ " this_retinal_aligned_image = this_retinal_aligned_image[:, video_width-video_height:(video_width-video_height)+2*video_height,:]\n",+ "\n",+ " this_retinal_aligned_image = cv2.line(this_retinal_aligned_image, \n",+ " (int(video_height), 0), \n",+ " (int(video_height), int(video_height)*2), \n",+ " (0,0,0,255), \n",+ " 5)\n",+ " this_retinal_aligned_image = cv2.line(this_retinal_aligned_image, \n",+ " (0, int(video_height)), \n",+ " (int(video_width)*2, \n",+ " int(video_height)), \n",+ " (0,0,0,255), \n",+ " 5)\n",+ " this_retinal_aligned_image = cv2.line(this_retinal_aligned_image, \n",+ " (int(video_height), 0), \n",+ " (int(video_height), int(video_height)*2), \n",+ " (255,255,255,255), \n",+ " 1,)\n",+ " this_retinal_aligned_image = cv2.line(this_retinal_aligned_image, \n",+ " (0, int(video_height)), \n",+ " (int(video_width)*2, \n",+ " int(video_height)), \n",+ " (255,255,255,255), \n",+ " 1)\n",+ " this_retinal_aligned_image = cv2.circle(this_retinal_aligned_image, \n",+ " (int(video_height), int(video_height)),\n",+ " 20,\n",+ " (0,0,0,255), \n",+ " 5)\n",+ " this_retinal_aligned_image = cv2.circle(this_retinal_aligned_image, \n",+ " (int(video_height), int(video_height)),\n",+ " 20,\n",+ " (255,255,255,255), \n",+ " 1)\n",+ "\n",+ " # plt.cla()\n",+ " # plt.imshow(this_retinal_aligned_image[:,:,:3]) \n",+ " # plt.show()\n",+ " # plt.pause(0.1)\n",+ "\n",+ " retinal_video_writer.write(this_retinal_aligned_image) \n",+ "\n",+ " if frame_number%100 == 0:\n",+ " print(f'frame_number: {frame_number}')\n",+ "\n",+ "retinal_video_writer.release()"+ ]+ }+ ],+ "metadata": {+ "interpreter": {+ "hash": "122c9931a57dd91c579842728ee2ee1c7ffefd770186c077f442b22e7e57f48e"+ },+ "kernelspec": {+ "display_name": "Python 3.7.13 ('freemocap-env')",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.7.13"+ },+ "orig_nbformat": 4+ },+ "nbformat": 4,+ "nbformat_minor": 2+}