Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

nice retinal centered video output :D

Jon9eb230f22b0c
diff --git a/jon_scratch/create_retina_centered_video_from_pupil_labs_data.ipynb b/jon_scratch/create_retina_centered_video_from_pupil_labs_data.ipynb
new file mode 100644
index 00000000..0f92b8cf
--- /dev/null
+++ b/jon_scratch/create_retina_centered_video_from_pupil_labs_data.ipynb
@@ -0,0 +1,1197 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# a notebook to make a quick and dirty retina centered video from standard Pupil Player export video\n",
+ "\n",
+ "It'll make one retina centered video from a standard Pupil Payer export video\n",
+ "\n",
+ "and also a video of the gaze trace\n",
+ "\n",
+ "I'll put them together in Adobe Premier or, via this script. We'll see lol"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import cv2\n",
+ "from pathlib import Path\n",
+ "import matplotlib.pyplot as plt\n",
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "from PIL import Image\n",
+ "from scipy import signal"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Using matplotlib backend: TkAgg\n"
+ ]
+ }
+ ],
+ "source": [
+ "%matplotlib "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Getcher paths straight"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# session_id = 'sesh_2022-05-07_17_15_05_pupil_wobble_juggle_0'\n",
+ "\n",
+ "# freemocap_data_folder = Path('C:/Users/jonma/Dropbox/FreeMoCapProject/FreeMocap_Data')\n",
+ "\n",
+ "# session_folder_path = freemocap_data_folder / session_id\n",
+ "\n",
+ "# pupil_data_path = session_folder_path / 'pupil_000'\n",
+ "# pupil_data_exports_path = pupil_data_path / 'exports' / '000'\n",
+ "# pupil_export_video_path = pupil_data_exports_path / 'world.mp4'\n",
+ "# pupil_world_video_path = pupil_data_path / 'world.mp4'\n",
+ "\n",
+ "pupil_data_path = Path(r'H:\\Other computers\\My Computer_MocapComputer\\Wirth_ARGP\\ARGP_Main\\data\\2022_04_29\\pupil_data')\n",
+ "pupil_data_exports_path = pupil_data_path / 'exports' / 'Wirth_Pilot_ARGP_2022-04-09_export'\n",
+ "pupil_export_video_path = pupil_data_exports_path / 'argp_pilot_May2022.mp4'\n",
+ "\n",
+ "pupil_world_video_path = pupil_data_path / 'world.mp4'\n",
+ "\n",
+ "\n",
+ "gaze_positions_path = pupil_data_exports_path / 'gaze_positions.csv'\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "load stuff "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#open cv video capture object\n",
+ "pupil_export_video_cap_object = cv2.VideoCapture(str(pupil_export_video_path))\n",
+ "\n",
+ "#pupil labs - gaze data\n",
+ "pupil_dataframe = pd.read_csv(gaze_positions_path)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "get some info from the video cap"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "video_width : 1280\n",
+ "video_height: 720\n",
+ "video_framerate: 29\n"
+ ]
+ }
+ ],
+ "source": [
+ "video_width = int(pupil_export_video_cap_object.get(cv2.CAP_PROP_FRAME_WIDTH))\n",
+ "video_height = int(pupil_export_video_cap_object.get(cv2.CAP_PROP_FRAME_HEIGHT))\n",
+ "video_framerate = int(pupil_export_video_cap_object.get(cv2.CAP_PROP_FPS))\n",
+ "print(f'video_width : {video_width }')\n",
+ "print(f'video_height: {video_height}')\n",
+ "print(f'video_framerate: {video_framerate}')\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "gaze locations"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "norm_pos_x = pupil_dataframe.norm_pos_x\n",
+ "norm_pos_y = pupil_dataframe.norm_pos_y\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# fig = plt.Figure(123)\n",
+ "# fig.plot(norm_pos_x, label = 'norm_pos_x')\n",
+ "# fig.plot(norm_pos_y, label ='norm_pos_x')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "\n",
+ "gaze_on_screen_x = norm_pos_x * video_width\n",
+ "gaze_on_screen_y = norm_pos_y * video_height\n",
+ "\n",
+ "world_camera_frame_index_all = pupil_dataframe.world_index"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "zero-lag 4th order butterworth filter with a 7Hz cutoff"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "nans in gaze_on_screen_x: 0\n",
+ "nans in gaze_on_screen_y: 0\n",
+ "nans in gaze_on_screen_x_filtered: 0\n",
+ "nans in gaze_on_screen_y_filtered: 0\n"
+ ]
+ }
+ ],
+ "source": [
+ "\n",
+ "\n",
+ "order = 5\n",
+ "cutoff=10**-1\n",
+ "b, a = signal.butter(order, cutoff)\n",
+ "gaze_on_screen_x_filtered = signal.filtfilt(b, a, gaze_on_screen_x)\n",
+ "gaze_on_screen_y_filtered = signal.filtfilt(b, a, gaze_on_screen_y)\n",
+ "\n",
+ "print(f\"nans in gaze_on_screen_x: {np.isnan(gaze_on_screen_x).sum()}\")\n",
+ "print(f\"nans in gaze_on_screen_y: {np.isnan(gaze_on_screen_y).sum()}\")\n",
+ "print(f\"nans in gaze_on_screen_x_filtered: {np.isnan(gaze_on_screen_x_filtered).sum()}\")\n",
+ "print(f\"nans in gaze_on_screen_y_filtered: {np.isnan(gaze_on_screen_y_filtered).sum()}\")\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "array([617.18147505, 617.38013879, 617.58062962, ..., 764.46021398,\n",
+ " 790.05560885, 815.63926686])"
+ ]
+ },
+ "execution_count": 11,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "gaze_on_screen_x_filtered"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "<matplotlib.legend.Legend at 0x1ba33e1af48>"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "c:\\Users\\jonma\\miniconda3\\envs\\freemocap-env\\lib\\tkinter\\__init__.py:749: UserWarning: Creating legend with loc=\"best\" can be slow with large amounts of data.\n",
+ " func(*args)\n",
+ "c:\\Users\\jonma\\miniconda3\\envs\\freemocap-env\\lib\\tkinter\\__init__.py:1705: UserWarning: Creating legend with loc=\"best\" can be slow with large amounts of data.\n",
+ " return self.func(*args)\n"
+ ]
+ }
+ ],
+ "source": [
+ "\n",
+ "plt.cla()\n",
+ "fig2 = plt.Figure()\n",
+ "ax = plt.gca()\n",
+ "ax.plot(gaze_on_screen_x, label = 'gaze_on_screen_x_raw')\n",
+ "ax.plot(gaze_on_screen_y, label = 'gaze_on_screen_y_raw')\n",
+ "ax.plot(gaze_on_screen_x_filtered, label = 'gaze_on_screen_x_filtered')\n",
+ "ax.plot(gaze_on_screen_y_filtered, label = 'gaze_on_screen_y_filtered')\n",
+ "ax.set_ylim((0-100, video_width+100))\n",
+ "ax.legend()\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "squash data down so each frame is the average of all data recorded on that frame"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "number_of_frames = np.max(world_camera_frame_index_all)\n",
+ "number_of_frames\n",
+ "\n",
+ "gaze_on_screen_x_downsampled = []\n",
+ "gaze_on_screen_y_downsampled = []\n",
+ "\n",
+ "for this_world_frame_index in range(number_of_frames):\n",
+ " this_frame_data = world_camera_frame_index_all == this_world_frame_index\n",
+ " this_frame_x_data = gaze_on_screen_x_filtered[this_frame_data]\n",
+ " this_frame_y_data = gaze_on_screen_y_filtered[this_frame_data]\n",
+ " gaze_on_screen_x_downsampled.append(np.nanmedian(this_frame_x_data))\n",
+ " gaze_on_screen_y_downsampled.append(np.nanmedian(this_frame_y_data))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "nans in gaze_on_screen_x: 0\n",
+ "nans in gaze_on_screen_y: 0\n",
+ "nans in gaze_on_screen_x_filtered: 0\n",
+ "nans in gaze_on_screen_y_filtered: 0\n",
+ "nans in gaze_on_screen_x_downsampled: 0\n",
+ "nans in gaze_on_screen_y_downsampled: 0\n"
+ ]
+ }
+ ],
+ "source": [
+ "\n",
+ "print(f\"nans in gaze_on_screen_x: {np.isnan(gaze_on_screen_x).sum()}\")\n",
+ "print(f\"nans in gaze_on_screen_y: {np.isnan(gaze_on_screen_y).sum()}\")\n",
+ "print(f\"nans in gaze_on_screen_x_filtered: {np.isnan(gaze_on_screen_x_filtered).sum()}\")\n",
+ "print(f\"nans in gaze_on_screen_y_filtered: {np.isnan(gaze_on_screen_y_filtered).sum()}\")\n",
+ "print(f\"nans in gaze_on_screen_x_downsampled: {np.isnan(gaze_on_screen_x_downsampled).sum()}\")\n",
+ "print(f\"nans in gaze_on_screen_y_downsampled: {np.isnan(gaze_on_screen_y_downsampled).sum()}\")\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "nans in gaze_on_screen_x: 0\n",
+ "nans in gaze_on_screen_y: 0\n",
+ "nans in gaze_on_screen_x_filtered: 0\n",
+ "nans in gaze_on_screen_y_filtered: 0\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "<matplotlib.legend.Legend at 0x1ba35d67c88>"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "c:\\Users\\jonma\\miniconda3\\envs\\freemocap-env\\lib\\tkinter\\__init__.py:749: UserWarning: Creating legend with loc=\"best\" can be slow with large amounts of data.\n",
+ " func(*args)\n",
+ "c:\\Users\\jonma\\miniconda3\\envs\\freemocap-env\\lib\\tkinter\\__init__.py:1705: UserWarning: Creating legend with loc=\"best\" can be slow with large amounts of data.\n",
+ " return self.func(*args)\n"
+ ]
+ }
+ ],
+ "source": [
+ "\n",
+ "# order = 4\n",
+ "# cutoff=10**-1\n",
+ "# b, a = signal.butter(order, cutoff)\n",
+ "# gaze_on_screen_x_filtered = signal.filtfilt(b, a, gaze_on_screen_x_downsampled)\n",
+ "# gaze_on_screen_y_filtered = signal.filtfilt(b, a, gaze_on_screen_y_downsampled)\n",
+ "\n",
+ "print(f\"nans in gaze_on_screen_x: {np.isnan(gaze_on_screen_x).sum()}\")\n",
+ "print(f\"nans in gaze_on_screen_y: {np.isnan(gaze_on_screen_y).sum()}\")\n",
+ "print(f\"nans in gaze_on_screen_x_filtered: {np.isnan(gaze_on_screen_x_filtered).sum()}\")\n",
+ "print(f\"nans in gaze_on_screen_y_filtered: {np.isnan(gaze_on_screen_y_filtered).sum()}\")\n",
+ "\n",
+ "plt.close('all')\n",
+ "fig2 = plt.Figure()\n",
+ "ax = plt.gca()\n",
+ "# ax.plot(gaze_on_screen_x_downsampled, label = 'gaze_on_screen_x_downsampled')\n",
+ "# ax.plot(gaze_on_screen_y_downsampled, label = 'gaze_on_screen_y_downsampled')\n",
+ "ax.plot(gaze_on_screen_x_filtered, label = 'gaze_on_screen_x_filtered')\n",
+ "ax.plot(gaze_on_screen_y_filtered, label = 'gaze_on_screen_y_filtered')\n",
+ "ax.set_ylim((0-100, video_width+100))\n",
+ "ax.legend()\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "<matplotlib.legend.Legend at 0x1ba35da7f08>"
+ ]
+ },
+ "execution_count": 16,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "plt.close(\"all\")\n",
+ "fig2 = plt.Figure()\n",
+ "ax = plt.gca()\n",
+ "ax.plot(gaze_on_screen_x_downsampled, '.-', label = 'gaze_on_screen_x_downsampled')\n",
+ "ax.plot(gaze_on_screen_y_downsampled, '.-',label = 'gaze_on_screen_y_downsampled')\n",
+ "ax.set_ylim((0-100, video_width+100))\n",
+ "ax.legend()\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "nans in gaze_on_screen_x_downsampled: 0\n",
+ "nans in gaze_on_screen_y_downsampled: 0\n",
+ "nans in gaze_on_screen_x_downsampled_gapfilled: 0\n",
+ "nans in gaze_on_screen_y_downsampled_gapfilled: 0\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(f\"nans in gaze_on_screen_x_downsampled: {np.isnan(gaze_on_screen_x_downsampled).sum()}\")\n",
+ "print(f\"nans in gaze_on_screen_y_downsampled: {np.isnan(gaze_on_screen_y_downsampled).sum()}\")\n",
+ "\n",
+ "gaze_x_df = pd.DataFrame(gaze_on_screen_x_downsampled)\n",
+ "gaze_x_df.interpolate(method = 'linear', inplace = True)\n",
+ "gaze_on_screen_x_downsampled_gapfilled = gaze_x_df.to_numpy()\n",
+ "\n",
+ "gaze_y_df = pd.DataFrame(gaze_on_screen_y_downsampled)\n",
+ "gaze_y_df.interpolate(method = 'linear', inplace = True)\n",
+ "gaze_on_screen_y_downsampled_gapfilled = gaze_y_df.to_numpy()\n",
+ "\n",
+ "print(f\"nans in gaze_on_screen_x_downsampled_gapfilled: {np.isnan(gaze_on_screen_x_downsampled_gapfilled).sum()}\")\n",
+ "print(f\"nans in gaze_on_screen_y_downsampled_gapfilled: {np.isnan(gaze_on_screen_y_downsampled_gapfilled).sum()}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "\n",
+ "\n",
+ "\n",
+ "big_blank_image = np.ones((int(video_height)*2, int(video_width)*2, 3), dtype=np.uint8)\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# pupil_world_video_cap_object = cv2.VideoCapture(str(pupil_world_video_path))\n",
+ "\n",
+ "# frame_num = 0\n",
+ "# success = True\n",
+ "# while success: \n",
+ "# success, image_raw = pupil_world_video_cap_object.read()\n",
+ "# if not success:\n",
+ "# break\n",
+ "# frame_num += 1\n",
+ "# print(f\"frame_num: {frame_num}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "create circular mask\n",
+ "\n",
+ "from =- https://stackoverflow.com/questions/61516526/how-to-use-opencv-to-crop-circular-image"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#define circles\n",
+ "radius = video_height #pixels\n",
+ "x_center = video_width\n",
+ "y_center = video_height\n",
+ "\n",
+ "# draw filled circles in white on black background as masks\n",
+ "circular_mask = np.zeros_like(big_blank_image)\n",
+ "circular_mask = cv2.circle(circular_mask, (x_center,y_center), radius, (255,255,255), -1)\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "video_framerate"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "frame_number: 0\n",
+ "frame_number: 100\n",
+ "frame_number: 200\n",
+ "frame_number: 300\n",
+ "frame_number: 400\n",
+ "frame_number: 500\n",
+ "frame_number: 600\n",
+ "frame_number: 700\n",
+ "frame_number: 800\n",
+ "frame_number: 900\n",
+ "frame_number: 1000\n",
+ "frame_number: 1100\n",
+ "frame_number: 1200\n",
+ "frame_number: 1300\n",
+ "frame_number: 1400\n",
+ "frame_number: 1500\n",
+ "frame_number: 1600\n",
+ "frame_number: 1700\n",
+ "frame_number: 1800\n",
+ "frame_number: 1900\n",
+ "frame_number: 2000\n",
+ "frame_number: 2100\n",
+ "frame_number: 2200\n",
+ "frame_number: 2300\n",
+ "frame_number: 2400\n",
+ "frame_number: 2500\n",
+ "frame_number: 2600\n",
+ "frame_number: 2700\n",
+ "frame_number: 2800\n",
+ "frame_number: 2900\n",
+ "frame_number: 3000\n",
+ "frame_number: 3100\n",
+ "frame_number: 3200\n",
+ "frame_number: 3300\n",
+ "frame_number: 3400\n",
+ "frame_number: 3500\n",
+ "frame_number: 3600\n",
+ "frame_number: 3700\n",
+ "frame_number: 3800\n",
+ "frame_number: 3900\n",
+ "frame_number: 4000\n",
+ "frame_number: 4100\n",
+ "frame_number: 4200\n",
+ "frame_number: 4300\n",
+ "frame_number: 4400\n",
+ "frame_number: 4500\n",
+ "frame_number: 4600\n",
+ "frame_number: 4700\n",
+ "frame_number: 4800\n",
+ "frame_number: 4900\n",
+ "frame_number: 5000\n",
+ "frame_number: 5100\n",
+ "frame_number: 5200\n",
+ "frame_number: 5300\n",
+ "frame_number: 5400\n",
+ "frame_number: 5500\n",
+ "frame_number: 5600\n",
+ "frame_number: 5700\n",
+ "frame_number: 5800\n",
+ "frame_number: 5900\n",
+ "frame_number: 6000\n",
+ "frame_number: 6100\n",
+ "frame_number: 6200\n",
+ "frame_number: 6300\n",
+ "frame_number: 6400\n",
+ "frame_number: 6500\n",
+ "frame_number: 6600\n",
+ "frame_number: 6700\n",
+ "frame_number: 6800\n",
+ "frame_number: 6900\n",
+ "frame_number: 7000\n",
+ "frame_number: 7100\n",
+ "frame_number: 7200\n",
+ "frame_number: 7300\n",
+ "frame_number: 7400\n",
+ "frame_number: 7500\n",
+ "frame_number: 7600\n",
+ "frame_number: 7700\n",
+ "frame_number: 7800\n",
+ "frame_number: 7900\n",
+ "frame_number: 8000\n",
+ "frame_number: 8100\n",
+ "frame_number: 8200\n",
+ "frame_number: 8300\n",
+ "frame_number: 8400\n",
+ "frame_number: 8500\n",
+ "frame_number: 8600\n",
+ "frame_number: 8700\n",
+ "frame_number: 8800\n",
+ "frame_number: 8900\n",
+ "frame_number: 9000\n",
+ "frame_number: 9100\n",
+ "frame_number: 9200\n",
+ "frame_number: 9300\n",
+ "frame_number: 9400\n",
+ "frame_number: 9500\n",
+ "frame_number: 9600\n",
+ "frame_number: 9700\n",
+ "frame_number: 9800\n",
+ "frame_number: 9900\n",
+ "frame_number: 10000\n",
+ "frame_number: 10100\n",
+ "frame_number: 10200\n",
+ "frame_number: 10300\n",
+ "frame_number: 10400\n",
+ "frame_number: 10500\n",
+ "frame_number: 10600\n",
+ "frame_number: 10700\n",
+ "frame_number: 10800\n",
+ "frame_number: 10900\n",
+ "frame_number: 11000\n",
+ "frame_number: 11100\n",
+ "frame_number: 11200\n",
+ "frame_number: 11300\n",
+ "frame_number: 11400\n",
+ "frame_number: 11500\n",
+ "frame_number: 11600\n",
+ "frame_number: 11700\n",
+ "frame_number: 11800\n",
+ "frame_number: 11900\n",
+ "frame_number: 12000\n",
+ "frame_number: 12100\n",
+ "frame_number: 12200\n",
+ "frame_number: 12300\n",
+ "frame_number: 12400\n",
+ "frame_number: 12500\n",
+ "frame_number: 12600\n",
+ "frame_number: 12700\n",
+ "frame_number: 12800\n",
+ "frame_number: 12900\n",
+ "frame_number: 13000\n",
+ "frame_number: 13100\n",
+ "frame_number: 13200\n",
+ "frame_number: 13300\n",
+ "frame_number: 13400\n",
+ "frame_number: 13500\n",
+ "frame_number: 13600\n",
+ "frame_number: 13700\n",
+ "frame_number: 13800\n",
+ "frame_number: 13900\n",
+ "frame_number: 14000\n",
+ "frame_number: 14100\n",
+ "frame_number: 14200\n",
+ "frame_number: 14300\n",
+ "frame_number: 14400\n",
+ "frame_number: 14500\n",
+ "frame_number: 14600\n",
+ "frame_number: 14700\n",
+ "frame_number: 14800\n",
+ "frame_number: 14900\n",
+ "frame_number: 15000\n",
+ "frame_number: 15100\n",
+ "frame_number: 15200\n",
+ "frame_number: 15300\n",
+ "frame_number: 15400\n",
+ "frame_number: 15500\n",
+ "frame_number: 15600\n",
+ "frame_number: 15700\n",
+ "frame_number: 15800\n",
+ "frame_number: 15900\n",
+ "frame_number: 16000\n",
+ "frame_number: 16100\n",
+ "frame_number: 16200\n",
+ "frame_number: 16300\n",
+ "frame_number: 16400\n",
+ "frame_number: 16500\n",
+ "frame_number: 16600\n",
+ "frame_number: 16700\n",
+ "frame_number: 16800\n",
+ "frame_number: 16900\n",
+ "frame_number: 17000\n",
+ "frame_number: 17100\n",
+ "frame_number: 17200\n",
+ "frame_number: 17300\n",
+ "frame_number: 17400\n",
+ "frame_number: 17500\n",
+ "frame_number: 17600\n",
+ "frame_number: 17700\n",
+ "frame_number: 17800\n",
+ "frame_number: 17900\n",
+ "frame_number: 18000\n",
+ "frame_number: 18100\n",
+ "frame_number: 18200\n",
+ "frame_number: 18300\n",
+ "frame_number: 18400\n",
+ "frame_number: 18500\n",
+ "frame_number: 18600\n",
+ "frame_number: 18700\n",
+ "frame_number: 18800\n",
+ "frame_number: 18900\n",
+ "frame_number: 19000\n",
+ "frame_number: 19100\n",
+ "frame_number: 19200\n",
+ "frame_number: 19300\n",
+ "frame_number: 19400\n",
+ "frame_number: 19500\n",
+ "frame_number: 19600\n",
+ "frame_number: 19700\n",
+ "frame_number: 19800\n",
+ "frame_number: 19900\n",
+ "frame_number: 20000\n",
+ "frame_number: 20100\n",
+ "frame_number: 20200\n",
+ "frame_number: 20300\n",
+ "frame_number: 20400\n",
+ "frame_number: 20500\n",
+ "frame_number: 20600\n",
+ "frame_number: 20700\n",
+ "frame_number: 20800\n",
+ "frame_number: 20900\n",
+ "frame_number: 21000\n",
+ "frame_number: 21100\n",
+ "frame_number: 21200\n",
+ "frame_number: 21300\n",
+ "frame_number: 21400\n",
+ "frame_number: 21500\n",
+ "frame_number: 21600\n",
+ "frame_number: 21700\n",
+ "frame_number: 21800\n",
+ "frame_number: 21900\n",
+ "frame_number: 22000\n",
+ "frame_number: 22100\n",
+ "frame_number: 22200\n",
+ "frame_number: 22300\n",
+ "frame_number: 22400\n",
+ "frame_number: 22500\n",
+ "frame_number: 22600\n",
+ "frame_number: 22700\n",
+ "frame_number: 22800\n",
+ "frame_number: 22900\n",
+ "frame_number: 23000\n",
+ "frame_number: 23100\n",
+ "frame_number: 23200\n",
+ "frame_number: 23300\n",
+ "frame_number: 23400\n",
+ "frame_number: 23500\n",
+ "frame_number: 23600\n",
+ "frame_number: 23700\n",
+ "frame_number: 23800\n",
+ "frame_number: 23900\n",
+ "frame_number: 24000\n",
+ "frame_number: 24100\n",
+ "frame_number: 24200\n",
+ "frame_number: 24300\n",
+ "frame_number: 24400\n",
+ "frame_number: 24500\n",
+ "frame_number: 24600\n",
+ "frame_number: 24700\n",
+ "frame_number: 24800\n",
+ "frame_number: 24900\n",
+ "frame_number: 25000\n",
+ "frame_number: 25100\n",
+ "frame_number: 25200\n",
+ "frame_number: 25300\n",
+ "frame_number: 25400\n",
+ "frame_number: 25500\n",
+ "frame_number: 25600\n",
+ "frame_number: 25700\n",
+ "frame_number: 25800\n",
+ "frame_number: 25900\n",
+ "frame_number: 26000\n",
+ "frame_number: 26100\n",
+ "frame_number: 26200\n",
+ "frame_number: 26300\n",
+ "frame_number: 26400\n",
+ "frame_number: 26500\n",
+ "frame_number: 26600\n",
+ "frame_number: 26700\n",
+ "frame_number: 26800\n",
+ "frame_number: 26900\n",
+ "frame_number: 27000\n",
+ "frame_number: 27100\n",
+ "frame_number: 27200\n",
+ "frame_number: 27300\n",
+ "frame_number: 27400\n",
+ "frame_number: 27500\n",
+ "frame_number: 27600\n",
+ "frame_number: 27700\n",
+ "frame_number: 27800\n",
+ "frame_number: 27900\n",
+ "frame_number: 28000\n",
+ "frame_number: 28100\n",
+ "frame_number: 28200\n",
+ "frame_number: 28300\n",
+ "frame_number: 28400\n",
+ "frame_number: 28500\n",
+ "frame_number: 28600\n",
+ "frame_number: 28700\n",
+ "frame_number: 28800\n",
+ "frame_number: 28900\n",
+ "frame_number: 29000\n",
+ "frame_number: 29100\n",
+ "frame_number: 29200\n",
+ "frame_number: 29300\n",
+ "frame_number: 29400\n",
+ "frame_number: 29500\n",
+ "frame_number: 29600\n",
+ "frame_number: 29700\n",
+ "frame_number: 29800\n",
+ "frame_number: 29900\n",
+ "frame_number: 30000\n",
+ "frame_number: 30100\n",
+ "frame_number: 30200\n",
+ "frame_number: 30300\n",
+ "frame_number: 30400\n",
+ "frame_number: 30500\n",
+ "frame_number: 30600\n",
+ "frame_number: 30700\n",
+ "frame_number: 30800\n",
+ "frame_number: 30900\n",
+ "frame_number: 31000\n",
+ "frame_number: 31100\n",
+ "frame_number: 31200\n",
+ "frame_number: 31300\n",
+ "frame_number: 31400\n",
+ "frame_number: 31500\n",
+ "frame_number: 31600\n",
+ "frame_number: 31700\n",
+ "frame_number: 31800\n",
+ "frame_number: 31900\n",
+ "frame_number: 32000\n",
+ "frame_number: 32100\n",
+ "frame_number: 32200\n",
+ "frame_number: 32300\n",
+ "frame_number: 32400\n",
+ "frame_number: 32500\n",
+ "frame_number: 32600\n",
+ "frame_number: 32700\n",
+ "frame_number: 32800\n",
+ "frame_number: 32900\n",
+ "frame_number: 33000\n",
+ "frame_number: 33100\n",
+ "frame_number: 33200\n",
+ "frame_number: 33300\n",
+ "frame_number: 33400\n",
+ "frame_number: 33500\n",
+ "frame_number: 33600\n",
+ "frame_number: 33700\n",
+ "frame_number: 33800\n",
+ "frame_number: 33900\n",
+ "frame_number: 34000\n",
+ "frame_number: 34100\n",
+ "frame_number: 34200\n",
+ "frame_number: 34300\n",
+ "frame_number: 34400\n",
+ "frame_number: 34500\n",
+ "frame_number: 34600\n",
+ "frame_number: 34700\n",
+ "frame_number: 34800\n",
+ "frame_number: 34900\n",
+ "frame_number: 35000\n",
+ "frame_number: 35100\n",
+ "frame_number: 35200\n",
+ "frame_number: 35300\n",
+ "frame_number: 35400\n",
+ "frame_number: 35500\n",
+ "frame_number: 35600\n",
+ "frame_number: 35700\n",
+ "frame_number: 35800\n",
+ "frame_number: 35900\n",
+ "frame_number: 36000\n",
+ "frame_number: 36100\n",
+ "frame_number: 36200\n",
+ "frame_number: 36300\n",
+ "frame_number: 36400\n",
+ "frame_number: 36500\n",
+ "frame_number: 36600\n",
+ "frame_number: 36700\n",
+ "frame_number: 36800\n",
+ "frame_number: 36900\n",
+ "frame_number: 37000\n",
+ "frame_number: 37100\n",
+ "frame_number: 37200\n",
+ "frame_number: 37300\n",
+ "frame_number: 37400\n",
+ "frame_number: 37500\n",
+ "frame_number: 37600\n",
+ "frame_number: 37700\n",
+ "frame_number: 37800\n",
+ "frame_number: 37900\n",
+ "frame_number: 38000\n",
+ "frame_number: 38100\n",
+ "frame_number: 38200\n",
+ "frame_number: 38300\n",
+ "frame_number: 38400\n",
+ "frame_number: 38500\n",
+ "frame_number: 38600\n",
+ "frame_number: 38700\n",
+ "frame_number: 38800\n",
+ "frame_number: 38900\n",
+ "frame_number: 39000\n",
+ "frame_number: 39100\n",
+ "frame_number: 39200\n",
+ "frame_number: 39300\n",
+ "frame_number: 39400\n",
+ "frame_number: 39500\n",
+ "frame_number: 39600\n",
+ "frame_number: 39700\n",
+ "frame_number: 39800\n",
+ "frame_number: 39900\n",
+ "frame_number: 40000\n",
+ "frame_number: 40100\n",
+ "frame_number: 40200\n",
+ "frame_number: 40300\n",
+ "frame_number: 40400\n",
+ "frame_number: 40500\n",
+ "frame_number: 40600\n",
+ "frame_number: 40700\n",
+ "frame_number: 40800\n",
+ "frame_number: 40900\n",
+ "frame_number: 41000\n",
+ "frame_number: 41100\n",
+ "frame_number: 41200\n",
+ "frame_number: 41300\n",
+ "frame_number: 41400\n",
+ "frame_number: 41500\n",
+ "frame_number: 41600\n",
+ "frame_number: 41700\n",
+ "frame_number: 41800\n",
+ "frame_number: 41900\n",
+ "frame_number: 42000\n",
+ "frame_number: 42100\n",
+ "frame_number: 42200\n",
+ "frame_number: 42300\n",
+ "frame_number: 42400\n",
+ "frame_number: 42500\n",
+ "frame_number: 42600\n",
+ "frame_number: 42700\n",
+ "frame_number: 42800\n",
+ "frame_number: 42900\n",
+ "frame_number: 43000\n",
+ "frame_number: 43100\n",
+ "frame_number: 43200\n",
+ "frame_number: 43300\n",
+ "frame_number: 43400\n",
+ "frame_number: 43500\n",
+ "frame_number: 43600\n",
+ "frame_number: 43700\n",
+ "frame_number: 43800\n",
+ "frame_number: 43900\n",
+ "frame_number: 44000\n",
+ "frame_number: 44100\n",
+ "frame_number: 44200\n",
+ "frame_number: 44300\n",
+ "frame_number: 44400\n",
+ "frame_number: 44500\n",
+ "frame_number: 44600\n",
+ "frame_number: 44700\n",
+ "frame_number: 44800\n",
+ "frame_number: 44900\n",
+ "frame_number: 45000\n",
+ "frame_number: 45100\n",
+ "frame_number: 45200\n",
+ "frame_number: 45300\n",
+ "frame_number: 45400\n",
+ "frame_number: 45500\n",
+ "frame_number: 45600\n",
+ "frame_number: 45700\n",
+ "frame_number: 45800\n",
+ "frame_number: 45900\n",
+ "frame_number: 46000\n",
+ "frame_number: 46100\n",
+ "frame_number: 46200\n",
+ "frame_number: 46300\n",
+ "frame_number: 46400\n",
+ "frame_number: 46500\n",
+ "frame_number: 46600\n",
+ "frame_number: 46700\n",
+ "frame_number: 46800\n",
+ "frame_number: 46900\n",
+ "frame_number: 47000\n",
+ "frame_number: 47100\n",
+ "frame_number: 47200\n",
+ "frame_number: 47300\n",
+ "frame_number: 47400\n",
+ "frame_number: 47500\n",
+ "frame_number: 47600\n",
+ "frame_number: 47700\n",
+ "frame_number: 47800\n",
+ "frame_number: 47900\n",
+ "frame_number: 48000\n",
+ "frame_number: 48100\n",
+ "frame_number: 48200\n",
+ "frame_number: 48300\n",
+ "frame_number: 48400\n",
+ "frame_number: 48500\n",
+ "frame_number: 48600\n",
+ "frame_number: 48700\n",
+ "frame_number: 48800\n",
+ "frame_number: 48900\n",
+ "frame_number: 49000\n",
+ "frame_number: 49100\n",
+ "frame_number: 49200\n",
+ "frame_number: 49300\n",
+ "frame_number: 49400\n",
+ "frame_number: 49500\n",
+ "frame_number: 49600\n",
+ "frame_number: 49700\n",
+ "frame_number: 49800\n",
+ "frame_number: 49900\n",
+ "frame_number: 50000\n",
+ "frame_number: 50100\n",
+ "frame_number: 50200\n",
+ "frame_number: 50300\n",
+ "frame_number: 50400\n",
+ "frame_number: 50500\n",
+ "frame_number: 50600\n",
+ "frame_number: 50700\n",
+ "frame_number: 50800\n",
+ "frame_number: 50900\n",
+ "frame_number: 51000\n",
+ "frame_number: 51100\n",
+ "frame_number: 51200\n",
+ "frame_number: 51300\n",
+ "frame_number: 51400\n",
+ "frame_number: 51500\n",
+ "frame_number: 51600\n",
+ "frame_number: 51700\n",
+ "frame_number: 51800\n",
+ "frame_number: 51900\n",
+ "frame_number: 52000\n",
+ "frame_number: 52100\n",
+ "frame_number: 52200\n",
+ "frame_number: 52300\n",
+ "frame_number: 52400\n",
+ "frame_number: 52500\n",
+ "frame_number: 52600\n",
+ "frame_number: 52700\n",
+ "frame_number: 52800\n",
+ "frame_number: 52900\n",
+ "frame_number: 53000\n",
+ "frame_number: 53100\n",
+ "frame_number: 53200\n",
+ "frame_number: 53300\n",
+ "frame_number: 53400\n"
+ ]
+ }
+ ],
+ "source": [
+ "pupil_world_video_cap_object = cv2.VideoCapture(str(pupil_world_video_path))\n",
+ "\n",
+ "this_retinal_aligned_image = big_blank_image.copy()\n",
+ "this_retinal_aligned_image[:, video_width-video_height:(video_width-video_height)+2*video_height,:]\n",
+ "\n",
+ "retinal_video_save_path = pupil_data_exports_path / 'retinal_aligned_video.mp4'\n",
+ "retinal_video_writer = cv2.VideoWriter(str(retinal_video_save_path),\n",
+ " cv2.VideoWriter_fourcc(*\"MP4V\"),\n",
+ " video_framerate, \n",
+ " (this_retinal_aligned_image.shape[0], this_retinal_aligned_image.shape[0]))\n",
+ "\n",
+ "success = True\n",
+ "for frame_number in range(gaze_on_screen_x_downsampled_gapfilled.shape[0]):\n",
+ " success, image= pupil_world_video_cap_object.read()\n",
+ " if not success:\n",
+ " print('failed to read frame')\n",
+ " break\n",
+ "\n",
+ " # image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n",
+ " this_retinal_aligned_image = big_blank_image.copy()\n",
+ "\n",
+ " #pull out gaze data to align image to retinal coordinates\n",
+ " gx = int(gaze_on_screen_x_downsampled_gapfilled[frame_number])\n",
+ " gy = int(gaze_on_screen_y_downsampled_gapfilled[frame_number])\n",
+ "\n",
+ " bottom_edge_row = int(video_height)*2-(int(video_height)-gy)\n",
+ " top_edge_row = int(video_height) -(int(video_height)-gy)\n",
+ " left_col = (int(video_width)-gx)\n",
+ " right_col = int(video_width)+(int(video_width)-gx)\n",
+ "\n",
+ " cols = np.arange(top_edge_row,bottom_edge_row)\n",
+ "\n",
+ " #if things are going off screen, use whatever gaze data worked most recently\n",
+ " if top_edge_row < 0:\n",
+ " gy = previous_gy\n",
+ " top_edge_row = int(video_height) -(int(video_height)-gy)\n",
+ " bottom_edge_row = int(video_height)*2-(int(video_height)-gy)\n",
+ "\n",
+ " if bottom_edge_row > int(video_height)*2:\n",
+ " gy = previous_gy\n",
+ " top_edge_row = int(video_height) -(int(video_height)-gy)\n",
+ " bottom_edge_row = int(video_height)*2-(int(video_height)-gy)\n",
+ " \n",
+ " if left_col < 0:\n",
+ " gx = previous_gx\n",
+ " right_col = int(video_width)+(int(video_width)-gx)\n",
+ " left_col = (int(video_width)-gx)\n",
+ " \n",
+ " if right_col > int(video_width)*2:\n",
+ " gx = previous_gx\n",
+ " right_col = int(video_width)+(int(video_width)-gx)\n",
+ " left_col = (int(video_width)-gx)\n",
+ "\n",
+ " previous_gx = gx \n",
+ " previous_gy = gy\n",
+ " \n",
+ " this_retinal_aligned_image[top_edge_row:bottom_edge_row, left_col:right_col,:] = image\n",
+ "\n",
+ " this_retinal_aligned_image[circular_mask==0] = 0\n",
+ "\n",
+ "\n",
+ " this_retinal_aligned_image = this_retinal_aligned_image[:, video_width-video_height:(video_width-video_height)+2*video_height,:]\n",
+ "\n",
+ " this_retinal_aligned_image = cv2.line(this_retinal_aligned_image, \n",
+ " (int(video_height), 0), \n",
+ " (int(video_height), int(video_height)*2), \n",
+ " (0,0,0,255), \n",
+ " 5)\n",
+ " this_retinal_aligned_image = cv2.line(this_retinal_aligned_image, \n",
+ " (0, int(video_height)), \n",
+ " (int(video_width)*2, \n",
+ " int(video_height)), \n",
+ " (0,0,0,255), \n",
+ " 5)\n",
+ " this_retinal_aligned_image = cv2.line(this_retinal_aligned_image, \n",
+ " (int(video_height), 0), \n",
+ " (int(video_height), int(video_height)*2), \n",
+ " (255,255,255,255), \n",
+ " 1,)\n",
+ " this_retinal_aligned_image = cv2.line(this_retinal_aligned_image, \n",
+ " (0, int(video_height)), \n",
+ " (int(video_width)*2, \n",
+ " int(video_height)), \n",
+ " (255,255,255,255), \n",
+ " 1)\n",
+ " this_retinal_aligned_image = cv2.circle(this_retinal_aligned_image, \n",
+ " (int(video_height), int(video_height)),\n",
+ " 20,\n",
+ " (0,0,0,255), \n",
+ " 5)\n",
+ " this_retinal_aligned_image = cv2.circle(this_retinal_aligned_image, \n",
+ " (int(video_height), int(video_height)),\n",
+ " 20,\n",
+ " (255,255,255,255), \n",
+ " 1)\n",
+ "\n",
+ " # plt.cla()\n",
+ " # plt.imshow(this_retinal_aligned_image[:,:,:3]) \n",
+ " # plt.show()\n",
+ " # plt.pause(0.1)\n",
+ "\n",
+ " retinal_video_writer.write(this_retinal_aligned_image) \n",
+ "\n",
+ " if frame_number%100 == 0:\n",
+ " print(f'frame_number: {frame_number}')\n",
+ "\n",
+ "retinal_video_writer.release()"
+ ]
+ }
+ ],
+ "metadata": {
+ "interpreter": {
+ "hash": "122c9931a57dd91c579842728ee2ee1c7ffefd770186c077f442b22e7e57f48e"
+ },
+ "kernelspec": {
+ "display_name": "Python 3.7.13 ('freemocap-env')",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.13"
+ },
+ "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}