Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

pretty much functional staged pipeline :D

jon matthisafec081321fa
diff --git a/freemocap/dev_scratchpad/dev_alpha/FMC_MultiCamera.py b/freemocap/dev_scratchpad/dev_alpha/FMC_MultiCamera.py
index 16bce12e..9362073d 100644
--- a/freemocap/dev_scratchpad/dev_alpha/FMC_MultiCamera.py
+++ b/freemocap/dev_scratchpad/dev_alpha/FMC_MultiCamera.py
@@ -51,7 +51,7 @@ class FMC_MultiCamera:
def __init__(
self,
rec_name = None,
- save_path = None,
+ freemocap_data_folder = None,
num_cams = 0,
cams_to_use_list = None,
rotation_codes_list = None,
@@ -70,12 +70,12 @@ class FMC_MultiCamera:
self._rec_name = rec_name
- if save_path is None:
+ if freemocap_data_folder is None:
self._save_path = Path('data/' + self._rec_name)
else:
- self._save_path = Path(save_path) / self._rec_name
+ self._save_path = Path(freemocap_data_folder) / self._rec_name
- if save_path or save_multi_cam_to_mp4 or save_to_h5 or save_log_file or save_each_cam_to_mp4:
+ if freemocap_data_folder or save_multi_cam_to_mp4 or save_to_h5 or save_log_file or save_each_cam_to_mp4:
self._save_path.mkdir(parents=True, exist_ok=True)
self._path_to_log_file = Path(str(self._save_path / self._rec_name) + '_log.txt')
@@ -438,7 +438,7 @@ class FMC_MultiCamera:
self._each_cam_video_writer_object_list=[None]*self.num_cams
self._each_cam_video_filename_list=[None]*self.num_cams
- self.syncedVidsPath = self._save_path / 'SyncedVideos'
+ self.syncedVidsPath = self._save_path / 'Synchronized_Videos'
self.syncedVidsPath.mkdir(parents=True)
for this_cam_num in range(self.num_cams):
@@ -450,7 +450,7 @@ class FMC_MultiCamera:
self._cam_image_size = ( self._cam_image_width, self._cam_image_height)
fourcc = cv2.VideoWriter_fourcc(*'DIVX')
- this_cam_vid_filename = 'Cam_'+str(this_cam_num)+'_synchronized.mp4'
+ this_cam_vid_filename = self._rec_name + '_Camera_'+str(this_cam_num)+'_synchronized.mp4'
self._each_cam_video_filename_list[this_cam_num] = self.syncedVidsPath / this_cam_vid_filename
fps = 25 #this is a bad and stupid guess, but it's actually kinda tricky to guess what is the right thing to put here. I'll fix this later and expect videos to be a bit faster than usual :(
@@ -556,7 +556,7 @@ if __name__ == '__main__':
try:
- multi_cam = FMC_MultiCamera(save_path=str(freemocap_data_path), rotation_codes_list=in_rotation_codes_list)
+ multi_cam = FMC_MultiCamera(freemocap_data_folder=str(freemocap_data_path), rotation_codes_list=in_rotation_codes_list)
multi_cam.start(standalone_mode=True)
except Exception:
diff --git a/freemocap/dev_scratchpad/dev_alpha/FMC_Session.py b/freemocap/dev_scratchpad/dev_alpha/FMC_Session.py
index 2d9928e5..b8d7a727 100644
--- a/freemocap/dev_scratchpad/dev_alpha/FMC_Session.py
+++ b/freemocap/dev_scratchpad/dev_alpha/FMC_Session.py
@@ -9,6 +9,8 @@
## https://patorjk.com/software/taag/#p=display&f=ANSI%20Regular&t=FMC%20Session
from FMC_MultiCamera import FMC_MultiCamera
+from FMC_Visualizer import FMC_Visualizer
+
import datetime
from pathlib import Path
import pickle
@@ -34,13 +36,10 @@ class FMC_Session:
##
##
- def __init_(self):
+ def __init__(self):
+
+ self.sessionID = datetime.datetime.now().strftime("FreeMoCap_Session_%Y-%m-%d_%H_%M_%S")
- self.sessionID = datetime.datetime.now().strftime("FreeMoCap_Session_%Y-%b-%d_%H_%M_%S")
- self.save_path = Path('data/' + self._rec_name)
- self.save_path.mkdir(parents=True)
- self.path_to_log_file = Path(str(self.save_path / self.sessionID) + '_log.txt')
-
##
## β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆβ–ˆ β–ˆβ–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆβ–ˆ β–ˆβ–ˆβ–ˆ
## β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆ
@@ -50,8 +49,8 @@ class FMC_Session:
##
##
- def record_multi_cam(self,save_path=None, rotation_codes_list=None):
- self.multi_cam = FMC_MultiCamera(save_path=str(freemocap_data_path), rotation_codes_list=in_rotation_codes_list )
+ def record_multi_cam(self,freemocap_data_folder=None, rotation_codes_list=None):
+ self.multi_cam = FMC_MultiCamera(freemocap_data_folder=str(freemocap_data_path), rotation_codes_list=in_rotation_codes_list, rec_name=self.sessionID )
self.multi_cam.start()
##
@@ -80,14 +79,14 @@ class FMC_Session:
- video_paths_list = self.multi_cam._each_cam_video_filename_list
+ self.synched_video_path_list = self.multi_cam._each_cam_video_filename_list
video_paths_list_of_list_of_strings = []
- for this_path in video_paths_list:
+ for this_path in self.synched_video_path_list:
video_paths_list_of_list_of_strings.append([str(this_path)])#anipose needs this to be a list of lists (which is annoying by whatevs)
- cam_names = ['cam_'+str(cam_num) for cam_num in range(len(video_paths_list))]
+ cam_names = ['cam_'+str(cam_num) for cam_num in range(len(self.synched_video_path_list))]
self.anipose_camera_calibration = fmc_anipose.CameraGroup.from_names(cam_names, fisheye=True ) # Looking through their code... it looks lke the 'fisheye=True' doesn't do much (see 2020-03-29 obsidian note)
error,charuco_frame_data, charuco_frame_numbers = self.anipose_camera_calibration.calibrate_videos(video_paths_list_of_list_of_strings, board)
@@ -127,7 +126,7 @@ class FMC_Session:
####
####
- def reconstruct3D(self, data_nCams_nFrames_nImgPts_XYC, calibration_toml_path = None, confidence_threshold=0):
+ def reconstruct3D(self, data_nCams_nFrames_nImgPts_XYC, calibration_toml_path = None, confidence_threshold=0.3):
if calibration_toml_path: #if the user specified a calibration toml (from a previous Anipose-based calibration) - load it here. Otherwise, assume the calibration is included in `self`
self.anipose_camera_calibration = fmc_anipose.CameraGroup.load(str(calibration_toml_path))
@@ -164,6 +163,36 @@ class FMC_Session:
dataReprojErr = dataReprojerr_flat.reshape(num_frames, num_img_points)
return data_fr_mar_xyz, dataReprojErr
+
+ ###
+ ###
+ ### β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ
+ ### β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ
+ ### β–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆβ–ˆ
+ ### β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ
+ ### β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ
+ ###
+ ###
+ def run_tracking_methods(self, use_media_pipe_bool=True, reconstruct3D_bool = True, save_to_npy_bool=True):
+
+ if use_media_pipe_bool:
+ from tracking_methods.fmc_mediapipe_alpha import run_mediapipe
+
+ self.mediapipe_nCams_nFrames_nImgPts_XY = run_mediapipe(self.synched_video_path_list)
+
+ if reconstruct3D_bool:
+ self.mediapipe_nFrames_nTrackedPoints_XYZ, self.mediapipe_reprojection_error = self.reconstruct3D(self.mediapipe_nCams_nFrames_nImgPts_XY)
+
+ if save_to_npy_bool:
+ mediapipe3d_filename = self.multi_cam._save_path/'mediapipe_3d_points.npy'
+ np.save(mediapipe3d_filename,self.medipipe_nFrames_nTrackedPoints_XYZ)
+
+ f=9
+
+
+
+
+
###
### β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ
### β–ˆβ–ˆ β–ˆβ–ˆ β–ˆβ–ˆ
@@ -210,11 +239,16 @@ if __name__ == "__main__":
try:
sesh = FMC_Session()
- sesh.record_multi_cam(save_path=str(freemocap_data_path), rotation_codes_list=in_rotation_codes_list)
+ sesh.record_multi_cam(freemocap_data_folder=str(freemocap_data_path), rotation_codes_list=in_rotation_codes_list)
sesh.calibrate_capture_volume(charucoSquareSize = this_charucoSquareSize)
sesh.charuco_nFrames_nTrackedPoints_XYZ, sesh.charuco_reprojection_error = sesh.reconstruct3D(sesh.charuco_nCams_nFrames_nImgPts_XY)
- f=9
+
charuco3d_filename = sesh.multi_cam._save_path/'charuco_3d_points.npy'
np.save(charuco3d_filename,sesh.charuco_nFrames_nTrackedPoints_XYZ)
+
+ sesh.run_tracking_methods()
+ visualizer = FMC_Visualizer(fmc_session_obj=sesh)
+ visualizer.start()
+
except:
console.print_exception()
\ No newline at end of file
diff --git a/freemocap/dev_scratchpad/dev_alpha/FMC_Visualizer.py b/freemocap/dev_scratchpad/dev_alpha/FMC_Visualizer.py
index e434e3c0..a4d41006 100644
--- a/freemocap/dev_scratchpad/dev_alpha/FMC_Visualizer.py
+++ b/freemocap/dev_scratchpad/dev_alpha/FMC_Visualizer.py
@@ -1,59 +1,88 @@
# -*- coding: utf-8 -*-
"""
-Demonstrates use of GLScatterPlotItem with rapidly-updating plots.
-
+built off the '3D Graphics- Scatter Plot' example from `python -m pyqtgraph.examples`
"""
+from pathlib import Path
+import sys
import pyqtgraph as pg
from pyqtgraph import functions as fn
from pyqtgraph.Qt import QtCore, QtGui
import pyqtgraph.opengl as gl
import numpy as np
+from rich.traceback import Frame
-app = pg.mkQApp("GLScatterPlotItem Example")
-w = gl.GLViewWidget()
-w.show()
-w.setWindowTitle('pyqtgraph example: GLScatterPlotItem')
-w.setCameraPosition(distance=20)
-g = gl.GLGridItem()
-w.addItem(g)
+class FMC_Visualizer:
+ def __init__(self, fmc_session_obj = None, recording_session_path = None):
-phase = 0.
-##
-## Third example shows a grid of points with rapidly updating position
-## and pxMode = False
-##
+ #setup QT app window
+ self.app = pg.mkQApp("FreeMoCap Data Visualizer")
+ self.gl_view_widget = gl.GLViewWidget()
+ self.gl_view_widget.show()
+ self.gl_view_widget.setWindowTitle('FreeMoCap Data Visualizer')
+ self.gl_view_widget.setCameraPosition(distance=10)
-pos3 = np.zeros((100,100,3))
-pos3[:,:,:2] = np.mgrid[:100, :100].transpose(1,2,0) * [-0.1,0.1]
-pos3 = pos3.reshape(10000,3)
-d3 = (pos3**2).sum(axis=1)**0.5
+ gl_grid_item = gl.GLGridItem()
+ self.gl_view_widget.addItem(gl_grid_item)
-sp3 = gl.GLScatterPlotItem(pos=pos3, color=(1,1,1,.3), size=0.1, pxMode=False)
+
+ if fmc_session_obj is None:
+ if recording_session_path:
+ self.recording_session_path = Path(recording_session_path)
+ else:
+ pass
+ #JSM NOTE - add something to load the last recorded session if nothing is specified
+
+ # load charuco data
+ self.charuco_3d_path = self.recording_session_path / "charuco_3d_points.npy"
+ self.charuco_nFrames_nTrackedPoints_XYZ = np.load(str(self.charuco_3d_path))
+ self.charuco_nFrames_nTrackedPoints_XYZ = self.charuco_nFrames_nTrackedPoints_XYZ/1000 #convert to meters
-w.addItem(sp3)
+ # load mediapipe data
+ self.mediapipe_3d_path = self.recording_session_path / "mediapipe_3d_points.npy"
+ self.mediapipe_nFrames_nTrackedPoints_XYZ = np.load(str(self.mediapipe_3d_path))
+ self.mediapipe_nFrames_nTrackedPoints_XYZ = self.mediapipe_nFrames_nTrackedPoints_XYZ/1000 #convert to meters
+ else:
+ # load charuco data
+ self.charuco_nFrames_nTrackedPoints_XYZ = fmc_session_obj.charuco_nFrames_nTrackedPoints_XYZ/1000 #convert to meters
+ # load mediapipe data
+ self.mediapipe_nFrames_nTrackedPoints_XYZ = fmc_session_obj.mediapipe_nFrames_nTrackedPoints_XYZ/1000 #convert to meters
-def update():
- ## update volume colors
- global phase
- phase -= 0.1
-
- ## update surface positions and colors
- global sp3, d3, pos3
- z = -np.cos(d3*2+phase)
- pos3[:,2] = z
- color = np.zeros((len(d3),4), dtype=np.float32)
- color[:,3] = .3
- color[:,0] = pos3[:,2]
- color[:,1] = .5
- # color[:,2] = pos3[:,2]
- sp3.setData(pos=pos3, color=color)
+ self.frame_num = 0
+ self.num_frames = self.charuco_nFrames_nTrackedPoints_XYZ.shape[0]
+
+
+ self.charuco_gl_scatter_item = gl.GLScatterPlotItem(pos=self.charuco_nFrames_nTrackedPoints_XYZ[self.frame_num,:,:], color=(1,1,1,.3), size=.1, pxMode=False)
+ self.gl_view_widget.addItem(self.charuco_gl_scatter_item)
+
+ self.mediapipe_gl_scatter_item = gl.GLScatterPlotItem(pos=self.mediapipe_nFrames_nTrackedPoints_XYZ[self.frame_num,:,:], color=(1,0,1,.3), size=.1, pxMode=False)
+ self.gl_view_widget.addItem(self.mediapipe_gl_scatter_item)
+
+ self.qt_timer = QtCore.QTimer()
+ self.qt_timer.timeout.connect(self.update)
+ self.qt_timer.start(50)
+
+
+
+ def update(self):
+
+ self.frame_num +=1
+ if self.frame_num > self.num_frames:
+ self.frame_num = 0
+
+
+ self.charuco_gl_scatter_item.setData(pos=self.charuco_nFrames_nTrackedPoints_XYZ[self.frame_num,:,:])
+ self.mediapipe_gl_scatter_item.setData(pos=self.mediapipe_nFrames_nTrackedPoints_XYZ[self.frame_num,:,:])
+
+ if self.frame_num % 10 == 0:
+ print("Frame Num : " + str(self.frame_num))
-t = QtCore.QTimer()
-t.timeout.connect(update)
-t.start(50)
+ def start(self):
+ if (sys.flags.interactive != 1) or not hasattr(QtCore, "PYQT_VERSION"):
+ QtGui.QApplication.instance().exec_()
+
if __name__ == '__main__':
pg.exec()
diff --git a/freemocap/dev_scratchpad/dev_alpha/__init__.py b/freemocap/dev_scratchpad/dev_alpha/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/freemocap/dev_scratchpad/dev_alpha/fmc_alpha_runme.py b/freemocap/dev_scratchpad/dev_alpha/fmc_alpha_runme.py
index 2fe7fbdc..f1e016fd 100644
--- a/freemocap/dev_scratchpad/dev_alpha/fmc_alpha_runme.py
+++ b/freemocap/dev_scratchpad/dev_alpha/fmc_alpha_runme.py
@@ -7,7 +7,7 @@ from pathos.helpers import mp as pathos_mp_helper
if __name__ =='__main__':
pathos_mp_helper.freeze_support()
- multi_cam = FMC_MultiCamera(save_path='C:/Users/WindowsPC_Hallway/Dropbox/FreeMoCapProject/FreeMocap_Data')
+ multi_cam = FMC_MultiCamera(freemocap_data_folder='C:/Users/WindowsPC_Hallway/Dropbox/FreeMoCapProject/FreeMocap_Data')
sessionID = multi_cam._rec_name
multi_cam.start(standalone_mode=True)
f=9
diff --git a/freemocap/dev_scratchpad/dev_alpha/freemocap_alpha_runme_script.py b/freemocap/dev_scratchpad/dev_alpha/freemocap_alpha_runme_script.py
index d237fe5a..806fdad3 100644
--- a/freemocap/dev_scratchpad/dev_alpha/freemocap_alpha_runme_script.py
+++ b/freemocap/dev_scratchpad/dev_alpha/freemocap_alpha_runme_script.py
@@ -26,7 +26,7 @@ if __name__ == '__main__':
try:
- multi_cam = FMC_MultiCamera(save_path=str(freemocap_data_path), rotation_codes_list=in_rotation_codes_list)
+ multi_cam = FMC_MultiCamera(freemocap_data_folder=str(freemocap_data_path), rotation_codes_list=in_rotation_codes_list)
multi_cam.start(standalone_mode=True)
except Exception:
console.print_exception()
diff --git a/freemocap/dev_scratchpad/dev_alpha/tracking_methods/__init__.py b/freemocap/dev_scratchpad/dev_alpha/tracking_methods/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/freemocap/dev_scratchpad/dev_alpha/tracking_methods/fmc_mediapipe_alpha.py b/freemocap/dev_scratchpad/dev_alpha/tracking_methods/fmc_mediapipe_alpha.py
new file mode 100644
index 00000000..9371bb6f
--- /dev/null
+++ b/freemocap/dev_scratchpad/dev_alpha/tracking_methods/fmc_mediapipe_alpha.py
@@ -0,0 +1,175 @@
+from pathlib import Path
+
+import numpy as np
+from rich.progress import Progress
+import cv2
+import mediapipe as mp
+
+# from numba import jit
+
+mp_drawing = mp.solutions.drawing_utils
+mp_drawing_styles = mp.solutions.drawing_styles
+mp_holistic = mp.solutions.holistic
+
+
+def run_mediapipe(synched_video_path_list, dummyRun=False):
+ """
+ Run MediaPipe on synced videos, and save body tracking data to be parsed
+ """
+ num_vids = len(synched_video_path_list)
+
+ with mp_holistic.Holistic(
+ static_image_mode = False, #use 'static image mode' to avoid system getting 'stuck' on ghost skeletons?
+ model_complexity = 2, #in this house, we turn the Speed/Accuracy dial all the way towards accuracy \o/)
+ ) as holistic:
+
+ mediapipe_results_from_all_videos_list = [] # Create an empty list that holds each cameras data
+ each_video_resolution_height = [] # mediapipe gives results in normalized screen coordinate, so we need this to convert to pixels
+ each_video_resolution_width = []
+
+ for this_vid_path in ( synched_video_path_list): # Run MediaPipe 'Holistic' (body, hands, face) tracker on each video in the list of synched videos
+ mediaPipe_results_from_this_video_list = [] # Create an empty list for mediapipes data
+ if not dummyRun:
+ cap = cv2.VideoCapture(str(this_vid_path))
+
+ frame_num = -1
+ num_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+ success, image_raw = cap.read() # load first image from video
+
+ with Progress() as progress:
+ progressBar = progress.add_task("[magenta]MediaPiping: {}".format(this_vid_path.name),total=num_frames,) # make progressbar
+
+ while success:
+
+ if frame_num % 5 == 0:
+ progress.update( progressBar, advance=5 ) # increment progress bar everyh 5th frame
+
+ frame_num += 1
+
+ image_height, image_width, _ = image_raw.shape
+ image = cv2.cvtColor(image_raw, cv2.COLOR_BGR2RGB), #Convert the BGR image to RGB before processing
+ image = image[0] #unpack the tuple into a straight up image
+ results = holistic.process( image) # NOTE: THIS IS WHERE THE MAGIC HAPENS
+
+ mediaPipe_results_from_this_video_list.append( results ) # Append data to mediapipe data list
+
+ # load the next image (will break `while-loop` on the last frame)
+ success, image_raw = cap.read() # load next image from video
+
+ each_video_resolution_height.append(image_height)
+ each_video_resolution_width.append(image_width)
+
+ mediapipe_results_from_all_videos_list.append( mediaPipe_results_from_this_video_list) # Append that cameras data for every frame to the camera datalist
+
+ ##
+ ##
+ ## #Parse MediaPipe into FMC Format
+ ##
+ ##
+
+ #standard MediaPipe output
+ numBodyPoints = 33
+ numFacePoints = 468 #ton of face points because it's a mesh (rather than individually tracked points)
+ numHandPoints = 21 #remember, your average person has almost exactly 2 hands
+
+ numTrackedPoints = (numBodyPoints + numHandPoints * 2 + numFacePoints ) # Get total points
+
+ # Create array of nans the size of number of cams, frame, points, XYC
+ mediaPipeData_nCams_nFrames_nImgPts_XYC = np.empty((int(num_vids), int(num_frames), int(numTrackedPoints), 3)) # create empty array
+ mediaPipeData_nCams_nFrames_nImgPts_XYC[:] = np.NaN # Fill it with NaNs!
+
+ list_of_frames_with_full_data = []
+
+ for vid_num in range(num_vids): # Loop through each camera
+ this_vid_mediapipe_results_per_frame = mediapipe_results_from_all_videos_list[vid_num]
+ for frame_num in range(num_frames): # Loop through each frame
+
+ this_frame_mediapipe_results = this_vid_mediapipe_results_per_frame[frame_num]
+
+ thisFrame_X_body = np.empty(numBodyPoints)
+ thisFrame_X_body[:] = np.nan
+ thisFrame_Y_body = thisFrame_X_body.copy()
+ thisFrame_C_body = thisFrame_X_body.copy() #'C' is for 'condfidence' (mediapipe calls it 'visibility')
+
+ thisFrame_X_face = np.empty(numFacePoints)
+ thisFrame_X_face[:] = np.nan
+ thisFrame_Y_face = thisFrame_X_face.copy()
+ thisFrame_C_face = thisFrame_X_face.copy()
+
+ thisFrame_hands = np.empty(numHandPoints)
+ thisFrame_hands[:]= np.nan
+
+ thisFrame_X_lefthand = thisFrame_hands.copy()
+ thisFrame_Y_lefthand = thisFrame_hands.copy()
+ thisFrame_C_lefthand = thisFrame_hands.copy()
+
+ thisFrame_X_righthand = thisFrame_hands.copy()
+ thisFrame_Y_righthand = thisFrame_hands.copy()
+ thisFrame_C_righthand = thisFrame_hands.copy()
+
+ fullFrame = True #keep track of which frames have 'full' data (all body, all hands, all face data present)
+ try:
+ # pull out ThisFrame's mediapipe data (`mpData.pose_landmarks.landmark` returns something iterable Β―\_(ツ)_/Β―)
+ thisFrame_poseDataLandMarks = this_frame_mediapipe_results.pose_landmarks.landmark # body ('pose') data
+ # stuff body data into pre-allocated nan array
+ thisFrame_X_body[:numBodyPoints] = [pp.x for pp in thisFrame_poseDataLandMarks] # PoseX data - Normalized screen coords (in range [0, 1]) - need multiply by image resultion for pixels
+ thisFrame_Y_body[:numBodyPoints] = [pp.y for pp in thisFrame_poseDataLandMarks] # PoseY data
+ thisFrame_C_body[:numBodyPoints] = [pp.visibility for pp in thisFrame_poseDataLandMarks] #'visibility' is MediaPose's 'confidence' measure in range [0,1]
+ except:
+ fullFrame = False
+
+ # Right hand data
+ try:
+ thisFrame_rHandDataLandMarks = this_frame_mediapipe_results.right_hand_landmarks.landmark # right hand data
+ thisFrame_X_righthand[:numHandPoints] = [pp.x for pp in thisFrame_rHandDataLandMarks] # PoseX data - Normalized screen coords (in range [0, 1]) - need multiply by image resultion for pixels
+ thisFrame_Y_righthand[:numHandPoints] = [pp.y for pp in thisFrame_rHandDataLandMarks] # PoseY data
+ thisFrame_C_righthand[:numHandPoints] = [pp.visibility for pp in thisFrame_rHandDataLandMarks] #'visibility' is MediaPose's 'confidence' measure in range [0,1]
+ except:
+ fullFrame = False
+
+ # Left hand data
+ try:
+ thisFrame_lHandDataLandMarks = this_frame_mediapipe_results.left_hand_landmarks.landmark # left hand data
+ thisFrame_X_lefthand[:numHandPoints ] = [pp.x for pp in thisFrame_lHandDataLandMarks] # PoseX data - Normalized screen coords (in range [0, 1]) - need multiply by image resultion for pixels
+ thisFrame_Y_lefthand[:numHandPoints] = [pp.y for pp in thisFrame_lHandDataLandMarks] # PoseY data
+ thisFrame_C_lefthand[:numHandPoints] = [pp.visibility for pp in thisFrame_lHandDataLandMarks] #'visibility' is MediaPose's 'confidence' measure in range [0,1]
+ except:
+ fullFrame = False
+
+ # FaceMeshData
+ try:
+ thisFrame_faceDataLandMarks = this_frame_mediapipe_results.face_landmarks.landmark # face (mesh) data
+ thisFrame_X_face[:numFacePoints] = [pp.x for pp in thisFrame_faceDataLandMarks] # PoseX data - Normalized screen coords (in range [0, 1]) - need multiply by image resultion for pixels
+ thisFrame_Y_face[:numFacePoints] = [pp.y for pp in thisFrame_faceDataLandMarks] # PoseY data
+ # NOTE - There's also Z data in here, but we're gonna ignore it
+ thisFrame_C_face[:numFacePoints] = [ pp.visibility for pp in thisFrame_faceDataLandMarks ] #'visibility' is MediaPose's 'confidence' measure in range [0,1]
+ except:
+ fullFrame = False
+
+ if fullFrame:
+ list_of_frames_with_full_data.append(frame_num)
+
+ thisFrame_X = np.concatenate((thisFrame_X_body,thisFrame_X_righthand,thisFrame_X_lefthand,thisFrame_X_face))
+ thisFrame_Y = np.concatenate((thisFrame_Y_body,thisFrame_Y_righthand,thisFrame_Y_lefthand,thisFrame_Y_face))
+ thisFrame_C = np.concatenate((thisFrame_C_body,thisFrame_C_righthand,thisFrame_C_lefthand,thisFrame_C_face))
+
+ # stuff this frame's data into pre-allocated mediaPipeData_.... array
+ mediaPipeData_nCams_nFrames_nImgPts_XYC[vid_num, frame_num, :, 0] = thisFrame_X *each_video_resolution_width[vid_num]# convert from normalized screen coordinates to pixel coordinates
+ mediaPipeData_nCams_nFrames_nImgPts_XYC[vid_num, frame_num, :, 1] = thisFrame_Y *each_video_resolution_height[vid_num]
+ mediaPipeData_nCams_nFrames_nImgPts_XYC[vid_num, frame_num, :, 2] = thisFrame_C
+
+
+ mediaPipeData_nCams_nFrames_nImgPts_XYC[:, :, 34:, 2] = 1 #hand and face data don't get 'visibility'/'confidence' scores, so we'll just set them all to 1
+
+ return mediaPipeData_nCams_nFrames_nImgPts_XYC
+
+
+if __name__ == "__main__":
+ synched_vid_folder_path = Path(r"C:\Users\jonma\Dropbox\FreeMoCapProject\FreeMocap_Data\sesh_2021-12-06_08_52_10_mc\SyncedVideos")
+ synched_video_path_list = [vid_path for vid_path in synched_vid_folder_path.glob("*.mp4")]
+
+ mediaPipeData_nCams_nFrames_nImgPts_XYC = run_mediapipe(synched_video_path_list=synched_video_path_list)
+ mediapipe2d_filename = synched_vid_folder_path.parent/'mediapipe_2d_points.npy'
+ np.save(mediapipe2d_filename, mediaPipeData_nCams_nFrames_nImgPts_XYC)
+
+
\ No newline at end of file