Free Motion Capture for Everyone πβ¨
TypeScript
MediaPipe! and now 'reconstructionThreshold' actually works! and 'skel_fr_mar_dim' >> 'skel_fr_mar_xyz'
jon matthisd650630ee98e
diff --git a/freemocap/dev_scratchpad/balls_scratchpad_lol.py b/freemocap/analysis/balls_scratchpad.pysimilarity index 90%rename from freemocap/dev_scratchpad/balls_scratchpad_lol.pyrename to freemocap/analysis/balls_scratchpad.pyindex a8e473c6..54b75f83 100644--- a/freemocap/dev_scratchpad/balls_scratchpad_lol.py+++ b/freemocap/analysis/balls_scratchpad.py@@ -1,4 +1,4 @@-#%%+#%% lolimport numpy as npimport matplotlib.pyplot as plt@@ -24,11 +24,11 @@ from rich.markdown import MarkdowndlcDataPath = Path(r"C:\Users\jonma\Dropbox\GitKrakenRepos\freemocap - Copy\Data\sesh_21-07-20_165209\DataArrays\deepLabCut_3d.npy")openPoseDataPath = Path(r"C:\Users\jonma\Dropbox\GitKrakenRepos\freemocap - Copy\Data\sesh_21-07-20_165209\DataArrays\openPoseSkel_3d.npy")-dlc_fr_mar_dim = np.load(dlcDataPath)-dlc_trajectories = [dlc_fr_mar_dim[:,markerNum,:] for markerNum in range(dlc_fr_mar_dim.shape[1])]+dlc_fr_mar_xyz = np.load(dlcDataPath)+dlc_trajectories = [dlc_fr_mar_xyz[:,markerNum,:] for markerNum in range(dlc_fr_mar_xyz.shape[1])]-skel_fr_mar_dim = np.load(openPoseDataPath)-skel_trajectories = [skel_fr_mar_dim[:,markerNum,:] for markerNum in range(skel_fr_mar_dim.shape[1])]+skel_fr_mar_xyz = np.load(openPoseDataPath)+skel_trajectories = [skel_fr_mar_xyz[:,markerNum,:] for markerNum in range(skel_fr_mar_xyz.shape[1])]# %% hand/ball vs timefig1 = plt.figure()diff --git a/freemocap/analysis/skel_analisys.py b/freemocap/analysis/skel_analisys.pynew file mode 100644index 00000000..a8f80401--- /dev/null+++ b/freemocap/analysis/skel_analisys.py@@ -0,0 +1,57 @@+#%%+import numpy as np++import matplotlib.pyplot as plt+import mpl_toolkits.mplot3d.axes3d as p3+import matplotlib.animation as animation+from matplotlib.widgets import Slider+import cv2++import copy+from pathlib import Path++#RICH CONSOLE STUFF+from rich import pretty+pretty.install() #makes all print statement output pretty+from rich import inspect+from rich.console import Console+console = Console()+from rich.traceback import install as rich_traceback_install+from rich.markdown import Markdown++# %% load dlc data++openPose3dDataPath = Path(r"C:\Users\jonma\Dropbox\FreeMoCapProject\FreeMocap_Data\sesh_2021-08-27_17_31_30\DataArrays\openPoseSkel_3d.npy")++mediaPipe3dDataPath = Path(r"C:\Users\jonma\Dropbox\FreeMoCapProject\FreeMocap_Data\sesh_2021-08-27_17_31_30\DataArrays\mediaPipeSkel_3d.npy")++openPose2dDataPath = Path(r"C:\Users\jonma\Dropbox\FreeMoCapProject\FreeMocap_Data\sesh_2021-08-27_17_31_30\DataArrays\openPoseData_2d.npy")++mediaPipe2dDataPath = Path(r"C:\Users\jonma\Dropbox\FreeMoCapProject\FreeMocap_Data\sesh_2021-08-27_17_31_30\DataArrays\mediaPipe_2d.npy")++#%%+mediaPipe_nCams_nFrames_nImgPts_XYC = np.load(mediaPipe2dDataPath)++mp_rToe_conf = mediaPipe_nCams_nFrames_nImgPts_XYC[:,:,32,2].transpose()+mp_nose_conf = mediaPipe_nCams_nFrames_nImgPts_XYC[:,:,0,2].transpose()+++# %% hand/ball vs time+fig1 = plt.figure()+axToe = fig1.add_subplot(1,2,1)++axToe.plot(mp_rToe_conf, label='Right Toe')+axToe.set_xlabel('Time (frames)')+axToe.set_ylabel('Confidence')+axToe.legend()++axNose = fig1.add_subplot(1,2,2)++axNose.plot(mp_nose_conf, label='Nose')+axNose.set_xlabel('Time (frames)')+axNose.set_ylabel('Confidence')+axNose.legend()++++# %%diff --git a/freemocap/calibrate.py b/freemocap/calibrate.pyindex ad56ec67..9c5675f6 100644--- a/freemocap/calibrate.py+++ b/freemocap/calibrate.py@@ -81,18 +81,18 @@ def CalibrateCaptureVolume(session,board, calVideoFrameLength = 120):session.charuco_nCams_nFrames_nImgPts_XY = charuco_nCams_nFrames_nImgPts_XY- charuco_fr_mar_dim = reconstruct3D.reconstruct3D(+ charuco_fr_mar_xyz, charuco_reprojErr= reconstruct3D.reconstruct3D(session, charuco_nCams_nFrames_nImgPts_XY)- mean_charuco_fr_mar_dim = np.nanmean(charuco_fr_mar_dim, axis=0)+ mean_charuco_fr_mar_xyz = np.nanmean(charuco_fr_mar_xyz, axis=0)# charry_reshaped = charucoarray.reshape(session.numCams, -1, 2)# char_flat = cgroup.triangulate(charry_reshaped, progress=True)# charReprojerr_flat = cgroup.reprojection_error(char_flat, charry_reshaped, mean=True)- # char_fr_mar_dim = char_flat.reshape(n_frames, n_trackedPoints, 3)+ # char_fr_mar_xyz = char_flat.reshape(n_frames, n_trackedPoints, 3)# charReprojErr_fr_mar_err = charReprojerr_flat.reshape(n_frames, n_trackedPoints)if session.debug:@@ -100,9 +100,9 @@ def CalibrateCaptureVolume(session,board, calVideoFrameLength = 120):# mean charuco positionax1 = fig.add_subplot(111, projection="3d")ax1.cla()- x = mean_charuco_fr_mar_dim[:][:, 0]- y = mean_charuco_fr_mar_dim[:][:, 1]- z = mean_charuco_fr_mar_dim[:][:, 2]+ x = mean_charuco_fr_mar_xyz[:][:, 0]+ y = mean_charuco_fr_mar_xyz[:][:, 1]+ z = mean_charuco_fr_mar_xyz[:][:, 2]mx = np.nanmean(x)my = np.nanmean(y)mz = np.nanmean(z)@@ -135,15 +135,16 @@ def CalibrateCaptureVolume(session,board, calVideoFrameLength = 120):# #charuco points over time# ax2 = fig.add_subplot(122)# ax2.cla()- # numPts = charuco_fr_mar_dim.shape[1]+ # numPts = charuco_fr_mar_xyz.shape[1]# for pp in range(numPts):- # ax2.plot(charuco_fr_mar_dim[:,pp,:])+ # ax2.plot(charuco_fr_mar_xyz[:,pp,:])# ax2.set_title('Charuco Point positions over time')plt.show()- path_to_charuco_array = session.dataArrayPath/'charuco_3d_points.npy'- np.save(path_to_charuco_array, charuco_fr_mar_dim)- return cgroup, mean_charuco_fr_mar_dim+ np.save(session.dataArrayPath/'charuco_3d_points.npy', charuco_fr_mar_xyz)+ np.save(session.dataArrayPath/'charuco_3d_reprojErr.npy', charuco_reprojErr)++ return cgroup, charuco_fr_mar_xyzdef createCalibrationVideos(session, calVideoFrameLength):diff --git a/freemocap/fmc_openpose.py b/freemocap/fmc_openpose.pyindex 8a0b64ec..fd71e0e3 100644--- a/freemocap/fmc_openpose.py+++ b/freemocap/fmc_openpose.py@@ -10,11 +10,11 @@ from pathlib import Path# from numba import jit-def runOpenPose(session, dummyRun=False):+def runOpenPose(session, runOpenPose=False):"""Run OpenPose on synced videos, and save bdoy tracking data to be parsed"""- if not dummyRun:+ if runOpenPose:os.chdir(session.openPoseExePath)openPose_jsonPathList = [] # list to hold the paths to the json files@@ -42,7 +42,7 @@ def runOpenPose(session, dummyRun=False):openPose_imgPathList.append(imgPath)openPose_imgPathList_yaml.append(str(imgPath))openPose_jsonPathList_yaml.append(str(jsonPath))- if not dummyRun:+ if runOpenPose:subprocess.run(["bin\OpenPoseDemo.exe",diff --git a/freemocap/fmc_pyqtgraph.py b/freemocap/fmc_pyqtgraph.pyindex 2354969d..eab43d4a 100644--- a/freemocap/fmc_pyqtgraph.py+++ b/freemocap/fmc_pyqtgraph.py@@ -24,21 +24,21 @@ class PlaySkeletonWidget:# load data to be plottedself.useOpenPose = session.useOpenPoseif self.useOpenPose:- self.skel_fr_mar_dim = np.load(+ self.skel_fr_mar_xyz = np.load(session.dataArrayPath / "openPoseSkel_3d.npy")self.skelScatterItem = gl.GLScatterPlotItem(- pos=self.skel_fr_mar_dim, color=(1, 0, 1, 0.8), size=10, pxMode=False+ pos=self.skel_fr_mar_xyz, color=(1, 0, 1, 0.8), size=10, pxMode=False)self.Skel3dViewWidget.addItem(self.skelScatterItem)elif session.useMediaPipe:- self.skel_fr_mar_dim = np.load(+ self.skel_fr_mar_xyz = np.load(session.dataArrayPath / "mediaPipeSkel_3d.npy")self.skelScatterItem = gl.GLScatterPlotItem(- pos=self.skel_fr_mar_dim, color=(1, 0, 1, 0.8), size=10, pxMode=False+ pos=self.skel_fr_mar_xyz, color=(1, 0, 1, 0.8), size=10, pxMode=False)self.Skel3dViewWidget.addItem(self.skelScatterItem)@@ -97,7 +97,7 @@ class PlaySkeletonWidget:# self.lHand = self.rHand + lHandIDstart# self.headLineItem = gl.GLLinePlotItem(- # pos=self.skel_fr_mar_dim[:, self.head, :],+ # pos=self.skel_fr_mar_xyz[:, self.head, :],# color=(1, 1, 1, 0.2),# width=4.0,# antialias=True,@@ -105,7 +105,7 @@ class PlaySkeletonWidget:# self.Skel3dViewWidget.addItem(self.headLineItem)# self.spineLineItem = gl.GLLinePlotItem(- # pos=self.skel_fr_mar_dim[:, self.spine, :],+ # pos=self.skel_fr_mar_xyz[:, self.spine, :],# color=(1, 1, 1, 0.9),# width=4.0,# antialias=True,@@ -113,7 +113,7 @@ class PlaySkeletonWidget:# self.Skel3dViewWidget.addItem(self.spineLineItem)# self.rArmLineItem = gl.GLLinePlotItem(- # pos=self.skel_fr_mar_dim[:, self.rArm, :],+ # pos=self.skel_fr_mar_xyz[:, self.rArm, :],# color=(1, 0.2, 0.15, 0.9),# width=4.0,# antialias=True,@@ -121,7 +121,7 @@ class PlaySkeletonWidget:# self.Skel3dViewWidget.addItem(self.rArmLineItem)# self.lArmLineItem = gl.GLLinePlotItem(- # pos=self.skel_fr_mar_dim[:, self.lArm, :],+ # pos=self.skel_fr_mar_xyz[:, self.lArm, :],# color=(0.4, 0.6, 1, 0.9),# width=4.0,# antialias=True,@@ -129,7 +129,7 @@ class PlaySkeletonWidget:# self.Skel3dViewWidget.addItem(self.lArmLineItem)# self.rLegLineItem = gl.GLLinePlotItem(- # pos=self.skel_fr_mar_dim[:, self.rLeg, :],+ # pos=self.skel_fr_mar_xyz[:, self.rLeg, :],# color=(1, 0.2, 0.15, 0.9),# width=4.0,# antialias=True,@@ -137,7 +137,7 @@ class PlaySkeletonWidget:# self.Skel3dViewWidget.addItem(self.rLegLineItem)# self.lLegLineItem = gl.GLLinePlotItem(- # pos=self.skel_fr_mar_dim[:, self.lLeg, :],+ # pos=self.skel_fr_mar_xyz[:, self.lLeg, :],# color=(0.4, 0.6, 1, 0.9),# width=4.0,# antialias=True,@@ -145,7 +145,7 @@ class PlaySkeletonWidget:# self.Skel3dViewWidget.addItem(self.lLegLineItem)# self.rFootLineItem = gl.GLLinePlotItem(- # pos=self.skel_fr_mar_dim[:, self.rFoot, :],+ # pos=self.skel_fr_mar_xyz[:, self.rFoot, :],# color=(1, 0.2, 0.15, 0.9),# width=4.0,# antialias=True,@@ -153,7 +153,7 @@ class PlaySkeletonWidget:# self.Skel3dViewWidget.addItem(self.rFootLineItem)# self.lFootLineItem = gl.GLLinePlotItem(- # pos=self.skel_fr_mar_dim[:, self.lFoot, :],+ # pos=self.skel_fr_mar_xyz[:, self.lFoot, :],# color=(0.4, 0.6, 1, 0.9),# width=4.0,# antialias=True,@@ -162,7 +162,7 @@ class PlaySkeletonWidget:# # plot handybois# self.rHandLineItem = gl.GLLinePlotItem(- # pos=self.skel_fr_mar_dim[:, self.rHand, :],+ # pos=self.skel_fr_mar_xyz[:, self.rHand, :],# color=(0.2, 0.4, 1, 0.9),# width=1.0,# antialias=True,@@ -170,7 +170,7 @@ class PlaySkeletonWidget:# # self.Skel3dViewWidget.addItem(self.rHandLineItem)# self.lHandLineItem = gl.GLLinePlotItem(- # pos=self.skel_fr_mar_dim[:, self.lHand, :],+ # pos=self.skel_fr_mar_xyz[:, self.lHand, :],# color=(1, 0.1, 0.05, 0.9),# width=1.0,# antialias=True,@@ -179,27 +179,27 @@ class PlaySkeletonWidget:self.useDLC = session.useDLCif self.useDLC:- # self.dlc_fr_mar_dim = np.load(session.dataArrayPath/'mediaPipeSkel_3d.npy')- self.dlc_fr_mar_dim = np.load(session.dataArrayPath / "deepLabCut_3d.npy")+ # self.dlc_fr_mar_xyz = np.load(session.dataArrayPath/'mediaPipeSkel_3d.npy')+ self.dlc_fr_mar_xyz = np.load(session.dataArrayPath / "deepLabCut_3d.npy")self.dlcScatterItem = gl.GLScatterPlotItem(- pos=self.dlc_fr_mar_dim, color=(1, 0, 0, 0.8), size=60, pxMode=False+ pos=self.dlc_fr_mar_xyz, color=(1, 0, 0, 0.8), size=60, pxMode=False)self.Skel3dViewWidget.addItem(self.dlcScatterItem)self.dlc0LineItem = gl.GLLinePlotItem(- pos=self.dlc_fr_mar_dim[:, 0, :],+ pos=self.dlc_fr_mar_xyz[:, 0, :],color=(0, 1, 1, 0.5),width=4.0,antialias=True,)self.dlc1LineItem = gl.GLLinePlotItem(- pos=self.dlc_fr_mar_dim[:, 1, :],+ pos=self.dlc_fr_mar_xyz[:, 1, :],color=(1, 0, 1, 0.5),width=4.0,antialias=True,)self.dlc2LineItem = gl.GLLinePlotItem(- pos=self.dlc_fr_mar_dim[:, 2, :],+ pos=self.dlc_fr_mar_xyz[:, 2, :],color=(1, 1, 0, 0.5),width=4.0,antialias=True,@@ -232,47 +232,47 @@ class PlaySkeletonWidget:# Plots the dotsself.skelScatterItem.setData(- pos=self.skel_fr_mar_dim[self.currentFrameNumber, :, :]+ pos=self.skel_fr_mar_xyz[self.currentFrameNumber, :, :])# connect the dottosif self.useOpenPose:self.headLineItem.setData(- pos=self.skel_fr_mar_dim[self.currentFrameNumber, self.head, :]+ pos=self.skel_fr_mar_xyz[self.currentFrameNumber, self.head, :])self.spineLineItem.setData(- pos=self.skel_fr_mar_dim[self.currentFrameNumber, self.spine, :]+ pos=self.skel_fr_mar_xyz[self.currentFrameNumber, self.spine, :])self.rArmLineItem.setData(- pos=self.skel_fr_mar_dim[self.currentFrameNumber, self.rArm, :]+ pos=self.skel_fr_mar_xyz[self.currentFrameNumber, self.rArm, :])self.lArmLineItem.setData(- pos=self.skel_fr_mar_dim[self.currentFrameNumber, self.lArm, :]+ pos=self.skel_fr_mar_xyz[self.currentFrameNumber, self.lArm, :])self.rLegLineItem.setData(- pos=self.skel_fr_mar_dim[self.currentFrameNumber, self.rLeg, :]+ pos=self.skel_fr_mar_xyz[self.currentFrameNumber, self.rLeg, :])self.lLegLineItem.setData(- pos=self.skel_fr_mar_dim[self.currentFrameNumber, self.lLeg, :]+ pos=self.skel_fr_mar_xyz[self.currentFrameNumber, self.lLeg, :])self.rFootLineItem.setData(- pos=self.skel_fr_mar_dim[self.currentFrameNumber, self.rFoot, :]+ pos=self.skel_fr_mar_xyz[self.currentFrameNumber, self.rFoot, :])self.lFootLineItem.setData(- pos=self.skel_fr_mar_dim[self.currentFrameNumber, self.lFoot, :]+ pos=self.skel_fr_mar_xyz[self.currentFrameNumber, self.lFoot, :])self.rHandLineItem.setData(- pos=self.skel_fr_mar_dim[self.currentFrameNumber, self.rHand, :]+ pos=self.skel_fr_mar_xyz[self.currentFrameNumber, self.rHand, :])self.lHandLineItem.setData(- pos=self.skel_fr_mar_dim[self.currentFrameNumber, self.lHand, :]+ pos=self.skel_fr_mar_xyz[self.currentFrameNumber, self.lHand, :])if self.useDLC:# Plot DeepLabCut dataself.dlcScatterItem.setData(- pos=self.dlc_fr_mar_dim[self.currentFrameNumber, :, :]+ pos=self.dlc_fr_mar_xyz[self.currentFrameNumber, :, :])if self.currentFrameNumber > self.tailLength:@@ -280,17 +280,17 @@ class PlaySkeletonWidget:self.currentFrameNumber - self.tailLength, self.currentFrameNumber)self.dlc0LineItem.setData(- pos=self.dlc_fr_mar_dim[self.tailIndexes, 0, :]+ pos=self.dlc_fr_mar_xyz[self.tailIndexes, 0, :])self.dlc1LineItem.setData(- pos=self.dlc_fr_mar_dim[self.tailIndexes, 1, :]+ pos=self.dlc_fr_mar_xyz[self.tailIndexes, 1, :])self.dlc2LineItem.setData(- pos=self.dlc_fr_mar_dim[self.tailIndexes, 2, :]+ pos=self.dlc_fr_mar_xyz[self.tailIndexes, 2, :])self.currentFrameNumber += 1- if self.currentFrameNumber >= self.skel_fr_mar_dim.shape[0]:+ if self.currentFrameNumber >= self.skel_fr_mar_xyz.shape[0]:self.currentFrameNumber = 0def animate(self):diff --git a/freemocap/fmc_runme.py b/freemocap/fmc_runme.pyindex 444a61d2..b0fe4e3e 100644--- a/freemocap/fmc_runme.py+++ b/freemocap/fmc_runme.py@@ -28,9 +28,10 @@ from ruamel.yaml import YAMLdef RunMe(sessionID=None,stage=1,useOpenPose=True,- openPoseDummyRun = False,+ runOpenPose = False,useMediaPipe=False,useDLC=False,+ runMediaPipe=False,dlcConfigPath=None,debug=False,setDataPath = False,@@ -202,7 +203,7 @@ def RunMe(sessionID=None,if stage <= 3:print()print('Starting Calibration')- sesh.cgroup, sesh.mean_charuco_fr_mar_dim = calibrate.CalibrateCaptureVolume(sesh,board, calVideoFrameLength)+ sesh.cgroup, sesh.mean_charuco_fr_mar_xyz = calibrate.CalibrateCaptureVolume(sesh,board, calVideoFrameLength)else:print('Skipping Calibration')@@ -211,17 +212,21 @@ def RunMe(sessionID=None,print('Starting Track Image Points')if sesh.useMediaPipe:- fmc_mediapipe.runMediaPipe(sesh)+ if runMediaPipe:+ fmc_mediapipe.runMediaPipe(sesh)+sesh.mediaPipeData_nCams_nFrames_nImgPts_XYC = fmc_mediapipe.parseMediaPipe(sesh)- sesh.mediaPipeSkel_fr_mar_dim = reconstruct3D.reconstruct3D(sesh,sesh.mediaPipeData_nCams_nFrames_nImgPts_XYC, confidenceThreshold=reconstructionConfidenceThreshold)- np.save(sesh.dataArrayPath/'mediaPipeSkel_3d.npy', sesh.mediaPipeSkel_fr_mar_dim) #save data to npy+ sesh.mediaPipeSkel_fr_mar_xyz, sesh.mediaPipe_reprojErr = reconstruct3D.reconstruct3D(sesh,sesh.mediaPipeData_nCams_nFrames_nImgPts_XYC, confidenceThreshold=reconstructionConfidenceThreshold)+ np.save(sesh.dataArrayPath/'mediaPipeSkel_3d.npy', sesh.mediaPipeSkel_fr_mar_xyz) #save data to npy+ np.save(sesh.dataArrayPath/'mediaPipeSkel_reprojErr.npy', sesh.mediaPipe_reprojErr) #save data to npysesh.save_session()if sesh.useOpenPose:- fmc_openpose.runOpenPose(sesh, dummyRun=openPoseDummyRun)+ fmc_openpose.runOpenPose(sesh, runOpenPose=runOpenPose)sesh.openPoseData_nCams_nFrames_nImgPts_XYC = fmc_openpose.parseOpenPose(sesh)- sesh.openPoseSkel_fr_mar_dim = reconstruct3D.reconstruct3D(sesh,sesh.openPoseData_nCams_nFrames_nImgPts_XYC, confidenceThreshold=reconstructionConfidenceThreshold)- np.save(sesh.dataArrayPath/'openPoseSkel_3d.npy', sesh.openPoseSkel_fr_mar_dim) #save data to npy+ sesh.openPoseSkel_fr_mar_xyz, sesh.openPose_reprojErr = reconstruct3D.reconstruct3D(sesh,sesh.openPoseData_nCams_nFrames_nImgPts_XYC, confidenceThreshold=reconstructionConfidenceThreshold)+ np.save(sesh.dataArrayPath/'openPoseSkel_3d.npy', sesh.openPoseSkel_fr_mar_xyz) #save data to npy+ np.save(sesh.dataArrayPath/'openPoseSkel_reprojErr.npy', sesh.openPoseSkel_reprojErr) #save data to npysesh.save_session()sesh.syncedVidList = []if sesh.useDLC:@@ -231,8 +236,9 @@ def RunMe(sessionID=None,for config_path in dlc_config_paths:dlc.analyze_videos(config_path,sesh.syncedVidList, destfolder= sesh.dlcDataPath, save_as_csv=True)sesh.dlcData_nCams_nFrames_nImgPts_XYC = fmc_deeplabcut.parseDeepLabCut(sesh, config_path)- sesh.dlc_fr_mar_dim = reconstruct3D.reconstruct3D(sesh,sesh.dlcData_nCams_nFrames_nImgPts_XYC, confidenceThreshold=reconstructionConfidenceThreshold)- np.save(sesh.dataArrayPath/'deepLabCut_3d.npy', sesh.dlc_fr_mar_dim) #save data to npy+ sesh.dlc_fr_mar_xyz, sesh.dlc_reprojErr = reconstruct3D.reconstruct3D(sesh,sesh.dlcData_nCams_nFrames_nImgPts_XYC, confidenceThreshold=reconstructionConfidenceThreshold)+ np.save(sesh.dataArrayPath/'deepLabCut_3d.npy', sesh.dlc_fr_mar_xyz) #save data to npy+ np.save(sesh.dataArrayPath/'deepLabCut_reprojErr.npy', sesh.dlc_reprojErr) #save data to npysesh.save_session()else:print('Skipping Run MediaPipe')@@ -245,9 +251,9 @@ def RunMe(sessionID=None,startFrame=0,azimuth=-90,elevation=-81,- useOpenPose=True,- useMediaPipe=False,- useDLC=False,+ useOpenPose=useOpenPose,+ useMediaPipe=useMediaPipe,+ useDLC=useDLC,recordVid = recordVid)# print ('Starting PyQT Animation')diff --git a/freemocap/play_skeleton_animation.py b/freemocap/play_skeleton_animation.pyindex 91853084..8830704e 100644--- a/freemocap/play_skeleton_animation.py+++ b/freemocap/play_skeleton_animation.py@@ -81,8 +81,13 @@ def PlaySkeletonAnimation(Called by matplotlib animator for each frame."""- skel_dottos = matplotlib_artist_objs['skel_dottos']- skel_trajectories = figure_data['skel_trajectories|mar|fr_dim']+ if useOpenPose:+ op_skel_dottos = matplotlib_artist_objs['op_skel_dottos']+ op_skel_trajectories = figure_data['openPose_skel_trajectories|mar|fr_dim']++ if useMediaPipe:+ mp_skel_dottos = matplotlib_artist_objs['mp_skel_dottos']+ mp_skel_trajectories = figure_data['mediaPipe_skel_trajectories|mar|fr_dim']charuco_dottos = matplotlib_artist_objs['charuco_dottos']charuco_trajectories = figure_data['charuco_trajectories|mar|fr_dim']@@ -92,34 +97,48 @@ def PlaySkeletonAnimation(dlc_trajectories = figure_data['dlc_trajectories|mar|fr_dim']#function to update the lines for each 3d body segment- def update_3d_skel_segments(key, data_fr_mar_dim):+ def update_3d_skel_segments(key, data_fr_mar_xyz):"""updates the Artist of each body segment with the current frame data."""- try:+ split = key.split('_')++ if split[0] == 'op':dict_of_segments_idxs = dict_of_openPoseSegmentIdx_dicts[key]- except:+ if split[0] == 'mp':+ dict_of_segments_idxs = dict_of_mediaPipeSegmentIdx_dicts[key]+ elif split[0] == 'dlc':dict_of_segments_idxs = dict_of_dlcSegmentIdx_dicts[key]- dict_of_artists = matplotlib_artist_objs[key]+ try:+ dict_of_artists = matplotlib_artist_objs[key]+ except:+ print('KeyError:', key)+for thisItem in dict_of_segments_idxs.items():segName = thisItem[0]segArtist = dict_of_artists[segName]- segArtist.set_data((data_fr_mar_dim[frameNum, dict_of_segments_idxs[segName], 0], data_fr_mar_dim[frameNum, dict_of_segments_idxs[segName], 1]))- segArtist.set_3d_properties(data_fr_mar_dim[frameNum, dict_of_segments_idxs[segName], 2])-- update_3d_skel_segments('body', skel_fr_mar_dim)- update_3d_skel_segments('rHand', skel_fr_mar_dim)- update_3d_skel_segments('lHand', skel_fr_mar_dim)- update_3d_skel_segments('face', skel_fr_mar_dim)+ segArtist.set_data((data_fr_mar_xyz[frameNum, dict_of_segments_idxs[segName], 0], data_fr_mar_xyz[frameNum, dict_of_segments_idxs[segName], 1]))+ segArtist.set_3d_properties(data_fr_mar_xyz[frameNum, dict_of_segments_idxs[segName], 2])++ if useOpenPose:+ update_3d_skel_segments('op_body', openPose_skel_fr_mar_xyz)+ update_3d_skel_segments('op_rHand', openPose_skel_fr_mar_xyz)+ update_3d_skel_segments('op_lHand', openPose_skel_fr_mar_xyz)+ update_3d_skel_segments('op_face', openPose_skel_fr_mar_xyz)++ if useMediaPipe:+ update_3d_skel_segments('mp_body', mediaPipe_skel_fr_mar_xyz)+ update_3d_skel_segments('mp_rHand', mediaPipe_skel_fr_mar_xyz)+ update_3d_skel_segments('mp_lHand', mediaPipe_skel_fr_mar_xyz)if useDLC:- update_3d_skel_segments('pinkBall', dlc_fr_mar_dim)- update_3d_skel_segments('redBall', dlc_fr_mar_dim)- update_3d_skel_segments('greenBall', dlc_fr_mar_dim)- update_3d_skel_segments('wobbleBoard', dlc_fr_mar_dim)- update_3d_skel_segments('wobbleWheel', dlc_fr_mar_dim)+ update_3d_skel_segments('dlc_pinkBall', dlc_fr_mar_xyz)+ update_3d_skel_segments('dlc_redBall', dlc_fr_mar_xyz)+ update_3d_skel_segments('dlc_greenBall', dlc_fr_mar_xyz)+ update_3d_skel_segments('dlc_wobbleBoard', dlc_fr_mar_xyz)+ update_3d_skel_segments('dlc_wobbleWheel', dlc_fr_mar_xyz)for bb in range(3):thisTailArtist = matplotlib_artist_objs['ball_tails'][bb]@@ -138,15 +157,26 @@ def PlaySkeletonAnimation(#Plots the dots!#openpose data- marNum = -1- for thisSkelDotto, thisTraj in zip(skel_dottos,skel_trajectories):- marNum+=1- # NOTE: there is no .set_data() for 3 dim data...- thisSkelDotto.set_data(thisTraj[ frameNum, 0:2])- thisSkelDotto.set_3d_properties(thisTraj[ frameNum, 2])-- #charuco data+ if useOpenPose:+ marNum = -1+ for thisSkelDotto, thisTraj in zip(op_skel_dottos,op_skel_trajectories):+ marNum+=1+ # NOTE: there is no .set_data() for 3 dim data...+ thisSkelDotto.set_data(thisTraj[ frameNum, 0:2])+ thisSkelDotto.set_3d_properties(thisTraj[ frameNum, 2])+ #mediapipe data+ if useMediaPipe:+ marNum = -1+ for thisSkelDotto, thisTraj in zip(mp_skel_dottos,mp_skel_trajectories):+ marNum+=1+ # NOTE: there is no .set_data() for 3 dim data...+ thisSkelDotto.set_data(thisTraj[ frameNum, 0:2])+ thisSkelDotto.set_3d_properties(thisTraj[ frameNum, 2])++++ #charuco datafor thisDotto, thisTraj in zip(charuco_dottos,charuco_trajectories):if frameNum<charuco_trajectories[0].shape[0]:@@ -163,45 +193,59 @@ def PlaySkeletonAnimation("""updates the Artist of each body segment with the current frame data."""- dict_of_segments_idxs = dict_of_openPoseSegmentIdx_dicts[key]- dict_of_artists = thisVidOpenPoseArtist_dict[key]+ split = key.split('_')+ if split[0] == 'op':+ dict_of_segments_idxs = dict_of_openPoseSegmentIdx_dicts[key]+ dict_of_artists = thisVidOpenPoseArtist_dict[key]+ elif split[0] == 'mp':+ dict_of_segments_idxs = dict_of_mediaPipeSegmentIdx_dicts[key]+ dict_of_artists = thisVidMediaPipeArtist_dict[key]for thisItem in dict_of_segments_idxs.items():segName = thisItem[0]segArtist = dict_of_artists[segName]-- xData = openPoseData_nCams_nFrames_nImgPts_XYC[vidNum, frameNum, dict_of_segments_idxs[segName],0]- yData = openPoseData_nCams_nFrames_nImgPts_XYC[vidNum, frameNum, dict_of_segments_idxs[segName],1]++ if split[0] == 'op':+ xData = openPose_nCams_nFrames_nImgPts_XYC[vidNum, frameNum, dict_of_segments_idxs[segName],0]+ yData = openPose_nCams_nFrames_nImgPts_XYC[vidNum, frameNum, dict_of_segments_idxs[segName],1]+ elif split[0] == 'mp':+ xData = mediaPipe_nCams_nFrames_nImgPts_XYC[vidNum, frameNum, dict_of_segments_idxs[segName],0]+ yData = mediaPipe_nCams_nFrames_nImgPts_XYC[vidNum, frameNum, dict_of_segments_idxs[segName],1]xDataMasked = np.ma.masked_array(xData, mask=(xData==0))yDataMasked = np.ma.masked_array(yData, mask=(xData==0))segArtist.set_data(xDataMasked,yDataMasked)-- vidNum = -1- # for thisVidArtist, thisVidDLCArtist, thisVidOpenPoseArtist_dict in zip(vidAristList, vidDLCArtistList, list_of_vidOpenPoseArtistdicts): #JSM NOTE - This for loop needs to be fixed to work when DLC is true- for thisVidArtist, thisVidOpenPoseArtist_dict in zip(vidAristList, list_of_vidOpenPoseArtistdicts):- vidNum +=1+ #update video frames+ for vidNum, thisVidArtist in enumerate(vidAristList):vidCapObjList[vidNum].set(cv2.CAP_PROP_POS_FRAMES, frameNum)success, image = vidCapObjList[vidNum].read()+if success:thisVidArtist.set_array(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))else:vidCapObjList[vidNum].set(cv2.CAP_PROP_POS_FRAMES, 0)success, image = vidCapObjList[vidNum].read()- if useDLC:+ if useOpenPose:+ thisVidOpenPoseArtist_dict = list_of_vidOpenPoseArtistdicts[vidNum]+ update_2d_skel_segments(vidNum,'op_body')+ update_2d_skel_segments(vidNum,'op_rHand')+ update_2d_skel_segments(vidNum,'op_lHand')+ update_2d_skel_segments(vidNum,'op_face')++ if useMediaPipe:+ thisVidMediaPipeArtist_dict = list_of_vidMediaPipeArtistdicts[vidNum]+ update_2d_skel_segments(vidNum,'mp_body')+ update_2d_skel_segments(vidNum,'mp_rHand')+ update_2d_skel_segments(vidNum,'mp_lHand')++ if useDLC: #JSM NOTE - this one won't work, we'll need to fix it the next time we get DLC datathisVidDLCArtist[0].set_data( dlcData_nCams_nFrames_nImgPts_XYC[vidNum,frameNum,:,0],dlcData_nCams_nFrames_nImgPts_XYC[vidNum,frameNum,:,1],)- update_2d_skel_segments(vidNum,'body')- update_2d_skel_segments(vidNum,'rHand')- update_2d_skel_segments(vidNum,'lHand')- update_2d_skel_segments(vidNum,'face')- # thisVidOpenPoseArtist_dict[0].set_data( openPoseData_nCams_nFrames_nImgPts_XYC[vidNum,frameNum,:,0],- # openPoseData_nCams_nFrames_nImgPts_XYC[vidNum,frameNum,:,1],)#shift timeseries xlims along with the frameNumxTimeSeriesAx.set_xlim([(frameNum/fps)-timeRange, (frameNum/fps)+timeRange])@@ -255,60 +299,83 @@ def PlaySkeletonAnimation(ax3d.tick_params(length=0) # WHY DOESNT THIS WORK? I HATE THOSE TICK MARKS >:(- try:- skel_fr_mar_dim = np.load(session.dataArrayPath / 'openPoseSkel_3d.npy')- openPoseData_nCams_nFrames_nImgPts_XYC = np.load(session.dataArrayPath / 'openPoseData_2d.npy')- except:- console.warning('No openPose data found. This iteration requires OpenPose data')+ smoothData=True+ smoothWinLength = 5+ smoothOrder = 3++ if useOpenPose:+ try:+ openPose_skel_fr_mar_xyz = np.load(session.dataArrayPath / 'openPoseSkel_3d.npy')+ openPose_nCams_nFrames_nImgPts_XYC = np.load(session.dataArrayPath / 'openPoseData_2d.npy')+ except:+ print('No openPose data found.')++ if smoothData:+ for dim in range(openPose_skel_fr_mar_xyz.shape[2]):+ for mm in range(openPose_skel_fr_mar_xyz.shape[1]):+ openPose_skel_fr_mar_xyz[:,mm,dim] = savgol_filter(openPose_skel_fr_mar_xyz[:,mm,dim], smoothWinLength, smoothOrder)++ if useMediaPipe:+ try:+ mediaPipe_skel_fr_mar_xyz = np.load(session.dataArrayPath / 'mediaPipeSkel_3d.npy')+ mediaPipe_nCams_nFrames_nImgPts_XYC = np.load(session.dataArrayPath / 'mediaPipe_2d.npy')+ except:+ print('No mediaPipe data found.')++ if smoothData:+ for dim in range(mediaPipe_skel_fr_mar_xyz.shape[2]):+ for mm in range(mediaPipe_skel_fr_mar_xyz.shape[1]):+ mediaPipe_skel_fr_mar_xyz[:,mm,dim] = savgol_filter(mediaPipe_skel_fr_mar_xyz[:,mm,dim], smoothWinLength, smoothOrder)try:- charuco_fr_mar_dim = np.load(session.dataArrayPath / 'charuco_3d_points.npy')+ charuco_fr_mar_xyz = np.load(session.dataArrayPath / 'charuco_3d_points.npy')except:console.warning('No charuco data found')- smoothData=True- if smoothData:- # smoothThese = np.arange(67, skel_fr_mar_dim.shape[1])- smoothThese = np.arange(0, skel_fr_mar_dim.shape[1])- for mm in smoothThese:- if mm > 24 and mm < 67: #don't smooth the hands, or they disappear! :O- pass- else:- for dim in range(skel_fr_mar_dim.shape[2]):- skel_fr_mar_dim[:,mm,dim] = savgol_filter(skel_fr_mar_dim[:,mm,dim], 5, 3)++figure_data = dict()+ if useOpenPose:+ figure_data['openPose_skel_trajectories|mar|fr_dim'] = [openPose_skel_fr_mar_xyz[:,markerNum,:] for markerNum in range(openPose_skel_fr_mar_xyz.shape[1])]+ figure_data['openPose_skel_fr_mar_xyz'] = openPose_skel_fr_mar_xyz++ dict_of_openPoseSegmentIdx_dicts, dict_of_op_skel_lineColor = formatOpenPoseStickIndices() #these will help us draw body and hands stick figures- skel_trajectories = [skel_fr_mar_dim[:,markerNum,:] for markerNum in range(skel_fr_mar_dim.shape[1])]- figure_data['skel_trajectories|mar|fr_dim'] = skel_trajectories- figure_data['skel_fr_mar_dim'] = skel_fr_mar_dim- charuco_trajectories = [charuco_fr_mar_dim[:,markerNum,:] for markerNum in range(charuco_fr_mar_dim.shape[1])]+ if useMediaPipe:+ mediaPipe_trajectories = [mediaPipe_skel_fr_mar_xyz[:,markerNum,:] for markerNum in range(mediaPipe_skel_fr_mar_xyz.shape[1])]+ figure_data['mediaPipe_skel_trajectories|mar|fr_dim'] = [mediaPipe_skel_fr_mar_xyz[:,markerNum,:] for markerNum in range(mediaPipe_skel_fr_mar_xyz.shape[1])]+ figure_data['mediaPipe_skel_fr_mar_xyz'] = mediaPipe_skel_fr_mar_xyz++ dict_of_mediaPipeSegmentIdx_dicts, dict_of_mp_skel_lineColor = formatMediaPipeStickIndices() #these will help us draw body and hands stick figures+++ charuco_trajectories = [charuco_fr_mar_xyz[:,markerNum,:] for markerNum in range(charuco_fr_mar_xyz.shape[1])]figure_data['charuco_trajectories|mar|fr_dim'] = charuco_trajectories- figure_data['charuco_fr_mar_dim'] = charuco_fr_mar_dim+ figure_data['charuco_fr_mar_xyz'] = charuco_fr_mar_xyz- dict_of_openPoseSegmentIdx_dicts, dict_of_skel_lineColor = formatOpenPoseStickIndices() #these will help us draw body and hands stick figuresif useDLC:- dlc_fr_mar_dim = np.load(session.dataArrayPath / "deepLabCut_3d.npy")+ dlc_fr_mar_xyz = np.load(session.dataArrayPath / "deepLabCut_3d.npy")dlcData_nCams_nFrames_nImgPts_XYC = np.load(session.dataArrayPath / "deepLabCutData_2d.npy")- for mm in range(dlc_fr_mar_dim.shape[1]):- for dim in range(dlc_fr_mar_dim.shape[2]):- dlc_fr_mar_dim[:,mm,dim] = savgol_filter(dlc_fr_mar_dim[:,mm,dim], 11, 3)+ for mm in range(dlc_fr_mar_xyz.shape[1]):+ for dim in range(dlc_fr_mar_xyz.shape[2]):+ dlc_fr_mar_xyz[:,mm,dim] = savgol_filter(dlc_fr_mar_xyz[:,mm,dim], 11, 3)ballTailLen = 6if ballTailLen > startFrame:startFrame = ballTailLen+1 #gotta do this, or the tail will index negative frames (without requiring annoying checks later)- dlc_trajectories = [dlc_fr_mar_dim[:,markerNum,:] for markerNum in range(dlc_fr_mar_dim.shape[1])]+ dlc_trajectories = [dlc_fr_mar_xyz[:,markerNum,:] for markerNum in range(dlc_fr_mar_xyz.shape[1])]figure_data['dlc_trajectories|mar|fr_dim'] = dlc_trajectories- figure_data['dlc_fr_mar_dim'] = dlc_fr_mar_dim+ figure_data['dlc_fr_mar_xyz'] = dlc_fr_mar_xyz- def build_3d_segment_artist_dict(data_fr_mar_dim,+ def build_3d_segment_artist_dict(data_fr_mar_xyz,dict_of_list_of_segment_idxs,segColor = 'k',lineWidth = 1,@@ -356,9 +423,9 @@ def PlaySkeletonAnimation(dict_of_artist_objects[segName] = ax3d.plot(- data_fr_mar_dim[startFrame, idxs ,0],- data_fr_mar_dim[startFrame, idxs ,1],- data_fr_mar_dim[startFrame, idxs ,2],+ data_fr_mar_xyz[startFrame, idxs ,0],+ data_fr_mar_xyz[startFrame, idxs ,1],+ data_fr_mar_xyz[startFrame, idxs ,2],linestyle=lineStyle,linewidth=lineWidth,markerSize = marSize,@@ -371,11 +438,18 @@ def PlaySkeletonAnimation(matplotlib_artist_objs = dict()- matplotlib_artist_objs['body'] = build_3d_segment_artist_dict(skel_fr_mar_dim, dict_of_openPoseSegmentIdx_dicts['body'], segColor = dict_of_skel_lineColor)- matplotlib_artist_objs['rHand'] = build_3d_segment_artist_dict(skel_fr_mar_dim, dict_of_openPoseSegmentIdx_dicts['rHand'], segColor=np.append(humon_red, 1), markerType='.', markerEdgeColor = humon_red, lineWidth=1, marSize = 2)- matplotlib_artist_objs['lHand'] = build_3d_segment_artist_dict(skel_fr_mar_dim, dict_of_openPoseSegmentIdx_dicts['lHand'], segColor=np.append(humon_blue, 1), markerType='.', markerEdgeColor = humon_blue, lineWidth=1, marSize = 2)- matplotlib_artist_objs['face'] = build_3d_segment_artist_dict(skel_fr_mar_dim, dict_of_openPoseSegmentIdx_dicts['face'], segColor='k', lineWidth=.5)-++ if useOpenPose:+ matplotlib_artist_objs['op_body'] = build_3d_segment_artist_dict(openPose_skel_fr_mar_xyz, dict_of_openPoseSegmentIdx_dicts['op_body'], segColor = dict_of_op_skel_lineColor)+ matplotlib_artist_objs['op_rHand'] = build_3d_segment_artist_dict(openPose_skel_fr_mar_xyz, dict_of_openPoseSegmentIdx_dicts['op_rHand'], segColor=np.append(humon_red, 1), markerType='.', markerEdgeColor = humon_red, lineWidth=1, marSize = 2)+ matplotlib_artist_objs['op_lHand'] = build_3d_segment_artist_dict(openPose_skel_fr_mar_xyz, dict_of_openPoseSegmentIdx_dicts['op_lHand'], segColor=np.append(humon_blue, 1), markerType='.', markerEdgeColor = humon_blue, lineWidth=1, marSize = 2)+ matplotlib_artist_objs['op_face'] = build_3d_segment_artist_dict(openPose_skel_fr_mar_xyz, dict_of_openPoseSegmentIdx_dicts['op_face'], segColor='k', lineWidth=.5)++ if useMediaPipe:+ matplotlib_artist_objs['mp_body'] = build_3d_segment_artist_dict(mediaPipe_skel_fr_mar_xyz, dict_of_mediaPipeSegmentIdx_dicts['mp_body'], segColor = dict_of_mp_skel_lineColor)+ matplotlib_artist_objs['mp_rHand'] = build_3d_segment_artist_dict(mediaPipe_skel_fr_mar_xyz, dict_of_mediaPipeSegmentIdx_dicts['mp_rHand'], segColor=np.append(humon_red, 1), markerType='.', markerEdgeColor = humon_red, lineWidth=1, marSize = 2)+ matplotlib_artist_objs['mp_lHand'] = build_3d_segment_artist_dict(mediaPipe_skel_fr_mar_xyz, dict_of_mediaPipeSegmentIdx_dicts['mp_lHand'], segColor=np.append(humon_blue, 1), markerType='.', markerEdgeColor = humon_blue, lineWidth=1, marSize = 2)+if useDLC:dict_of_dlcSegmentIdx_dicts = dict()dict_of_dlcSegmentIdx_dicts['pinkBall'] = {'pinkBall': [0]}@@ -386,30 +460,34 @@ def PlaySkeletonAnimation(ballColor = ['darkviolet', 'forestgreen', 'xkcd:goldenrod']- matplotlib_artist_objs['pinkBall'] = build_3d_segment_artist_dict(dlc_fr_mar_dim, dict_of_dlcSegmentIdx_dicts['pinkBall'], segColor=ballColor[0], markerType = 'o', marSize = 6, markerEdgeColor = 'indigo')- matplotlib_artist_objs['redBall'] = build_3d_segment_artist_dict(dlc_fr_mar_dim, dict_of_dlcSegmentIdx_dicts['redBall'], segColor=ballColor[2], markerType = 'o', marSize = 6, markerEdgeColor ='orangered')- matplotlib_artist_objs['greenBall'] = build_3d_segment_artist_dict(dlc_fr_mar_dim, dict_of_dlcSegmentIdx_dicts['greenBall'], segColor=ballColor[1], markerType = 'o', marSize = 6, markerEdgeColor ='darkgreen')-- matplotlib_artist_objs['wobbleBoard'] = build_3d_segment_artist_dict(dlc_fr_mar_dim, dict_of_dlcSegmentIdx_dicts['wobbleBoard'], segColor='k', lineWidth=1)- matplotlib_artist_objs['wobbleWheel'] = build_3d_segment_artist_dict(dlc_fr_mar_dim, dict_of_dlcSegmentIdx_dicts['wobbleWheel'], segColor='k', lineWidth=1)-- skel_dottos = []- for mm in range(67): #getcher dottos off my face!- thisTraj = skel_fr_mar_dim[:, mm, :]- if mm==15:- col = 'r'- markerSize = 2- elif mm == 16:- col = 'b'- markerSize = 2- else:- col = 'k'- markerSize = 1-- thisDotto =ax3d.plot(thisTraj[0, 0:1], thisTraj[1, 0:1], thisTraj[2, 0:1][0],'.', markersize=markerSize, color = col)- skel_dottos.append(thisDotto[0])-- matplotlib_artist_objs['skel_dottos'] = skel_dottos+ matplotlib_artist_objs['pinkBall'] = build_3d_segment_artist_dict(dlc_fr_mar_xyz, dict_of_dlcSegmentIdx_dicts['pinkBall'], segColor=ballColor[0], markerType = 'o', marSize = 6, markerEdgeColor = 'indigo')+ matplotlib_artist_objs['redBall'] = build_3d_segment_artist_dict(dlc_fr_mar_xyz, dict_of_dlcSegmentIdx_dicts['redBall'], segColor=ballColor[2], markerType = 'o', marSize = 6, markerEdgeColor ='orangered')+ matplotlib_artist_objs['greenBall'] = build_3d_segment_artist_dict(dlc_fr_mar_xyz, dict_of_dlcSegmentIdx_dicts['greenBall'], segColor=ballColor[1], markerType = 'o', marSize = 6, markerEdgeColor ='darkgreen')++ matplotlib_artist_objs['wobbleBoard'] = build_3d_segment_artist_dict(dlc_fr_mar_xyz, dict_of_dlcSegmentIdx_dicts['wobbleBoard'], segColor='k', lineWidth=1)+ matplotlib_artist_objs['wobbleWheel'] = build_3d_segment_artist_dict(dlc_fr_mar_xyz, dict_of_dlcSegmentIdx_dicts['wobbleWheel'], segColor='k', lineWidth=1)++ if useOpenPose:+ op_skel_dottos = []+ for mm in range(67): #getcher dottos off my face!+ thisTraj = openPose_skel_fr_mar_xyz[:, mm, :]+ if mm==15:+ col = 'r'+ markerSize = 2+ elif mm == 16:+ col = 'b'+ markerSize = 2+ else:+ col = 'k'+ markerSize = 1++ thisDotto =ax3d.plot(thisTraj[0, 0:1], thisTraj[1, 0:1], thisTraj[2, 0:1][0],'.', markersize=markerSize, color = col)+ op_skel_dottos.append(thisDotto[0])++ matplotlib_artist_objs['op_skel_dottos'] = op_skel_dottos++ if useMediaPipe:+ matplotlib_artist_objs['mp_skel_dottos'] = [ax3d.plot(thisTraj[0, 0:1], thisTraj[1, 0:1], thisTraj[2, 0:1],'m.', markersize=1)[0] for thisTraj in mediaPipe_trajectories]matplotlib_artist_objs['charuco_dottos'] = [ax3d.plot(thisTraj[0, 0:1], thisTraj[1, 0:1], thisTraj[2, 0:1],'m.', markersize=1)[0] for thisTraj in charuco_trajectories]@@ -424,22 +502,26 @@ def PlaySkeletonAnimation('-', color=ballColor[bb])[0] for bb in range(3)]- numFrames = skel_fr_mar_dim.shape[0]-+ if useOpenPose:+ numFrames = openPose_skel_fr_mar_xyz.shape[0]+ mx = np.nanmean(openPose_skel_fr_mar_xyz[int(numFrames/2),:,0])+ my = np.nanmean(openPose_skel_fr_mar_xyz[int(numFrames/2),:,1])+ mz = np.nanmean(openPose_skel_fr_mar_xyz[int(numFrames/2),:,2])+ if useMediaPipe:+ numFrames = mediaPipe_skel_fr_mar_xyz.shape[0]+ mx = np.nanmean(mediaPipe_skel_fr_mar_xyz[int(numFrames/2),:,0])+ my = np.nanmean(mediaPipe_skel_fr_mar_xyz[int(numFrames/2),:,1])+ mz = np.nanmean(mediaPipe_skel_fr_mar_xyz[int(numFrames/2),:,2])-- mx = np.nanmean(skel_fr_mar_dim[int(numFrames/2),:,0])- my = np.nanmean(skel_fr_mar_dim[int(numFrames/2),:,1])- mz = np.nanmean(skel_fr_mar_dim[int(numFrames/2),:,2])-+# groundX = np.arange(mx-100,mx+100)# groundZ = np.arange(my-100,my+100)# groundXX, groundZZ = np.meshgrid(groundX, groundZ)# groundYY = np.zeros_like(groundXX)# groundMesh = ax3d.plot_surface(groundXX, groundZZ, groundYY, color='k', alpha=.5)- axRange = 300#session.board.square_length * 10+ axRange = 500#session.board.square_length * 10# Setting the axes propertiesax3d.set_xlim3d([mx-axRange, mx+axRange])@@ -476,6 +558,7 @@ def PlaySkeletonAnimation(vidCapObjList = []list_of_vidOpenPoseArtistdicts = []+ list_of_vidMediaPipeArtistdicts = []if useDLC: vidDLCArtistList = []@@ -570,12 +653,20 @@ def PlaySkeletonAnimation(if useOpenPose:vidOpenPoseArtist_dict = dict()- vidOpenPoseArtist_dict['body'] = build_2d_segment_artist_dict(thisVidNum, openPoseData_nCams_nFrames_nImgPts_XYC, dict_of_openPoseSegmentIdx_dicts['body'], segColor = dict_of_skel_lineColor)- vidOpenPoseArtist_dict['rHand'] = build_2d_segment_artist_dict(thisVidNum, openPoseData_nCams_nFrames_nImgPts_XYC, dict_of_openPoseSegmentIdx_dicts['rHand'], segColor=np.append(humon_red, .75), lineWidth=.5)- vidOpenPoseArtist_dict['lHand'] = build_2d_segment_artist_dict(thisVidNum, openPoseData_nCams_nFrames_nImgPts_XYC, dict_of_openPoseSegmentIdx_dicts['lHand'], segColor=np.append(humon_blue, .75), lineWidth=.5)- vidOpenPoseArtist_dict['face'] = build_2d_segment_artist_dict(thisVidNum, openPoseData_nCams_nFrames_nImgPts_XYC, dict_of_openPoseSegmentIdx_dicts['face'], segColor = np.array([1.,1.,1.,1]), lineWidth=.25)+ vidOpenPoseArtist_dict['op_body'] = build_2d_segment_artist_dict(thisVidNum, openPose_nCams_nFrames_nImgPts_XYC, dict_of_openPoseSegmentIdx_dicts['op_body'], segColor = dict_of_op_skel_lineColor)+ vidOpenPoseArtist_dict['op_rHand'] = build_2d_segment_artist_dict(thisVidNum, openPose_nCams_nFrames_nImgPts_XYC, dict_of_openPoseSegmentIdx_dicts['op_rHand'], segColor=np.append(humon_red, .75), lineWidth=.5)+ vidOpenPoseArtist_dict['op_lHand'] = build_2d_segment_artist_dict(thisVidNum, openPose_nCams_nFrames_nImgPts_XYC, dict_of_openPoseSegmentIdx_dicts['op_lHand'], segColor=np.append(humon_blue, .75), lineWidth=.5)+ vidOpenPoseArtist_dict['op_face'] = build_2d_segment_artist_dict(thisVidNum, openPose_nCams_nFrames_nImgPts_XYC, dict_of_openPoseSegmentIdx_dicts['op_face'], segColor = np.array([1.,1.,1.,1]), lineWidth=.25)list_of_vidOpenPoseArtistdicts.append(vidOpenPoseArtist_dict)+ if useMediaPipe:+ vidMediaPipeArtist_dict = dict()+ vidMediaPipeArtist_dict['mp_body'] = build_2d_segment_artist_dict(thisVidNum, mediaPipe_nCams_nFrames_nImgPts_XYC, dict_of_mediaPipeSegmentIdx_dicts['mp_body'], segColor = 'g')+ vidMediaPipeArtist_dict['mp_rHand'] = build_2d_segment_artist_dict(thisVidNum, mediaPipe_nCams_nFrames_nImgPts_XYC, dict_of_mediaPipeSegmentIdx_dicts['mp_rHand'], segColor=np.append(humon_red, .75), lineWidth=.5)+ vidMediaPipeArtist_dict['mp_lHand'] = build_2d_segment_artist_dict(thisVidNum, mediaPipe_nCams_nFrames_nImgPts_XYC, dict_of_mediaPipeSegmentIdx_dicts['mp_lHand'], segColor=np.append(humon_blue, .75), lineWidth=.5)+ list_of_vidMediaPipeArtistdicts.append(vidMediaPipeArtist_dict)++if useDLC:vidDLCArtistList.append(thisVidAxis.plot(@@ -586,6 +677,7 @@ def PlaySkeletonAnimation(markersize = 1,color='w',markerfacecolor='none' ))+######### βββ βββ βββββ ββ ββ βββββββ ββ ββ βββ ββ βββββββ ββββββ ββ ββββββ ββββββββ βββββ ββ ββ βββββββ βββββββ@@ -599,11 +691,17 @@ def PlaySkeletonAnimation(fps=30 #NOTE - This should be saved in the session Class some howtime = np.arange(0, numFrames)/fps-- rHandX = skel_trajectories[4][:,0]/1000 #convert mm to meters- rHandY = -skel_trajectories[4][:,1]/1000- lHandX = skel_trajectories[7][:,0]/1000- lHandY = -skel_trajectories[7][:,1]/1000+ if useMediaPipe:+ rHandX = mediaPipe_skel_fr_mar_xyz[:,4,0]/1000 #convert mm to meters+ rHandY = -mediaPipe_skel_fr_mar_xyz[:,4,1]/1000+ lHandX = mediaPipe_skel_fr_mar_xyz[:,7,0]/1000+ lHandY = -mediaPipe_skel_fr_mar_xyz[:,7,1]/1000++ if useOpenPose:+ rHandX = openPose_skel_fr_mar_xyz[:,4,0]/1000 #convert mm to meters+ rHandY = -openPose_skel_fr_mar_xyz[:,4,1]/1000+ lHandX = openPose_skel_fr_mar_xyz[:,7,0]/1000+ lHandY = -openPose_skel_fr_mar_xyz[:,7,1]/1000linePlotWidth = .45@@ -745,29 +843,29 @@ def formatOpenPoseStickIndices():returns:openPoseBodySegmentIds= a dictionary of arrays containing indices of individual body segments (Note, a lot of markerless mocap comp sci types like to say 'pose' instead of 'body'. They also use 'pose' to refer to camera 6 DoF position sometimes. Comp sci is frustrating like that lol)- openPoseHandIds = a dictionary of arrays containing indices of individual hand segments, along with offset to know where to start in the 'skel_fr_mar_dim.shape[1]' part of the array- dict_of_skel_lineColor = a dictionary of arrays, each containing the color (RGBA) to use for a given body segment+ openPoseHandIds = a dictionary of arrays containing indices of individual hand segments, along with offset to know where to start in the 'skel_fr_mar_xyz.shape[1]' part of the array+ dict_of_op_skel_lineColor = a dictionary of arrays, each containing the color (RGBA) to use for a given body segment"""dict_of_openPoseSegmentIdx_dicts = dict()#make body dictionaryopenPoseBodySegmentIds = dict()- openPoseBodySegmentIds['head'] = [17, 15, 0, 1,0, 16, 18, ]- openPoseBodySegmentIds['spine'] = [1,8,5,1, 2, 12, 8, 9, 5, 1, 2, 8]- openPoseBodySegmentIds['rArm'] = [1, 2, 3, 4, ]- openPoseBodySegmentIds['lArm'] = [1, 5, 6, 7, ]- openPoseBodySegmentIds['rLeg'] = [8, 9, 10, 11, 22, 23, 11, 24, ]- openPoseBodySegmentIds['lLeg'] = [8,12, 13, 14, 19, 20, 14, 21,]- dict_of_openPoseSegmentIdx_dicts['body'] = openPoseBodySegmentIds+ openPoseBodySegmentIds['op_head'] = [17, 15, 0, 1,0, 16, 18, ]+ openPoseBodySegmentIds['op_spine'] = [1,8,5,1, 2, 12, 8, 9, 5, 1, 2, 8]+ openPoseBodySegmentIds['op_rArm'] = [1, 2, 3, 4, ]+ openPoseBodySegmentIds['op_lArm'] = [1, 5, 6, 7, ]+ openPoseBodySegmentIds['op_rLeg'] = [8, 9, 10, 11, 22, 23, 11, 24, ]+ openPoseBodySegmentIds['op_lLeg'] = [8,12, 13, 14, 19, 20, 14, 21,]+ dict_of_openPoseSegmentIdx_dicts['op_body'] = openPoseBodySegmentIds- dict_of_skel_lineColor = dict()- dict_of_skel_lineColor['head'] = np.append(humon_dark, 0.5)- dict_of_skel_lineColor['spine'] = np.append(humon_dark, 1)- dict_of_skel_lineColor['rArm'] = np.append(humon_red, 1)- dict_of_skel_lineColor['lArm'] = np.append(humon_blue, 1)- dict_of_skel_lineColor['rLeg'] = np.append(humon_red, 1)- dict_of_skel_lineColor['lLeg'] = np.append(humon_blue, 1)+ dict_of_op_skel_lineColor = dict()+ dict_of_op_skel_lineColor['op_head'] = np.append(humon_dark, 0.5)+ dict_of_op_skel_lineColor['op_spine'] = np.append(humon_dark, 1)+ dict_of_op_skel_lineColor['op_rArm'] = np.append(humon_red, 1)+ dict_of_op_skel_lineColor['op_lArm'] = np.append(humon_blue, 1)+ dict_of_op_skel_lineColor['op_rLeg'] = np.append(humon_red, 1)+ dict_of_op_skel_lineColor['op_lLeg'] = np.append(humon_blue, 1)# Make some handy maps ;D@@ -775,11 +873,11 @@ def formatOpenPoseStickIndices():rHandIDstart = 25lHandIDstart = rHandIDstart + 21- openPoseHandIds['thumb'] = np.array([0, 1, 2, 3, 4, ])- openPoseHandIds['index'] = np.array([0, 5, 6, 7, 8, ])- openPoseHandIds['bird']= np.array([0, 9, 10, 11, 12, ])- openPoseHandIds['ring']= np.array([0, 13, 14, 15, 16, ])- openPoseHandIds['pinky'] = np.array([0, 17, 18, 19, 20, ])+ openPoseHandIds['op_thumb'] = np.array([0, 1, 2, 3, 4, ])+ openPoseHandIds['op_index'] = np.array([0, 5, 6, 7, 8, ])+ openPoseHandIds['op_bird']= np.array([0, 9, 10, 11, 12, ])+ openPoseHandIds['op_ring']= np.array([0, 13, 14, 15, 16, ])+ openPoseHandIds['op_pinky'] = np.array([0, 17, 18, 19, 20, ])rHand_dict = copy.deepcopy(openPoseHandIds.copy()) #copy.deepcopy() is necessary to make sure the dicts are independent of each other@@ -789,30 +887,114 @@ def formatOpenPoseStickIndices():rHand_dict[key] += rHandIDstartlHand_dict[key] += lHandIDstart- dict_of_openPoseSegmentIdx_dicts['rHand'] = rHand_dict- dict_of_openPoseSegmentIdx_dicts['lHand'] = lHand_dict+ dict_of_openPoseSegmentIdx_dicts['op_rHand'] = rHand_dict+ dict_of_openPoseSegmentIdx_dicts['op_lHand'] = lHand_dict#how to face --> :D <--openPoseFaceIDs = dict()faceIDStart = 67#define face parts- openPoseFaceIDs['jaw'] = np.arange(0,16) + faceIDStart- openPoseFaceIDs['rBrow'] = np.arange(17,21) + faceIDStart- openPoseFaceIDs['lBrow'] = np.arange(22,26) + faceIDStart- openPoseFaceIDs['noseRidge'] = np.arange(27,30) + faceIDStart- openPoseFaceIDs['noseBot'] = np.arange(31,35) + faceIDStart- openPoseFaceIDs['rEye'] = np.concatenate((np.arange(36,41), [36])) + faceIDStart- openPoseFaceIDs['lEye'] = np.concatenate((np.arange(42,47), [42])) + faceIDStart- openPoseFaceIDs['upperLip'] = np.concatenate((np.arange(48,54), np.flip(np.arange(60, 64)), [48])) + faceIDStart- openPoseFaceIDs['lowerLip'] = np.concatenate(([60], np.arange(64,67), np.arange(54, 59), [48], [60])) + faceIDStart- openPoseFaceIDs['rPupil'] = np.array([68]) + faceIDStart- openPoseFaceIDs['lPupil'] = np.array([69]) + faceIDStart #nice-- dict_of_openPoseSegmentIdx_dicts['face'] = openPoseFaceIDs-- return dict_of_openPoseSegmentIdx_dicts, dict_of_skel_lineColor+ openPoseFaceIDs['op_jaw'] = np.arange(0,16) + faceIDStart+ openPoseFaceIDs['op_rBrow'] = np.arange(17,21) + faceIDStart+ openPoseFaceIDs['op_lBrow'] = np.arange(22,26) + faceIDStart+ openPoseFaceIDs['op_noseRidge'] = np.arange(27,30) + faceIDStart+ openPoseFaceIDs['op_noseBot'] = np.arange(31,35) + faceIDStart+ openPoseFaceIDs['op_rEye'] = np.concatenate((np.arange(36,41), [36])) + faceIDStart+ openPoseFaceIDs['op_lEye'] = np.concatenate((np.arange(42,47), [42])) + faceIDStart+ openPoseFaceIDs['op_upperLip'] = np.concatenate((np.arange(48,54), np.flip(np.arange(60, 64)), [48])) + faceIDStart+ openPoseFaceIDs['op_lowerLip'] = np.concatenate(([60], np.arange(64,67), np.arange(54, 59), [48], [60])) + faceIDStart+ openPoseFaceIDs['op_rPupil'] = np.array([68]) + faceIDStart+ openPoseFaceIDs['op_lPupil'] = np.array([69]) + faceIDStart #nice++ dict_of_openPoseSegmentIdx_dicts['op_face'] = openPoseFaceIDs++ return dict_of_openPoseSegmentIdx_dicts, dict_of_op_skel_lineColor++++###+### βββββββ ββββββ ββββββ βββ βββ βββββ ββββββββ βββ βββ βββββββ ββββββ ββ βββββ ββββββ ββ ββββββ βββββββ βββββββ ββ ββ βββββββ ββ+### ββ ββ ββ ββ ββ ββββ ββββ ββ ββ ββ ββββ ββββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ+### βββββ ββ ββ ββββββ ββ ββββ ββ βββββββ ββ ββ ββββ ββ βββββ ββ ββ ββ βββββββ ββββββ ββ ββββββ βββββ βββββββ βββββ βββββ ββ+### ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ ββ+### ββ ββββββ ββ ββ ββ ββ ββ ββ ββ ββ ββ βββββββ ββββββ ββ ββ ββ ββ ββ ββ βββββββ βββββββ ββ ββ βββββββ βββββββ+###+###+def formatMediaPipeStickIndices():+ """+ generate dictionary of arrays, each containing the 'connect-the-dots' order to draw a given body segment++ returns:+ mediaPipeBodySegmentIds= a dictionary of arrays containing indices of individual body segments (Note, a lot of markerless mocap comp sci types like to say 'pose' instead of 'body'. They also use 'pose' to refer to camera 6 DoF position sometimes. Comp sci is frustrating like that lol)+ mediaPipeHandIds = a dictionary of arrays containing indices of individual hand segments, along with offset to know where to start in the 'skel_fr_mar_xyz.shape[1]' part of the array+ dict_of_mp_skel_lineColor = a dictionary of arrays, each containing the color (RGBA) to use for a given body segment+ """+ dict_of_mediaPipeSegmentIdx_dicts = dict()++ #make body dictionary+ mediaPipeBodySegmentIds = dict()+ mediaPipeBodySegmentIds['mp_head'] = [8, 6, 5, 4, 0, 10, 9, 0, 1, 2, 3, 7 ]+ mediaPipeBodySegmentIds['mp_tors'] = [12, 11, 24, 23, 12]+ mediaPipeBodySegmentIds['mp_rArm'] = [12, 14, 16, 18, 20, 16, 22 ]+ mediaPipeBodySegmentIds['mp_lArm'] = [11, 13, 15, 17, 19, 15, 21]+ mediaPipeBodySegmentIds['mp_rLeg'] = [24, 26, 28, 30, 32, 28, ]+ mediaPipeBodySegmentIds['mp_lLeg'] = [23, 25, 27, 29, 31, 27, ]+ dict_of_mediaPipeSegmentIdx_dicts['mp_body'] = mediaPipeBodySegmentIds+++ dict_of_mp_skel_lineColor = dict()+ greenshift = [0, .2, 0]+ dict_of_mp_skel_lineColor['mp_head'] = np.append(humon_dark+greenshift, 1)+ dict_of_mp_skel_lineColor['mp_tors'] = np.append(humon_dark+greenshift, 1)+ dict_of_mp_skel_lineColor['mp_rArm'] = np.append(humon_red+greenshift, 1)+ dict_of_mp_skel_lineColor['mp_lArm'] = np.append([humon_blue+greenshift], 1)+ dict_of_mp_skel_lineColor['mp_rLeg'] = np.append(humon_red+greenshift, 1)+ dict_of_mp_skel_lineColor['mp_lLeg'] = np.append(humon_blue+greenshift, 1)+++ # Make some handy maps ;D+ mediaPipeHandIds = dict()+ rHandIDstart = 33+ lHandIDstart = rHandIDstart + 21++ mediaPipeHandIds['mp_thumb'] = np.array([0, 1, 2, 3, 4, ])+ mediaPipeHandIds['mp_index'] = np.array([0, 5, 6, 7, 8, ])+ mediaPipeHandIds['mp_bird']= np.array([0, 9, 10, 11, 12, ])+ mediaPipeHandIds['mp_ring']= np.array([0, 13, 14, 15, 16, ])+ mediaPipeHandIds['mp_pinky'] = np.array([0, 17, 18, 19, 20, ])++ rHand_dict = copy.deepcopy(mediaPipeHandIds.copy()) #copy.deepcopy() is necessary to make sure the dicts are independent of each other+ lHand_dict = copy.deepcopy(rHand_dict)++ for key in rHand_dict:+ rHand_dict[key] += rHandIDstart+ lHand_dict[key] += lHandIDstart++ dict_of_mediaPipeSegmentIdx_dicts['mp_rHand'] = rHand_dict+ dict_of_mediaPipeSegmentIdx_dicts['mp_lHand'] = lHand_dict+++ # #how to face --> :D <--+ # mediaPipeFaceIDs = dict()+ # faceIDStart = 67+ # #define face parts+ # mediaPipeFaceIDs['mp_jaw'] = np.arange(0,16) + faceIDStart+ # mediaPipeFaceIDs['mp_rBrow'] = np.arange(17,21) + faceIDStart+ # mediaPipeFaceIDs['mp_lBrow'] = np.arange(22,26) + faceIDStart+ # mediaPipeFaceIDs['mp_noseRidge'] = np.arange(27,30) + faceIDStart+ # mediaPipeFaceIDs['mp_noseBot'] = np.arange(31,35) + faceIDStart+ # mediaPipeFaceIDs['mp_rEye'] = np.concatenate((np.arange(36,41), [36])) + faceIDStart+ # mediaPipeFaceIDs['mp_lEye'] = np.concatenate((np.arange(42,47), [42])) + faceIDStart+ # mediaPipeFaceIDs['mp_upperLip'] = np.concatenate((np.arange(48,54), np.flip(np.arange(60, 64)), [48])) + faceIDStart+ # mediaPipeFaceIDs['mp_lowerLip'] = np.concatenate(([60], np.arange(64,67), np.arange(54, 59), [48], [60])) + faceIDStart+ # mediaPipeFaceIDs['mp_rPupil'] = np.array([68]) + faceIDStart+ # mediaPipeFaceIDs['mp_lPupil'] = np.array([69]) + faceIDStart #nice++ # dict_of_mediaPipeSegmentIdx_dicts['mp_face'] = mediaPipeFaceIDs++ return dict_of_mediaPipeSegmentIdx_dicts, dict_of_mp_skel_lineColorif __name__ == '__main__':diff --git a/freemocap/playskeleton.py b/freemocap/playskeleton.pyindex a0c67c46..57dc24e5 100644--- a/freemocap/playskeleton.py+++ b/freemocap/playskeleton.py@@ -38,21 +38,21 @@ def ReplaySkeleton_matplotlib():- mean_charuco_fr_mar_dim = np.load(session.dataArrayPath/'charuco_3d_points.npy')+ mean_charuco_fr_mar_xyz = np.load(session.dataArrayPath/'charuco_3d_points.npy')if useOpenPose:- skel_fr_mar_dim = np.load(session.dataArrayPath / "openPoseSkel_3d.npy")+ skel_fr_mar_xyz = np.load(session.dataArrayPath / "openPoseSkel_3d.npy")if useMediaPipe:- mediaPipe_skel_fr_mar_dim = np.load(+ mediaPipe_skel_fr_mar_xyz = np.load(session.dataArrayPath / "mediaPipeSkel_3d.npy")if useDLC:- dlc_fr_mar_dim = np.load(session.dataArrayPath / "deepLabCut_3d.npy")- dlc0 = np.squeeze(dlc_fr_mar_dim[:, 0, :])- dlc1 = np.squeeze(dlc_fr_mar_dim[:, 1, :])- dlc2 = np.squeeze(dlc_fr_mar_dim[:, 2, :])+ dlc_fr_mar_xyz = np.load(session.dataArrayPath / "deepLabCut_3d.npy")+ dlc0 = np.squeeze(dlc_fr_mar_xyz[:, 0, :])+ dlc1 = np.squeeze(dlc_fr_mar_xyz[:, 1, :])+ dlc2 = np.squeeze(dlc_fr_mar_xyz[:, 2, :])ballTrailLen = 4if useOpenPose:@@ -106,15 +106,15 @@ def ReplaySkeleton_matplotlib(if vidType == 1:axCam1.cla()- char_x = mean_charuco_fr_mar_dim[:][:, 0]- char_y = mean_charuco_fr_mar_dim[:][:, 1]- char_z = mean_charuco_fr_mar_dim[:][:, 2]+ char_x = mean_charuco_fr_mar_xyz[:][:, 0]+ char_y = mean_charuco_fr_mar_xyz[:][:, 1]+ char_z = mean_charuco_fr_mar_xyz[:][:, 2]# plot skeleton (openPose by default)- sk_x = skel_fr_mar_dim[fr, :, 0] # skeleton x data- sk_y = skel_fr_mar_dim[fr, :, 1] # skeleton y data- sk_z = skel_fr_mar_dim[fr, :, 2] # skeleton z data+ sk_x = skel_fr_mar_xyz[fr, :, 0] # skeleton x data+ sk_y = skel_fr_mar_xyz[fr, :, 1] # skeleton y data+ sk_z = skel_fr_mar_xyz[fr, :, 2] # skeleton z dataaxMain.scatter3D(sk_x, sk_y, sk_z, marker=".", color="k", s=4.0)axMain.plot(@@ -243,16 +243,16 @@ def ReplaySkeleton_matplotlib()# plot mediapipe- mp_sk_x = mediaPipe_skel_fr_mar_dim[fr, :, 0] # skeleton x data- mp_sk_y = mediaPipe_skel_fr_mar_dim[fr, :, 1] # skeleton y data- mp_sk_z = mediaPipe_skel_fr_mar_dim[fr, :, 2] # skeleton z data+ mp_sk_x = mediaPipe_skel_fr_mar_xyz[fr, :, 0] # skeleton x data+ mp_sk_y = mediaPipe_skel_fr_mar_xyz[fr, :, 1] # skeleton y data+ mp_sk_z = mediaPipe_skel_fr_mar_xyz[fr, :, 2] # skeleton z dataaxMain.scatter3D(mp_sk_x, mp_sk_y, mp_sk_z, marker=".", color="g", s=8.0)# plot deeplabcut- dlc_x = dlc_fr_mar_dim[fr, :, 0]- dlc_y = dlc_fr_mar_dim[fr, :, 1]- dlc_z = dlc_fr_mar_dim[fr, :, 2]+ dlc_x = dlc_fr_mar_xyz[fr, :, 0]+ dlc_y = dlc_fr_mar_xyz[fr, :, 1]+ dlc_z = dlc_fr_mar_xyz[fr, :, 2]axMain.scatter3D(dlc_x, dlc_y, dlc_z, marker="o", color="r", s=24.0)@@ -314,24 +314,24 @@ def ReplaySkeleton_matplotlib( session,useDLC=False):session.imOutPath.mkdir(exist_ok = True)- mean_charuco_fr_mar_dim = np.load(session.dataArrayPath/'charuco_3d_points.npy')+ mean_charuco_fr_mar_xyz = np.load(session.dataArrayPath/'charuco_3d_points.npy')if session.useMediaPipe:- mediaPipe_skel_fr_mar_dim = np.load(session.dataArrayPath/'mediaPipeSkel_3d.npy')+ mediaPipe_skel_fr_mar_xyz = np.load(session.dataArrayPath/'mediaPipeSkel_3d.npy')imgPathList = session.session_settings['mediaPipe_imgPathList']#session.numCams = len(session.mediaPipe_imgPathList)if session.useOpenPose:- skel_fr_mar_dim = np.load(session.dataArrayPath/'openPoseSkel_3d.npy')+ skel_fr_mar_xyz = np.load(session.dataArrayPath/'openPoseSkel_3d.npy')imgPathList = session.session_settings['openPose_imgPathList']#session.numCams = len(session.openPose_imgPathList)if session.useDLC:- dlc_fr_mar_dim = np.load(session.dataArrayPath / "deepLabCut_3d.npy")- dlc0 = np.squeeze(dlc_fr_mar_dim[:, 0, :])- dlc1 = np.squeeze(dlc_fr_mar_dim[:, 1, :])- dlc2 = np.squeeze(dlc_fr_mar_dim[:, 2, :])+ dlc_fr_mar_xyz = np.load(session.dataArrayPath / "deepLabCut_3d.npy")+ dlc0 = np.squeeze(dlc_fr_mar_xyz[:, 0, :])+ dlc1 = np.squeeze(dlc_fr_mar_xyz[:, 1, :])+ dlc2 = np.squeeze(dlc_fr_mar_xyz[:, 2, :])ballTrailLen = 4camImgPathList = {}@@ -379,15 +379,15 @@ def ReplaySkeleton_matplotlib( session,if vidType == 1:axCam1.cla()- char_x = mean_charuco_fr_mar_dim[:][:, 0]- char_y = mean_charuco_fr_mar_dim[:][:, 1]- char_z = mean_charuco_fr_mar_dim[:][:, 2]+ char_x = mean_charuco_fr_mar_xyz[:][:, 0]+ char_y = mean_charuco_fr_mar_xyz[:][:, 1]+ char_z = mean_charuco_fr_mar_xyz[:][:, 2]# plot skeleton (openPose by default)if session.useOpenPose:- sk_x = skel_fr_mar_dim[fr, :, 0] # skeleton x data- sk_y = skel_fr_mar_dim[fr, :, 1] # skeleton y data- sk_z = skel_fr_mar_dim[fr, :, 2] # skeleton z data+ sk_x = skel_fr_mar_xyz[fr, :, 0] # skeleton x data+ sk_y = skel_fr_mar_xyz[fr, :, 1] # skeleton y data+ sk_z = skel_fr_mar_xyz[fr, :, 2] # skeleton z dataaxMain.scatter3D(sk_x, sk_y, sk_z, marker=".", color="k", s=4.0)axMain.plot(@@ -542,16 +542,16 @@ def ReplaySkeleton_matplotlib( session,# plot mediapipeif session.useMediaPipe:- mp_sk_x = mediaPipe_skel_fr_mar_dim[fr, :, 0] # skeleton x data- mp_sk_y = mediaPipe_skel_fr_mar_dim[fr, :, 1] # skeleton y data- mp_sk_z = mediaPipe_skel_fr_mar_dim[fr, :, 2] # skeleton z data+ mp_sk_x = mediaPipe_skel_fr_mar_xyz[fr, :, 0] # skeleton x data+ mp_sk_y = mediaPipe_skel_fr_mar_xyz[fr, :, 1] # skeleton y data+ mp_sk_z = mediaPipe_skel_fr_mar_xyz[fr, :, 2] # skeleton z dataaxMain.scatter3D(mp_sk_x, mp_sk_y, mp_sk_z, marker=".", color="g", s=8.0)# plot deeplabcutif session.useDLC:- dlc_x = dlc_fr_mar_dim[fr, :, 0]- dlc_y = dlc_fr_mar_dim[fr, :, 1]- dlc_z = dlc_fr_mar_dim[fr, :, 2]+ dlc_x = dlc_fr_mar_xyz[fr, :, 0]+ dlc_y = dlc_fr_mar_xyz[fr, :, 1]+ dlc_z = dlc_fr_mar_xyz[fr, :, 2]axMain.scatter3D(dlc_x, dlc_y, dlc_z, marker="o", color="r", s=24.0)diff --git a/freemocap/reconstruct3D.py b/freemocap/reconstruct3D.pyindex 13e8232e..cdb7dbce 100644--- a/freemocap/reconstruct3D.py+++ b/freemocap/reconstruct3D.py@@ -19,39 +19,45 @@ def reconstruct3D(session, data_nCams_nFrames_nImgPts_XYC, confidenceThreshold=0nCams, nFrames, nImgPts, nDims = data_nCams_nFrames_nImgPts_XYC.shapeif nDims == 3:+ dataOG = data_nCams_nFrames_nImgPts_XYC.copy()++ for camNum in range(nCams):++ thisCamX = data_nCams_nFrames_nImgPts_XYC[camNum, :, :,0 ]+ thisCamY = data_nCams_nFrames_nImgPts_XYC[camNum, :, :,1 ]+ thisCamConf = data_nCams_nFrames_nImgPts_XYC[camNum, :, :, 2]++ thisCamX[thisCamConf < confidenceThreshold] = np.nan+ thisCamY[thisCamConf < confidenceThreshold] = np.nan++ if session.debug:+ import matplotlib.pyplot as plt+ fig = plt.figure(8000+camNum)+ fig.suptitle("3d reconstruction Confidence thresholding - cam{}".format(camNum))+ axOG = fig.add_subplot(1,2,1)+ axOG.imshow(dataOG[0,:,:,0])+ axOG.set_title("Original Data")+ axTh = fig.add_subplot(1,2,2)+ axTh.imshow(thisCamX)+ axTh.set_title("Thresholded Data (there should be NEW and EXCITING gaps :O")+++- data_nCams_nFrames_nImgPts_X = np.squeeze(- data_nCams_nFrames_nImgPts_XYC[:, :, :, 0].copy()- )- data_nCams_nFrames_nImgPts_Y = np.squeeze(- data_nCams_nFrames_nImgPts_XYC[:, :, :, 1].copy()- )- confidence = np.squeeze(data_nCams_nFrames_nImgPts_XYC[:, :, :, 2].copy())-- data_nCams_nFrames_nImgPts_X[- confidence < confidenceThreshold- ] = np.nan # replace low confidence points with 'nan'- data_nCams_nFrames_nImgPts_Y[confidence < confidenceThreshold] = np.nan- data_nCams_nFrames_nImgPts_XY = np.stack(- (data_nCams_nFrames_nImgPts_X, data_nCams_nFrames_nImgPts_Y), axis=3- )if nDims == 2:data_nCams_nFrames_nImgPts_XY = data_nCams_nFrames_nImgPts_XYC+ elif nDims == 3:+ data_nCams_nFrames_nImgPts_XY = np.squeeze(data_nCams_nFrames_nImgPts_XYC[:, :, :, 0:2])++ dataFlat_nCams_nTotalPoints_XY = data_nCams_nFrames_nImgPts_XY.reshape(nCams, -1, 2) # reshape data to collapse across 'frames' so it becomes [numCams, numFrames*numPoints, XY]- dataFlat_nCams_nTotalPoints_XY = data_nCams_nFrames_nImgPts_XY.reshape(- nCams, -1, 2- ) # reshape data to collapse across 'frames' so it becomes [numCams, numFrames*numPoints, XY]+ data3d_flat = session.cgroup.triangulate(dataFlat_nCams_nTotalPoints_XY, progress=True)- data3d_flat = session.cgroup.triangulate(- dataFlat_nCams_nTotalPoints_XY, progress=True- )- dataReprojerr_flat = session.cgroup.reprojection_error(- data3d_flat, dataFlat_nCams_nTotalPoints_XY, mean=True- )+ dataReprojerr_flat = session.cgroup.reprojection_error( data3d_flat, dataFlat_nCams_nTotalPoints_XY, mean=True)##return:- data_fr_mar_dim = data3d_flat.reshape(nFrames, nImgPts, 3)- dataReprojErr_fr_mar_err = dataReprojerr_flat.reshape(nFrames, nImgPts)+ data_fr_mar_xyz = data3d_flat.reshape(nFrames, nImgPts, 3)+ dataReprojErr = dataReprojerr_flat.reshape(nFrames, nImgPts)- return data_fr_mar_dim+ return data_fr_mar_xyz, dataReprojErrdiff --git a/freemocap/session.py b/freemocap/session.pyindex 32728337..5bf60b81 100644--- a/freemocap/session.py+++ b/freemocap/session.py@@ -25,8 +25,8 @@ class Session: #self like "recording self"self.cgroup = None- self.mean_charuco_fr_mar_dim = None- self.skel_fr_mar_dim = None+ self.mean_charuco_fr_mar_xyz = None+ self.skel_fr_mar_xyz = Noneself.mediaPipe_imgPathList = Noneself.openPose_imgPathList = Nonediff --git a/freemocap/user_preferences.yaml b/freemocap/user_preferences.yamlindex 2d944229..c7d4a449 100644--- a/freemocap/user_preferences.yaml+++ b/freemocap/user_preferences.yaml@@ -10,3 +10,13 @@ default:codec: DIVXsaved:path_to_save: C:/Users/jonma/Dropbox/FreeMoCapProject+ rotations:+ 0: 0+ 2: 0+ 3: 0+ parameters:+ exposure: -7+ resWidth: 640+ resHeight: 480+ framerate: 30+ codec: DIVXdiff --git a/freemocap_blender_addon/freemocap_blender_addon/load_marker_empties.py b/freemocap_blender_addon/freemocap_blender_addon/load_marker_empties.pyindex 4aa481ca..32b01a78 100644--- a/freemocap_blender_addon/freemocap_blender_addon/load_marker_empties.py+++ b/freemocap_blender_addon/freemocap_blender_addon/load_marker_empties.py@@ -29,10 +29,10 @@ class FMC_OT_loadMarkerEmpties(bpy.types.Operator): #setting the type as "FMC" fdataPath = sessionPath / 'outData' / 'saveData.npz'sessionData = np.load(str(dataPath))- skel_fr_mar_dim = sessionData['skel_fr_mar_dim']+ skel_fr_mar_xyz = sessionData['skel_fr_mar_xyz']startFr = 0- numFrames = skel_fr_mar_dim.shape[0]-1+ numFrames = skel_fr_mar_xyz.shape[0]-1#names of markers@@ -44,8 +44,8 @@ class FMC_OT_loadMarkerEmpties(bpy.types.Operator): #setting the type as "FMC" f#%% _______________________________________________________________________# Skreleton data!print('Loading Skeleton Markers!')- for marNum in range(len(skel_fr_mar_dim[startFr,:,0])):- thisMarLoc = skel_fr_mar_dim[startFr,marNum,:]+ for marNum in range(len(skel_fr_mar_xyz[startFr,:,0])):+ thisMarLoc = skel_fr_mar_xyz[startFr,marNum,:]#these will define the size of teh body, hand, and face markersbms = self.emptySize@@ -75,6 +75,6 @@ class FMC_OT_loadMarkerEmpties(bpy.types.Operator): #setting the type as "FMC" f#loop through each frame (after the first [0th] frame) to set the keyframes for this markerfor fr in range(1, numFrames):- thisMarker.location = skel_fr_mar_dim[fr,marNum,:]+ thisMarker.location = skel_fr_mar_xyz[fr,marNum,:]thisMarker.keyframe_insert(data_path="location", frame=fr)return{'FINISHED'}diff --git a/freemocap_blender_addon/freemocap_blender_addon/old_loadOpenMocapSession.py b/freemocap_blender_addon/freemocap_blender_addon/old_loadOpenMocapSession.pyindex 2d092f77..5555eb3a 100644--- a/freemocap_blender_addon/freemocap_blender_addon/old_loadOpenMocapSession.py+++ b/freemocap_blender_addon/freemocap_blender_addon/old_loadOpenMocapSession.py@@ -69,12 +69,12 @@ class OMC_OT_loadOpenMoCapSession(bpy.types.Operator): #setting the type as "OMC# Load in dataloadedData = np.load(r"C:\Users\jonma\Dropbox\GitKrakenRepos\OpenMoCap\Data\test6_01_21a\outData\saveData.npz")- skel_fr_mar_dim = loadedData['skel_fr_mar_dim'] #OpenPose skeleton data- dlcPts_fr_mar_dim = loadedData['dlcPts_fr_mar_dim']+ skel_fr_mar_xyz = loadedData['skel_fr_mar_xyz'] #OpenPose skeleton data+ dlcPts_fr_mar_xyz = loadedData['dlcPts_fr_mar_xyz']charucoCorners_pt_XYZ = loadedData['charucoCornersXYZ']startFr = 0- numFrames = skel_fr_mar_dim.shape[0]-1+ numFrames = skel_fr_mar_xyz.shape[0]-1#names of markersskel_markerID = ["Nose", "Neck", "RShoulder", "RElbow", "RWrist", "LShoulder",@@ -115,8 +115,8 @@ class OMC_OT_loadOpenMoCapSession(bpy.types.Operator): #setting the type as "OMC# C.scene.collection.children.link(charColl) #link the newly created collection to the Blender Scene (so it will show up in the outliner)print('Loading DLC Markers')- for marNum in range(len(dlcPts_fr_mar_dim[0,:,0])):- thisMarLoc = dlcPts_fr_mar_dim[0,marNum,:]+ for marNum in range(len(dlcPts_fr_mar_xyz[0,:,0])):+ thisMarLoc = dlcPts_fr_mar_xyz[0,marNum,:]ms = self.dlcMar_sizeif self.meshType == 'uv_sphere':@@ -129,7 +129,7 @@ class OMC_OT_loadOpenMoCapSession(bpy.types.Operator): #setting the type as "OMC#loop through each frame (after the first [0th] frame) to set the keyframes for this markerfor fr in range(1, numFrames):- thisMarker.location = dlcPts_fr_mar_dim[fr,marNum,:]+ thisMarker.location = dlcPts_fr_mar_xyz[fr,marNum,:]thisMarker.keyframe_insert(data_path="location", frame=fr)@@ -142,8 +142,8 @@ class OMC_OT_loadOpenMoCapSession(bpy.types.Operator): #setting the type as "OMC# C.scene.collection.children.link(charColl) #link the newly created collection to the Blender Scene (so it will show up in the outliner)print('Loading Skeleton Markers!')- for marNum in range(len(skel_fr_mar_dim[startFr,:,0])):- thisMarLoc = skel_fr_mar_dim[startFr,marNum,:]+ for marNum in range(len(skel_fr_mar_xyz[startFr,:,0])):+ thisMarLoc = skel_fr_mar_xyz[startFr,marNum,:]#these will define the size of teh body, hand, and face markersbms = self.bodyMar_size@@ -177,7 +177,7 @@ class OMC_OT_loadOpenMoCapSession(bpy.types.Operator): #setting the type as "OMC#loop through each frame (after the first [0th] frame) to set the keyframes for this markerfor fr in range(1, numFrames):- thisMarker.location = skel_fr_mar_dim[fr,marNum,:]+ thisMarker.location = skel_fr_mar_xyz[fr,marNum,:]thisMarker.keyframe_insert(data_path="location", frame=fr)return{'FINISHED'}diff --git a/freemocap_runme_script.py b/freemocap_runme_script.pyindex 5d3a0195..684b5ce7 100644--- a/freemocap_runme_script.py+++ b/freemocap_runme_script.py@@ -1,13 +1,18 @@+from freemocap.fmc_mediapipe import runMediaPipeimport freemocap as fmcfrom pathlib import Pathimport socket--fmc.RunMe( stage=5,- useOpenPose=True,- openPoseDummyRun=True,- useMediaPipe=False,- useDLC=False,- recordVid = True,- reconstructionConfidenceThreshold = 0.2,- calVideoFrameLength = -1)+try:+ fmc.RunMe( stage=5,+ useOpenPose=False,+ runOpenPose=True,+ useMediaPipe=True,+ runMediaPipe = True,+ useDLC=False,+ recordVid = True,+ reconstructionConfidenceThreshold = 0.5,+ calVideoFrameLength = -1)+except:+ print("Unexpected error:", sys.exc_info()[0])+ raise\ No newline at end of file