Free Motion Capture for Everyone πβ¨
TypeScript
I thought I already pushed these changes wtf?
jon matthis2687fe8fbdae
diff --git a/freemocap/fmc_mediapipe.py b/freemocap/fmc_mediapipe.pyindex 4ad1e4ff..255a66cb 100644--- a/freemocap/fmc_mediapipe.py+++ b/freemocap/fmc_mediapipe.py@@ -1,15 +1,13 @@-import osfrom pathlib import Path-import glob-import json+import numpy as npfrom rich.progress import Progressfrom ruamel.yaml import YAMLimport cv2-import h5pyimport mediapipe as mp-from pathlib import Path+# from numba import jit+mp_drawing = mp.solutions.drawing_utilsmp_holistic = mp.solutions.holistic@@ -113,7 +111,7 @@ def runMediaPipe(session, dummyRun=False):yaml.dump(data, session.yamlPath)session.mediaPipeData = eachCamerasData-+# @jit(nopython=True)def parseMediaPipe(session):numCams = len(session.mediaPipeData) #Get number of camerasnumFrames = len(session.mediaPipeData[0]) #Get number of framesdiff --git a/freemocap/fmc_openpose.py b/freemocap/fmc_openpose.pyindex 09a4998a..7294161d 100644--- a/freemocap/fmc_openpose.py+++ b/freemocap/fmc_openpose.py@@ -1,12 +1,12 @@import osimport subprocess+import json+import numpy as npfrom rich.progress import trackfrom ruamel.yaml import YAMLfrom pathlib import Path--import glob-import json+# from numba import jitdef runOpenPose(session, dummyRun=False):@@ -45,7 +45,7 @@ def runOpenPose(session, dummyRun=False):os.chdir(session.sessionPath)-+# @jit(nopython=True)def parseOpenPose(session):thisCamNum = -1diff --git a/freemocap/playskeleton.py b/freemocap/playskeleton.pyindex 153565c2..85a3f8b4 100644--- a/freemocap/playskeleton.py+++ b/freemocap/playskeleton.py@@ -8,142 +8,205 @@ Created on Fri Feb 26 12:51:21 2021import numpy as np#from mpl_toolkits.mplot3d import Axes3Dimport matplotlib.pyplot as plt-import matplotlib as mplimport cv2from pathlib import Path-#from glob import glob+import pyqtgraph as pg++########################################################################################################################################+########################################################################################################################################+##+## ββββ ββββ ββββββ ββββββββββββββββ βββ βββββββ ββββββββββββ ββββββββββ+## βββββ βββββββββββββββββββββββββββββββββ βββββββββββββββββββββ βββββββββββ+## βββββββββββββββββββ βββ βββββββββββ βββ βββ βββ βββ βββββββββββ+## βββββββββββββββββββ βββ βββββββ βββ βββ βββ βββ βββ βββββββββββ+## βββ βββ ββββββ βββ βββ βββ βββββββββββββββββ βββ βββββββββββββββββββ+## βββ ββββββ βββ βββ βββ ββββββββ βββββββ βββ ββββββββββββββββββ+########################################################################################################################################+########################################################################################################################################++def ReplaySkeleton_matplotlib( session,+ vidType=1,+ startFrame=40,+ azimuth=-90,+ elevation=-80,+ useOpenPose=True,+ useMediaPipe=False,+ useDLC=False):-# def PlaySkeleton(session, vidType):++ mean_charuco_fr_mar_dim = np.load(session.dataArrayPath/'charuco_points.npy')-# # vidType = 0 #No Cam Ims-# # vidType = 1 #Only Cam1 im-# # vidType = 2 #All Cam Ims+ if useOpenPose:+ skel_fr_mar_dim = np.load(session.dataArrayPath/'openPoseSkel_3d.npy')-# #where to put the output images (later will be justa video)-+ if useMediaPipe:+ mediaPipe_skel_fr_mar_dim = np.load(session.dataArrayPath/'mediaPipeSkel_3d.npy')-# #os.chdir(imOutPath)+ if useDLC:+ dlc_fr_mar_dim = np.load(session.dataArrayPath/'deepLabCut_3d.npy')+ dlc0 = np.squeeze(dlc_fr_mar_dim[:,0,:])+ dlc1 = np.squeeze(dlc_fr_mar_dim[:,1,:])+ dlc2 = np.squeeze(dlc_fr_mar_dim[:,2,:])+ ballTrailLen = 4+ if useOpenPose:+ imgPathList = session.config_settings['openPose_imgPathList']+ session.numCams = len(session.openPose_imgPathList)++ camImgPathList = {}+ for cam in range(session.numCams):+ camImgPathList[cam] = list(sorted(Path(imgPathList[cam]).glob('*.png')))+ session.numFrames = len(camImgPathList[cam]) #will need to perhaps put a check in on whether numFrames between cameras are the same+++# define Skeleton connections+# head = [17, 15, 0, 16, 18]+ head = [ 15, 0, 16]+ spine = [0,1,8]+ rArm = [17, 15, 0, 16, 18]+ rArm = [4 ,3 ,2 ,1]+ lArm = [1, 5, 6, 7]+ rLeg = [11 ,10, 9, 8]+ lLeg = [14 ,13 ,12, 8]+ rFoot = [11, 23,22, 11, 24]+ lFoot = [14, 20, 19, 14, 21]++ #Make some handy maps ;D+ rHandIDstart = 25+ lHandIDstart = rHandIDstart+21++ thumb = np.array([0,1,2,3,4])+ index = np.array([0, 5,6,7,8])+ bird = np.array([0, 9,10,11,12])+ ring = np.array([0, 13,14,15,16])+ pinky = np.array([0, 17,18,19,20])+++ fig = plt.figure(figsize=(11,8))++ if vidType == 0:+ axMain = fig.add_subplot(111,projection='3d')+ elif vidType == 1:+ axMain = fig.add_subplot(121,projection='3d')+ axCam1 = fig.add_subplot(122)+ # fig.tight_layout(h_pad = 5)+ axMain.view_init(azim = azimuth, elev= elevation)+ plt.ion()+ for fr in range(startFrame,session.numFrames):+ fig.suptitle(['Frame# - ', str(fr)])+ axMain.cla()++ if vidType == 1:+ axCam1.cla()+++ char_x = mean_charuco_fr_mar_dim[:][:,0]+ char_y = mean_charuco_fr_mar_dim[:][:,1]+ char_z = mean_charuco_fr_mar_dim[:][:,2]+++ #plot skeleton (openPose by default)-# #set axis limits based on charuco board-# # mx = np.mean(charuco_x)-# # my = np.mean(charuco_y)-# # mz = np.mean(charuco_z)--# #set up figure-# # fig = plt.figure(figsize=([11,8]))--# # if vidType==0:-# # axMain = fig.add_subplot(111,projection='3d')-# # elif vidType == 1:-# # axMain = fig.add_subplot(121,projection='3d')-# # axCam1 = fig.add_subplot(122)-# # elif vidType == 2:-# # figGridSpec = fig.add_gridspec(2,4)-# # axMain = fig.add_subplot(figGridSpec[:2,:2],projection='3d')-# # axCam1 = fig.add_subplot(figGridSpec[0,2])-# # axCam2 = fig.add_subplot(figGridSpec[0,3])-# # axCam3 = fig.add_subplot(figGridSpec[1,2])-# # axCam4 = fig.add_subplot(figGridSpec[1,3])-----# ##############################################################################################-# ##Frame-by-frame animation loop starts here-# ##############################################################################################--# # fig = plt.figure()-# # ax = fig.add_subplot(111,projection='3d')-# # ax.cla()-# # x = session.mean_charuco_fr_mar_dim[:][:,0]-# # y = session.mean_charuco_fr_mar_dim[:][:,1]-# # z = session.mean_charuco_fr_mar_dim[:][:,2]-# # ax.scatter(x,y,z, marker='o')-# # plt.show()--# # define Skeleton connections-# # head = [17, 15, 0, 16, 18]-# head = [ 15, 0, 16]-# spine = [0,1,8]-# rArm = [17, 15, 0, 16, 18]-# rArm = [4 ,3 ,2 ,1]-# lArm = [1, 5, 6, 7]-# rLeg = [11 ,10, 9, 8]-# lLeg = [14 ,13 ,12, 8]-# rFoot = [11, 23,22, 11, 24]-# lFoot = [14, 20, 19, 14, 21]--# #Make some handy maps ;D-# rHandIDstart = 25-# lHandIDstart = rHandIDstart+21--# thumb = np.array([0,1,2,3,4])-# index = np.array([0, 5,6,7,8])-# bird = np.array([0, 9,10,11,12])-# ring = np.array([0, 13,14,15,16])-# pinky = np.array([0, 17,18,19,20])---# fig = plt.figure(figsize=(8,6))-# axMain = fig.add_subplot(111,projection='3d')-# plt.ion()-# for fr in range(session.numFrames):-# fig.suptitle(['Frame# - ', str(fr)])-# axMain.cla()-# char_x = session.mean_charuco_fr_mar_dim[:][:,0]-# char_y = session.mean_charuco_fr_mar_dim[:][:,1]-# char_z = session.mean_charuco_fr_mar_dim[:][:,2]--# if session.useOpenPose:-# sk_x = session.skel_fr_mar_dim[fr,:,0] #skeleton x data-# sk_y = session.skel_fr_mar_dim[fr,:,1] #skeleton y data-# sk_z = session.skel_fr_mar_dim[fr,:,2] #skeleton z data--# axMain.scatter3D(sk_x,sk_y,sk_z, marker='.',color = 'k', s=4.)-# axMain.plot(sk_x[head],sk_y[head],sk_z[head], linestyle='-', color='g', linewidth = 1.)-# axMain.plot(sk_x[spine],sk_y[spine],sk_z[spine], linestyle='-', color = 'g', linewidth = 1.)-# axMain.plot(sk_x[rArm],sk_y[rArm],sk_z[rArm], linestyle='-', color = 'r', linewidth = 1.)-# axMain.plot(sk_x[lArm],sk_y[lArm],sk_z[lArm], linestyle='-', color = 'b', linewidth = 1.)-# axMain.plot(sk_x[rLeg],sk_y[rLeg],sk_z[rLeg], linestyle='-', color = 'r', linewidth = 1.)-# axMain.plot(sk_x[lLeg],sk_y[lLeg],sk_z[lLeg], linestyle='-', color = 'b', linewidth = 1.)-# axMain.plot(sk_x[rFoot],sk_y[rFoot],sk_z[rFoot], linestyle='-', color = 'r', linewidth = 1.)-# axMain.plot(sk_x[lFoot],sk_y[lFoot],sk_z[lFoot], linestyle='-', color = 'b', linewidth = 1.)--# # plot handybois-# # right hand-# axMain.plot(sk_x[thumb+rHandIDstart],sk_y[thumb+rHandIDstart],sk_z[thumb+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)-# axMain.plot(sk_x[index+rHandIDstart],sk_y[index+rHandIDstart],sk_z[index+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)-# axMain.plot(sk_x[bird+rHandIDstart],sk_y[bird+rHandIDstart],sk_z[bird+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)-# axMain.plot(sk_x[ring+rHandIDstart],sk_y[ring+rHandIDstart],sk_z[ring+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)-# axMain.plot(sk_x[pinky+rHandIDstart],sk_y[pinky+rHandIDstart],sk_z[pinky+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)--# #left hand-# axMain.plot(sk_x[thumb+lHandIDstart],sk_y[thumb+lHandIDstart],sk_z[thumb+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)-# axMain.plot(sk_x[index+lHandIDstart],sk_y[index+lHandIDstart],sk_z[index+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)-# axMain.plot(sk_x[bird+lHandIDstart],sk_y[bird+lHandIDstart],sk_z[bird+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)-# axMain.plot(sk_x[ring+lHandIDstart],sk_y[ring+lHandIDstart],sk_z[ring+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)-# axMain.plot(sk_x[pinky+lHandIDstart],sk_y[pinky+lHandIDstart],sk_z[pinky+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)+ sk_x = skel_fr_mar_dim[fr,:,0] #skeleton x data+ sk_y = skel_fr_mar_dim[fr,:,1] #skeleton y data+ sk_z = skel_fr_mar_dim[fr,:,2] #skeleton z data++ axMain.scatter3D(sk_x,sk_y,sk_z, marker='.',color = 'k', s=4.)+ axMain.plot(sk_x[head],sk_y[head],sk_z[head], linestyle='-', color='g', linewidth = 1.)+ axMain.plot(sk_x[spine],sk_y[spine],sk_z[spine], linestyle='-', color = 'g', linewidth = 1.)+ axMain.plot(sk_x[rArm],sk_y[rArm],sk_z[rArm], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[lArm],sk_y[lArm],sk_z[lArm], linestyle='-', color = 'b', linewidth = 1.)+ axMain.plot(sk_x[rLeg],sk_y[rLeg],sk_z[rLeg], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[lLeg],sk_y[lLeg],sk_z[lLeg], linestyle='-', color = 'b', linewidth = 1.)+ axMain.plot(sk_x[rFoot],sk_y[rFoot],sk_z[rFoot], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[lFoot],sk_y[lFoot],sk_z[lFoot], linestyle='-', color = 'b', linewidth = 1.)++ # plot handybois+ # right hand+ axMain.plot(sk_x[thumb+rHandIDstart],sk_y[thumb+rHandIDstart],sk_z[thumb+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[index+rHandIDstart],sk_y[index+rHandIDstart],sk_z[index+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[bird+rHandIDstart],sk_y[bird+rHandIDstart],sk_z[bird+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[ring+rHandIDstart],sk_y[ring+rHandIDstart],sk_z[ring+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[pinky+rHandIDstart],sk_y[pinky+rHandIDstart],sk_z[pinky+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)++ #left hand+ axMain.plot(sk_x[thumb+lHandIDstart],sk_y[thumb+lHandIDstart],sk_z[thumb+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)+ axMain.plot(sk_x[index+lHandIDstart],sk_y[index+lHandIDstart],sk_z[index+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)+ axMain.plot(sk_x[bird+lHandIDstart],sk_y[bird+lHandIDstart],sk_z[bird+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)+ axMain.plot(sk_x[ring+lHandIDstart],sk_y[ring+lHandIDstart],sk_z[ring+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)+ axMain.plot(sk_x[pinky+lHandIDstart],sk_y[pinky+lHandIDstart],sk_z[pinky+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)-# axMain.scatter(char_x,char_y,char_z, marker='o')-# axMain.set_xbound(lower=-150, upper=150)-# axMain.set_ybound(lower=-150, upper=150)-# axMain.set_zbound(lower=-150, upper=150)+ #plot mediapipe+ mp_sk_x = mediaPipe_skel_fr_mar_dim[fr,:,0] #skeleton x data+ mp_sk_y = mediaPipe_skel_fr_mar_dim[fr,:,1] #skeleton y data+ mp_sk_z = mediaPipe_skel_fr_mar_dim[fr,:,2] #skeleton z data+ axMain.scatter3D(mp_sk_x, mp_sk_y, mp_sk_z, marker='.',color = 'g', s=8.)++ #plot deeplabcut++ dlc_x = dlc_fr_mar_dim[fr,:,0]+ dlc_y = dlc_fr_mar_dim[fr,:,1]+ dlc_z = dlc_fr_mar_dim[fr,:,2]++ axMain.scatter3D(dlc_x,dlc_y,dlc_z, marker='o',color = 'r', s=24.)++ #plot charuco grid+ axMain.scatter(char_x,char_y,char_z, marker='o')++ # #plot camera positions (I think...)+ # for camNum in range(len(session.cgroup.cameras)):+ # axMain.scatter(session.cgroup.cameras[camNum].tvec[0], session.cgroup.cameras[camNum].tvec[1], session.cgroup.cameras[camNum].tvec[2], marker = 'p')+ axRange = session.board.square_length*10+ mx = np.nanmean(char_x)+ my = np.nanmean(char_y)+ mz = np.nanmean(char_z)++ axMain.set_xlim(mx-axRange,mx+axRange)+ axMain.set_ylim(my-axRange,my+axRange)+ axMain.set_zlim(mz-axRange,mz+axRange)++ axMain.set_xlabel('x')+ axMain.set_ylabel('y')+ axMain.set_zlabel('z')++++ if vidType ==0:+ pass+ elif vidType == 1:+ axCam1.imshow(cv2.cvtColor(cv2.imread(str(camImgPathList[0][fr])),cv2.COLOR_BGR2RGB))+ axCam1.axis('off')++ plt.pause(0.01)+ plt.show()++ frameName = str(fr).zfill(6)+ saveFrameName = '{}.png'.format(frameName)+ saveFramePath = session.imOutPath/saveFrameName+ fig.savefig(saveFramePath )++-# #imName = [session.sessionID + '_frame' + fr.zfill(6)]-# frameName = str(fr).zfill(6)-# saveFrameName = '{}.png'.format(frameName)-# saveFramePath = session.imOutPath/saveFrameName-# fig.savefig(saveFramePath)# , bbox_inches = 'tight')-# plt.pause(0.01)-# plt.show()+########################################################################################################################################+########################################################################################################################################+### βββββββ βββββββ ββββββββββββ βββ βββββββ βββ+### ββββββββββββββββββββββββββββββ βββ ββββββββ βββ+### βββ βββββββββββββββββ ββββββ βββ βββ βββββββ+### βββ ββββββββββ ββββββ ββββββββββ βββ ββββββ+### ββββββββββββ βββββββββββ ββββββ βββββββββββββββββ+########################################################################################################################################+########################################################################################################################################-def ReplaySkeleton( session,+++++def ReplaySkeleton_matplotlib( session,vidType=1,startFrame=40,azimuth=-90,@@ -183,7 +246,6 @@ def ReplaySkeleton( session,# head = [17, 15, 0, 16, 18]head = [ 15, 0, 16]spine = [0,1,8]- rArm = [17, 15, 0, 16, 18]rArm = [4 ,3 ,2 ,1]lArm = [1, 5, 6, 7]rLeg = [11 ,10, 9, 8]diff --git a/freemocap/reconstruct3D.py b/freemocap/reconstruct3D.pyindex 9edadceb..838ed202 100644--- a/freemocap/reconstruct3D.py+++ b/freemocap/reconstruct3D.py@@ -1,6 +1,8 @@from freemocap import fmc_aniposeimport numpy as np+# from numba import jit+# @jit(nopython=True)def reconstruct3D(session, data_nCams_nFrames_nImgPts_XYC, confidenceThreshold = .3):if session.cgroup is None: #load the calibration settings into the session class if it hasn't been alreadydiff --git a/runme_fmc_staged.py b/runme_fmc_staged.pyindex 1a29880b..3539b1da 100644--- a/runme_fmc_staged.py+++ b/runme_fmc_staged.py@@ -1,4 +1,5 @@-from freemocap import createvideo, initialization, runcams, calibrate, fmc_mediapipe, fmc_openpose, fmc_deeplabcut, reconstruct3D, playskeleton, session+from freemocap import createvideo, initialization, runcams, calibrate, fmc_mediapipe, fmc_openpose, fmc_deeplabcut, reconstruct3D, playskeleton, session+from freemocap.fmc_pyqtgraph import PlaySkeletonfrom pathlib import Pathimport os@@ -21,7 +22,7 @@ if useDLC:# %% Inputs to edit-stage = 3 #set your starting stage here (stage = 1 will run the pipeline from webcams)+stage = 7 #set your starting stage here (stage = 1 will run the pipeline from webcams)sesh.debug = Falsesesh.sessionID = 'sesh_21-05-31_111833' #fill in if you are running from Stage 2 onwards@@ -82,17 +83,18 @@ if stage <= 4:np.save(sesh.dataArrayPath/'mediaPipeSkel_3d.npy', sesh.mediaPipeSkel_fr_mar_dim) #save data to npyif useOpenPose:- fmc_openpose.runOpenPose(sesh, dummyRun=False)+ fmc_openpose.runOpenPose(sesh, dummyRun=True)sesh.openPoseData_nCams_nFrames_nImgPts_XYC = fmc_openpose.parseOpenPose(sesh)sesh.openPoseskel_fr_mar_dim = reconstruct3D.reconstruct3D(sesh,sesh.openPoseData_nCams_nFrames_nImgPts_XYC, confidenceThreshold=.1)np.save(sesh.dataArrayPath/'openPoseSkel_3d.npy', sesh.openPoseskel_fr_mar_dim) #save data to npyif useDLC:- syncedVidList = []+ sesh.syncedVidList = []for vid in sesh.syncedVidPath.glob('*.mp4'):- syncedVidList.append(str(vid))+ sesh.syncedVidList.append(str(vid))- dlc.analyze_videos(sesh.dlcConfigPath,syncedVidList, destfolder= sesh.dlcDataPath, save_as_csv=True)+ dlc.analyze_videos(sesh.dlcConfigPath,sesh.syncedVidList, destfolder= sesh.dlcDataPath, save_as_csv=True)+sesh.dlcData_nCams_nFrames_nImgPts_XYC = fmc_deeplabcut.parseDeepLabCut(sesh)sesh.dlc_fr_mar_dim = reconstruct3D.reconstruct3D(sesh,sesh.dlcData_nCams_nFrames_nImgPts_XYC, confidenceThreshold=.95)np.save(sesh.dataArrayPath/'deepLabCut_3d.npy', sesh.dlc_fr_mar_dim) #save data to npy@@ -125,15 +127,28 @@ else:# %% Stage Sevenif stage <= 7:print('Starting Skeleton Plotting')- playskeleton.ReplaySkeleton(- sesh,- vidType=1,- startFrame=40,- azimuth=-90,- elevation=-80,- useOpenPose=useOpenPose,- useMediaPipe=useMediaPipe,- useDLC=useDLC)+ # playskeleton.ReplaySkeleton_matplotlib(+ # sesh,+ # vidType=1,+ # startFrame=40,+ # azimuth=-90,+ # elevation=-80,+ # useOpenPose=useOpenPose,+ # useMediaPipe=useMediaPipe,+ # useDLC=useDLC)+ # fmc_pyqtgraph.PlaySkeleton(+ # sesh,+ # vidType=1,+ # startFrame=40,+ # azimuth=-90,+ # elevation=-80,+ # useOpenPose=useOpenPose,+ # useMediaPipe=useMediaPipe,+ # useDLC=useDLC)++ playSkel = PlaySkeleton(sesh)+ playSkel.animate()+else:print('Skipping Skeleton Plotting')