Free Motion Capture for Everyone πβ¨
TypeScript
Separating the mediapipe/openpose pipelines
Aaron Cherian15df094bfb8d
diff --git a/freemocap/calibrate.py b/freemocap/calibrate.pyindex 9ea6de6d..74f3e8dc 100644--- a/freemocap/calibrate.py+++ b/freemocap/calibrate.py@@ -79,6 +79,7 @@ def CalibrateCaptureVolume(session,board):## if you need to save and load## example saving and loading for later+ if not session.cameraCalFilePath.is_file():error,merged = cgroup.calibrate_videos(vidnames, board)cgroup.dump(session.cameraCalFilePath) #JSM NOTE - let's just use .yaml's unless there is some reason to use .tomlmergename = session.sessionPath/'merged.npy'@@ -86,6 +87,7 @@ def CalibrateCaptureVolume(session,board):else:cgroup = CameraGroup.load(session.cameraCalFilePath) #load previous calibration config+session.cgroup = cgroupn_frames= 40startframe = 0diff --git a/freemocap/initialization.py b/freemocap/initialization.pyindex 32fe0cc8..ccc9a92e 100644--- a/freemocap/initialization.py+++ b/freemocap/initialization.py@@ -174,8 +174,13 @@ def initialize(session, stage, board):session.config_settings = recordingconfig.load_config_yaml(session.yamlPath) #config settings = paths and camera parameter inputsrecordingconfig.load_session_paths(session,session.config_settings) #add paths to session class++ session.mediaPipe_imgPathList = session.config_settings['mediaPipe_imgPathList']session.openPose_imgPathList = session.config_settings['openPose_imgPathList']- session.numCams = len(session.openPose_imgPathList)+++ #session.openPose_imgPathList = session.config_settings['openPose_imgPathList']+ #session.numCams = len(session.openPose_imgPathList)diff --git a/freemocap/playskeleton.py b/freemocap/playskeleton.pyindex 153565c2..1144197a 100644--- a/freemocap/playskeleton.py+++ b/freemocap/playskeleton.py@@ -155,23 +155,27 @@ def ReplaySkeleton( session,mean_charuco_fr_mar_dim = np.load(session.dataArrayPath/'charuco_points.npy')- if useOpenPose:- skel_fr_mar_dim = np.load(session.dataArrayPath/'openPoseSkel_3d.npy')- if useMediaPipe:+ if session.useMediaPipe:mediaPipe_skel_fr_mar_dim = np.load(session.dataArrayPath/'mediaPipeSkel_3d.npy')+ imgPathList = session.config_settings['mediaPipe_imgPathList']+ session.numCams = len(session.mediaPipe_imgPathList)+++ if session.useOpenPose:+ skel_fr_mar_dim = np.load(session.dataArrayPath/'openPoseSkel_3d.npy')+ imgPathList = session.config_settings['openPose_imgPathList']+ session.numCams = len(session.openPose_imgPathList)- if useDLC:+ if session.useDLC:dlc_fr_mar_dim = np.load(session.dataArrayPath/'deepLabCut_3d.npy')dlc0 = np.squeeze(dlc_fr_mar_dim[:,0,:])dlc1 = np.squeeze(dlc_fr_mar_dim[:,1,:])dlc2 = np.squeeze(dlc_fr_mar_dim[:,2,:])ballTrailLen = 4-- if useOpenPose:- imgPathList = session.config_settings['openPose_imgPathList']- session.numCams = len(session.openPose_imgPathList)++camImgPathList = {}for cam in range(session.numCams):@@ -227,50 +231,51 @@ def ReplaySkeleton( session,#plot skeleton (openPose by default)+ if session.useOpenPose:+ sk_x = skel_fr_mar_dim[fr,:,0] #skeleton x data+ sk_y = skel_fr_mar_dim[fr,:,1] #skeleton y data+ sk_z = skel_fr_mar_dim[fr,:,2] #skeleton z data++ axMain.scatter3D(sk_x,sk_y,sk_z, marker='.',color = 'k', s=4.)+ axMain.plot(sk_x[head],sk_y[head],sk_z[head], linestyle='-', color='g', linewidth = 1.)+ axMain.plot(sk_x[spine],sk_y[spine],sk_z[spine], linestyle='-', color = 'g', linewidth = 1.)+ axMain.plot(sk_x[rArm],sk_y[rArm],sk_z[rArm], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[lArm],sk_y[lArm],sk_z[lArm], linestyle='-', color = 'b', linewidth = 1.)+ axMain.plot(sk_x[rLeg],sk_y[rLeg],sk_z[rLeg], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[lLeg],sk_y[lLeg],sk_z[lLeg], linestyle='-', color = 'b', linewidth = 1.)+ axMain.plot(sk_x[rFoot],sk_y[rFoot],sk_z[rFoot], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[lFoot],sk_y[lFoot],sk_z[lFoot], linestyle='-', color = 'b', linewidth = 1.)++ # plot handybois+ # right hand+ axMain.plot(sk_x[thumb+rHandIDstart],sk_y[thumb+rHandIDstart],sk_z[thumb+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[index+rHandIDstart],sk_y[index+rHandIDstart],sk_z[index+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[bird+rHandIDstart],sk_y[bird+rHandIDstart],sk_z[bird+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[ring+rHandIDstart],sk_y[ring+rHandIDstart],sk_z[ring+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)+ axMain.plot(sk_x[pinky+rHandIDstart],sk_y[pinky+rHandIDstart],sk_z[pinky+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)++ #left hand+ axMain.plot(sk_x[thumb+lHandIDstart],sk_y[thumb+lHandIDstart],sk_z[thumb+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)+ axMain.plot(sk_x[index+lHandIDstart],sk_y[index+lHandIDstart],sk_z[index+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)+ axMain.plot(sk_x[bird+lHandIDstart],sk_y[bird+lHandIDstart],sk_z[bird+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)+ axMain.plot(sk_x[ring+lHandIDstart],sk_y[ring+lHandIDstart],sk_z[ring+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)+ axMain.plot(sk_x[pinky+lHandIDstart],sk_y[pinky+lHandIDstart],sk_z[pinky+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)- sk_x = skel_fr_mar_dim[fr,:,0] #skeleton x data- sk_y = skel_fr_mar_dim[fr,:,1] #skeleton y data- sk_z = skel_fr_mar_dim[fr,:,2] #skeleton z data-- axMain.scatter3D(sk_x,sk_y,sk_z, marker='.',color = 'k', s=4.)- axMain.plot(sk_x[head],sk_y[head],sk_z[head], linestyle='-', color='g', linewidth = 1.)- axMain.plot(sk_x[spine],sk_y[spine],sk_z[spine], linestyle='-', color = 'g', linewidth = 1.)- axMain.plot(sk_x[rArm],sk_y[rArm],sk_z[rArm], linestyle='-', color = 'r', linewidth = 1.)- axMain.plot(sk_x[lArm],sk_y[lArm],sk_z[lArm], linestyle='-', color = 'b', linewidth = 1.)- axMain.plot(sk_x[rLeg],sk_y[rLeg],sk_z[rLeg], linestyle='-', color = 'r', linewidth = 1.)- axMain.plot(sk_x[lLeg],sk_y[lLeg],sk_z[lLeg], linestyle='-', color = 'b', linewidth = 1.)- axMain.plot(sk_x[rFoot],sk_y[rFoot],sk_z[rFoot], linestyle='-', color = 'r', linewidth = 1.)- axMain.plot(sk_x[lFoot],sk_y[lFoot],sk_z[lFoot], linestyle='-', color = 'b', linewidth = 1.)-- # plot handybois- # right hand- axMain.plot(sk_x[thumb+rHandIDstart],sk_y[thumb+rHandIDstart],sk_z[thumb+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)- axMain.plot(sk_x[index+rHandIDstart],sk_y[index+rHandIDstart],sk_z[index+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)- axMain.plot(sk_x[bird+rHandIDstart],sk_y[bird+rHandIDstart],sk_z[bird+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)- axMain.plot(sk_x[ring+rHandIDstart],sk_y[ring+rHandIDstart],sk_z[ring+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)- axMain.plot(sk_x[pinky+rHandIDstart],sk_y[pinky+rHandIDstart],sk_z[pinky+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)-- #left hand- axMain.plot(sk_x[thumb+lHandIDstart],sk_y[thumb+lHandIDstart],sk_z[thumb+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)- axMain.plot(sk_x[index+lHandIDstart],sk_y[index+lHandIDstart],sk_z[index+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)- axMain.plot(sk_x[bird+lHandIDstart],sk_y[bird+lHandIDstart],sk_z[bird+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)- axMain.plot(sk_x[ring+lHandIDstart],sk_y[ring+lHandIDstart],sk_z[ring+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)- axMain.plot(sk_x[pinky+lHandIDstart],sk_y[pinky+lHandIDstart],sk_z[pinky+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)-#plot mediapipe- mp_sk_x = mediaPipe_skel_fr_mar_dim[fr,:,0] #skeleton x data- mp_sk_y = mediaPipe_skel_fr_mar_dim[fr,:,1] #skeleton y data- mp_sk_z = mediaPipe_skel_fr_mar_dim[fr,:,2] #skeleton z data- axMain.scatter3D(mp_sk_x, mp_sk_y, mp_sk_z, marker='.',color = 'g', s=8.)+ if session.useMediaPipe:+ mp_sk_x = mediaPipe_skel_fr_mar_dim[fr,:,0] #skeleton x data+ mp_sk_y = mediaPipe_skel_fr_mar_dim[fr,:,1] #skeleton y data+ mp_sk_z = mediaPipe_skel_fr_mar_dim[fr,:,2] #skeleton z data+ axMain.scatter3D(mp_sk_x, mp_sk_y, mp_sk_z, marker='.',color = 'g', s=8.)#plot deeplabcut-- dlc_x = dlc_fr_mar_dim[fr,:,0]- dlc_y = dlc_fr_mar_dim[fr,:,1]- dlc_z = dlc_fr_mar_dim[fr,:,2]-- axMain.scatter3D(dlc_x,dlc_y,dlc_z, marker='o',color = 'r', s=24.)+ if session.useDLC:+ dlc_x = dlc_fr_mar_dim[fr,:,0]+ dlc_y = dlc_fr_mar_dim[fr,:,1]+ dlc_z = dlc_fr_mar_dim[fr,:,2]++ axMain.scatter3D(dlc_x,dlc_y,dlc_z, marker='o',color = 'r', s=24.)#plot charuco gridaxMain.scatter(char_x,char_y,char_z, marker='o')diff --git a/freemocap/recordingconfig.py b/freemocap/recordingconfig.pyindex 74d0645d..bddfcbcc 100644--- a/freemocap/recordingconfig.py+++ b/freemocap/recordingconfig.py@@ -36,7 +36,7 @@ def createSession(session,filepath, existing_data = False):session.openPoseDataPath = session.sessionPath/'OpenPoseData'session.openPoseDataPath.mkdir(exist_ok = True)- pathList.append(session.mediaPipeDataPath)+ pathList.append(session.openPoseDataPath)session.mediaPipeDataPath = session.sessionPath/'MediaPipeData'session.mediaPipeDataPath.mkdir(exist_ok = True)@@ -59,6 +59,8 @@ def createSession(session,filepath, existing_data = False):config_settings,config_yaml = create_config_yaml()+ session.config_settings = config_settings+yaml_name = session.sessionPath/'{}_config.yaml'.format(session.sessionID)session.yamlPath = yaml_name# print(yaml_name)@@ -93,6 +95,7 @@ def create_config_yaml():rawVidPath:syncedVidPath:calVidPath:+ openPoseDataPath:mediaPipeDataPath:dlcDataPath:imOutPath:@@ -101,6 +104,7 @@ def create_config_yaml():"""config_yaml = YAML()config_settings = config_yaml.load(yaml_str)+return config_settings,config_yamldiff --git a/freemocap/session.py b/freemocap/session.pyindex ec7b4754..7975528d 100644--- a/freemocap/session.py+++ b/freemocap/session.py@@ -16,4 +16,5 @@ class Session: #session like "recording session"self.mean_charuco_fr_mar_dim = Noneself.skel_fr_mar_dim = None- self.mediaPipe_imgPathList = None\ No newline at end of file+ self.mediaPipe_imgPathList = None+ self.openPose_imgPathList = None\ No newline at end of filediff --git a/runme_fmc_staged.py b/runme_fmc_staged.pyindex 1a29880b..7fda929c 100644--- a/runme_fmc_staged.py+++ b/runme_fmc_staged.py@@ -1,4 +1,4 @@-from freemocap import createvideo, initialization, runcams, calibrate, fmc_mediapipe, fmc_openpose, fmc_deeplabcut, reconstruct3D, playskeleton, session+from freemocap import createvideo, initialization, runcams, calibrate, fmc_mediapipe, fmc_openpose, fmc_deeplabcut, reconstruct3D, playskeleton, recordingconfig,sessionfrom pathlib import Pathimport os@@ -10,21 +10,21 @@ import numpy as npsesh = session.Session()-useOpenPose= True-useMediaPipe = True-useDLC=True+sesh.useOpenPose= True+sesh.useMediaPipe = True+sesh.useDLC=False-if useDLC:+if sesh.useDLC:import deeplabcut as dlcsesh.dlcConfigPath = Path("C:\\Users\\jonma\\Dropbox\\GitKrakenRepos\\freemocap\\DLC_Models\\PinkGreenRedJugglingBalls-JSM-2021-05-31\\config.yaml")# %% Inputs to edit-stage = 3 #set your starting stage here (stage = 1 will run the pipeline from webcams)+stage = 4 #set your starting stage here (stage = 1 will run the pipeline from webcams)sesh.debug = False-sesh.sessionID = 'sesh_21-05-31_111833' #fill in if you are running from Stage 2 onwards+sesh.sessionID = '' #fill in if you are running from Stage 2 onwardsif not sesh.sessionID:dataFolder = Path.cwd()/'Data'subfolders = [ f.path for f in os.scandir(dataFolder) if f.is_dir() ] #copy-pasta from who knows where@@ -75,19 +75,19 @@ else:# %% Stage Fourif stage <= 4:print('Starting Track Image Points')- if useMediaPipe:+ if sesh.useMediaPipe:fmc_mediapipe.runMediaPipe(sesh)sesh.mediaPipeData_nCams_nFrames_nImgPts_XYC = fmc_mediapipe.parseMediaPipe(sesh)sesh.mediaPipeSkel_fr_mar_dim = reconstruct3D.reconstruct3D(sesh,sesh.mediaPipeData_nCams_nFrames_nImgPts_XYC, confidenceThreshold=.1)np.save(sesh.dataArrayPath/'mediaPipeSkel_3d.npy', sesh.mediaPipeSkel_fr_mar_dim) #save data to npy- if useOpenPose:+ if sesh.useOpenPose:fmc_openpose.runOpenPose(sesh, dummyRun=False)sesh.openPoseData_nCams_nFrames_nImgPts_XYC = fmc_openpose.parseOpenPose(sesh)sesh.openPoseskel_fr_mar_dim = reconstruct3D.reconstruct3D(sesh,sesh.openPoseData_nCams_nFrames_nImgPts_XYC, confidenceThreshold=.1)np.save(sesh.dataArrayPath/'openPoseSkel_3d.npy', sesh.openPoseskel_fr_mar_dim) #save data to npy- if useDLC:+ if sesh.useDLC:syncedVidList = []for vid in sesh.syncedVidPath.glob('*.mp4'):syncedVidList.append(str(vid))@@ -122,6 +122,9 @@ else:+#reupdate the config_settings with mediapipe and openpose image paths+sesh.config_settings = recordingconfig.load_config_yaml(sesh.yamlPath)+# %% Stage Sevenif stage <= 7:print('Starting Skeleton Plotting')@@ -131,9 +134,9 @@ if stage <= 7:startFrame=40,azimuth=-90,elevation=-80,- useOpenPose=useOpenPose,- useMediaPipe=useMediaPipe,- useDLC=useDLC)+ useOpenPose=sesh.useOpenPose,+ useMediaPipe=sesh.useMediaPipe,+ useDLC=sesh.useDLC)else:print('Skipping Skeleton Plotting')