Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

Separating the mediapipe/openpose pipelines

Aaron Cherian15df094bfb8d
diff --git a/freemocap/calibrate.py b/freemocap/calibrate.py
index 9ea6de6d..74f3e8dc 100644
--- a/freemocap/calibrate.py
+++ b/freemocap/calibrate.py
@@ -79,6 +79,7 @@ def CalibrateCaptureVolume(session,board):
## if you need to save and load
## example saving and loading for later
+ if not session.cameraCalFilePath.is_file():
error,merged = cgroup.calibrate_videos(vidnames, board)
cgroup.dump(session.cameraCalFilePath) #JSM NOTE - let's just use .yaml's unless there is some reason to use .toml
mergename = session.sessionPath/'merged.npy'
@@ -86,6 +87,7 @@ def CalibrateCaptureVolume(session,board):
else:
cgroup = CameraGroup.load(session.cameraCalFilePath) #load previous calibration config
+
session.cgroup = cgroup
n_frames= 40
startframe = 0
diff --git a/freemocap/initialization.py b/freemocap/initialization.py
index 32fe0cc8..ccc9a92e 100644
--- a/freemocap/initialization.py
+++ b/freemocap/initialization.py
@@ -174,8 +174,13 @@ def initialize(session, stage, board):
session.config_settings = recordingconfig.load_config_yaml(session.yamlPath) #config settings = paths and camera parameter inputs
recordingconfig.load_session_paths(session,session.config_settings) #add paths to session class
+
+ session.mediaPipe_imgPathList = session.config_settings['mediaPipe_imgPathList']
session.openPose_imgPathList = session.config_settings['openPose_imgPathList']
- session.numCams = len(session.openPose_imgPathList)
+
+
+ #session.openPose_imgPathList = session.config_settings['openPose_imgPathList']
+ #session.numCams = len(session.openPose_imgPathList)
diff --git a/freemocap/playskeleton.py b/freemocap/playskeleton.py
index 153565c2..1144197a 100644
--- a/freemocap/playskeleton.py
+++ b/freemocap/playskeleton.py
@@ -155,23 +155,27 @@ def ReplaySkeleton( session,
mean_charuco_fr_mar_dim = np.load(session.dataArrayPath/'charuco_points.npy')
- if useOpenPose:
- skel_fr_mar_dim = np.load(session.dataArrayPath/'openPoseSkel_3d.npy')
- if useMediaPipe:
+ if session.useMediaPipe:
mediaPipe_skel_fr_mar_dim = np.load(session.dataArrayPath/'mediaPipeSkel_3d.npy')
+ imgPathList = session.config_settings['mediaPipe_imgPathList']
+ session.numCams = len(session.mediaPipe_imgPathList)
+
+
+ if session.useOpenPose:
+ skel_fr_mar_dim = np.load(session.dataArrayPath/'openPoseSkel_3d.npy')
+ imgPathList = session.config_settings['openPose_imgPathList']
+ session.numCams = len(session.openPose_imgPathList)
- if useDLC:
+ if session.useDLC:
dlc_fr_mar_dim = np.load(session.dataArrayPath/'deepLabCut_3d.npy')
dlc0 = np.squeeze(dlc_fr_mar_dim[:,0,:])
dlc1 = np.squeeze(dlc_fr_mar_dim[:,1,:])
dlc2 = np.squeeze(dlc_fr_mar_dim[:,2,:])
ballTrailLen = 4
-
- if useOpenPose:
- imgPathList = session.config_settings['openPose_imgPathList']
- session.numCams = len(session.openPose_imgPathList)
+
+
camImgPathList = {}
for cam in range(session.numCams):
@@ -227,50 +231,51 @@ def ReplaySkeleton( session,
#plot skeleton (openPose by default)
+ if session.useOpenPose:
+ sk_x = skel_fr_mar_dim[fr,:,0] #skeleton x data
+ sk_y = skel_fr_mar_dim[fr,:,1] #skeleton y data
+ sk_z = skel_fr_mar_dim[fr,:,2] #skeleton z data
+
+ axMain.scatter3D(sk_x,sk_y,sk_z, marker='.',color = 'k', s=4.)
+ axMain.plot(sk_x[head],sk_y[head],sk_z[head], linestyle='-', color='g', linewidth = 1.)
+ axMain.plot(sk_x[spine],sk_y[spine],sk_z[spine], linestyle='-', color = 'g', linewidth = 1.)
+ axMain.plot(sk_x[rArm],sk_y[rArm],sk_z[rArm], linestyle='-', color = 'r', linewidth = 1.)
+ axMain.plot(sk_x[lArm],sk_y[lArm],sk_z[lArm], linestyle='-', color = 'b', linewidth = 1.)
+ axMain.plot(sk_x[rLeg],sk_y[rLeg],sk_z[rLeg], linestyle='-', color = 'r', linewidth = 1.)
+ axMain.plot(sk_x[lLeg],sk_y[lLeg],sk_z[lLeg], linestyle='-', color = 'b', linewidth = 1.)
+ axMain.plot(sk_x[rFoot],sk_y[rFoot],sk_z[rFoot], linestyle='-', color = 'r', linewidth = 1.)
+ axMain.plot(sk_x[lFoot],sk_y[lFoot],sk_z[lFoot], linestyle='-', color = 'b', linewidth = 1.)
+
+ # plot handybois
+ # right hand
+ axMain.plot(sk_x[thumb+rHandIDstart],sk_y[thumb+rHandIDstart],sk_z[thumb+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
+ axMain.plot(sk_x[index+rHandIDstart],sk_y[index+rHandIDstart],sk_z[index+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
+ axMain.plot(sk_x[bird+rHandIDstart],sk_y[bird+rHandIDstart],sk_z[bird+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
+ axMain.plot(sk_x[ring+rHandIDstart],sk_y[ring+rHandIDstart],sk_z[ring+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
+ axMain.plot(sk_x[pinky+rHandIDstart],sk_y[pinky+rHandIDstart],sk_z[pinky+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
+
+ #left hand
+ axMain.plot(sk_x[thumb+lHandIDstart],sk_y[thumb+lHandIDstart],sk_z[thumb+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
+ axMain.plot(sk_x[index+lHandIDstart],sk_y[index+lHandIDstart],sk_z[index+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
+ axMain.plot(sk_x[bird+lHandIDstart],sk_y[bird+lHandIDstart],sk_z[bird+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
+ axMain.plot(sk_x[ring+lHandIDstart],sk_y[ring+lHandIDstart],sk_z[ring+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
+ axMain.plot(sk_x[pinky+lHandIDstart],sk_y[pinky+lHandIDstart],sk_z[pinky+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
- sk_x = skel_fr_mar_dim[fr,:,0] #skeleton x data
- sk_y = skel_fr_mar_dim[fr,:,1] #skeleton y data
- sk_z = skel_fr_mar_dim[fr,:,2] #skeleton z data
-
- axMain.scatter3D(sk_x,sk_y,sk_z, marker='.',color = 'k', s=4.)
- axMain.plot(sk_x[head],sk_y[head],sk_z[head], linestyle='-', color='g', linewidth = 1.)
- axMain.plot(sk_x[spine],sk_y[spine],sk_z[spine], linestyle='-', color = 'g', linewidth = 1.)
- axMain.plot(sk_x[rArm],sk_y[rArm],sk_z[rArm], linestyle='-', color = 'r', linewidth = 1.)
- axMain.plot(sk_x[lArm],sk_y[lArm],sk_z[lArm], linestyle='-', color = 'b', linewidth = 1.)
- axMain.plot(sk_x[rLeg],sk_y[rLeg],sk_z[rLeg], linestyle='-', color = 'r', linewidth = 1.)
- axMain.plot(sk_x[lLeg],sk_y[lLeg],sk_z[lLeg], linestyle='-', color = 'b', linewidth = 1.)
- axMain.plot(sk_x[rFoot],sk_y[rFoot],sk_z[rFoot], linestyle='-', color = 'r', linewidth = 1.)
- axMain.plot(sk_x[lFoot],sk_y[lFoot],sk_z[lFoot], linestyle='-', color = 'b', linewidth = 1.)
-
- # plot handybois
- # right hand
- axMain.plot(sk_x[thumb+rHandIDstart],sk_y[thumb+rHandIDstart],sk_z[thumb+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
- axMain.plot(sk_x[index+rHandIDstart],sk_y[index+rHandIDstart],sk_z[index+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
- axMain.plot(sk_x[bird+rHandIDstart],sk_y[bird+rHandIDstart],sk_z[bird+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
- axMain.plot(sk_x[ring+rHandIDstart],sk_y[ring+rHandIDstart],sk_z[ring+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
- axMain.plot(sk_x[pinky+rHandIDstart],sk_y[pinky+rHandIDstart],sk_z[pinky+rHandIDstart], linestyle='-', color = 'r', linewidth = 1.)
-
- #left hand
- axMain.plot(sk_x[thumb+lHandIDstart],sk_y[thumb+lHandIDstart],sk_z[thumb+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
- axMain.plot(sk_x[index+lHandIDstart],sk_y[index+lHandIDstart],sk_z[index+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
- axMain.plot(sk_x[bird+lHandIDstart],sk_y[bird+lHandIDstart],sk_z[bird+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
- axMain.plot(sk_x[ring+lHandIDstart],sk_y[ring+lHandIDstart],sk_z[ring+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
- axMain.plot(sk_x[pinky+lHandIDstart],sk_y[pinky+lHandIDstart],sk_z[pinky+lHandIDstart], linestyle='-', color = 'b', linewidth = 1.)
-
#plot mediapipe
- mp_sk_x = mediaPipe_skel_fr_mar_dim[fr,:,0] #skeleton x data
- mp_sk_y = mediaPipe_skel_fr_mar_dim[fr,:,1] #skeleton y data
- mp_sk_z = mediaPipe_skel_fr_mar_dim[fr,:,2] #skeleton z data
- axMain.scatter3D(mp_sk_x, mp_sk_y, mp_sk_z, marker='.',color = 'g', s=8.)
+ if session.useMediaPipe:
+ mp_sk_x = mediaPipe_skel_fr_mar_dim[fr,:,0] #skeleton x data
+ mp_sk_y = mediaPipe_skel_fr_mar_dim[fr,:,1] #skeleton y data
+ mp_sk_z = mediaPipe_skel_fr_mar_dim[fr,:,2] #skeleton z data
+ axMain.scatter3D(mp_sk_x, mp_sk_y, mp_sk_z, marker='.',color = 'g', s=8.)
#plot deeplabcut
-
- dlc_x = dlc_fr_mar_dim[fr,:,0]
- dlc_y = dlc_fr_mar_dim[fr,:,1]
- dlc_z = dlc_fr_mar_dim[fr,:,2]
-
- axMain.scatter3D(dlc_x,dlc_y,dlc_z, marker='o',color = 'r', s=24.)
+ if session.useDLC:
+ dlc_x = dlc_fr_mar_dim[fr,:,0]
+ dlc_y = dlc_fr_mar_dim[fr,:,1]
+ dlc_z = dlc_fr_mar_dim[fr,:,2]
+
+ axMain.scatter3D(dlc_x,dlc_y,dlc_z, marker='o',color = 'r', s=24.)
#plot charuco grid
axMain.scatter(char_x,char_y,char_z, marker='o')
diff --git a/freemocap/recordingconfig.py b/freemocap/recordingconfig.py
index 74d0645d..bddfcbcc 100644
--- a/freemocap/recordingconfig.py
+++ b/freemocap/recordingconfig.py
@@ -36,7 +36,7 @@ def createSession(session,filepath, existing_data = False):
session.openPoseDataPath = session.sessionPath/'OpenPoseData'
session.openPoseDataPath.mkdir(exist_ok = True)
- pathList.append(session.mediaPipeDataPath)
+ pathList.append(session.openPoseDataPath)
session.mediaPipeDataPath = session.sessionPath/'MediaPipeData'
session.mediaPipeDataPath.mkdir(exist_ok = True)
@@ -59,6 +59,8 @@ def createSession(session,filepath, existing_data = False):
config_settings,config_yaml = create_config_yaml()
+ session.config_settings = config_settings
+
yaml_name = session.sessionPath/'{}_config.yaml'.format(session.sessionID)
session.yamlPath = yaml_name
# print(yaml_name)
@@ -93,6 +95,7 @@ def create_config_yaml():
rawVidPath:
syncedVidPath:
calVidPath:
+ openPoseDataPath:
mediaPipeDataPath:
dlcDataPath:
imOutPath:
@@ -101,6 +104,7 @@ def create_config_yaml():
"""
config_yaml = YAML()
config_settings = config_yaml.load(yaml_str)
+
return config_settings,config_yaml
diff --git a/freemocap/session.py b/freemocap/session.py
index ec7b4754..7975528d 100644
--- a/freemocap/session.py
+++ b/freemocap/session.py
@@ -16,4 +16,5 @@ class Session: #session like "recording session"
self.mean_charuco_fr_mar_dim = None
self.skel_fr_mar_dim = None
- self.mediaPipe_imgPathList = None
\ No newline at end of file
+ self.mediaPipe_imgPathList = None
+ self.openPose_imgPathList = None
\ No newline at end of file
diff --git a/runme_fmc_staged.py b/runme_fmc_staged.py
index 1a29880b..7fda929c 100644
--- a/runme_fmc_staged.py
+++ b/runme_fmc_staged.py
@@ -1,4 +1,4 @@
-from freemocap import createvideo, initialization, runcams, calibrate, fmc_mediapipe, fmc_openpose, fmc_deeplabcut, reconstruct3D, playskeleton, session
+from freemocap import createvideo, initialization, runcams, calibrate, fmc_mediapipe, fmc_openpose, fmc_deeplabcut, reconstruct3D, playskeleton, recordingconfig,session
from pathlib import Path
import os
@@ -10,21 +10,21 @@ import numpy as np
sesh = session.Session()
-useOpenPose= True
-useMediaPipe = True
-useDLC=True
+sesh.useOpenPose= True
+sesh.useMediaPipe = True
+sesh.useDLC=False
-if useDLC:
+if sesh.useDLC:
import deeplabcut as dlc
sesh.dlcConfigPath = Path("C:\\Users\\jonma\\Dropbox\\GitKrakenRepos\\freemocap\\DLC_Models\\PinkGreenRedJugglingBalls-JSM-2021-05-31\\config.yaml")
# %% Inputs to edit
-stage = 3 #set your starting stage here (stage = 1 will run the pipeline from webcams)
+stage = 4 #set your starting stage here (stage = 1 will run the pipeline from webcams)
sesh.debug = False
-sesh.sessionID = 'sesh_21-05-31_111833' #fill in if you are running from Stage 2 onwards
+sesh.sessionID = '' #fill in if you are running from Stage 2 onwards
if not sesh.sessionID:
dataFolder = Path.cwd()/'Data'
subfolders = [ f.path for f in os.scandir(dataFolder) if f.is_dir() ] #copy-pasta from who knows where
@@ -75,19 +75,19 @@ else:
# %% Stage Four
if stage <= 4:
print('Starting Track Image Points')
- if useMediaPipe:
+ if sesh.useMediaPipe:
fmc_mediapipe.runMediaPipe(sesh)
sesh.mediaPipeData_nCams_nFrames_nImgPts_XYC = fmc_mediapipe.parseMediaPipe(sesh)
sesh.mediaPipeSkel_fr_mar_dim = reconstruct3D.reconstruct3D(sesh,sesh.mediaPipeData_nCams_nFrames_nImgPts_XYC, confidenceThreshold=.1)
np.save(sesh.dataArrayPath/'mediaPipeSkel_3d.npy', sesh.mediaPipeSkel_fr_mar_dim) #save data to npy
- if useOpenPose:
+ if sesh.useOpenPose:
fmc_openpose.runOpenPose(sesh, dummyRun=False)
sesh.openPoseData_nCams_nFrames_nImgPts_XYC = fmc_openpose.parseOpenPose(sesh)
sesh.openPoseskel_fr_mar_dim = reconstruct3D.reconstruct3D(sesh,sesh.openPoseData_nCams_nFrames_nImgPts_XYC, confidenceThreshold=.1)
np.save(sesh.dataArrayPath/'openPoseSkel_3d.npy', sesh.openPoseskel_fr_mar_dim) #save data to npy
- if useDLC:
+ if sesh.useDLC:
syncedVidList = []
for vid in sesh.syncedVidPath.glob('*.mp4'):
syncedVidList.append(str(vid))
@@ -122,6 +122,9 @@ else:
+#reupdate the config_settings with mediapipe and openpose image paths
+sesh.config_settings = recordingconfig.load_config_yaml(sesh.yamlPath)
+
# %% Stage Seven
if stage <= 7:
print('Starting Skeleton Plotting')
@@ -131,9 +134,9 @@ if stage <= 7:
startFrame=40,
azimuth=-90,
elevation=-80,
- useOpenPose=useOpenPose,
- useMediaPipe=useMediaPipe,
- useDLC=useDLC)
+ useOpenPose=sesh.useOpenPose,
+ useMediaPipe=sesh.useMediaPipe,
+ useDLC=sesh.useDLC)
else:
print('Skipping Skeleton Plotting')