Free Motion Capture for Everyone πβ¨
TypeScript
Added prettier Rich prints - And iterative Anipose solver
jon matthis67b4e0003d08
diff --git a/freemocap/calibrate.py b/freemocap/calibrate.pyindex 9c5675f6..d5c57ae9 100644--- a/freemocap/calibrate.py+++ b/freemocap/calibrate.py@@ -1,3 +1,6 @@+import os+import copy+from aniposelib.cameras import CameraGroupfrom aniposelib.utils import load_pose2d_fnamesimport numpy as np@@ -6,26 +9,34 @@ import cv2import matplotlib.pyplot as pltfrom mpl_toolkits.mplot3d import Axes3Dfrom freemocap import reconstruct3D, fmc_anipose-import osfrom rich.progress import track+from rich import print--def CalibrateCaptureVolume(session,board, calVideoFrameLength = 120):+def CalibrateCaptureVolume(session,board, calVideoFrameLength = .25, spoof_anipose_fail_bool=False):"""Check if a previous calibration yaml exists, and if not, create a set of shortened calibration videos and run Anipose functionsto create a calibration yaml. Takes the 2D charuco board points and reconstructs them into 3D points that are saved outinto the DataArrays folder++ note `debugAniposeIter` spoofs a failed anipose calibration to make it easier to test the thing where we iterate through with different frame windows to try to find one that works"""+session.calVidPath.mkdir(exist_ok = True)session.dataArrayPath.mkdir(exist_ok = True)+ calibrationVideoPath = session.calVidPath- if calVideoFrameLength < 0:- calVideoFrameLength = session.numFrames- calibrationVideoPath = session.syncedVidPath- else:- createCalibrationVideos(session, calVideoFrameLength)+ if calVideoFrameLength < 0: # if '-1' use the whole video+ calVideoFrameLength = [0, session.numFrames]calibrationVideoPath = session.calVidPath+ else:+ if calVideoFrameLength>0 and calVideoFrameLength<1: #if between 0 and 1, use as a percentage of the total video length+ calVideoFrameLength = [0, round(calVideoFrameLength * session.numFrames)]+ else: #otherwise, just use the input value as the number of frames to use+ calVideoFrameLength = [0, calVideoFrameLength]+ createCalibrationVideos(session, calVideoFrameLength)++ calVideoFrameLength_og = copy.copy(calVideoFrameLength)vidnames = []cam_names = []@@ -34,36 +45,65 @@ def CalibrateCaptureVolume(session,board, calVideoFrameLength = 120):cam_names.append(str(count))session.numCams = count- cgroup = fmc_anipose.CameraGroup.from_names(- cam_names, fisheye=True- ) # Looking through their code... it looks lke the 'fisheye=True' doesn't do much (see 2020-03-29 obsidian note)+ cgroup = fmc_anipose.CameraGroup.from_names( cam_names, fisheye=True ) # Looking through their code... it looks lke the 'fisheye=True' doesn't do much (see 2020-03-29 obsidian note)calibrationFile = "{}_calibration.yaml".format(session.sessionID)session.cameraCalFilePath = session.sessionPath / calibrationFile- forceRunCameraExtrinsicsCalibration = True # set this to True to re-run the camera calibration and saveover the existing config file-- # if not session.cameraCalFilePath.exists() or forceRunCameraExtrinsicsCalibration: #if that config file doesn't exist, run the camera calibration whosists. If it does, then just load the toml- ## this will take a few minutes- ## it will detect the charuco board in the videos,- ## then calibrate the cameras based on the detections, using iterative bundle adjustment- # error,all_rows,merged = cgroup.calibrate_videos(vidnames, board) #JSM NOTE - in its original form, this method doesn't throw an error when a video fails to load. This is an easy fix!++ try:+ error,charuco_data, charuco_frames = cgroup.calibrate_videos(vidnames, board)+ if not spoof_anipose_fail_bool:+ anipose_success = True+ else:+ anipose_success = False+ except:+ anipose_success = False+ print('Anipose Calibration Failed for frame range {} - {}'.format(calVideoFrameLength[0], calVideoFrameLength[1]))++ anipose_iter = -1+ if not anipose_success: #%% run anipose calibration, and if it fails, re-make calibration vids with different frame range and try again++ frame_window_size = round(session.numFrames*.5)+ cal_video_frame_range = [0, frame_window_size]+ while not anipose_success:+ anipose_iter += 1+ print('Anipose - Reprocessing - Iteration #{}'.format(anipose_iter))++ if cal_video_frame_range[1] > session.numFrames:+ print('We moved the window all the way through the videos and Anipose still isn\'t happy, starting again from the begining with a shorter frame window')+ frame_window_size = round(frame_window_size/2)+ cal_video_frame_range = [0, frame_window_size]++ if round(cal_video_frame_range[-1] * session.numFrames) < 1:+ Exception('Sorry, we weren\'t able to get AniPose to run successfully :( - Make sure your Charuco board was visible to each camera (not necessarily at the same time) and that there wasn\'t, like, a bunch of glare on it')+ break++ print('Trying Anipose calibration again with frame range {} - {}'.format(cal_video_frame_range[0], cal_video_frame_range[1]))+ createCalibrationVideos(session, cal_video_frame_range)++ try:+ error,charuco_data, charuco_frames = cgroup.calibrate_videos(vidnames, board)++ if not spoof_anipose_fail_bool:+ anipose_success=True+ else:+ cal_video_frame_range[0] = cal_video_frame_range[0] + round(session.numFrames*.1)+ cal_video_frame_range[1] = cal_video_frame_range[1] + round(session.numFrames*.1)+ except:+ cal_video_frame_range[0] = cal_video_frame_range[0] + round(session.numFrames*.1)+ cal_video_frame_range[1] = cal_video_frame_range[1] + round(session.numFrames*.1)- #%% This next bit is a copy-paste of the inner bits of `cgroup.calibrate_videos` - Part 1 of 2- """Takes as input a list of list of video filenames, one list of each camera.- Also takes a board which specifies what should be detected in the videos"""- error,charuco_data, charuco_frames = cgroup.calibrate_videos(vidnames, board)cgroup.dump(session.cameraCalFilePath) #JSM NOTE - let's just use .yaml's unless there is some reason to use .tomlsession.cgroup = cgroup- n_frames = calVideoFrameLength+ n_frames = calVideoFrameLength[1]-calVideoFrameLength[0]startframe = 0n_trackedPoints = 24- framelist = range(startframe, startframe + n_frames)charuco_nCams_nFrames_nImgPts_XY = np.empty([session.numCams, n_frames, n_trackedPoints, 2])charuco_nCams_nFrames_nImgPts_XY[:] = np.nan@@ -153,7 +193,13 @@ def createCalibrationVideos(session, calVideoFrameLength):to that specific length"""vidList = os.listdir(session.syncedVidPath)- framelist = list(range(calVideoFrameLength))+ if len(calVideoFrameLength)==1:+ framelist = list(range(calVideoFrameLength))+ elif len(calVideoFrameLength)==2:+ framelist = list(range(calVideoFrameLength[0], calVideoFrameLength[1]))+ else:+ Exception('calVideoFrameLength must be either 1 or 2 elements long')+codec = "DIVX"for count, vid in enumerate(vidList, start=1):cam_name = "Cam{}".format(count)diff --git a/freemocap/fmc_runme.py b/freemocap/fmc_runme.pyindex 05b5d571..aa7697f0 100644--- a/freemocap/fmc_runme.py+++ b/freemocap/fmc_runme.py@@ -49,6 +49,7 @@ def RunMe(sessionID=None,setDataPath = False,userDataPath = None,recordVid = True,+ showAnimation = True,reconstructionConfidenceThreshold = .7,charucoSquareSize = 36,#mm - ~the size of the squares when printed on 8.5x11" paper based on parameters in ReadMe.mdcalVideoFrameLength = -1,@@ -109,6 +110,10 @@ def RunMe(sessionID=None,marker_bits=4, dict_size=250)sesh.board = board+ console.rule()+ console.rule('Finding available webcams')+ console.rule()+# %% Initializationif stage == 1:camera_settings.initialize(sesh,stage,board)@@ -119,18 +124,22 @@ def RunMe(sessionID=None,# %% Stage Oneif stage <= 1:- print()- print('Starting Video Recordings')+ console.rule()+ console.rule('Starting Video Recordings')+ console.rule()+runcams.RecordCams(sesh, sesh.cam_inputs, sesh.parameterDictionary, sesh.rotationInputs)sesh.save_session()else:- print('Skipping Video Recording')+ console.rule('Skipping Video Recording')# %% Stage Twoif stage <= 2:- print()- print('Starting Video Syncing')+ console.rule()+ console.rule('Synchronizing Recorded Videos')+ console.rule()+runcams.SyncCams(sesh, sesh.timeStampData,sesh.numCamRange,sesh.vidNames,sesh.camIDs)sesh.save_session()else:@@ -138,17 +147,30 @@ def RunMe(sessionID=None,# %% Stage Threeif stage <= 3:- print()- print('Starting Calibration')- sesh.cgroup, sesh.mean_charuco_fr_mar_xyz = calibrate.CalibrateCaptureVolume(sesh,board, calVideoFrameLength)+ console.rule()+ console.rule('Starting Anipose Calibration')+ console.rule('Identifying 6 DoF position of each camera based on detected charuco boards')+ console.rule()++ print('See https://anipose.org for details')+ sesh.cgroup, sesh.mean_charuco_fr_mar_xyz = calibrate.CalibrateCaptureVolume(sesh,board, calVideoFrameLength, spoof_anipose_fail_bool=False)+ print('Anipose Calibration Successful!')else:print('Skipping Calibration')# %% Stage Fourif stage <= 4:+ console.rule()+ console.rule('Starting 2D Point Trackers')+ console.rule('This step impolements the machine learning driven computer vision convolutional neural networks that track the skeleton, etc')+ console.rule('This part is crazy future tech sci fi stuff. Seriously unbelievable this kind of thing is possible.')+ console.rule()- print('Starting Track Image Points')if sesh.useMediaPipe:+ console.rule()+ console.rule('Running MediaPipe skeleton tracker - https://google.github.io/mediapipe')+ console.rule()+if runMediaPipe:fmc_mediapipe.runMediaPipe(sesh)@@ -159,6 +181,10 @@ def RunMe(sessionID=None,sesh.save_session()if sesh.useOpenPose:+ console.rule()+ console.rule('Running OpenPose skeleton tracker - https://github.com/CMU-Perceptual-Computing-Lab/openpose')+ console.rule()+fmc_openpose.runOpenPose(sesh, runOpenPose=runOpenPose)sesh.openPoseData_nCams_nFrames_nImgPts_XYC = fmc_openpose.parseOpenPose(sesh)sesh.openPoseSkel_fr_mar_xyz, sesh.openPoseSkel_reprojErr = reconstruct3D.reconstruct3D(sesh,sesh.openPoseData_nCams_nFrames_nImgPts_XYC, confidenceThreshold=reconstructionConfidenceThreshold)@@ -166,7 +192,13 @@ def RunMe(sessionID=None,np.save(sesh.dataArrayPath/'openPoseSkel_reprojErr.npy', sesh.openPoseSkel_reprojErr) #save data to npysesh.save_session()sesh.syncedVidList = []+if sesh.useDLC:++ console.rule()+ console.rule('Running DeepLabCut :mouse: - https://deeplabcut.org')+ console.rule()+for vid in sesh.syncedVidPath.glob('*.mp4'):sesh.syncedVidList.append(str(vid))@@ -178,10 +210,11 @@ def RunMe(sessionID=None,np.save(sesh.dataArrayPath/'deepLabCut_reprojErr.npy', sesh.dlc_reprojErr) #save data to npysesh.save_session()else:- print('Skipping Run MediaPipe')+ print('Skipping 2d point tracking')+<<<<<<< Updated upstream# if stage <=5:# if useBlender == True:# #blenderExePath = Path('C:\Program Files\Blender Foundation\Blender 2.93')@@ -192,6 +225,29 @@ def RunMe(sessionID=None,# %% Stage Five - Make Skreleton Animationif stage <= 5:print('Starting Skeleton Plotting')+=======+ if stage <=5:+ if useBlender == True:++ console.rule()+ console.rule('Exporting Files...')+ console.rule('Hijacking Blender\'s file format converters to export FreeMoCap data as various file format (.blend, .usd, .gltf, .fbx)')+ console.rule()++ #blenderExePath = Path('C:\Program Files\Blender Foundation\Blender 2.93')+ #os.chdir(blenderExePath)+ output = subprocess.run([blenderEXEpath, "--background", "--python", str(subprocessPath), "--", str(sesh.dataArrayPath/'mediaPipeSkel_3d.npy')], capture_output=True, text=True, check=True)+ print(output)++ # %% Stage Five - Make Skreleton Animation+ if stage <= 6:++ console.rule()+ console.rule('Creating the Skelton animation!')+ console.rule()('The video creation is very slow. This whole animation maker is crazy slow, tbh. Sorry about that. Future iterations will be better :sweat_smile:')+ console.rule()++>>>>>>> Stashed changesplay_skeleton_animation.PlaySkeletonAnimation(sesh,startFrame=sesh.startFrame,@@ -200,7 +256,8 @@ def RunMe(sessionID=None,useOpenPose=useOpenPose,useMediaPipe=useMediaPipe,useDLC=useDLC,- recordVid = recordVid+ recordVid = recordVid,+ showAnimation=showAnimation,)# print ('Starting PyQT Animation')# createvideo.createBodyTrackingVideos(sesh)@@ -214,5 +271,7 @@ def RunMe(sessionID=None,print('Skipping Skeleton Plotting')-+ console.rule('All Done! Data folder is at: ' + str(sesh.basePath))++ console.print(Markdown("""# β¨πβ¨ """))diff --git a/freemocap/play_skeleton_animation.py b/freemocap/play_skeleton_animation.pyindex 77706315..9812d77e 100644--- a/freemocap/play_skeleton_animation.py+++ b/freemocap/play_skeleton_animation.py@@ -67,7 +67,8 @@ def PlaySkeletonAnimation(useOpenPose=True,useMediaPipe=False,useDLC=False,- recordVid = True+ recordVid = True,+ showAnimation =True,):@@ -845,14 +846,15 @@ def PlaySkeletonAnimation(try:- plt.pause(0.1)- plt.draw()+ if showAnimation:+ plt.pause(0.1)+ plt.draw()except:pass- console.print(":sparkle: :skull: :sparkle:")+###diff --git a/freemocap_runme_script.py b/freemocap_runme_script.pyindex c7e38098..104eb169 100644--- a/freemocap_runme_script.py+++ b/freemocap_runme_script.py@@ -1,3 +1,3 @@import freemocap as fmc-fmc.RunMe( )+fmc.RunMe()diff --git a/requirements.txt b/requirements.txtindex 86e34d22..dd3b3708 100644--- a/requirements.txt+++ b/requirements.txt@@ -3,11 +3,12 @@ h5py>=3.1.0imutils~=0.5.4matplotlib~=3.1.3mediapipe==0.8.8-opencv-contrib-python~=3.4.14.51+opencv-contrib-python~=3.4pandas~=1.2.5rich~=10.4.0tqdm~=4.61.1ruamel.yaml>=0.15.0numpy<=1.19typed-ast<1.5,>=1.4.0-moviepy ~=1.0.3\ No newline at end of file+moviepy ~=1.0.3+ipython\ No newline at end of file