Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

Added prettier Rich prints - And iterative Anipose solver

jon matthis67b4e0003d08
diff --git a/freemocap/calibrate.py b/freemocap/calibrate.py
index 9c5675f6..d5c57ae9 100644
--- a/freemocap/calibrate.py
+++ b/freemocap/calibrate.py
@@ -1,3 +1,6 @@
+import os
+import copy
+
from aniposelib.cameras import CameraGroup
from aniposelib.utils import load_pose2d_fnames
import numpy as np
@@ -6,26 +9,34 @@ import cv2
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from freemocap import reconstruct3D, fmc_anipose
-import os
from rich.progress import track
+from rich import print
-
-def CalibrateCaptureVolume(session,board, calVideoFrameLength = 120):
+def CalibrateCaptureVolume(session,board, calVideoFrameLength = .25, spoof_anipose_fail_bool=False):
"""
Check if a previous calibration yaml exists, and if not, create a set of shortened calibration videos and run Anipose functions
to create a calibration yaml. Takes the 2D charuco board points and reconstructs them into 3D points that are saved out
into the DataArrays folder
+
+ note `debugAniposeIter` spoofs a failed anipose calibration to make it easier to test the thing where we iterate through with different frame windows to try to find one that works
"""
+
session.calVidPath.mkdir(exist_ok = True)
session.dataArrayPath.mkdir(exist_ok = True)
+ calibrationVideoPath = session.calVidPath
- if calVideoFrameLength < 0:
- calVideoFrameLength = session.numFrames
- calibrationVideoPath = session.syncedVidPath
- else:
- createCalibrationVideos(session, calVideoFrameLength)
+ if calVideoFrameLength < 0: # if '-1' use the whole video
+ calVideoFrameLength = [0, session.numFrames]
calibrationVideoPath = session.calVidPath
+ else:
+ if calVideoFrameLength>0 and calVideoFrameLength<1: #if between 0 and 1, use as a percentage of the total video length
+ calVideoFrameLength = [0, round(calVideoFrameLength * session.numFrames)]
+ else: #otherwise, just use the input value as the number of frames to use
+ calVideoFrameLength = [0, calVideoFrameLength]
+ createCalibrationVideos(session, calVideoFrameLength)
+
+ calVideoFrameLength_og = copy.copy(calVideoFrameLength)
vidnames = []
cam_names = []
@@ -34,36 +45,65 @@ def CalibrateCaptureVolume(session,board, calVideoFrameLength = 120):
cam_names.append(str(count))
session.numCams = count
- cgroup = fmc_anipose.CameraGroup.from_names(
- cam_names, fisheye=True
- ) # Looking through their code... it looks lke the 'fisheye=True' doesn't do much (see 2020-03-29 obsidian note)
+ cgroup = fmc_anipose.CameraGroup.from_names( cam_names, fisheye=True ) # Looking through their code... it looks lke the 'fisheye=True' doesn't do much (see 2020-03-29 obsidian note)
calibrationFile = "{}_calibration.yaml".format(session.sessionID)
session.cameraCalFilePath = session.sessionPath / calibrationFile
- forceRunCameraExtrinsicsCalibration = True # set this to True to re-run the camera calibration and saveover the existing config file
-
- # if not session.cameraCalFilePath.exists() or forceRunCameraExtrinsicsCalibration: #if that config file doesn't exist, run the camera calibration whosists. If it does, then just load the toml
- ## this will take a few minutes
- ## it will detect the charuco board in the videos,
- ## then calibrate the cameras based on the detections, using iterative bundle adjustment
- # error,all_rows,merged = cgroup.calibrate_videos(vidnames, board) #JSM NOTE - in its original form, this method doesn't throw an error when a video fails to load. This is an easy fix!
+
+ try:
+ error,charuco_data, charuco_frames = cgroup.calibrate_videos(vidnames, board)
+ if not spoof_anipose_fail_bool:
+ anipose_success = True
+ else:
+ anipose_success = False
+ except:
+ anipose_success = False
+ print('Anipose Calibration Failed for frame range {} - {}'.format(calVideoFrameLength[0], calVideoFrameLength[1]))
+
+ anipose_iter = -1
+ if not anipose_success: #%% run anipose calibration, and if it fails, re-make calibration vids with different frame range and try again
+
+ frame_window_size = round(session.numFrames*.5)
+ cal_video_frame_range = [0, frame_window_size]
+ while not anipose_success:
+ anipose_iter += 1
+ print('Anipose - Reprocessing - Iteration #{}'.format(anipose_iter))
+
+ if cal_video_frame_range[1] > session.numFrames:
+ print('We moved the window all the way through the videos and Anipose still isn\'t happy, starting again from the begining with a shorter frame window')
+ frame_window_size = round(frame_window_size/2)
+ cal_video_frame_range = [0, frame_window_size]
+
+ if round(cal_video_frame_range[-1] * session.numFrames) < 1:
+ Exception('Sorry, we weren\'t able to get AniPose to run successfully :( - Make sure your Charuco board was visible to each camera (not necessarily at the same time) and that there wasn\'t, like, a bunch of glare on it')
+ break
+
+ print('Trying Anipose calibration again with frame range {} - {}'.format(cal_video_frame_range[0], cal_video_frame_range[1]))
+ createCalibrationVideos(session, cal_video_frame_range)
+
+ try:
+ error,charuco_data, charuco_frames = cgroup.calibrate_videos(vidnames, board)
+
+ if not spoof_anipose_fail_bool:
+ anipose_success=True
+ else:
+ cal_video_frame_range[0] = cal_video_frame_range[0] + round(session.numFrames*.1)
+ cal_video_frame_range[1] = cal_video_frame_range[1] + round(session.numFrames*.1)
+ except:
+ cal_video_frame_range[0] = cal_video_frame_range[0] + round(session.numFrames*.1)
+ cal_video_frame_range[1] = cal_video_frame_range[1] + round(session.numFrames*.1)
- #%% This next bit is a copy-paste of the inner bits of `cgroup.calibrate_videos` - Part 1 of 2
- """Takes as input a list of list of video filenames, one list of each camera.
- Also takes a board which specifies what should be detected in the videos"""
- error,charuco_data, charuco_frames = cgroup.calibrate_videos(vidnames, board)
cgroup.dump(session.cameraCalFilePath) #JSM NOTE - let's just use .yaml's unless there is some reason to use .toml
session.cgroup = cgroup
- n_frames = calVideoFrameLength
+ n_frames = calVideoFrameLength[1]-calVideoFrameLength[0]
startframe = 0
n_trackedPoints = 24
- framelist = range(startframe, startframe + n_frames)
charuco_nCams_nFrames_nImgPts_XY = np.empty([session.numCams, n_frames, n_trackedPoints, 2])
charuco_nCams_nFrames_nImgPts_XY[:] = np.nan
@@ -153,7 +193,13 @@ def createCalibrationVideos(session, calVideoFrameLength):
to that specific length
"""
vidList = os.listdir(session.syncedVidPath)
- framelist = list(range(calVideoFrameLength))
+ if len(calVideoFrameLength)==1:
+ framelist = list(range(calVideoFrameLength))
+ elif len(calVideoFrameLength)==2:
+ framelist = list(range(calVideoFrameLength[0], calVideoFrameLength[1]))
+ else:
+ Exception('calVideoFrameLength must be either 1 or 2 elements long')
+
codec = "DIVX"
for count, vid in enumerate(vidList, start=1):
cam_name = "Cam{}".format(count)
diff --git a/freemocap/fmc_runme.py b/freemocap/fmc_runme.py
index 05b5d571..aa7697f0 100644
--- a/freemocap/fmc_runme.py
+++ b/freemocap/fmc_runme.py
@@ -49,6 +49,7 @@ def RunMe(sessionID=None,
setDataPath = False,
userDataPath = None,
recordVid = True,
+ showAnimation = True,
reconstructionConfidenceThreshold = .7,
charucoSquareSize = 36,#mm - ~the size of the squares when printed on 8.5x11" paper based on parameters in ReadMe.md
calVideoFrameLength = -1,
@@ -109,6 +110,10 @@ def RunMe(sessionID=None,
marker_bits=4, dict_size=250)
sesh.board = board
+ console.rule()
+ console.rule('Finding available webcams')
+ console.rule()
+
# %% Initialization
if stage == 1:
camera_settings.initialize(sesh,stage,board)
@@ -119,18 +124,22 @@ def RunMe(sessionID=None,
# %% Stage One
if stage <= 1:
- print()
- print('Starting Video Recordings')
+ console.rule()
+ console.rule('Starting Video Recordings')
+ console.rule()
+
runcams.RecordCams(sesh, sesh.cam_inputs, sesh.parameterDictionary, sesh.rotationInputs)
sesh.save_session()
else:
- print('Skipping Video Recording')
+ console.rule('Skipping Video Recording')
# %% Stage Two
if stage <= 2:
- print()
- print('Starting Video Syncing')
+ console.rule()
+ console.rule('Synchronizing Recorded Videos')
+ console.rule()
+
runcams.SyncCams(sesh, sesh.timeStampData,sesh.numCamRange,sesh.vidNames,sesh.camIDs)
sesh.save_session()
else:
@@ -138,17 +147,30 @@ def RunMe(sessionID=None,
# %% Stage Three
if stage <= 3:
- print()
- print('Starting Calibration')
- sesh.cgroup, sesh.mean_charuco_fr_mar_xyz = calibrate.CalibrateCaptureVolume(sesh,board, calVideoFrameLength)
+ console.rule()
+ console.rule('Starting Anipose Calibration')
+ console.rule('Identifying 6 DoF position of each camera based on detected charuco boards')
+ console.rule()
+
+ print('See https://anipose.org for details')
+ sesh.cgroup, sesh.mean_charuco_fr_mar_xyz = calibrate.CalibrateCaptureVolume(sesh,board, calVideoFrameLength, spoof_anipose_fail_bool=False)
+ print('Anipose Calibration Successful!')
else:
print('Skipping Calibration')
# %% Stage Four
if stage <= 4:
+ console.rule()
+ console.rule('Starting 2D Point Trackers')
+ console.rule('This step impolements the machine learning driven computer vision convolutional neural networks that track the skeleton, etc')
+ console.rule('This part is crazy future tech sci fi stuff. Seriously unbelievable this kind of thing is possible.')
+ console.rule()
- print('Starting Track Image Points')
if sesh.useMediaPipe:
+ console.rule()
+ console.rule('Running MediaPipe skeleton tracker - https://google.github.io/mediapipe')
+ console.rule()
+
if runMediaPipe:
fmc_mediapipe.runMediaPipe(sesh)
@@ -159,6 +181,10 @@ def RunMe(sessionID=None,
sesh.save_session()
if sesh.useOpenPose:
+ console.rule()
+ console.rule('Running OpenPose skeleton tracker - https://github.com/CMU-Perceptual-Computing-Lab/openpose')
+ console.rule()
+
fmc_openpose.runOpenPose(sesh, runOpenPose=runOpenPose)
sesh.openPoseData_nCams_nFrames_nImgPts_XYC = fmc_openpose.parseOpenPose(sesh)
sesh.openPoseSkel_fr_mar_xyz, sesh.openPoseSkel_reprojErr = reconstruct3D.reconstruct3D(sesh,sesh.openPoseData_nCams_nFrames_nImgPts_XYC, confidenceThreshold=reconstructionConfidenceThreshold)
@@ -166,7 +192,13 @@ def RunMe(sessionID=None,
np.save(sesh.dataArrayPath/'openPoseSkel_reprojErr.npy', sesh.openPoseSkel_reprojErr) #save data to npy
sesh.save_session()
sesh.syncedVidList = []
+
if sesh.useDLC:
+
+ console.rule()
+ console.rule('Running DeepLabCut :mouse: - https://deeplabcut.org')
+ console.rule()
+
for vid in sesh.syncedVidPath.glob('*.mp4'):
sesh.syncedVidList.append(str(vid))
@@ -178,10 +210,11 @@ def RunMe(sessionID=None,
np.save(sesh.dataArrayPath/'deepLabCut_reprojErr.npy', sesh.dlc_reprojErr) #save data to npy
sesh.save_session()
else:
- print('Skipping Run MediaPipe')
+ print('Skipping 2d point tracking')
+<<<<<<< Updated upstream
# if stage <=5:
# if useBlender == True:
# #blenderExePath = Path('C:\Program Files\Blender Foundation\Blender 2.93')
@@ -192,6 +225,29 @@ def RunMe(sessionID=None,
# %% Stage Five - Make Skreleton Animation
if stage <= 5:
print('Starting Skeleton Plotting')
+=======
+ if stage <=5:
+ if useBlender == True:
+
+ console.rule()
+ console.rule('Exporting Files...')
+ console.rule('Hijacking Blender\'s file format converters to export FreeMoCap data as various file format (.blend, .usd, .gltf, .fbx)')
+ console.rule()
+
+ #blenderExePath = Path('C:\Program Files\Blender Foundation\Blender 2.93')
+ #os.chdir(blenderExePath)
+ output = subprocess.run([blenderEXEpath, "--background", "--python", str(subprocessPath), "--", str(sesh.dataArrayPath/'mediaPipeSkel_3d.npy')], capture_output=True, text=True, check=True)
+ print(output)
+
+ # %% Stage Five - Make Skreleton Animation
+ if stage <= 6:
+
+ console.rule()
+ console.rule('Creating the Skelton animation!')
+ console.rule()('The video creation is very slow. This whole animation maker is crazy slow, tbh. Sorry about that. Future iterations will be better :sweat_smile:')
+ console.rule()
+
+>>>>>>> Stashed changes
play_skeleton_animation.PlaySkeletonAnimation(
sesh,
startFrame=sesh.startFrame,
@@ -200,7 +256,8 @@ def RunMe(sessionID=None,
useOpenPose=useOpenPose,
useMediaPipe=useMediaPipe,
useDLC=useDLC,
- recordVid = recordVid
+ recordVid = recordVid,
+ showAnimation=showAnimation,
)
# print ('Starting PyQT Animation')
# createvideo.createBodyTrackingVideos(sesh)
@@ -214,5 +271,7 @@ def RunMe(sessionID=None,
print('Skipping Skeleton Plotting')
-
+ console.rule('All Done! Data folder is at: ' + str(sesh.basePath))
+
+ console.print(Markdown("""# βœ¨πŸ’€βœ¨ """))
diff --git a/freemocap/play_skeleton_animation.py b/freemocap/play_skeleton_animation.py
index 77706315..9812d77e 100644
--- a/freemocap/play_skeleton_animation.py
+++ b/freemocap/play_skeleton_animation.py
@@ -67,7 +67,8 @@ def PlaySkeletonAnimation(
useOpenPose=True,
useMediaPipe=False,
useDLC=False,
- recordVid = True
+ recordVid = True,
+ showAnimation =True,
):
@@ -845,14 +846,15 @@ def PlaySkeletonAnimation(
try:
- plt.pause(0.1)
- plt.draw()
+ if showAnimation:
+ plt.pause(0.1)
+ plt.draw()
except:
pass
- console.print(":sparkle: :skull: :sparkle:")
+
###
diff --git a/freemocap_runme_script.py b/freemocap_runme_script.py
index c7e38098..104eb169 100644
--- a/freemocap_runme_script.py
+++ b/freemocap_runme_script.py
@@ -1,3 +1,3 @@
import freemocap as fmc
-fmc.RunMe( )
+fmc.RunMe()
diff --git a/requirements.txt b/requirements.txt
index 86e34d22..dd3b3708 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,11 +3,12 @@ h5py>=3.1.0
imutils~=0.5.4
matplotlib~=3.1.3
mediapipe==0.8.8
-opencv-contrib-python~=3.4.14.51
+opencv-contrib-python~=3.4
pandas~=1.2.5
rich~=10.4.0
tqdm~=4.61.1
ruamel.yaml>=0.15.0
numpy<=1.19
typed-ast<1.5,>=1.4.0
-moviepy ~=1.0.3
\ No newline at end of file
+moviepy ~=1.0.3
+ipython
\ No newline at end of file