Free Motion Capture for Everyone πβ¨
TypeScript
Merge branch 'jon/pupil' into main
jonmatthis61c0a18a1dbb
diff --cc README.mdindex e9068d09,6cb200f1..e0d6afeb--- a/README.md+++ b/README.md@@@ -1,11 -1,8 +1,12 @@@-https://user-images.githubusercontent.com/15314521/124694557-8069ea00-deaf-11eb-9328-3be27a4b1ea4.mp4+++https://user-images.githubusercontent.com/15314521/164949971-099c0d8c-7ff4-4e64-a707-999eac0a6ecb.mp4+# This is all very much a work in progress! More to come!- ## (As of April 2022) We're currently working on building a proper API with documentation, but for now enjoy this pile of semi-spaghettified code and sloppy ReadMe π+++ ## (As of May 2022) We're currently working on building a proper API with documentation, but for now enjoy this pile of semi-spaghettified code and sloppy ReadMe π______________________________________@@@ -18,7 -15,7 +19,9 @@@* Two or more USB webcams attached to viable USB ports* ~~USB hubs typically don't work~~ I think they do now?- * Note that two cameras is the minimum reuired for 3d reconstruction. However, with just two views, many points will be occluded/not visible to both cameras. For better performance, use three or four cameras+++ * Note that two cameras is the minimum required for 3d reconstruction. However, with just two views, many points will be occluded/not visible to both cameras. For better performance, use three or four cameras++* Each recording must (for now) an unobstructed view of a Charuco board within the first few seconds of recording (See below).________diff --cc freemocap/fmc_runme.pyindex bddb358c,c972b9ce..c872a11f--- a/freemocap/fmc_runme.py+++ b/freemocap/fmc_runme.py@@@ -213,19 -180,17 +179,19 @@@ def RunMe(sessionID=None# %% Stage Fourif stage <= 4:thisStage=4- console.rule(style="color({})".format(thisStage))- console.rule('Starting 2D Point Trackers'.upper(),style="color({})".format(thisStage))+ thisStageColor=12+ console.rule(style="color({})".format(thisStageColor))+ console.rule('Starting 2D Point Trackers'.upper(),style="color({})".format(thisStageColor))stage4_msg ='This step implements various computer vision that track the skeleton (and other objects) in the 2d videos, to produce the data that will be combined with the `camera projection matrices` from the calibration stage to produce the estimates of 3d movement. \n \n Each algorithm is different, but most involve using [bold magenta] convolutional neural networks [/bold magenta] trained from labeled videos to produce a 2d probability map of the likelihood that the tracked bodypart/object/feature (e.g. \'LeftElbow\') is in a given location. \n \n The peak of that distrubtion on each frame is recorded as the pixel-location of that item on that frame (e.g. \'LeftElbow(pixel-x, pixel-y, confidence\') where the a confidence value proportional to the underlying probability distribution (i.e. tall peaks in the probablitiy distribution indicate high confidence that the LeftElbow actually is at this pixel-x, pixel-y location) \n \nThis part is crazy future tech sci fi stuff. Seriously unbelievable this kind of thing is possible β¨'- console.print(Padding(stage4_msg, (1,4)), overflow="fold", justify='center',style="color({})".format(thisStage))- console.rule(style="color({})".format(thisStage))-+ console.print(Padding(stage4_msg, (1,4)), overflow="fold", justify='center',style="color({})".format(thisStageColor))+ console.rule(style="color({})".format(thisStageColor))+if sesh.useMediaPipe:- console.rule(style="color({})".format(thisStage))- console.rule('Running MediaPipe skeleton tracker - https://google.github.io/mediapipe', style="color({})".format(thisStage))- console.rule(style="color({})".format(thisStage))++ console.rule(style="color({})".format(thisStage))+ console.rule('Running MediaPipe skeleton tracker - https://google.github.io/mediapipe', style="color({})".format(thisStage))+ console.rule(style="color({})".format(thisStage))if runMediaPipe:fmc_mediapipe.runMediaPipe(sesh)@@@ -234,15 -199,26 +200,26 @@@else:print('`runMediaPipe` set to False, so we\'re loading MediaPipe data from npy file')sesh.mediaPipeData_nCams_nFrames_nImgPts_XYC = np.load(sesh.dataArrayPath/'mediaPipeData_2d.npy', allow_pickle=True)-+sesh.mediaPipeSkel_fr_mar_xyz, sesh.mediaPipeSkel_reprojErr = reconstruct3D.reconstruct3D(sesh,sesh.mediaPipeData_nCams_nFrames_nImgPts_XYC, confidenceThreshold=reconstructionConfidenceThreshold)-++ bundle_adjust_3d_points=True+ if bundle_adjust_3d_points:+ sesh.mediaPipeSkel_fr_mar_xyz_og = sesh.mediaPipeSkel_fr_mar_xyz.copy()+ np.save(sesh.dataArrayPath/'mediaPipeSkel_3d_raw.npy', sesh.mediaPipeSkel_fr_mar_xyz_og) #save data to npy++ from mediapipe.python.solutions import holistic as mp_holistic+ mediapipe_body_pose_connections = [this_connection for this_connection in mp_holistic.POSE_CONNECTIONS]+ print('Running bundle adjustment optimization on 3d points...')+ sesh.mediaPipeSkel_fr_mar_xyz = sesh.cgroup.optim_points(sesh.mediaPipeData_nCams_nFrames_nImgPts_XYC[:,:,:,:2],+ sesh.mediaPipeSkel_fr_mar_xyz,+ constraints=mediapipe_body_pose_connections,+ verbose=True)+ print('Done adjusting bundles!')+np.save(sesh.dataArrayPath/'mediaPipeSkel_3d.npy', sesh.mediaPipeSkel_fr_mar_xyz) #save data to npy- np.save(sesh.dataArrayPath/'mediaPipeSkel_reprojErr.npy', sesh.mediaPipeSkel_reprojErr) #save data to npy-+ np.save(sesh.dataArrayPath/'mediaPipeSkel_reprojErr.npy', sesh.mediaPipeSkel_reprojErr) #save data to npy+#smoooooooooth, just a bitsmoothWinLength = 5smoothOrder = 3