Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

Merge branch 'jon/pupil' into main

jonmatthis61c0a18a1dbb
diff --cc README.md
index e9068d09,6cb200f1..e0d6afeb
--- a/README.md
+++ b/README.md
@@@ -1,11 -1,8 +1,12 @@@
-https://user-images.githubusercontent.com/15314521/124694557-8069ea00-deaf-11eb-9328-3be27a4b1ea4.mp4
+
+
+https://user-images.githubusercontent.com/15314521/164949971-099c0d8c-7ff4-4e64-a707-999eac0a6ecb.mp4
+
# This is all very much a work in progress! More to come!
- ## (As of April 2022) We're currently working on building a proper API with documentation, but for now enjoy this pile of semi-spaghettified code and sloppy ReadMe πŸ˜…
++
+ ## (As of May 2022) We're currently working on building a proper API with documentation, but for now enjoy this pile of semi-spaghettified code and sloppy ReadMe πŸ˜…
___________________________________
___
@@@ -18,7 -15,7 +19,9 @@@
* Two or more USB webcams attached to viable USB ports
* ~~USB hubs typically don't work~~ I think they do now?
- * Note that two cameras is the minimum reuired for 3d reconstruction. However, with just two views, many points will be occluded/not visible to both cameras. For better performance, use three or four cameras
++
+ * Note that two cameras is the minimum required for 3d reconstruction. However, with just two views, many points will be occluded/not visible to both cameras. For better performance, use three or four cameras
++
* Each recording must (for now) an unobstructed view of a Charuco board within the first few seconds of recording (See below).
____
____
diff --cc freemocap/fmc_runme.py
index bddb358c,c972b9ce..c872a11f
--- a/freemocap/fmc_runme.py
+++ b/freemocap/fmc_runme.py
@@@ -213,19 -180,17 +179,19 @@@ def RunMe(sessionID=None
# %% Stage Four
if stage <= 4:
thisStage=4
- console.rule(style="color({})".format(thisStage))
- console.rule('Starting 2D Point Trackers'.upper(),style="color({})".format(thisStage))
+ thisStageColor=12
+ console.rule(style="color({})".format(thisStageColor))
+ console.rule('Starting 2D Point Trackers'.upper(),style="color({})".format(thisStageColor))
stage4_msg ='This step implements various computer vision that track the skeleton (and other objects) in the 2d videos, to produce the data that will be combined with the `camera projection matrices` from the calibration stage to produce the estimates of 3d movement. \n \n Each algorithm is different, but most involve using [bold magenta] convolutional neural networks [/bold magenta] trained from labeled videos to produce a 2d probability map of the likelihood that the tracked bodypart/object/feature (e.g. \'LeftElbow\') is in a given location. \n \n The peak of that distrubtion on each frame is recorded as the pixel-location of that item on that frame (e.g. \'LeftElbow(pixel-x, pixel-y, confidence\') where the a confidence value proportional to the underlying probability distribution (i.e. tall peaks in the probablitiy distribution indicate high confidence that the LeftElbow actually is at this pixel-x, pixel-y location) \n \nThis part is crazy future tech sci fi stuff. Seriously unbelievable this kind of thing is possible ✨'
- console.print(Padding(stage4_msg, (1,4)), overflow="fold", justify='center',style="color({})".format(thisStage))
- console.rule(style="color({})".format(thisStage))
-
+ console.print(Padding(stage4_msg, (1,4)), overflow="fold", justify='center',style="color({})".format(thisStageColor))
+ console.rule(style="color({})".format(thisStageColor))
+
if sesh.useMediaPipe:
- console.rule(style="color({})".format(thisStage))
- console.rule('Running MediaPipe skeleton tracker - https://google.github.io/mediapipe', style="color({})".format(thisStage))
- console.rule(style="color({})".format(thisStage))
+
+ console.rule(style="color({})".format(thisStage))
+ console.rule('Running MediaPipe skeleton tracker - https://google.github.io/mediapipe', style="color({})".format(thisStage))
+ console.rule(style="color({})".format(thisStage))
if runMediaPipe:
fmc_mediapipe.runMediaPipe(sesh)
@@@ -234,15 -199,26 +200,26 @@@
else:
print('`runMediaPipe` set to False, so we\'re loading MediaPipe data from npy file')
sesh.mediaPipeData_nCams_nFrames_nImgPts_XYC = np.load(sesh.dataArrayPath/'mediaPipeData_2d.npy', allow_pickle=True)
-
+
sesh.mediaPipeSkel_fr_mar_xyz, sesh.mediaPipeSkel_reprojErr = reconstruct3D.reconstruct3D(sesh,sesh.mediaPipeData_nCams_nFrames_nImgPts_XYC, confidenceThreshold=reconstructionConfidenceThreshold)
-
+
+ bundle_adjust_3d_points=True
+ if bundle_adjust_3d_points:
+ sesh.mediaPipeSkel_fr_mar_xyz_og = sesh.mediaPipeSkel_fr_mar_xyz.copy()
+ np.save(sesh.dataArrayPath/'mediaPipeSkel_3d_raw.npy', sesh.mediaPipeSkel_fr_mar_xyz_og) #save data to npy
+
+ from mediapipe.python.solutions import holistic as mp_holistic
+ mediapipe_body_pose_connections = [this_connection for this_connection in mp_holistic.POSE_CONNECTIONS]
+ print('Running bundle adjustment optimization on 3d points...')
+ sesh.mediaPipeSkel_fr_mar_xyz = sesh.cgroup.optim_points(sesh.mediaPipeData_nCams_nFrames_nImgPts_XYC[:,:,:,:2],
+ sesh.mediaPipeSkel_fr_mar_xyz,
+ constraints=mediapipe_body_pose_connections,
+ verbose=True)
+ print('Done adjusting bundles!')
+
np.save(sesh.dataArrayPath/'mediaPipeSkel_3d.npy', sesh.mediaPipeSkel_fr_mar_xyz) #save data to npy
- np.save(sesh.dataArrayPath/'mediaPipeSkel_reprojErr.npy', sesh.mediaPipeSkel_reprojErr) #save data to npy
-
+ np.save(sesh.dataArrayPath/'mediaPipeSkel_reprojErr.npy', sesh.mediaPipeSkel_reprojErr) #save data to npy
+
#smoooooooooth, just a bit
smoothWinLength = 5
smoothOrder = 3