Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

Tweaks to make anipose and blender stuff smoother

jon matthis7daaceaa97d5
diff --git a/freemocap/dev_scratchpad/dev_alpha/capture_voulme_calibration.py b/freemocap/dev_scratchpad/dev_alpha/capture_voulme_calibration.py
new file mode 100644
index 00000000..66948f5c
--- /dev/null
+++ b/freemocap/dev_scratchpad/dev_alpha/capture_voulme_calibration.py
@@ -0,0 +1,120 @@
+from rich.console import Console
+from FMC_MultiCamera import FMC_MultiCamera
+import socket
+from pathlib import Path
+from pathos.helpers import mp as pathos_mp_helper
+import cv2
+
+aruco_dict = cv2.aruco.Dictionary_get(cv2.aruco.DICT_6X6_250)
+board = cv2.aruco.CharucoBoard_create(7, 5, 1, .8, aruco_dict)
+
+def detect_charuco_board(image):
+ """
+ Charuco base pose estimation.
+ more-or-less copied from - https://mecaruco2.readthedocs.io/en/latest/notebooks_rst/Aruco/sandbox/ludovic/aruco_calibration_rotation.html
+ """
+ allCorners = []
+ allIds = []
+ # SUB PIXEL CORNER DETECTION CRITERION
+ criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 100, 0.00001)
+
+ gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+ charuco_corners, charuco_corner_ids, rejectedImgPoints = cv2.aruco.detectMarkers(gray, aruco_dict)
+
+ if len(charuco_corners)>0:
+ # SUB PIXEL DETECTION
+ for this_corner in charuco_corners:
+ cv2.cornerSubPix(gray, this_corner,
+ winSize = (3,3),
+ zeroZone = (-1,-1),
+ criteria = criteria)
+ res2 = cv2.aruco.interpolateCornersCharuco(charuco_corners,charuco_corner_ids,gray,board)
+
+
+ return charuco_corners,charuco_corner_ids
+
+
+if __name__ == "__main__":
+ pathos_mp_helper.freeze_support()
+
+ console = Console() #create rich console to catch and print exceptions
+
+
+ this_computer_name = socket.gethostname()
+
+ freemocap_data_path=None
+ in_rotation_codes_list=None
+
+ if this_computer_name=='jon-hallway-XPS-8930':
+ freemocap_data_path = Path('/home/jon/Dropbox/FreeMoCapProject/FreeMocap_Data')
+ in_rotation_codes_list = ['cv2.ROTATE_90_COUNTERCLOCKWISE', 'cv2.ROTATE_90_COUNTERCLOCKWISE', 'cv2.ROTATE_90_CLOCKWISE', 'cv2.ROTATE_90_CLOCKWISE', 'cv2.ROTATE_90_CLOCKWISE', ]
+ elif this_computer_name == 'DESKTOP-DCG6K4F':
+ freemocap_data_path = Path(r'C:\Users\jonma\Dropbox\FreeMoCapProject\FreeMocap_Data')
+ elif this_computer_name == 'Jons-MacBook-Pro.local':
+ freemocap_data_path = Path('/Users/jon/Dropbox/FreeMoCapProject')
+
+ try:
+ multi_cam = FMC_MultiCamera(show_multi_cam_stream_bool = False, freemocap_data_folder=str(freemocap_data_path), rotation_codes_list=in_rotation_codes_list)
+ multi_cam.start()
+
+
+
+
+ console.rule('Launching Multi Cam Viewer')
+ frame_count = 0
+ while not multi_cam.exit_event.is_set():
+ if not multi_cam.multi_cam_tuple_queue.empty():
+
+ this_multi_cam_tuple = multi_cam.multi_cam_tuple_queue.get()
+ frame_count += 1
+
+ multi_cam.save_synchronized_videos(this_multi_cam_tuple)
+
+ if frame_count % 30 == 0:
+ for this_cam_idx in range(multi_cam.num_cams):
+ this_cam_tuple = this_multi_cam_tuple[this_cam_idx]
+ this_cam_image = this_cam_tuple[1]
+ charuco_corners,charuco_corner_ids = detect_charuco_board(this_cam_image)
+ f=9
+
+
+ this_multi_cam_image = multi_cam.stitch_multicam_image(this_multi_cam_tuple)
+
+ if multi_cam._save_multi_cam_to_mp4:
+ if multi_cam._output_multi_cam_video_object is None:
+ multi_cam.initialize_multi_cam_output_video(this_multi_cam_image)
+
+
+ multi_cam._output_multi_cam_video_object.write(this_multi_cam_image)
+
+ if multi_cam._save_each_cam_to_mp4:
+ if multi_cam._each_cam_video_writer_object_list is None:
+ multi_cam.initialize_each_cam_output_video(this_multi_cam_tuple)
+
+ for this_cam_num in range(multi_cam.num_cams):
+ multi_cam._each_cam_video_writer_object_list[this_cam_num].write(this_multi_cam_tuple[this_cam_num][1])
+
+ cv2.imshow(multi_cam._rec_name, this_multi_cam_image)
+ key = cv2.waitKey(1)
+
+ if key == 27: # exit on ESC
+ break
+
+ if cv2.getWindowProperty(multi_cam._rec_name, cv2.WND_PROP_VISIBLE) < 1: #break loop if window closed
+ break
+
+ cv2.destroyAllWindows()
+
+ if multi_cam._save_multi_cam_to_mp4:
+ multi_cam._output_multi_cam_video_object.release()
+
+ if multi_cam._save_each_cam_to_mp4:
+ for this_cam_num in range(multi_cam.num_cams):
+ multi_cam._each_cam_video_writer_object_list[this_cam_num].release()
+
+ console.rule('Shutting down MultiCamera Viewer')
+ multi_cam.exit_event.set() #send the 'Exit' signal to everyone.
+
+
+ except Exception:
+ console.print_exception()
diff --git a/freemocap/fmc_anipose.py b/freemocap/fmc_anipose.py
index 1ebfc128..c78ae383 100644
--- a/freemocap/fmc_anipose.py
+++ b/freemocap/fmc_anipose.py
@@ -12,9 +12,9 @@ from tqdm import trange
from pprint import pprint
import time
-from .boards import merge_rows, extract_points, \
+from aniposelib.boards import merge_rows, extract_points, \
extract_rtvecs, get_video_params
-from .utils import get_initial_extrinsics, make_M, get_rtvec, \
+from aniposelib.utils import get_initial_extrinsics, make_M, get_rtvec, \
get_connections
@jit(nopython=True, parallel=True)
diff --git a/freemocap/fmc_blender.py b/freemocap/fmc_blender.py
index 1f7732da..af9fe522 100644
--- a/freemocap/fmc_blender.py
+++ b/freemocap/fmc_blender.py
@@ -36,8 +36,18 @@ order_of_markers = []
bpy.context.scene.render.fps = 30
#dictionary corresponding to marker numbers and body parts
-body_dict = {
- 0:"head1",
+mediapipe_tracked_point_names = {
+ 0:"nose",
+ 1:"left_eye_inner",
+ 2:"left_eye",
+ 3:"left_eye_outer",
+ 4:"right_eye_inner",
+ 5:"right_eye",
+ 6:"right_eye_outer",
+ 7:"left_ear",
+ 8:"right_ear",
+ 9:"mouth_left",
+ 10:"mouth_right",
11:"shoulder_L",
12:"shoulder_R",
13:"elbow_L",
@@ -59,7 +69,7 @@ body_dict = {
bpy.context.scene.unit_settings.length_unit = 'METERS'
#create sub-collection of empties
-cname = "Empty_Objects"
+cname = "mediapipe_empty_objects"
empty_collection = bpy.data.collections.new(cname)
bpy.data.collections.get("Collection").children.link(empty_collection)
master_collection = bpy.data.collections.get("Collection")
@@ -75,15 +85,24 @@ for index, col in enumerate(markers_list):
col[1] = np.nan
if math.isnan(col[2]):
col[2] = np.nan
- coord = Vector(((float(col[0])*0.001), (float(col[2])*0.001), (float(col[1]))* -0.001))
+ coord = Vector(((float(col[0])*0.001), (float(col[1])*0.001), (float(col[2]))* 0.001))
#empties
bpy.ops.object.add(type='EMPTY', location=coord)
this_empty = bpy.context.active_object
- this_empty.name = "mt_" + str(index)
- if index in body_dict.keys():
- this_empty.name += "_" + str(body_dict[index])
+ this_empty.name = "empty_" + str(index)
+
+ if index in mediapipe_tracked_point_names.keys():
+ this_empty.name += "_" + str(mediapipe_tracked_point_names[index])
+ else:
+ if index > 32 and index < 54:
+ this_empty.name += "_" + str("right_hand")
+ elif index >= 54 and index < 76:
+ this_empty.name += "_" + str("left_hand")
+ else:
+ this_empty.name += "_" + str("face")
+
order_of_markers.append(this_empty)
#link this empty to the scene's Empty collection
@@ -143,12 +162,12 @@ def update_virtual_data(relationship, surrounding, vweights, vname):
#Neck Base: Halfway between order_of_markers[12] shoulder_L and order_of_markers[12] shoulder_R
l0 = [order_of_markers[12], order_of_markers[11]]
w0 = [0.5, 0.5]
-update_virtual_data("weight", l0, w0, "mt_neck")
+update_virtual_data("weight", l0, w0, "empty_neck")
#Waist Base: Halfway between order_of_markers[24] and order_of_markers[23]
l0 = [order_of_markers[24], order_of_markers[23]]
w0 = [0.5, 0.5]
-update_virtual_data("weight", l0, w0, "mt_waist")
+update_virtual_data("weight", l0, w0, "empty_waist")
#Update the location of virtual markers on each frame
def update_virtual_marker(index):
@@ -363,10 +382,12 @@ def my_handler(scene):
if math.isnan(col[0]) and math.isnan(col[1]) and math.isnan(col[2]):
#marker was not tracked, pass
+ nan_frame = True
pass
else:
#Flip y and z coordinates, and negate y to account for different coordinate system
- coord = Vector(((float(col[0])*0.001), (float(col[2])*0.001), (float(col[1]))* -0.001))
+ nan_frame = False
+ coord = Vector(((float(col[0])*0.001), (float(col[1])*0.001), (float(col[2]))* 0.001))
if len(order_of_markers) > 0:
empty = order_of_markers[current_marker]
empty.location = coord
@@ -377,12 +398,13 @@ def my_handler(scene):
#keyframe bones
#Goes through each bone
- for editBone in get_armature().data.edit_bones:
- boneName = editBone.name
- poseBone = arm.pose.bones[boneName]
- poseBone.keyframe_insert('rotation_euler', frame=scene.frame_current)
- poseBone.keyframe_insert('location', frame=scene.frame_current)
- poseBone.keyframe_insert('scale', frame=scene.frame_current)
+ if not nan_frame:
+ for editBone in get_armature().data.edit_bones:
+ boneName = editBone.name
+ poseBone = arm.pose.bones[boneName]
+ poseBone.keyframe_insert('rotation_euler', frame=scene.frame_current)
+ poseBone.keyframe_insert('location', frame=scene.frame_current)
+ poseBone.keyframe_insert('scale', frame=scene.frame_current)
#-----------------------------------------------------------------------------------
#script to create a mesh of the armature
diff --git a/freemocap/fmc_runme.py b/freemocap/fmc_runme.py
index b17b07c5..c54cee06 100644
--- a/freemocap/fmc_runme.py
+++ b/freemocap/fmc_runme.py
@@ -253,6 +253,15 @@ def RunMe(sessionID=None,
sesh.openPoseSkel_fr_mar_xyz, sesh.openPoseSkel_reprojErr = reconstruct3D.reconstruct3D(sesh,sesh.openPoseData_nCams_nFrames_nImgPts_XYC, confidenceThreshold=reconstructionConfidenceThreshold)
np.save(sesh.dataArrayPath/'openPoseSkel_3d.npy', sesh.openPoseSkel_fr_mar_xyz) #save data to npy
np.save(sesh.dataArrayPath/'openPoseSkel_reprojErr.npy', sesh.openPoseSkel_reprojErr) #save data to npy
+
+ smoothWinLength = 5
+ smoothOrder = 3
+ for dim in range(sesh.openPoseSkel_fr_mar_xyz.shape[2]):
+ for mm in range(sesh.openPoseSkel_fr_mar_xyz.shape[1]):
+ sesh.openPoseSkel_fr_mar_xyz[:,mm,dim] = savgol_filter(sesh.openPoseSkel_fr_mar_xyz[:,mm,dim], smoothWinLength, smoothOrder)
+
+ np.save(sesh.dataArrayPath/'openPoseSkel_3d_smoothed.npy', sesh.openPoseSkel_fr_mar_xyz) #save data to npy
+
sesh.save_session()
sesh.syncedVidList = []
@@ -292,8 +301,8 @@ def RunMe(sessionID=None,
#blenderPath = Path('C:\Program Files\Blender Foundation\Blender 2.93')
#os.chdir(blenderExePath)
- output = subprocess.run([str(blenderPath), "--background", "--python", "fmc_blender.py", "--", str(sesh.dataArrayPath/'mediaPipeSkel_3d.npy'), str(sesh.dataArrayPath), sesh.sessionID], capture_output=True, text=True, check=True)
- print(output)
+ output = subprocess.run([str(blenderPath), "--background", "--python", "freemocap/fmc_blender.py", "--", str(sesh.dataArrayPath/'mediaPipeSkel_3d_smoothed.npy'), str(sesh.dataArrayPath), sesh.sessionID], capture_output=True, text=True, check=True)
+ print(output)
except:
console.print_exception()