Free Motion Capture for Everyone πβ¨
TypeScript
Tweaks to make anipose and blender stuff smoother
jon matthis7daaceaa97d5
diff --git a/freemocap/dev_scratchpad/dev_alpha/capture_voulme_calibration.py b/freemocap/dev_scratchpad/dev_alpha/capture_voulme_calibration.pynew file mode 100644index 00000000..66948f5c--- /dev/null+++ b/freemocap/dev_scratchpad/dev_alpha/capture_voulme_calibration.py@@ -0,0 +1,120 @@+from rich.console import Console+from FMC_MultiCamera import FMC_MultiCamera+import socket+from pathlib import Path+from pathos.helpers import mp as pathos_mp_helper+import cv2++aruco_dict = cv2.aruco.Dictionary_get(cv2.aruco.DICT_6X6_250)+board = cv2.aruco.CharucoBoard_create(7, 5, 1, .8, aruco_dict)++def detect_charuco_board(image):+ """+ Charuco base pose estimation.+ more-or-less copied from - https://mecaruco2.readthedocs.io/en/latest/notebooks_rst/Aruco/sandbox/ludovic/aruco_calibration_rotation.html+ """+ allCorners = []+ allIds = []+ # SUB PIXEL CORNER DETECTION CRITERION+ criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 100, 0.00001)++ gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)+ charuco_corners, charuco_corner_ids, rejectedImgPoints = cv2.aruco.detectMarkers(gray, aruco_dict)++ if len(charuco_corners)>0:+ # SUB PIXEL DETECTION+ for this_corner in charuco_corners:+ cv2.cornerSubPix(gray, this_corner,+ winSize = (3,3),+ zeroZone = (-1,-1),+ criteria = criteria)+ res2 = cv2.aruco.interpolateCornersCharuco(charuco_corners,charuco_corner_ids,gray,board)+++ return charuco_corners,charuco_corner_ids+++if __name__ == "__main__":+ pathos_mp_helper.freeze_support()++ console = Console() #create rich console to catch and print exceptions+++ this_computer_name = socket.gethostname()++ freemocap_data_path=None+ in_rotation_codes_list=None++ if this_computer_name=='jon-hallway-XPS-8930':+ freemocap_data_path = Path('/home/jon/Dropbox/FreeMoCapProject/FreeMocap_Data')+ in_rotation_codes_list = ['cv2.ROTATE_90_COUNTERCLOCKWISE', 'cv2.ROTATE_90_COUNTERCLOCKWISE', 'cv2.ROTATE_90_CLOCKWISE', 'cv2.ROTATE_90_CLOCKWISE', 'cv2.ROTATE_90_CLOCKWISE', ]+ elif this_computer_name == 'DESKTOP-DCG6K4F':+ freemocap_data_path = Path(r'C:\Users\jonma\Dropbox\FreeMoCapProject\FreeMocap_Data')+ elif this_computer_name == 'Jons-MacBook-Pro.local':+ freemocap_data_path = Path('/Users/jon/Dropbox/FreeMoCapProject')++ try:+ multi_cam = FMC_MultiCamera(show_multi_cam_stream_bool = False, freemocap_data_folder=str(freemocap_data_path), rotation_codes_list=in_rotation_codes_list)+ multi_cam.start()+++++ console.rule('Launching Multi Cam Viewer')+ frame_count = 0+ while not multi_cam.exit_event.is_set():+ if not multi_cam.multi_cam_tuple_queue.empty():++ this_multi_cam_tuple = multi_cam.multi_cam_tuple_queue.get()+ frame_count += 1++ multi_cam.save_synchronized_videos(this_multi_cam_tuple)++ if frame_count % 30 == 0:+ for this_cam_idx in range(multi_cam.num_cams):+ this_cam_tuple = this_multi_cam_tuple[this_cam_idx]+ this_cam_image = this_cam_tuple[1]+ charuco_corners,charuco_corner_ids = detect_charuco_board(this_cam_image)+ f=9+++ this_multi_cam_image = multi_cam.stitch_multicam_image(this_multi_cam_tuple)++ if multi_cam._save_multi_cam_to_mp4:+ if multi_cam._output_multi_cam_video_object is None:+ multi_cam.initialize_multi_cam_output_video(this_multi_cam_image)+++ multi_cam._output_multi_cam_video_object.write(this_multi_cam_image)++ if multi_cam._save_each_cam_to_mp4:+ if multi_cam._each_cam_video_writer_object_list is None:+ multi_cam.initialize_each_cam_output_video(this_multi_cam_tuple)++ for this_cam_num in range(multi_cam.num_cams):+ multi_cam._each_cam_video_writer_object_list[this_cam_num].write(this_multi_cam_tuple[this_cam_num][1])++ cv2.imshow(multi_cam._rec_name, this_multi_cam_image)+ key = cv2.waitKey(1)++ if key == 27: # exit on ESC+ break++ if cv2.getWindowProperty(multi_cam._rec_name, cv2.WND_PROP_VISIBLE) < 1: #break loop if window closed+ break++ cv2.destroyAllWindows()++ if multi_cam._save_multi_cam_to_mp4:+ multi_cam._output_multi_cam_video_object.release()++ if multi_cam._save_each_cam_to_mp4:+ for this_cam_num in range(multi_cam.num_cams):+ multi_cam._each_cam_video_writer_object_list[this_cam_num].release()++ console.rule('Shutting down MultiCamera Viewer')+ multi_cam.exit_event.set() #send the 'Exit' signal to everyone.+++ except Exception:+ console.print_exception()diff --git a/freemocap/fmc_anipose.py b/freemocap/fmc_anipose.pyindex 1ebfc128..c78ae383 100644--- a/freemocap/fmc_anipose.py+++ b/freemocap/fmc_anipose.py@@ -12,9 +12,9 @@ from tqdm import trangefrom pprint import pprintimport time-from .boards import merge_rows, extract_points, \+from aniposelib.boards import merge_rows, extract_points, \extract_rtvecs, get_video_params-from .utils import get_initial_extrinsics, make_M, get_rtvec, \+from aniposelib.utils import get_initial_extrinsics, make_M, get_rtvec, \get_connections@jit(nopython=True, parallel=True)diff --git a/freemocap/fmc_blender.py b/freemocap/fmc_blender.pyindex 1f7732da..af9fe522 100644--- a/freemocap/fmc_blender.py+++ b/freemocap/fmc_blender.py@@ -36,8 +36,18 @@ order_of_markers = []bpy.context.scene.render.fps = 30#dictionary corresponding to marker numbers and body parts-body_dict = {- 0:"head1",+mediapipe_tracked_point_names = {+ 0:"nose",+ 1:"left_eye_inner",+ 2:"left_eye",+ 3:"left_eye_outer",+ 4:"right_eye_inner",+ 5:"right_eye",+ 6:"right_eye_outer",+ 7:"left_ear",+ 8:"right_ear",+ 9:"mouth_left",+ 10:"mouth_right",11:"shoulder_L",12:"shoulder_R",13:"elbow_L",@@ -59,7 +69,7 @@ body_dict = {bpy.context.scene.unit_settings.length_unit = 'METERS'#create sub-collection of empties-cname = "Empty_Objects"+cname = "mediapipe_empty_objects"empty_collection = bpy.data.collections.new(cname)bpy.data.collections.get("Collection").children.link(empty_collection)master_collection = bpy.data.collections.get("Collection")@@ -75,15 +85,24 @@ for index, col in enumerate(markers_list):col[1] = np.nanif math.isnan(col[2]):col[2] = np.nan- coord = Vector(((float(col[0])*0.001), (float(col[2])*0.001), (float(col[1]))* -0.001))+ coord = Vector(((float(col[0])*0.001), (float(col[1])*0.001), (float(col[2]))* 0.001))#emptiesbpy.ops.object.add(type='EMPTY', location=coord)this_empty = bpy.context.active_object- this_empty.name = "mt_" + str(index)- if index in body_dict.keys():- this_empty.name += "_" + str(body_dict[index])+ this_empty.name = "empty_" + str(index)++ if index in mediapipe_tracked_point_names.keys():+ this_empty.name += "_" + str(mediapipe_tracked_point_names[index])+ else:+ if index > 32 and index < 54:+ this_empty.name += "_" + str("right_hand")+ elif index >= 54 and index < 76:+ this_empty.name += "_" + str("left_hand")+ else:+ this_empty.name += "_" + str("face")+order_of_markers.append(this_empty)#link this empty to the scene's Empty collection@@ -143,12 +162,12 @@ def update_virtual_data(relationship, surrounding, vweights, vname):#Neck Base: Halfway between order_of_markers[12] shoulder_L and order_of_markers[12] shoulder_Rl0 = [order_of_markers[12], order_of_markers[11]]w0 = [0.5, 0.5]-update_virtual_data("weight", l0, w0, "mt_neck")+update_virtual_data("weight", l0, w0, "empty_neck")#Waist Base: Halfway between order_of_markers[24] and order_of_markers[23]l0 = [order_of_markers[24], order_of_markers[23]]w0 = [0.5, 0.5]-update_virtual_data("weight", l0, w0, "mt_waist")+update_virtual_data("weight", l0, w0, "empty_waist")#Update the location of virtual markers on each framedef update_virtual_marker(index):@@ -363,10 +382,12 @@ def my_handler(scene):if math.isnan(col[0]) and math.isnan(col[1]) and math.isnan(col[2]):#marker was not tracked, pass+ nan_frame = Truepasselse:#Flip y and z coordinates, and negate y to account for different coordinate system- coord = Vector(((float(col[0])*0.001), (float(col[2])*0.001), (float(col[1]))* -0.001))+ nan_frame = False+ coord = Vector(((float(col[0])*0.001), (float(col[1])*0.001), (float(col[2]))* 0.001))if len(order_of_markers) > 0:empty = order_of_markers[current_marker]empty.location = coord@@ -377,12 +398,13 @@ def my_handler(scene):#keyframe bones#Goes through each bone- for editBone in get_armature().data.edit_bones:- boneName = editBone.name- poseBone = arm.pose.bones[boneName]- poseBone.keyframe_insert('rotation_euler', frame=scene.frame_current)- poseBone.keyframe_insert('location', frame=scene.frame_current)- poseBone.keyframe_insert('scale', frame=scene.frame_current)+ if not nan_frame:+ for editBone in get_armature().data.edit_bones:+ boneName = editBone.name+ poseBone = arm.pose.bones[boneName]+ poseBone.keyframe_insert('rotation_euler', frame=scene.frame_current)+ poseBone.keyframe_insert('location', frame=scene.frame_current)+ poseBone.keyframe_insert('scale', frame=scene.frame_current)#-----------------------------------------------------------------------------------#script to create a mesh of the armaturediff --git a/freemocap/fmc_runme.py b/freemocap/fmc_runme.pyindex b17b07c5..c54cee06 100644--- a/freemocap/fmc_runme.py+++ b/freemocap/fmc_runme.py@@ -253,6 +253,15 @@ def RunMe(sessionID=None,sesh.openPoseSkel_fr_mar_xyz, sesh.openPoseSkel_reprojErr = reconstruct3D.reconstruct3D(sesh,sesh.openPoseData_nCams_nFrames_nImgPts_XYC, confidenceThreshold=reconstructionConfidenceThreshold)np.save(sesh.dataArrayPath/'openPoseSkel_3d.npy', sesh.openPoseSkel_fr_mar_xyz) #save data to npynp.save(sesh.dataArrayPath/'openPoseSkel_reprojErr.npy', sesh.openPoseSkel_reprojErr) #save data to npy++ smoothWinLength = 5+ smoothOrder = 3+ for dim in range(sesh.openPoseSkel_fr_mar_xyz.shape[2]):+ for mm in range(sesh.openPoseSkel_fr_mar_xyz.shape[1]):+ sesh.openPoseSkel_fr_mar_xyz[:,mm,dim] = savgol_filter(sesh.openPoseSkel_fr_mar_xyz[:,mm,dim], smoothWinLength, smoothOrder)++ np.save(sesh.dataArrayPath/'openPoseSkel_3d_smoothed.npy', sesh.openPoseSkel_fr_mar_xyz) #save data to npy+sesh.save_session()sesh.syncedVidList = []@@ -292,8 +301,8 @@ def RunMe(sessionID=None,#blenderPath = Path('C:\Program Files\Blender Foundation\Blender 2.93')#os.chdir(blenderExePath)- output = subprocess.run([str(blenderPath), "--background", "--python", "fmc_blender.py", "--", str(sesh.dataArrayPath/'mediaPipeSkel_3d.npy'), str(sesh.dataArrayPath), sesh.sessionID], capture_output=True, text=True, check=True)- print(output)+ output = subprocess.run([str(blenderPath), "--background", "--python", "freemocap/fmc_blender.py", "--", str(sesh.dataArrayPath/'mediaPipeSkel_3d_smoothed.npy'), str(sesh.dataArrayPath), sesh.sessionID], capture_output=True, text=True, check=True)+ print(output)except:console.print_exception()