Free Motion Capture for Everyone πβ¨
TypeScript
Tweaks to Blender output (#445)
jonmatthiscad73e2a4641
diff --git a/freemocap/data_layer/export_data/blender_stuff/blender_bpy_export_scripts/freemocap_blender_megascript_take2.py b/freemocap/data_layer/export_data/blender_stuff/blender_bpy_export_scripts/freemocap_blender_megascript_take2.pyindex 7619bd21..8eef08fc 100644--- a/freemocap/data_layer/export_data/blender_stuff/blender_bpy_export_scripts/freemocap_blender_megascript_take2.py+++ b/freemocap/data_layer/export_data/blender_stuff/blender_bpy_export_scripts/freemocap_blender_megascript_take2.py@@ -3,7 +3,7 @@ import loggingimport sysfrom copy import copyfrom pathlib import Path-from typing import List, Union, Dict+from typing import List, Union, Dict, Tupleimport addon_utilsimport bpy@@ -49,11 +49,17 @@ def main(recording_path: Union[str, Path],path_to_left_hand_npy = path_to_data_arrays_folder / "mediapipe_left_hand_3d_xyz.npy"path_to_face_npy = path_to_data_arrays_folder / "mediapipe_face_3d_xyz.npy"+ path_to_center_of_mass_npy = path_to_data_arrays_folder / "center_of_mass" / "total_body_center_of_mass_xyz.npy"+ path_to_body_segment_centers_npy = path_to_data_arrays_folder / "center_of_mass" / "segmentCOM_frame_joint_xyz.npy"+mediapipe_body_fr_mar_xyz = np.load(str(path_to_body_npy)) / 1000 # load and convert to metersmediapipe_right_hand_fr_mar_xyz = np.load(str(path_to_right_hand_npy)) / 1000mediapipe_left_hand_fr_mar_xyz = np.load(str(path_to_left_hand_npy)) / 1000mediapipe_face_fr_mar_xyz = np.load(str(path_to_face_npy)) / 1000+ center_of_mass_fr_xyz = np.load(str(path_to_center_of_mass_npy)) / 1000+ body_segment_centers_fr_mar_xyz = np.load(str(path_to_body_segment_centers_npy)) / 1000+# load reprojection errorpath_to_reprojection_error_npy = (path_to_data_arrays_folder / "raw_data" / "mediapipe3dData_numFrames_numTrackedPoints_reprojectionError.npy"@@ -111,61 +117,88 @@ def main(recording_path: Union[str, Path],# mediapipe_face_trajectory_names = [f"face_{number}:{empty_name}" for number, empty_name in# face_contour_marker_indices]- # create empty markers for body- body_empty_scale = 0.02- for marker_number in range(mediapipe_body_fr_mar_xyz.shape[1]):- trajectory_name = mediapipe_body_trajectory_names[marker_number]- trajectory_fr_xyz = mediapipe_body_fr_mar_xyz[:, marker_number, :]- create_keyframed_empty_from_3d_trajectory_data(- trajectory_fr_xyz=trajectory_fr_xyz,- trajectory_name=trajectory_name,- parent_origin=freemocap_origin_axes,- empty_scale=0.01,- empty_type="SPHERE",- )+ try: #load as much of the data as you can, but if there's an error keep going++ # create empty markers for body+ body_empty_scale = 0.03+ for marker_number in range(mediapipe_body_fr_mar_xyz.shape[1]):+ trajectory_name = mediapipe_body_trajectory_names[marker_number]+ trajectory_fr_xyz = mediapipe_body_fr_mar_xyz[:, marker_number, :]+ create_keyframed_empty_from_3d_trajectory_data(+ trajectory_fr_xyz=trajectory_fr_xyz,+ trajectory_name=trajectory_name,+ parent_origin=freemocap_origin_axes,+ empty_scale=body_empty_scale,+ empty_type="SPHERE",+ )- # create empty markers for right hand- hand_empty_scale = 0.01- for marker_number in range(mediapipe_right_hand_fr_mar_xyz.shape[1]):- trajectory_name = mediapipe_right_hand_trajectory_names[marker_number]- trajectory_fr_xyz = mediapipe_right_hand_fr_mar_xyz[:, marker_number, :]- create_keyframed_empty_from_3d_trajectory_data(- trajectory_fr_xyz=trajectory_fr_xyz,- trajectory_name=trajectory_name,- parent_origin=freemocap_origin_axes,- empty_scale=hand_empty_scale,- empty_type="PLAIN_AXES",- )+ # create empty markers for right hand+ hand_empty_scale = body_empty_scale * 0.5+ for marker_number in range(mediapipe_right_hand_fr_mar_xyz.shape[1]):+ trajectory_name = mediapipe_right_hand_trajectory_names[marker_number]+ trajectory_fr_xyz = mediapipe_right_hand_fr_mar_xyz[:, marker_number, :]+ create_keyframed_empty_from_3d_trajectory_data(+ trajectory_fr_xyz=trajectory_fr_xyz,+ trajectory_name=trajectory_name,+ parent_origin=freemocap_origin_axes,+ empty_scale=hand_empty_scale,+ empty_type="PLAIN_AXES",+ )- # create empty markers for left hand- for marker_number in range(mediapipe_left_hand_fr_mar_xyz.shape[1]):- trajectory_name = mediapipe_left_hand_trajectory_names[marker_number]- trajectory_fr_xyz = mediapipe_left_hand_fr_mar_xyz[:, marker_number, :]+ # create empty markers for left hand+ for marker_number in range(mediapipe_left_hand_fr_mar_xyz.shape[1]):+ trajectory_name = mediapipe_left_hand_trajectory_names[marker_number]+ trajectory_fr_xyz = mediapipe_left_hand_fr_mar_xyz[:, marker_number, :]+ create_keyframed_empty_from_3d_trajectory_data(+ trajectory_fr_xyz=trajectory_fr_xyz,+ trajectory_name=trajectory_name,+ parent_origin=freemocap_origin_axes,+ empty_scale=hand_empty_scale,+ empty_type="PLAIN_AXES",+ )++ # create empty markers for body segment centers+ body_segment_center_empty_scale = body_empty_scale * 0.7+ for marker_number in range(body_segment_centers_fr_mar_xyz.shape[1]):+ trajectory_name = BODY_SEGMENT_NAMES[marker_number]+ trajectory_fr_xyz = body_segment_centers_fr_mar_xyz[:, marker_number, :]+ create_keyframed_empty_from_3d_trajectory_data(+ trajectory_fr_xyz=trajectory_fr_xyz,+ trajectory_name=trajectory_name,+ parent_origin=freemocap_origin_axes,+ empty_scale=body_segment_center_empty_scale,+ empty_type="PLAIN_AXES",+ )++ # create empty markers for full body center of mass+ full_body_center_of_mass_empty_scale = body_empty_scale * 1.5create_keyframed_empty_from_3d_trajectory_data(- trajectory_fr_xyz=trajectory_fr_xyz,- trajectory_name=trajectory_name,- parent_origin=freemocap_origin_axes,- empty_scale=hand_empty_scale,- empty_type="PLAIN_AXES",- )+ trajectory_fr_xyz=center_of_mass_fr_xyz,+ trajectory_name=FULL_BODY_CENTER_OF_MASS_NAME,+ parent_origin=freemocap_origin_axes,+ empty_scale=full_body_center_of_mass_empty_scale,+ empty_type="SPHERE",+ )- # # create empty markers for face- # face_empty_scale = 0.005- # for marker_number in face_contour_marker_indices:- #- # if marker_number < len(mediapipe_face_trajectory_names):- # trajectory_name = mediapipe_face_trajectory_names[marker_number]- # else:- # trajectory_name = f"face_{marker_number}"- #- # trajectory_fr_xyz = mediapipe_face_fr_mar_xyz[:, marker_number, :]- # create_keyframed_empty_from_3d_trajectory_data(- # trajectory_fr_xyz=trajectory_fr_xyz,- # trajectory_name=trajectory_name,- # parent_origin=freemocap_origin_axes,- # empty_scale=face_empty_scale,- # empty_type="PLAIN_AXES",- # )+ # # create empty markers for face+ # face_empty_scale = 0.005+ # for marker_number in face_contour_marker_indices:+ #+ # if marker_number < len(mediapipe_face_trajectory_names):+ # trajectory_name = mediapipe_face_trajectory_names[marker_number]+ # else:+ # trajectory_name = f"face_{marker_number}"+ #+ # trajectory_fr_xyz = mediapipe_face_fr_mar_xyz[:, marker_number, :]+ # create_keyframed_empty_from_3d_trajectory_data(+ # trajectory_fr_xyz=trajectory_fr_xyz,+ # trajectory_name=trajectory_name,+ # parent_origin=freemocap_origin_axes,+ # empty_scale=face_empty_scale,+ # empty_type="PLAIN_AXES",+ # )+ except Exception as e:+ print(f"Error loading empty markers: {e}!")######################################################################## %% create virtual markers@@ -206,40 +239,67 @@ def main(recording_path: Union[str, Path],######################################################################## put sphere meshes on empty markers for visualization+ try: # try to create sphere meshes on empty markers, if it fails, print the error and continue+ # body+ put_sphere_meshes_on_empties(+ empty_names_list=mediapipe_body_trajectory_names,+ parent_object=freemocap_origin_axes,+ sphere_scale=body_empty_scale*0.8,+ marker_type="body",+ color = '#33FF66'+ )- # body- put_sphere_meshes_on_empties(- empty_names_list=mediapipe_body_trajectory_names,- parent_object=freemocap_origin_axes,- sphere_scale=body_empty_scale,- )+ # create_stick_meshes_between_emtpies(+ # body_frame_marker_xyz=mediapipe_body_fr_mar_xyz,+ # empty_names=mediapipe_body_trajectory_names,+ # body_connections=body_connections,+ # parent_object=freemocap_origin_axes,+ # )++ # right hand+ put_sphere_meshes_on_empties(+ empty_names_list=mediapipe_right_hand_trajectory_names,+ parent_object=freemocap_origin_axes,+ sphere_scale=hand_empty_scale*0.8,+ marker_type="right_hand",+ color='#FF0000'+ )- # create_stick_meshes_between_emtpies(- # body_frame_marker_xyz=mediapipe_body_fr_mar_xyz,- # empty_names=mediapipe_body_trajectory_names,- # body_connections=body_connections,- # parent_object=freemocap_origin_axes,- # )-- # right hand- put_sphere_meshes_on_empties(- empty_names_list=mediapipe_right_hand_trajectory_names,- parent_object=freemocap_origin_axes,- sphere_scale=hand_empty_scale,- )+ put_sphere_meshes_on_empties(+ empty_names_list=mediapipe_left_hand_trajectory_names,+ parent_object=freemocap_origin_axes,+ sphere_scale=hand_empty_scale*0.8,+ marker_type="left_hand",+ color='#0000FF'+ )- put_sphere_meshes_on_empties(- empty_names_list=mediapipe_left_hand_trajectory_names,- parent_object=freemocap_origin_axes,- sphere_scale=hand_empty_scale,- )+ # # face+ # put_sphere_meshes_on_empties(+ # empty_names_list=mediapipe_face_trajectory_names,+ # parent_object=freemocap_origin_axes,+ # sphere_scale=face_empty_scale,+ # )++ put_sphere_meshes_on_empties(+ empty_names_list=BODY_SEGMENT_NAMES,+ parent_object=freemocap_origin_axes,+ sphere_scale=body_segment_center_empty_scale*0.8,+ marker_type="body_segment_center",+ color='#CCCC00'- # # face- # put_sphere_meshes_on_empties(- # empty_names_list=mediapipe_face_trajectory_names,- # parent_object=freemocap_origin_axes,- # sphere_scale=face_empty_scale,- # )+ )++ put_sphere_meshes_on_empties(+ empty_names_list=[FULL_BODY_CENTER_OF_MASS_NAME],+ parent_object=freemocap_origin_axes,+ sphere_scale=body_empty_scale*1.25,+ marker_type="center_of_mass",+ color='#FF00FF',+ emission_strength=30,++ )+ except Exception as e:+ print(f"Error creating sphere meshes on empties: {e}")############################## Make stick figure@@ -290,6 +350,8 @@ def main(recording_path: Union[str, Path],print(f"Creating `rigify human meta-rig`")bpy.ops.object.armature_human_metarig_add()human_metarig = bpy.context.editable_objects[-1]+ human_metarig.data.display_type = "WIRE"+##################### Scale armature@@ -366,8 +428,8 @@ def main(recording_path: Union[str, Path],bpy.ops.wm.save_as_mainfile(filepath=str(blender_file_save_path))print(f"Saved .blend file to: {blender_file_save_path}")- print("try to hide the shameful metarig and then save again lol")- human_metarig.hide_set = True+ # print("try to hide the shameful metarig and then save again lol")+ # human_metarig.hide_set = Truebpy.ops.wm.save_as_mainfile(filepath=str(blender_file_save_path))@@ -445,25 +507,95 @@ def create_keyframed_empty_from_3d_trajectory_data(empty_object.keyframe_insert(data_path="location", frame=frame_number)++def create_material(name: str = "Generic",+ color: Union[str, Tuple, List] = "#00FFFF",+ emission_strength: float = 1.0,):+ """+ Create a material with the given name and color, with a strong emission.++ :param name: The name of the material.+ :param color: The color of the material, in hex format.+ :return: The created material.+ """+ material = bpy.data.materials.new(name=name)+ material.use_nodes = True+ nodes = material.node_tree.nodes+ links = material.node_tree.links++ # Clear all nodes to start clean+ nodes.clear()++ # Create necessary nodes+ output = nodes.new(type='ShaderNodeOutputMaterial')+ emission = nodes.new(type='ShaderNodeEmission')+ bsdf = nodes.new(type='ShaderNodeBsdfPrincipled')+ mix = nodes.new(type='ShaderNodeMixShader')++ # Set the locations+ output.location = (300, 0)+ mix.location = (100, 0)+ bsdf.location = (-100, 50)+ emission.location = (-100, -50)++ # Set node links+ links.new(output.inputs[0], mix.outputs[0])+ links.new(mix.inputs[1], bsdf.outputs[0])+ links.new(mix.inputs[2], emission.outputs[0])++ # Convert color from hex to RGB+ if isinstance(color, str) and color.startswith("#"):+ color_rgb = [int(color[i:i+2], 16) / 255 for i in (1, 3, 5)] # skips the "#" and separates R, G, and B+ elif isinstance(color, list) or isinstance(color, tuple):+ color_rgb = color++ # Set the colors+ bsdf.inputs[0].default_value = (*color_rgb, 1)+ emission.inputs[0].default_value = (*color_rgb, 1) # RGB + Alpha++ # Set the strength of the emission+ emission.inputs[1].default_value = emission_strength # The higher this value, the more the material will glow++ return material+def put_sphere_meshes_on_empties(+empty_names_list: list,parent_object: bpy.types.Object,sphere_scale: float = 0.02,+ material: Union[bpy.types.Material, str] = None,+ color: str = "#00FFFF",+ marker_type: str = "generic",+ emission_strength: float = 1.0,):"""- put uv sphere meshes on the empties in `empty_names_list` with a scale of `sphere_scale`+ Put uv sphere meshes on the empties in `empty_names_list` with a scale of `sphere_scale`.++ :param empty_names_list: List of empty object names.+ :param parent_object: The object to be set as parent for the created spheres.+ :param sphere_scale: The scale of the sphere.+ :param material: The material to be applied to the spheres, can be a bpy.types.Material object or a color string in hex format."""+ # If material is None, create a default material+ if material is None:+ material = create_material(color=color,+ name=marker_type,+ emission_strength=emission_strength,+ )++ # Ensure the provided material is a bpy.types.Material object+ assert isinstance(material, bpy.types.Material), "Material must be a bpy.types.Material object or a color string"for empty_name in empty_names_list:bpy.ops.mesh.primitive_uv_sphere_add(segments=8, ring_count=8, scale=(sphere_scale, sphere_scale, sphere_scale))sphere = bpy.context.editable_objects[-1]sphere.name = f"{empty_name}_sphere"sphere.parent = parent_object+ sphere.data.materials.append(material)constraint = sphere.constraints.new(type="COPY_LOCATION")constraint.target = bpy.data.objects[empty_name]-def create_stick_meshes_between_emtpies(body_frame_marker_xyz: np.ndarray,empty_names: List[str],body_connections: Dict[int, int],@@ -904,13 +1036,33 @@ rig_constraint_dict_of_dicts = {},}+BODY_SEGMENT_NAMES = [+ "head",+ "trunk",+ "right_upper_arm",+ "left_upper_arm",+ "right_forearm",+ "left_forearm",+ "right_hand",+ "left_hand",+ "right_thigh",+ "left_thigh",+ "right_shin",+ "left_shin",+ "right_foot",+ "left_foot",+]+++FULL_BODY_CENTER_OF_MASS_NAME = "_full_body_center_of_mass" #leading underscore to make it easier to find in the outliner+print(f"__name__: {__name__}")if __name__ == "__main__" or __name__ == "<run_path>":# this is the part that actually runs the scriptargv = sys.argv- create_rig_input = False+ create_rig_input = Trueprint(f"Running script to create Blender file from freemocap session data from script {__file__}")try:diff --git a/freemocap/gui/qt/main_window/freemocap_main_window.py b/freemocap/gui/qt/main_window/freemocap_main_window.pyindex c9163b47..e942b2b5 100644--- a/freemocap/gui/qt/main_window/freemocap_main_window.py+++ b/freemocap/gui/qt/main_window/freemocap_main_window.py@@ -23,8 +23,14 @@ from freemocap.data_layer.export_data.blender_stuff.export_to_blender import (export_to_blender,)from freemocap.data_layer.export_data.blender_stuff.get_best_guess_of_blender_path import get_best_guess_of_blender_path--from freemocap.data_layer.export_data.generate_jupyter_notebook.generate_jupyter_notebook import generate_jupyter_notebook+from freemocap.data_layer.export_data.generate_jupyter_notebook.generate_jupyter_notebook import \+ generate_jupyter_notebook+from freemocap.data_layer.recording_models.post_processing_parameter_models import (+ PostProcessingParameterModel,+)+from freemocap.data_layer.recording_models.recording_info_model import (+ RecordingInfoModel,+)from freemocap.gui.qt.actions_and_menus.actions import Actionsfrom freemocap.gui.qt.actions_and_menus.menu_bar import MenuBarfrom freemocap.gui.qt.style_sheet.css_file_watcher import CSSFileWatcher@@ -51,12 +57,6 @@ from freemocap.gui.qt.widgets.home_widget import ()from freemocap.gui.qt.widgets.import_videos_window import ImportVideosWizardfrom freemocap.gui.qt.widgets.log_view_widget import LogViewWidget-from freemocap.data_layer.recording_models.post_processing_parameter_models import (- PostProcessingParameterModel,-)-from freemocap.data_layer.recording_models.recording_info_model import (- RecordingInfoModel,-)# reboot GUI method based on this - https://stackoverflow.com/a/56563926/14662833from freemocap.system.open_file import open_filefrom freemocap.system.paths_and_filenames.file_and_folder_names import (@@ -82,7 +82,7 @@ class MainWindow(QMainWindow):):super().__init__(parent=parent)- self._log_view_widget = LogViewWidget(parent=self) #start this first so it will grab the setup logs+ self._log_view_widget = LogViewWidget(parent=self) # start this first so it will grab the setup logslogger.info("Initializing FreeMoCap MainWindow")self._size_main_window()@@ -145,7 +145,6 @@ class MainWindow(QMainWindow):# )# jupyter_console_dock_widget.setWidget(self._jupyter_console_widget)-log_view_dock_widget = QDockWidget("Log View", self)self.addDockWidget(Qt.DockWidgetArea.BottomDockWidgetArea, log_view_dock_widget)log_view_dock_widget.setWidget(self._log_view_widget)@@ -154,7 +153,7 @@ class MainWindow(QMainWindow):)logger.info("Finished initializing FreeMoCap MainWindow")- def _size_main_window(self, width_fraction:float=0.8, height_fraction:float=0.8):+ def _size_main_window(self, width_fraction: float = 0.8, height_fraction: float = 0.8):# Get screen sizescreen = QApplication.primaryScreen().availableGeometry()# Calculate width and height as a fraction of the screen size@@ -199,8 +198,6 @@ class MainWindow(QMainWindow):if self._controller_group_box.generate_jupyter_notebook_checked and not self._kill_thread_event.is_set():self._generate_jupyter_notebook()--def handle_start_new_session_action(self):# self._central_tab_widget.set_welcome_tab_enabled(True)self._central_tab_widget.set_camera_view_tab_enabled(True)@@ -219,7 +216,8 @@ class MainWindow(QMainWindow):def _set_up_stylesheet(self):apply_css_style_sheet(self, get_css_stylesheet_path())- scss_file_watcher = SCSSFileWatcher(path_to_scss_file=get_scss_stylesheet_path(), path_to_css_file=get_css_stylesheet_path(), parent=self)+ scss_file_watcher = SCSSFileWatcher(path_to_scss_file=get_scss_stylesheet_path(),+ path_to_css_file=get_css_stylesheet_path(), parent=self)css_file_watcher = CSSFileWatcher(path_to_css_file=get_css_stylesheet_path(), parent=self)return css_file_watcher@@ -313,7 +311,6 @@ class MainWindow(QMainWindow):self._generate_jupyter_notebook)-return ControlPanelWidget(camera_configuration_parameter_tree_widget=self._camera_configuration_parameter_tree_widget,# calibration_control_panel=self._calibration_control_panel,@@ -345,7 +342,8 @@ class MainWindow(QMainWindow):if Path(self._active_recording_info_widget.active_recording_info.blender_file_path).exists():open_file(self._active_recording_info_widget.active_recording_info.blender_file_path)else:- logger.error("Blender file does not exist! Did something go wrong in the `export_to_blender` call above?")+ logger.error(+ "Blender file does not exist! Did something go wrong in the `export_to_blender` call above?")def _generate_jupyter_notebook(self):logger.info("Exporting active recording to a Jupyter notebook...")@@ -384,20 +382,16 @@ class MainWindow(QMainWindow):except Exception as e:logger.error(e)--update_most_recent_recording_toml(recording_info_model=recording_info_model)def _update_skelly_viewer_widget(self):active_recording_info = self._active_recording_info_widget.active_recording_info-if active_recording_info.data3d_status_check:self._skelly_viewer_widget.load_skeleton_data(mediapipe_skeleton_npy_path=active_recording_info.mediapipe_3d_data_npy_file_path)-if active_recording_info.data2d_status_check:self._skelly_viewer_widget.generate_video_display(video_folder_path=active_recording_info.annotated_videos_folder_path@@ -409,9 +403,6 @@ class MainWindow(QMainWindow):video_folder_path=active_recording_info.synchronized_videos_folder_path)---def kill_running_threads_and_processes(self):logger.info("Killing running threads and processes... ")try:@@ -470,7 +461,6 @@ class MainWindow(QMainWindow):self._import_videos_window.folder_to_save_videos_to_selected.connect(self._handle_import_videos)text = self._import_videos_window.exec()-@pyqtSlot(list, str)def _handle_import_videos(self, video_paths: List[str], folder_to_save_videos: str):@@ -499,16 +489,16 @@ class MainWindow(QMainWindow):shutil.copy(video_path, destination_path)timestamps_copied = copy_directory_if_contains_timestamps(source_dir=Path(video_paths[0]).parent,- destination_dir=folder_to_save_videos)+ destination_dir=folder_to_save_videos)if timestamps_copied:logger.info(f"Copied timestamps from {Path(video_paths[0]).parent} to {folder_to_save_videos}")else:logger.info(f"No timestamps found in {Path(video_paths[0]).parent}")-self._active_recording_info_widget.set_active_recording(recording_folder_path=Path(folder_to_save_videos).parent)+def reboot_gui(self):logger.info("Rebooting GUI... ")get_qt_app().exit(EXIT_CODE_REBOOT)@@ -541,7 +531,6 @@ class MainWindow(QMainWindow):if __name__ == "__main__":import sys-app = QApplication(sys.argv)main_window = MainWindow(pipedream_pings=PipedreamPings(),freemocap_data_folder_path=get_freemocap_data_folder_path(),