Free Motion Capture for Everyone πβ¨
TypeScript
Console Log Fix (#434)
philipqueene3db71bb6553
diff --git a/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py b/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.pyindex 6087b204..a85b6fd2 100644--- a/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py+++ b/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py@@ -31,6 +31,7 @@ from freemocap.core_processes.post_process_skeleton_data.process_single_camera_sprocess_single_camera_skeleton_datafrom freemocap.data_layer.data_saver.data_saver import DataSaverfrom freemocap.data_layer.recording_models.post_processing_parameter_models import PostProcessingParameterModel+from freemocap.system.logging.queue_logger import DirectQueueHandlerfrom freemocap.system.paths_and_filenames.file_and_folder_names import RAW_DATA_FOLDER_NAME, \SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME, CENTER_OF_MASS_FOLDER_NAME, TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME, \MEDIAPIPE_BODY_3D_DATAFRAME_CSV_FILE_NAME@@ -40,24 +41,30 @@ from freemocap.tests.test_image_tracking_data_shape import (from freemocap.tests.test_mediapipe_skeleton_data_shape import test_mediapipe_skeleton_data_shapefrom freemocap.utilities.geometry.rotate_by_90_degrees_around_x_axis import rotate_by_90_degrees_around_x_axisfrom freemocap.utilities.save_dictionary_to_json import save_dictionary_to_json+from freemocap.system.logging.configure_logging import log_view_logging_format_stringlogger = logging.getLogger(__name__)-def process_recording_folder(recording_processing_parameter_model: PostProcessingParameterModel,kill_event: multiprocessing.Event = None,+ queue: multiprocessing.Queue = None,use_tqdm: bool = True,):"""Parameters----------- recording_processing_parameter_model : PostProcessingParameterModel+ recording_processing_parameter_model : RecordingProcessingParameterModelRecordingProcessingParameterModel object (contains all the paths and parameters necessary to process a session folder"""+ if queue:+ handler = DirectQueueHandler(queue)+ handler.setFormatter(logging.Formatter(fmt=log_view_logging_format_string, datefmt="%Y-%m-%dT%H:%M:%S"))+ logger.addHandler(handler)+rec = recording_processing_parameter_model # make it smolif not Path(rec.recording_info_model.synchronized_videos_folder_path).exists():@@ -71,7 +78,7 @@ def process_recording_folder(try:mediapipe_image_data_numCams_numFrames_numTrackedPts_XYZ = np.load(rec.recording_info_model.mediapipe_2d_data_npy_file_path)except Exception as e:- logger.error("Failed to load 2D data, cannot continue processing")+ logger.error("Failed to load 2D data, cannot continue processing", exc_info=True)returnelse:logger.info("Detecting 2d skeletons...")@@ -96,7 +103,7 @@ def process_recording_folder(image_tracking_data_file_path=rec.recording_info_model.mediapipe_2d_data_npy_file_path,)except AssertionError as error_message:- logger.error(error_message)+ logger.error(error_message, exc_info=True)# spoof 3D data if single cameraif mediapipe_image_data_numCams_numFrames_numTrackedPts_XYZ.shape[0] == 1:@@ -140,10 +147,11 @@ def process_recording_folder(reprojection_error_file_path=rec.recording_info_model.mediapipe_reprojection_error_data_npy_file_path,)except AssertionError as error_message:- logger.error(error_message)+ logger.error(error_message, exc_info=True)logger.info("Using SkellyForge Post-Processing to clean up data...")+rotated_raw_skel3d_frame_marker_xyz = rotate_by_90_degrees_around_x_axis(raw_skel3d_frame_marker_xyz)skel3d_frame_marker_xyz = post_process_data(@@ -177,7 +185,7 @@ def process_recording_folder(reprojection_error_file_path=rec.recording_info_model.mediapipe_reprojection_error_data_npy_file_path,)except AssertionError as error_message:- logger.error(error_message)+ logger.error(error_message, exc_info=True)logger.info("Breaking up big `npy` into smaller bits and converting to `csv`...")# break up big NPY and save out csv'sdiff --git a/freemocap/gui/qt/freemocap_main.py b/freemocap/gui/qt/freemocap_main.pyindex d38679de..f28fcab8 100644--- a/freemocap/gui/qt/freemocap_main.py+++ b/freemocap/gui/qt/freemocap_main.py@@ -6,9 +6,10 @@ from pathlib import Pathfrom PyQt6.QtCore import QTimerfrom PyQt6.QtWidgets import QApplication+from freemocap.gui.qt.main_window.freemocap_main_window import MainWindow, EXIT_CODE_REBOOT+logging.getLogger('matplotlib').setLevel(logging.WARNING)-from freemocap.gui.qt.main_window.freemocap_main_window import MainWindow, EXIT_CODE_REBOOTfrom freemocap.gui.qt.utilities.get_qt_app import get_qt_appfrom freemocap.system.paths_and_filenames.path_getters import get_freemocap_data_folder_pathfrom freemocap.system.user_data.pipedream_pings import PipedreamPingsdiff --git a/freemocap/gui/qt/main_window/freemocap_main_window.py b/freemocap/gui/qt/main_window/freemocap_main_window.pyindex b43fd5dd..8a7f747d 100644--- a/freemocap/gui/qt/main_window/freemocap_main_window.py+++ b/freemocap/gui/qt/main_window/freemocap_main_window.py@@ -2,7 +2,7 @@ import loggingimport multiprocessingimport shutilfrom pathlib import Path-from typing import Union, List+from typing import Union, List, Callablefrom PyQt6.QtCore import Qt, pyqtSlotfrom PyQt6.QtGui import QIcon@@ -15,13 +15,11 @@ from PyQt6.QtWidgets import (from skelly_viewer import SkellyViewerfrom skellycam import (SkellyCamParameterTreeWidget,- SkellyCamWidget,+ SkellyCamWidget, SkellyCamControllerWidget,)from tqdm import tqdm-from freemocap.data_layer.export_data.blender_stuff.export_to_blender import (- export_to_blender,-)+from freemocap.data_layer.export_data.blender_stuff.export_to_blender import export_to_blenderfrom freemocap.data_layer.export_data.blender_stuff.get_best_guess_of_blender_path import get_best_guess_of_blender_pathfrom freemocap.data_layer.export_data.generate_jupyter_notebook.generate_jupyter_notebook import \generate_jupyter_notebook@@ -81,8 +79,8 @@ from freemocap.system.paths_and_filenames.path_getters import ()from freemocap.system.paths_and_filenames.path_getters import get_recording_session_folder_path, \- get_css_stylesheet_path, get_scss_stylesheet_path, get_most_recent_recording_path, get_blender_file_path, \- get_freemocap_data_folder_path+ get_css_stylesheet_path, get_scss_stylesheet_path, get_blender_file_path, get_most_recent_recording_path+from freemocap.system.user_data.pipedream_pings import PipedreamPingsfrom freemocap.gui.qt.workers.download_sample_data_thread_worker import (DownloadSampleDataThreadWorker@@ -101,13 +99,12 @@ class MainWindow(QMainWindow):pipedream_pings: PipedreamPings,parent=None,):-super().__init__(parent=parent)self._log_view_widget = LogViewWidget(parent=self) # start this first so it will grab the setup logslogger.info("Initializing FreeMoCap MainWindow")- self._size_main_window()+ self.setGeometry(100, 100, 1280, 720)self.setWindowIcon(QIcon(PATH_TO_FREEMOCAP_LOGO_SVG))self.setWindowTitle(f"freemocap \U0001F480 \U00002728")@@ -155,7 +152,7 @@ class MainWindow(QMainWindow):self.addDockWidget(Qt.DockWidgetArea.RightDockWidgetArea, self._tools_dock_widget)# self._tools_dock_tab_widget = QTabWidget(self)- self._control_panel_widget = self._create_control_panel_widget()+ self._control_panel_widget = self._create_control_panel_widget(log_update=self._log_view_widget.add_log)self._tools_dock_widget.setWidget(self._control_panel_widget)# self._tools_dock_tab_widget.addTab(self._control_panel_widget, f"Control Panel")#@@ -314,8 +311,7 @@ class MainWindow(QMainWindow):# directory_view_widget=self._create_directory_view_widget(),# parent=self)- def _create_control_panel_widget(self):-+ def _create_control_panel_widget(self, log_update: Callable):self._camera_configuration_parameter_tree_widget = SkellyCamParameterTreeWidget(self._skellycam_widget)# self._calibration_control_panel = CalibrationControlPanel(@@ -327,6 +323,7 @@ class MainWindow(QMainWindow):get_active_recording_info=self._active_recording_info_widget.get_active_recording_info,gui_state=self._gui_state,kill_thread_event=self._kill_thread_event,+ log_update=log_update,)self._process_motion_capture_data_panel.processing_finished_signal.connect(self._handle_processing_finished_signal@@ -379,6 +376,7 @@ class MainWindow(QMainWindow):def _generate_jupyter_notebook(self):logger.info("Exporting active recording to a Jupyter notebook...")recording_path = self._active_recording_info_widget.get_active_recording_info(return_path=True)+ # TODO: Need to include jupyter notebook in recording files that we keep track of (2023-05-15)generate_jupyter_notebook(path_to_recording=recording_path)@@ -577,19 +575,31 @@ class MainWindow(QMainWindow):logger.error(f"Error while closing the viewer widget: {e}")super().closeEvent(a0)- for process in multiprocessing.active_children():- logger.info(f"Terminating process: {process}")- process.terminate()++def remove_empty_directories(root_dir: Union[str, Path]):+ """+ Recursively remove empty directories from the root directory+ :param root_dir: The root directory to start removing empty directories from+ """+ # logger.debug(f"Searching for empty directories in: {root_dir}")+ for path in Path(root_dir).rglob("*"):+ if path.is_dir() and not any(path.iterdir()):+ logger.info(f"Removing empty directory: {path}")+ path.rmdir()+ elif path.is_dir() and any(path.iterdir()):+ remove_empty_directories(path)+ else:+ continueif __name__ == "__main__":import sysapp = QApplication(sys.argv)- main_window = MainWindow(pipedream_pings=PipedreamPings(),- freemocap_data_folder_path=get_freemocap_data_folder_path(),- )+ main_window = MainWindow(pipedream_pings=PipedreamPings())main_window.show()app.exec()-+ for process in multiprocessing.active_children():+ logger.info(f"Terminating process: {process}")+ process.terminate()sys.exit()diff --git a/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/process_motion_capture_data_panel.py b/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/process_motion_capture_data_panel.pyindex 0399a8b4..b35acac8 100644--- a/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/process_motion_capture_data_panel.py+++ b/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/process_motion_capture_data_panel.py@@ -37,6 +37,7 @@ class ProcessMotionCaptureDataPanel(QWidget):recording_processing_parameters: PostProcessingParameterModel,get_active_recording_info: Callable,kill_thread_event: threading.Event,+ log_update: Callable,gui_state: GuiState,parent=None,):@@ -44,9 +45,10 @@ class ProcessMotionCaptureDataPanel(QWidget):self._kill_thread_event = kill_thread_eventself._get_active_recording_info = get_active_recording_info+ self._log_update = log_updateself._recording_processing_parameter_model = recording_processing_parametersself._recording_processing_parameter_model.recording_info_model = self._get_active_recording_info()-+self._layout = QVBoxLayout()self._layout.setAlignment(Qt.AlignmentFlag.AlignTop)self.setLayout(self._layout)@@ -209,6 +211,7 @@ class ProcessMotionCaptureDataPanel(QWidget):self._process_motion_capture_data_thread_worker = ProcessMotionCaptureDataThreadWorker(session_parameter_model,kill_event=self._kill_thread_event)+ self._process_motion_capture_data_thread_worker.in_progress.connect(self._log_update)self._process_motion_capture_data_thread_worker.start()self._process_motion_capture_data_thread_worker.finished.connect(self._handle_finished_signal)diff --git a/freemocap/gui/qt/widgets/log_view_widget.py b/freemocap/gui/qt/widgets/log_view_widget.pyindex 49aba76b..d15c28c7 100644--- a/freemocap/gui/qt/widgets/log_view_widget.py+++ b/freemocap/gui/qt/widgets/log_view_widget.py@@ -9,13 +9,10 @@ from PyQt6.QtCore import QThreadfrom PyQt6.QtWidgets import QApplication, QPlainTextEditfrom freemocap.gui.qt.utilities.colors import get_next_color, rgb_color_generator+from freemocap.system.logging.configure_logging import log_view_logging_format_stringlogger = logging.getLogger(__name__)-log_view_logging_format_string = (- "[%(asctime)s.%(msecs)04d] [%(levelname)4s] [%(name)s] [Process:%(process)d Thread:%(thread)d] ::::: [%(name)s:%(funcName)s():%(lineno)s] |||| %(message)s."-)-log_view_logging_formatter = logging.Formatter(fmt=log_view_logging_format_string, datefmt="%Y-%m-%dT%H:%M:%S")level_colors = {@@ -84,7 +81,7 @@ class LogViewWidget(QPlainTextEdit):self.show_code_path_info = Trueself._logging_queue_listener.start()- def add_log(self, record):+ def add_log(self, record: logging.LogRecord):level = record.levelname.ljust(7) # 7 characters long, right padded with spacestimestamp = f"[{record.asctime}.{record.msecs:04.0f}]".ljust(24) # 24 characters longprocess_id_str = f"{record.process}".rjust(6) # 5 characters long, left padded with spacesdiff --git a/freemocap/gui/qt/workers/process_motion_capture_data_thread_worker.py b/freemocap/gui/qt/workers/process_motion_capture_data_thread_worker.pyindex c43b244a..57f9e5c1 100644--- a/freemocap/gui/qt/workers/process_motion_capture_data_thread_worker.py+++ b/freemocap/gui/qt/workers/process_motion_capture_data_thread_worker.py@@ -1,12 +1,12 @@import loggingimport multiprocessing+import timefrom pathlib import Pathfrom PyQt6.QtCore import pyqtSignal, QThread-from freemocap.core_processes.process_motion_capture_videos.process_recording_folder import (- process_recording_folder,-)+from freemocap.core_processes.process_motion_capture_videos.process_recording_folder import process_recording_folder+from freemocap.data_layer.recording_models.post_processing_parameter_models import PostProcessingParameterModelfrom freemocap.system.paths_and_filenames.file_and_folder_names import RECORDING_PARAMETERS_JSON_FILE_NAMEfrom freemocap.utilities.save_dictionary_to_json import save_dictionary_to_json@@ -14,17 +14,24 @@ from freemocap.utilities.save_dictionary_to_json import save_dictionary_to_jsonlogger = logging.getLogger(__name__)+class ProcessMotionCaptureDataThreadWorker(QThread):finished = pyqtSignal()- in_progress = pyqtSignal(str)+ in_progress = pyqtSignal(object)def __init__(self, post_processing_parameters: PostProcessingParameterModel,kill_event: multiprocessing.Event, parent=None):super().__init__()self._post_processing_parameters = post_processing_parametersself._kill_event = kill_event- self._process = multiprocessing.Process(target=process_recording_folder, args=(self._post_processing_parameters,- self._kill_event))++ self._queue = multiprocessing.Queue()++ self._process = multiprocessing.Process(target=process_recording_folder,+ args=(self._post_processing_parameters,+ self._kill_event,+ self._queue))+def run(self):logger.info(@@ -35,6 +42,7 @@ class ProcessMotionCaptureDataThreadWorker(QThread):recording_info_dict = self._post_processing_parameters.dict(exclude={'recording_info_model'})Path(self._post_processing_parameters.recording_info_model.output_data_folder_path).mkdir(parents=True, exist_ok=True)+save_dictionary_to_json(save_path=self._post_processing_parameters.recording_info_model.output_data_folder_path,file_name=RECORDING_PARAMETERS_JSON_FILE_NAME,@@ -43,9 +51,22 @@ class ProcessMotionCaptureDataThreadWorker(QThread):try:self._process.start()- self._process.join()+ while self._process.is_alive():+ time.sleep(0.01)+ if self._queue.empty():+ continue+ else:+ record = self._queue.get()+ print(f"message: {record.msg}")+ self.in_progress.emit(record)++ while not self._queue.empty():+ record = self._queue.get()+ print(f"message: {record.msg}")+ self.in_progress.emit(record)+except Exception as e:- logger.error(f"Error processing motion capture data: {e}")+ logger.error(f"Error processing motion capture data: {str(e)}")logger.info("Finished processing session folder!")diff --git a/freemocap/system/logging/configure_logging.py b/freemocap/system/logging/configure_logging.pyindex cf08c590..8926c63d 100644--- a/freemocap/system/logging/configure_logging.py+++ b/freemocap/system/logging/configure_logging.py@@ -13,6 +13,10 @@ format_string = "[%(asctime)s.%(msecs)04d] [%(levelname)8s] [%(name)s] [%(funcNadefault_logging_formatter = logging.Formatter(fmt=format_string, datefmt="%Y-%m-%d %H:%M:%S")+log_view_logging_format_string = (+ "[%(asctime)s.%(msecs)04d] [%(levelname)4s] [%(name)s] [Process:%(process)d Thread:%(thread)d] ::::: [%(name)s:%(funcName)s():%(lineno)s] |||| %(message)s."+)+# logging.basicConfig(level=logging.WARNING)def get_logging_handlers():diff --git a/freemocap/system/logging/queue_logger.py b/freemocap/system/logging/queue_logger.pynew file mode 100644index 00000000..7916fe6f--- /dev/null+++ b/freemocap/system/logging/queue_logger.py@@ -0,0 +1,11 @@+import multiprocessing+from logging.handlers import QueueHandler++class DirectQueueHandler(QueueHandler):+ def __init__(self, queue: multiprocessing.Queue):+ super().__init__(queue)++ def enqueue(self, record):+ self.queue.put(record)++