Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

Console Log Fix (#434)

philipqueene3db71bb6553
diff --git a/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py b/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py
index 6087b204..a85b6fd2 100644
--- a/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py
+++ b/freemocap/core_processes/process_motion_capture_videos/process_recording_folder.py
@@ -31,6 +31,7 @@ from freemocap.core_processes.post_process_skeleton_data.process_single_camera_s
process_single_camera_skeleton_data
from freemocap.data_layer.data_saver.data_saver import DataSaver
from freemocap.data_layer.recording_models.post_processing_parameter_models import PostProcessingParameterModel
+from freemocap.system.logging.queue_logger import DirectQueueHandler
from freemocap.system.paths_and_filenames.file_and_folder_names import RAW_DATA_FOLDER_NAME, \
SEGMENT_CENTER_OF_MASS_NPY_FILE_NAME, CENTER_OF_MASS_FOLDER_NAME, TOTAL_BODY_CENTER_OF_MASS_NPY_FILE_NAME, \
MEDIAPIPE_BODY_3D_DATAFRAME_CSV_FILE_NAME
@@ -40,24 +41,30 @@ from freemocap.tests.test_image_tracking_data_shape import (
from freemocap.tests.test_mediapipe_skeleton_data_shape import test_mediapipe_skeleton_data_shape
from freemocap.utilities.geometry.rotate_by_90_degrees_around_x_axis import rotate_by_90_degrees_around_x_axis
from freemocap.utilities.save_dictionary_to_json import save_dictionary_to_json
+from freemocap.system.logging.configure_logging import log_view_logging_format_string
logger = logging.getLogger(__name__)
-
def process_recording_folder(
recording_processing_parameter_model: PostProcessingParameterModel,
kill_event: multiprocessing.Event = None,
+ queue: multiprocessing.Queue = None,
use_tqdm: bool = True,
):
"""
Parameters
----------
- recording_processing_parameter_model : PostProcessingParameterModel
+ recording_processing_parameter_model : RecordingProcessingParameterModel
RecordingProcessingParameterModel object (contains all the paths and parameters necessary to process a session folder
"""
+ if queue:
+ handler = DirectQueueHandler(queue)
+ handler.setFormatter(logging.Formatter(fmt=log_view_logging_format_string, datefmt="%Y-%m-%dT%H:%M:%S"))
+ logger.addHandler(handler)
+
rec = recording_processing_parameter_model # make it smol
if not Path(rec.recording_info_model.synchronized_videos_folder_path).exists():
@@ -71,7 +78,7 @@ def process_recording_folder(
try:
mediapipe_image_data_numCams_numFrames_numTrackedPts_XYZ = np.load(rec.recording_info_model.mediapipe_2d_data_npy_file_path)
except Exception as e:
- logger.error("Failed to load 2D data, cannot continue processing")
+ logger.error("Failed to load 2D data, cannot continue processing", exc_info=True)
return
else:
logger.info("Detecting 2d skeletons...")
@@ -96,7 +103,7 @@ def process_recording_folder(
image_tracking_data_file_path=rec.recording_info_model.mediapipe_2d_data_npy_file_path,
)
except AssertionError as error_message:
- logger.error(error_message)
+ logger.error(error_message, exc_info=True)
# spoof 3D data if single camera
if mediapipe_image_data_numCams_numFrames_numTrackedPts_XYZ.shape[0] == 1:
@@ -140,10 +147,11 @@ def process_recording_folder(
reprojection_error_file_path=rec.recording_info_model.mediapipe_reprojection_error_data_npy_file_path,
)
except AssertionError as error_message:
- logger.error(error_message)
+ logger.error(error_message, exc_info=True)
logger.info("Using SkellyForge Post-Processing to clean up data...")
+
rotated_raw_skel3d_frame_marker_xyz = rotate_by_90_degrees_around_x_axis(raw_skel3d_frame_marker_xyz)
skel3d_frame_marker_xyz = post_process_data(
@@ -177,7 +185,7 @@ def process_recording_folder(
reprojection_error_file_path=rec.recording_info_model.mediapipe_reprojection_error_data_npy_file_path,
)
except AssertionError as error_message:
- logger.error(error_message)
+ logger.error(error_message, exc_info=True)
logger.info("Breaking up big `npy` into smaller bits and converting to `csv`...")
# break up big NPY and save out csv's
diff --git a/freemocap/gui/qt/freemocap_main.py b/freemocap/gui/qt/freemocap_main.py
index d38679de..f28fcab8 100644
--- a/freemocap/gui/qt/freemocap_main.py
+++ b/freemocap/gui/qt/freemocap_main.py
@@ -6,9 +6,10 @@ from pathlib import Path
from PyQt6.QtCore import QTimer
from PyQt6.QtWidgets import QApplication
+from freemocap.gui.qt.main_window.freemocap_main_window import MainWindow, EXIT_CODE_REBOOT
+
logging.getLogger('matplotlib').setLevel(logging.WARNING)
-from freemocap.gui.qt.main_window.freemocap_main_window import MainWindow, EXIT_CODE_REBOOT
from freemocap.gui.qt.utilities.get_qt_app import get_qt_app
from freemocap.system.paths_and_filenames.path_getters import get_freemocap_data_folder_path
from freemocap.system.user_data.pipedream_pings import PipedreamPings
diff --git a/freemocap/gui/qt/main_window/freemocap_main_window.py b/freemocap/gui/qt/main_window/freemocap_main_window.py
index b43fd5dd..8a7f747d 100644
--- a/freemocap/gui/qt/main_window/freemocap_main_window.py
+++ b/freemocap/gui/qt/main_window/freemocap_main_window.py
@@ -2,7 +2,7 @@ import logging
import multiprocessing
import shutil
from pathlib import Path
-from typing import Union, List
+from typing import Union, List, Callable
from PyQt6.QtCore import Qt, pyqtSlot
from PyQt6.QtGui import QIcon
@@ -15,13 +15,11 @@ from PyQt6.QtWidgets import (
from skelly_viewer import SkellyViewer
from skellycam import (
SkellyCamParameterTreeWidget,
- SkellyCamWidget,
+ SkellyCamWidget, SkellyCamControllerWidget,
)
from tqdm import tqdm
-from freemocap.data_layer.export_data.blender_stuff.export_to_blender import (
- export_to_blender,
-)
+from freemocap.data_layer.export_data.blender_stuff.export_to_blender import export_to_blender
from freemocap.data_layer.export_data.blender_stuff.get_best_guess_of_blender_path import get_best_guess_of_blender_path
from freemocap.data_layer.export_data.generate_jupyter_notebook.generate_jupyter_notebook import \
generate_jupyter_notebook
@@ -81,8 +79,8 @@ from freemocap.system.paths_and_filenames.path_getters import (
)
from freemocap.system.paths_and_filenames.path_getters import get_recording_session_folder_path, \
- get_css_stylesheet_path, get_scss_stylesheet_path, get_most_recent_recording_path, get_blender_file_path, \
- get_freemocap_data_folder_path
+ get_css_stylesheet_path, get_scss_stylesheet_path, get_blender_file_path, get_most_recent_recording_path
+
from freemocap.system.user_data.pipedream_pings import PipedreamPings
from freemocap.gui.qt.workers.download_sample_data_thread_worker import (
DownloadSampleDataThreadWorker
@@ -101,13 +99,12 @@ class MainWindow(QMainWindow):
pipedream_pings: PipedreamPings,
parent=None,
):
-
super().__init__(parent=parent)
self._log_view_widget = LogViewWidget(parent=self) # start this first so it will grab the setup logs
logger.info("Initializing FreeMoCap MainWindow")
- self._size_main_window()
+ self.setGeometry(100, 100, 1280, 720)
self.setWindowIcon(QIcon(PATH_TO_FREEMOCAP_LOGO_SVG))
self.setWindowTitle(f"freemocap \U0001F480 \U00002728")
@@ -155,7 +152,7 @@ class MainWindow(QMainWindow):
self.addDockWidget(Qt.DockWidgetArea.RightDockWidgetArea, self._tools_dock_widget)
# self._tools_dock_tab_widget = QTabWidget(self)
- self._control_panel_widget = self._create_control_panel_widget()
+ self._control_panel_widget = self._create_control_panel_widget(log_update=self._log_view_widget.add_log)
self._tools_dock_widget.setWidget(self._control_panel_widget)
# self._tools_dock_tab_widget.addTab(self._control_panel_widget, f"Control Panel")
#
@@ -314,8 +311,7 @@ class MainWindow(QMainWindow):
# directory_view_widget=self._create_directory_view_widget(),
# parent=self)
- def _create_control_panel_widget(self):
-
+ def _create_control_panel_widget(self, log_update: Callable):
self._camera_configuration_parameter_tree_widget = SkellyCamParameterTreeWidget(self._skellycam_widget)
# self._calibration_control_panel = CalibrationControlPanel(
@@ -327,6 +323,7 @@ class MainWindow(QMainWindow):
get_active_recording_info=self._active_recording_info_widget.get_active_recording_info,
gui_state=self._gui_state,
kill_thread_event=self._kill_thread_event,
+ log_update=log_update,
)
self._process_motion_capture_data_panel.processing_finished_signal.connect(
self._handle_processing_finished_signal
@@ -379,6 +376,7 @@ class MainWindow(QMainWindow):
def _generate_jupyter_notebook(self):
logger.info("Exporting active recording to a Jupyter notebook...")
recording_path = self._active_recording_info_widget.get_active_recording_info(return_path=True)
+ # TODO: Need to include jupyter notebook in recording files that we keep track of (2023-05-15)
generate_jupyter_notebook(
path_to_recording=recording_path
)
@@ -577,19 +575,31 @@ class MainWindow(QMainWindow):
logger.error(f"Error while closing the viewer widget: {e}")
super().closeEvent(a0)
- for process in multiprocessing.active_children():
- logger.info(f"Terminating process: {process}")
- process.terminate()
+
+def remove_empty_directories(root_dir: Union[str, Path]):
+ """
+ Recursively remove empty directories from the root directory
+ :param root_dir: The root directory to start removing empty directories from
+ """
+ # logger.debug(f"Searching for empty directories in: {root_dir}")
+ for path in Path(root_dir).rglob("*"):
+ if path.is_dir() and not any(path.iterdir()):
+ logger.info(f"Removing empty directory: {path}")
+ path.rmdir()
+ elif path.is_dir() and any(path.iterdir()):
+ remove_empty_directories(path)
+ else:
+ continue
if __name__ == "__main__":
import sys
app = QApplication(sys.argv)
- main_window = MainWindow(pipedream_pings=PipedreamPings(),
- freemocap_data_folder_path=get_freemocap_data_folder_path(),
- )
+ main_window = MainWindow(pipedream_pings=PipedreamPings())
main_window.show()
app.exec()
-
+ for process in multiprocessing.active_children():
+ logger.info(f"Terminating process: {process}")
+ process.terminate()
sys.exit()
diff --git a/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/process_motion_capture_data_panel.py b/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/process_motion_capture_data_panel.py
index 0399a8b4..b35acac8 100644
--- a/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/process_motion_capture_data_panel.py
+++ b/freemocap/gui/qt/widgets/control_panel/process_mocap_data_panel/process_motion_capture_data_panel.py
@@ -37,6 +37,7 @@ class ProcessMotionCaptureDataPanel(QWidget):
recording_processing_parameters: PostProcessingParameterModel,
get_active_recording_info: Callable,
kill_thread_event: threading.Event,
+ log_update: Callable,
gui_state: GuiState,
parent=None,
):
@@ -44,9 +45,10 @@ class ProcessMotionCaptureDataPanel(QWidget):
self._kill_thread_event = kill_thread_event
self._get_active_recording_info = get_active_recording_info
+ self._log_update = log_update
self._recording_processing_parameter_model = recording_processing_parameters
self._recording_processing_parameter_model.recording_info_model = self._get_active_recording_info()
-
+
self._layout = QVBoxLayout()
self._layout.setAlignment(Qt.AlignmentFlag.AlignTop)
self.setLayout(self._layout)
@@ -209,6 +211,7 @@ class ProcessMotionCaptureDataPanel(QWidget):
self._process_motion_capture_data_thread_worker = ProcessMotionCaptureDataThreadWorker(session_parameter_model,
kill_event=self._kill_thread_event)
+ self._process_motion_capture_data_thread_worker.in_progress.connect(self._log_update)
self._process_motion_capture_data_thread_worker.start()
self._process_motion_capture_data_thread_worker.finished.connect(self._handle_finished_signal)
diff --git a/freemocap/gui/qt/widgets/log_view_widget.py b/freemocap/gui/qt/widgets/log_view_widget.py
index 49aba76b..d15c28c7 100644
--- a/freemocap/gui/qt/widgets/log_view_widget.py
+++ b/freemocap/gui/qt/widgets/log_view_widget.py
@@ -9,13 +9,10 @@ from PyQt6.QtCore import QThread
from PyQt6.QtWidgets import QApplication, QPlainTextEdit
from freemocap.gui.qt.utilities.colors import get_next_color, rgb_color_generator
+from freemocap.system.logging.configure_logging import log_view_logging_format_string
logger = logging.getLogger(__name__)
-log_view_logging_format_string = (
- "[%(asctime)s.%(msecs)04d] [%(levelname)4s] [%(name)s] [Process:%(process)d Thread:%(thread)d] ::::: [%(name)s:%(funcName)s():%(lineno)s] |||| %(message)s."
-)
-
log_view_logging_formatter = logging.Formatter(fmt=log_view_logging_format_string, datefmt="%Y-%m-%dT%H:%M:%S")
level_colors = {
@@ -84,7 +81,7 @@ class LogViewWidget(QPlainTextEdit):
self.show_code_path_info = True
self._logging_queue_listener.start()
- def add_log(self, record):
+ def add_log(self, record: logging.LogRecord):
level = record.levelname.ljust(7) # 7 characters long, right padded with spaces
timestamp = f"[{record.asctime}.{record.msecs:04.0f}]".ljust(24) # 24 characters long
process_id_str = f"{record.process}".rjust(6) # 5 characters long, left padded with spaces
diff --git a/freemocap/gui/qt/workers/process_motion_capture_data_thread_worker.py b/freemocap/gui/qt/workers/process_motion_capture_data_thread_worker.py
index c43b244a..57f9e5c1 100644
--- a/freemocap/gui/qt/workers/process_motion_capture_data_thread_worker.py
+++ b/freemocap/gui/qt/workers/process_motion_capture_data_thread_worker.py
@@ -1,12 +1,12 @@
import logging
import multiprocessing
+import time
from pathlib import Path
from PyQt6.QtCore import pyqtSignal, QThread
-from freemocap.core_processes.process_motion_capture_videos.process_recording_folder import (
- process_recording_folder,
-)
+from freemocap.core_processes.process_motion_capture_videos.process_recording_folder import process_recording_folder
+
from freemocap.data_layer.recording_models.post_processing_parameter_models import PostProcessingParameterModel
from freemocap.system.paths_and_filenames.file_and_folder_names import RECORDING_PARAMETERS_JSON_FILE_NAME
from freemocap.utilities.save_dictionary_to_json import save_dictionary_to_json
@@ -14,17 +14,24 @@ from freemocap.utilities.save_dictionary_to_json import save_dictionary_to_json
logger = logging.getLogger(__name__)
+
class ProcessMotionCaptureDataThreadWorker(QThread):
finished = pyqtSignal()
- in_progress = pyqtSignal(str)
+ in_progress = pyqtSignal(object)
def __init__(self, post_processing_parameters: PostProcessingParameterModel,
kill_event: multiprocessing.Event, parent=None):
super().__init__()
self._post_processing_parameters = post_processing_parameters
self._kill_event = kill_event
- self._process = multiprocessing.Process(target=process_recording_folder, args=(self._post_processing_parameters,
- self._kill_event))
+
+ self._queue = multiprocessing.Queue()
+
+ self._process = multiprocessing.Process(target=process_recording_folder,
+ args=(self._post_processing_parameters,
+ self._kill_event,
+ self._queue))
+
def run(self):
logger.info(
@@ -35,6 +42,7 @@ class ProcessMotionCaptureDataThreadWorker(QThread):
recording_info_dict = self._post_processing_parameters.dict(exclude={'recording_info_model'})
Path(self._post_processing_parameters.recording_info_model.output_data_folder_path).mkdir(parents=True, exist_ok=True)
+
save_dictionary_to_json(
save_path=self._post_processing_parameters.recording_info_model.output_data_folder_path,
file_name=RECORDING_PARAMETERS_JSON_FILE_NAME,
@@ -43,9 +51,22 @@ class ProcessMotionCaptureDataThreadWorker(QThread):
try:
self._process.start()
- self._process.join()
+ while self._process.is_alive():
+ time.sleep(0.01)
+ if self._queue.empty():
+ continue
+ else:
+ record = self._queue.get()
+ print(f"message: {record.msg}")
+ self.in_progress.emit(record)
+
+ while not self._queue.empty():
+ record = self._queue.get()
+ print(f"message: {record.msg}")
+ self.in_progress.emit(record)
+
except Exception as e:
- logger.error(f"Error processing motion capture data: {e}")
+ logger.error(f"Error processing motion capture data: {str(e)}")
logger.info("Finished processing session folder!")
diff --git a/freemocap/system/logging/configure_logging.py b/freemocap/system/logging/configure_logging.py
index cf08c590..8926c63d 100644
--- a/freemocap/system/logging/configure_logging.py
+++ b/freemocap/system/logging/configure_logging.py
@@ -13,6 +13,10 @@ format_string = "[%(asctime)s.%(msecs)04d] [%(levelname)8s] [%(name)s] [%(funcNa
default_logging_formatter = logging.Formatter(fmt=format_string, datefmt="%Y-%m-%d %H:%M:%S")
+log_view_logging_format_string = (
+ "[%(asctime)s.%(msecs)04d] [%(levelname)4s] [%(name)s] [Process:%(process)d Thread:%(thread)d] ::::: [%(name)s:%(funcName)s():%(lineno)s] |||| %(message)s."
+)
+
# logging.basicConfig(level=logging.WARNING)
def get_logging_handlers():
diff --git a/freemocap/system/logging/queue_logger.py b/freemocap/system/logging/queue_logger.py
new file mode 100644
index 00000000..7916fe6f
--- /dev/null
+++ b/freemocap/system/logging/queue_logger.py
@@ -0,0 +1,11 @@
+import multiprocessing
+from logging.handlers import QueueHandler
+
+class DirectQueueHandler(QueueHandler):
+ def __init__(self, queue: multiprocessing.Queue):
+ super().__init__(queue)
+
+ def enqueue(self, record):
+ self.queue.put(record)
+
+