Free Motion Capture for Everyone πβ¨
TypeScript
various bug fixes - functional camera view
Jon513b733b8961
diff --git a/src/api/services/user_config.py b/src/api/services/user_config.pyindex 3c9a5eaa..5737c77e 100644--- a/src/api/services/user_config.py+++ b/src/api/services/user_config.py@@ -7,18 +7,18 @@ from src.config.webcam_config import WebcamConfig@singletonclass UserConfigService:- _webcam_config: Dict[str, WebcamConfig] = {}+ _dict_of_webcam_configs: Dict[str, WebcamConfig] = {}@propertydef webcam_configs(self):- return self._webcam_config+ return self._dict_of_webcam_configs- def webcam_config_by_id(self, webcam_id: str):- if webcam_id in self._webcam_config:- return self._webcam_config[webcam_id]+ def webcam_config_by_id(self, webcam_id_as_str: str) -> WebcamConfig:+ if webcam_id_as_str in self._dict_of_webcam_configs:+ return self._dict_of_webcam_configs[webcam_id_as_str]else:- return WebcamConfig(webcam_id=int(webcam_id))+ return WebcamConfig(webcam_id=int(webcam_id_as_str))def set_webcam_config(self, webcam_config: WebcamConfig):as_str = str(webcam_config.webcam_id)- self._webcam_config[as_str] = webcam_config+ self._dict_of_webcam_configs[as_str] = webcam_configdiff --git a/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py b/src/cameras/capture/opencv_camera/camera_stream_thread_handler.pyindex a0c3e9cd..a2ff1479 100644--- a/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py+++ b/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py@@ -21,13 +21,6 @@ class VideoCaptureThread(threading.Thread):self._frame: FramePayload = FramePayload()self.setDaemon(True)- @property- def average_fps(self):- if self._elapsed <= 0:- return 0- if self._num_frames_processed <= 0:- return 0- return self._num_frames_processed / self._elapsed@propertydef latest_frame(self):diff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.pyindex 93a4160a..7e1c56ea 100644--- a/src/cameras/capture/opencv_camera/opencv_camera.py+++ b/src/cameras/capture/opencv_camera/opencv_camera.py@@ -76,13 +76,13 @@ class OpenCVCamera:self._apply_configuration()success, image = self._opencv_video_capture_object.read()- if not success:+ if not success or image is None:logger.error("Could not connect to a camera at port# {}".format(self._config.webcam_id))- return success+ return Falsetry:image_height = image.shape[0]@@ -157,11 +157,14 @@ class OpenCVCamera:if not self._opencv_video_capture_object.grab():return FramePayload(False, None, None)- timestamp_ns = time.time_ns()+ timestamp_ns_pre = time.perf_counter_ns()success, image = self._opencv_video_capture_object.retrieve()+ timestamp_ns_post = time.perf_counter_ns()++ timestamp_ns = (timestamp_ns_pre + timestamp_ns_post)/2self._new_frame_ready = success- return FramePayload(success, image, timestamp_ns)+ return FramePayload(success, image, timestamp_ns, self.webcam_id_as_str)def stop_frame_capture(self):self.close()diff --git a/src/cameras/detection/cam_detection.py b/src/cameras/detection/cam_detection.pyindex 14d3e7c8..2d7d81d0 100644--- a/src/cameras/detection/cam_detection.py+++ b/src/cameras/detection/cam_detection.py@@ -19,7 +19,7 @@ class DetectPossibleCameras:cap = cv2.VideoCapture(cam_id, cv2_backend)success, image = cap.read()- if success:+ if success and image is not None:try:cams_to_use_list.append(RawCamera(diff --git a/src/cameras/multicam_manager/cv_camera_manager.py b/src/cameras/multicam_manager/cv_camera_manager.pyindex 0bc7b9a0..a0efce0a 100644--- a/src/cameras/multicam_manager/cv_camera_manager.py+++ b/src/cameras/multicam_manager/cv_camera_manager.py@@ -33,7 +33,7 @@ class OpenCVCameraManager:if _open_cv_cameras is None:logger.info("Creating cams.")# we create the _cv_cams /once/, and reuse it for the lifetime of the session- _open_cv_cameras = self._create_opencv_cameras(calibrate_cameras=True)+ _open_cv_cameras = self._create_opencv_cameras()self._open_cv_camera_objects = _open_cv_cameras# self._timestamp_logger = TimestampLogger(self.available_webcam_ids, self.camera0_id)self._number_of_cameras = len(self._open_cv_camera_objects)@@ -82,7 +82,7 @@ class OpenCVCameraManager:return camreturn None- def _create_opencv_cameras(self, calibrate_cameras=True):+ def _create_opencv_cameras(self):raw_camera_objects = self._detected_cams_data.cameras_found_listopen_cv_cameras: List[OpenCVCamera] = []for this_raw_cam in raw_camera_objects:diff --git a/src/config/webcam_config.py b/src/config/webcam_config.pyindex 0443d0be..76a823d9 100644--- a/src/config/webcam_config.py+++ b/src/config/webcam_config.py@@ -2,9 +2,11 @@ from pathlib import Pathfrom pydantic import BaseModel+def _get_home_dir():return str(Path.home())+class WebcamConfig(BaseModel):webcam_id: int = 0exposure: int = -6@@ -13,4 +15,4 @@ class WebcamConfig(BaseModel):save_video: bool = True# fourcc: str = "MP4V"fourcc: str = "MJPG"- base_save_video_dir = _get_home_dir()\ No newline at end of file+ base_save_video_dir = _get_home_dir()diff --git a/src/core_processor/fps/timestamp_manager.py b/src/core_processor/fps/timestamp_manager.pyindex 4b1e6741..ef6dd1cc 100644--- a/src/core_processor/fps/timestamp_manager.py+++ b/src/core_processor/fps/timestamp_manager.py@@ -7,7 +7,8 @@ class TimestampLogger:_num_frames_processed: int_start_time: float- def __init__(self):+ def __init__(self, _session_start_time_unix_ns: int):+ self._session_start_time_unix_ns = _session_start_time_unix_nsself._num_frames_processed = 0self._timestamps = np.empty(0)@@ -18,7 +19,7 @@ class TimestampLogger:def median_frames_per_second(self):if self._num_frames_processed <= 10:return 0- return np.nanmedian(np.diff(self._timestamps)/1e9)**-1+ return np.nanmedian(np.diff(self._timestamps) / 1e9) ** -1@propertydef number_of_frames(self):@@ -26,37 +27,52 @@ class TimestampLogger:@propertydef timestamps(self):+ """return timestamps in nanoseconds in Unix epoch format (which is both the most useful, ugliest, AND funniest time format around. You can use `self.timestamps_in_seconds_from_zero()` for a more intuitive time format"""return self._timestamps+ @property+ def timestamps_in_seconds_from_session_start(self):+ """timestamps in seconds counting up from zero (where zero is the start time of the recording session)"""+ return (self._timestamps - self._session_start_time_unix_ns) / 1e9+@propertydef last_timestamp(self):if self._timestamps.shape[0] > 0:return self._timestamps[-1]return None- def increment_frame_processed(self, timestamp):+ def log_new_timestamp(self, timestamp):self._timestamps = np.append(self._timestamps, timestamp)self._num_frames_processed += 1class TimestampManager:- def __init__(self, webcam_ids: List[str]):+ def __init__(self, webcam_ids: List[str], session_start_time_unix_ns: int):+ self._session_start_time_unix_ns = session_start_time_unix_nsself._webcam_ids = webcam_ids- self._counters: Dict[str, TimestampLogger] = self._init_counters(webcam_ids)+ self._webcam_timestamp_loggers: Dict[str, TimestampLogger] = self._initialize_timestamp_loggers(webcam_ids)+ self._main_loop_timestamp_logger = TimestampLogger(self._session_start_time_unix_ns)- def _init_counters(self, webcam_ids: List[str]):- d = {}+ def _initialize_timestamp_loggers(self, webcam_ids: List[str]):+ dictionary_of_timestamp_logger_objects = {}for webcam_id in webcam_ids:- d[webcam_id] = TimestampLogger()- return d+ dictionary_of_timestamp_logger_objects[webcam_id] = TimestampLogger(self._session_start_time_unix_ns)+ return dictionary_of_timestamp_logger_objects++ def get_timestamps_from_camera_in_seconds_from_session_start(self, webcam_id):+ return self._webcam_timestamp_loggers[webcam_id].timestamps_in_seconds_from_session_start- def get_timestamps_from_camera(self, webcam_id):- return self._counters[webcam_id]+ def get_number_of_frames(self, webcam_id):+ return self._webcam_timestamp_loggers[webcam_id].number_of_frames- def increment_frame_processed_for(self, webcam_id, timestamp):- self._counters[webcam_id].increment_frame_processed(timestamp)+ def increment_main_loop_timestamp_logger(self, timestamp):+ self._main_loop_timestamp_logger.log_new_timestamp(timestamp)- def median_frames_per_second(self, webcam_id):- return self._counters[webcam_id].median_frames_per_second+ def median_frames_per_second_for_main_loop(self):+ return self._main_loop_timestamp_logger.median_frames_per_second+ def increment_frame_processed_for_webcam(self, webcam_id, timestamp):+ self._webcam_timestamp_loggers[webcam_id].log_new_timestamp(timestamp)+ def median_frames_per_second_for_webcam(self, webcam_id):+ return self._webcam_timestamp_loggers[webcam_id].median_frames_per_seconddiff --git a/src/core_processor/show_cam_window.py b/src/core_processor/show_cam_window.pyindex d84fa8a2..779b0ab8 100644--- a/src/core_processor/show_cam_window.py+++ b/src/core_processor/show_cam_window.py@@ -9,15 +9,6 @@ from src.core_processor.utils.image_fps_writer import write_fps_to_imagelogger = logging.getLogger(__name__)-def show_cam_window(webcam_id: str, image: np.array, fps_counter_cls: TimestampManager):- exit_key = cv2.waitKey(1)- if exit_key == 27:- logger.info("ESC has been pressed.")- return False-- write_fps_to_image(- image,- fps_counter_cls.median_frames_per_second(webcam_id),- )+def show_cam_window(webcam_id: str, image: np.array, timestamp_manager_cls: TimestampManager):cv2.imshow(webcam_id, image)return Truediff --git a/src/core_processor/utils/image_fps_writer.py b/src/core_processor/utils/image_fps_writer.pyindex bdfcba14..e1716379 100644--- a/src/core_processor/utils/image_fps_writer.py+++ b/src/core_processor/utils/image_fps_writer.py@@ -1,8 +1,10 @@+from typing import Union+import cv2import numpy as np-def write_fps_to_image(image: np.array, fps_number: int):+def write_fps_to_image(image: np.array, fps_number: Union[int,float]):cv2.putText(image,f"FPS: {str(fps_number)}",diff --git a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.pyindex 5b4cbd46..e34def08 100644--- a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py+++ b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py@@ -47,9 +47,9 @@ class QTVisualizerAndGui:self._setup_main_window()self._setup_control_panel()self._setup_camera_views_dock()- self._setup_timestamp_plot()- self._setup_time_difference_from_cam0_line_plot()- self._setup_time_difference_from_cam0_histogram_plot()+ # self._setup_timestamp_plot()+ # self._setup_time_difference_from_cam0_line_plot()+ # self._setup_time_difference_from_cam0_histogram_plot()self._setup_3d_viewport()logger.info('launching QT Visualizer and GUI window')self._main_window_widget.show()@@ -68,26 +68,26 @@ class QTVisualizerAndGui:def update_timestamp_plots(self, timestamp_manager: TimestampManager):if self.pause_button_pressed:return- if timestamp_manager.get_timestamps_from_camera(self._camera0_id).number_of_frames <= 0:+ if timestamp_manager.get_number_of_frames(self._camera0_id) <= 0:return- self._update_simple_timestamp_plot(timestamp_manager)- self._update_timestamp_difference_plot(timestamp_manager)- self._update_timestamp_difference_histogram(timestamp_manager)+ # self._update_simple_timestamp_plot(timestamp_manager)+ # self._update_timestamp_difference_plot(timestamp_manager)+ # self._update_timestamp_difference_histogram(timestamp_manager)def _update_simple_timestamp_plot(self, timestamp_manager: TimestampManager):for webcam_id in self._webcam_ids_list:plot_item = self._dict_of_simple_timestamp_line_plots[webcam_id]- plot_item.setData(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps)+ plot_item.setData(timestamp_manager.get_timestamps_from_camera_in_seconds_from_session_start(webcam_id))def _update_timestamp_difference_plot(self, timestamp_manager: TimestampManager):for webcam_id in self._webcam_ids_list:- plot_item = self._dict_of_simple_timestamp_line_plots[webcam_id]- plot_item.setData(np.diff(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps))+ plot_item = self._dict_of_timestamp_difference_line_plots[webcam_id]+ plot_item.setData(np.diff(timestamp_manager.get_timestamps_from_camera_in_seconds_from_session_start(webcam_id)))def _update_timestamp_difference_histogram(self, timestamp_manager: TimestampManager):for webcam_id in self._webcam_ids_list:- this_cam_timestamp_diffs = np.diff(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps)+ this_cam_timestamp_diffs = np.diff(timestamp_manager.get_timestamps_from_camera_in_seconds_from_session_start(webcam_id))num_samples = this_cam_timestamp_diffs.shape[0]counts, bin_edges = np.histogram(this_cam_timestamp_diffs,bins=np.linspace(0, 100, 100))@@ -100,7 +100,7 @@ class QTVisualizerAndGui:logger.warning(f'Could not find timestamp difference line plot widget for camera {webcam_id}')raise e- def _setup_main_window(self, window_width: int = 1000, window_height: int = 1000):+ def _setup_main_window(self, window_width: int = 1280, window_height: int = 720):"""This is the main main window for the GUI.Its structure is based loosely on the 'Dock Widgets' example from `python -m pyqtgraph.examples`@@ -270,4 +270,4 @@ class QTVisualizerAndGui:self.opengl_3d_plot_dock = Dock("3d View Port")self.opengl_3d_plot_dock.addWidget(self.opengl_3d_plot_widget)- self._main_dock_area.addDock(self.opengl_3d_plot_dock, 'right', self._timestamp_diff_histgoram_dock )+ self._main_dock_area.addDock(self.opengl_3d_plot_dock, 'right', self._camera_views_dock )diff --git a/src/session_pipeline/session_pipeline_orchestrator.py b/src/session_pipeline/session_pipeline_orchestrator.pyindex ccdfdc0e..05b1de9d 100644--- a/src/session_pipeline/session_pipeline_orchestrator.py+++ b/src/session_pipeline/session_pipeline_orchestrator.py@@ -12,23 +12,35 @@ from src.core_processor.camera_calibration.camera_calibrator import CameraCalibrfrom src.core_processor.fps.timestamp_manager import TimestampManagerfrom src.core_processor.mediapipe_skeleton_detection.mediapipe_skeleton_detection import MediaPipeSkeletonDetectionfrom src.core_processor.show_cam_window import show_cam_window+from src.core_processor.utils.image_fps_writer import write_fps_to_imagefrom src.qt_visualizer_and_gui.qt_visualizer_and_gui import QTVisualizerAndGuilogger = logging.getLogger(__name__)+def get_canonical_time_str():return time.strftime("%m-%d-%Y-%H_%M_%S")class SessionPipelineOrchestrator:- def __init__(self):++ def __init__(self, session_id: str = None):++ if session_id is not None:+ self._session_id = session_id+ else:+ self._session_id = 'session_' + time.strftime("%m-%d-%Y-%H_%M_%S")+self._session_start_time_unix_ns = time.time_ns()- self._session_id = 'session_' + time.strftime("%m-%d-%Y-%H_%M_%S")self._visualizer_gui = QTVisualizerAndGui()self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id)self._camera_calibrator = CameraCalibrator()self._mediapipe_skeleton_detector = MediaPipeSkeletonDetection(model_complexity=2)+ @property+ def session_id(self):+ return self._session_id+@propertydef session_folder_path(self):return freemocap_data_path / self._session_id@@ -65,7 +77,7 @@ class SessionPipelineOrchestrator:"charuco_board_detection",f"webcam_{cv_cam.webcam_id_as_str}",),- fps=fps_manager.median_frames_per_second(cv_cam.webcam_id_as_str),+ fps=fps_manager.median_frames_per_second_for_webcam(cv_cam.webcam_id_as_str),frame_width=cv_cam.image_width(),frame_height=cv_cam.image_height(),)@@ -88,11 +100,14 @@ class SessionPipelineOrchestrator:if show_visualizer_gui:self._visualizer_gui.setup_and_launch(self._open_cv_camera_manager.available_webcam_ids)- timestamp_manager = TimestampManager(self._open_cv_camera_manager.available_webcam_ids)+ timestamp_manager = TimestampManager(self._open_cv_camera_manager.available_webcam_ids,+ self._session_start_time_unix_ns)should_continue = Truewhile should_continue:+ timestamp_manager.increment_main_loop_timestamp_logger(time.perf_counter_ns())+for this_webcam_id, this_open_cv_camera in connected_cameras_dict.items():if not this_open_cv_camera.new_frame_ready:@@ -103,11 +118,11 @@ class SessionPipelineOrchestrator:if this_cam_latest_frame is None:continue- image_to_display = this_cam_latest_frame.image- this_cam_this_frame_timestamp = this_cam_latest_frame.timestamp+ image_to_display = this_cam_latest_frame.image.copy()+ this_cam_this_frame_timestamp_ns = this_cam_latest_frame.timestamp- timestamp_manager.increment_frame_processed_for(this_webcam_id,- this_cam_this_frame_timestamp)+ timestamp_manager.increment_frame_processed_for_webcam(this_webcam_id,+ this_cam_this_frame_timestamp_ns)if save_video:this_open_cv_camera.video_recorder.record(this_cam_latest_frame)@@ -126,8 +141,20 @@ class SessionPipelineOrchestrator:)if show_visualizer_gui:+ write_fps_to_image(+ image_to_display,+ timestamp_manager.median_frames_per_second_for_webcam(this_webcam_id),+ )self._visualizer_gui.update_camera_view_image(this_webcam_id, image_to_display)- # self._visualizer_gui.update_timestamp_plots(timestamp_manager)+ self._visualizer_gui.update_timestamp_plots(timestamp_manager)++ # exit loop when user presses ESC key+ exit_key = cv2.waitKey(1)+ if exit_key == 27:+ logger.info("ESC has been pressed.")+ should_continue = False++except:logger.error("Printing traceback")@@ -148,8 +175,8 @@ class SessionPipelineOrchestrator:if __name__ == "__main__":print('start main')- this_session = SessionPipelineOrchestrator()- this_session.run(save_video=False,- show_visualizer_gui=True,- calibrate_cameras=False,- detect_skeleton=True)+ this_session_orchestrator = SessionPipelineOrchestrator()+ this_session_orchestrator.run(save_video=False,+ show_visualizer_gui=True,+ calibrate_cameras=False,+ detect_skeleton=True)