Free Motion Capture for Everyone πβ¨
TypeScript
ok i think i saved christmas π
Joncdbaf8973603
diff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.pyindex 17e67e83..5656a825 100644--- a/src/cameras/capture/opencv_camera/opencv_camera.py+++ b/src/cameras/capture/opencv_camera/opencv_camera.py@@ -24,17 +24,11 @@ class OpenCVCamera:def __init__(self,config: WebcamConfig,- session_id: str = None,- timestamp_logger: TimestampLogger = None,- camera_view_update_function = None,+ session_id: str,+ session_start_time_perf_counter_ns:int,calibration_video_bool: bool = False):- if timestamp_logger is None:- self._timestamp_logger = TimestampLogger()- else:- self._timestamp_logger = timestamp_logger- self._camera_view_update_function = camera_view_update_functionself._config = configself._name = f"Camera_{self._config.webcam_id}"@@ -45,6 +39,7 @@ class OpenCVCamera:self._number_of_frames_recorded = 0self._calibration_video_bool = calibration_video_boolself._session_id = session_id+ self._session_start_time_perf_counter_ns = session_start_time_perf_counter_ns@propertydef session_id(self):@@ -190,6 +185,7 @@ class OpenCVCamera:### A - see -> https://stackoverflow.com/questions/57716962/difference-between-video-capture-read-and-grabself._opencv_video_capture_object.grab()success, image = self._opencv_video_capture_object.retrieve()+ this_frame_timestamp_perf_counter_ns = time.perf_counter_ns()-self._session_start_time_perf_counter_ns# timestamp_ns_post = time.perf_counter_ns()# it_took_this_many_seconds_to_grab_the_frame = (timestamp_ns_post-timestamp_ns_pre)/1e9@@ -197,9 +193,6 @@ class OpenCVCamera:logger.error(f"Failed to read frame from Camera: {self.webcam_id_as_str}")raise Exception- self._timestamp_logger.log_new_timestamp_perf_counter_ns(time.perf_counter_ns())-- self._timestamp_logger.log_new_timestamp_perf_counter_ns(time.perf_counter_ns())self._new_frame_ready = success@@ -210,7 +203,7 @@ class OpenCVCamera:return FramePayload(success=success,image=image,- timestamp_in_seconds_from_record_start=self._timestamp_logger.latest_timestamp_in_seconds_from_record_start,+ timestamp_in_seconds_from_record_start=this_frame_timestamp_perf_counter_ns/1e9,timestamp_unix_time_seconds = time.time(),frame_number=self.latest_frame_number,webcam_id=self.webcam_id_as_str)diff --git a/src/cameras/launch_camera_frame_loop.py b/src/cameras/launch_camera_frame_loop.pyindex 107e56c3..9d4cd4c0 100644--- a/src/cameras/launch_camera_frame_loop.py+++ b/src/cameras/launch_camera_frame_loop.py@@ -34,8 +34,8 @@ def launch_camera_frame_loop(- opencv_camera_manager = OpenCVCameraManager(session_id=session_id,- expected_framerate=None)+ opencv_camera_manager = OpenCVCameraManager(session_id=session_id)+with opencv_camera_manager.start_capture_session_all_cams(webcam_configs_dict=webcam_configs_dict,calibration_videos=calibration_videos_bool,@@ -53,12 +53,12 @@ def launch_camera_frame_loop(should_continue = Truewhile should_continue:- timestamp_manager.log_new_timestamp_for_main_loop_perf_coutner_ns(time.perf_counter_ns())if not opencv_camera_manager.new_multi_frame_ready():continuethis_multi_frame_payload = opencv_camera_manager.latest_multi_frame+ timestamp_manager.multi_frame_timestamp_logger.log_new_timestamp_seconds_from_unspecified_zero(this_multi_frame_payload.multi_frame_timestamp_seconds)for this_webcam_id, this_open_cv_camera in connected_cameras_dict.items():@@ -67,6 +67,9 @@ def launch_camera_frame_loop(if this_cam_latest_frame is None:continue+ this_cam_timestamp_logger = timestamp_manager.timestamp_logger_for_webcam_id(this_webcam_id)+ this_cam_timestamp_logger.log_new_timestamp_seconds_from_unspecified_zero(this_cam_latest_frame.timestamp_in_seconds_from_record_start)+# save frame to video fileif record_frames_bool:any_frames_recorded = True@@ -82,16 +85,13 @@ def launch_camera_frame_loop(image_to_display = write_fps_to_image(image_to_display,- timestamp_manager.median_frames_per_second_for_webcam(this_webcam_id),+ timestamp_manager.timestamp_logger_for_webcam_id(this_webcam_id).median_frames_per_second,)if show_camera_views_in_windows:- should_continue = show_cam_window(- this_webcam_id, image_to_display, timestamp_manager- )+ should_continue = show_cam_window(this_webcam_id, image_to_display )if show_visualizer_gui:-visualizer_gui.update_camera_view_image(this_webcam_id,image_to_display)diff --git a/src/cameras/multicam_manager/cv_camera_manager.py b/src/cameras/multicam_manager/cv_camera_manager.pyindex d9d10cc5..9c43ac6f 100644--- a/src/cameras/multicam_manager/cv_camera_manager.py+++ b/src/cameras/multicam_manager/cv_camera_manager.py@@ -29,7 +29,6 @@ class CamAndWriterResponse(BaseModel):class OpenCVCameraManager:def __init__(self,session_id: str = None,- expected_framerate: Union[int, float] = None,shut_down_event_bool:bool = None,):self._session_id = session_id@@ -39,8 +38,9 @@ class OpenCVCameraManager:self._timestamp_manager: Union[TimestampManager, None] = Noneself._connected_cameras_dict = {}self._available_cameras_dict = {}- self._expected_framerate = expected_framerateself._shut_down_event_bool = shut_down_event_bool+ self._number_of_multi_frames = 0+@propertydef timestamp_manager(self):@@ -60,27 +60,24 @@ class OpenCVCameraManager:logging.error("a multi_frame was requested before it was ready!")raise Exception+ this_multi_frame_timestamp_sec = (time.perf_counter_ns() - self._session_start_time_perf_counter_ns)/1e9+this_multi_frame_dict = {}+ each_cam_timestamp = []+for this_cam in self._connected_cameras_dict.values():if not this_cam.new_frame_ready:logger.error('It shouldnt be able to get here if the `new_multi_frame`')+this_multi_frame_dict[this_cam.webcam_id_as_str] = this_cam.latest_frame- self._timestamp_manager.log_new_multi_frame_timestamp_ns(time.perf_counter_ns())+ each_cam_timestamp.append(this_cam.latest_frame.timestamp_in_seconds_from_record_start)- if not self._timestamp_manager.verify_multi_frame_is_synchronized(this_multi_frame_dict,- self._expected_framerate):- m_f_interval = self._timestamp_manager.latest_multi_frame_interval- m_f_number = self._timestamp_manager.multi_frame_timestamp_logger.number_of_frames- logger.error(- f"Multi frame is not synchronized!! Skipping this one - multi_frame_frame_number|interval: {m_f_number}|{m_f_interval:.3f}")- # raise Exception- return None+ self._number_of_multi_frames +=1return MultiFramePayload(frames_dict=this_multi_frame_dict,- multi_frame_number=self._timestamp_manager.multi_frame_timestamp_logger.number_of_frames,- intra_frame_interval=self._timestamp_manager.latest_multi_frame_interval,- each_frame_timestamp=self._timestamp_manager.latest_multi_frame_timestamp_list,- multi_frame_timestamp=self._timestamp_manager.multi_frame_timestamp_logger.latest_timestamp_in_seconds_from_record_start,+ multi_frame_number=self._number_of_multi_frames,+ each_frame_timestamp=each_cam_timestamp,+ multi_frame_timestamp_seconds=this_multi_frame_timestamp_sec)def new_multi_frame_ready(self):@@ -129,13 +126,11 @@ class OpenCVCameraManager:self._initialize_timestamp_logger()open_cv_camera_objects = self._create_opencv_cameras(+ session_start_time_perf_counter_ns=self._session_start_time_perf_counter_ns,webcam_configs_dict=webcam_configs_dict,- timestamp_manager=self._timestamp_manager,- camera_view_update_function=camera_view_update_function,calibration_videos=calibration_videos, )try:-for cv_cam in open_cv_camera_objects:self._connected_cameras_dict[cv_cam.webcam_id_as_str] = cv_camself._start_frame_capture_on_cam_id(cv_cam)@@ -148,28 +143,26 @@ class OpenCVCameraManager:traceback.print_exc()def _create_opencv_cameras(self,- timestamp_manager: TimestampManager,+ session_start_time_perf_counter_ns: int,webcam_configs_dict: Dict[str, WebcamConfig] = None,- camera_view_update_function=None,- calibration_videos: bool = False,+ calibration_videos: bool = False,):raw_camera_objects = self._detected_cams_data.cameras_found_listopen_cv_cameras: List[OpenCVCamera] = []- if webcam_configs_dict is None:- webcam_configs_dict = {}+for this_raw_cam in raw_camera_objects:- this_cam_timestamp_logger = timestamp_manager.timestamp_logger_for_webcam_id(- webcam_id=this_raw_cam.webcam_id)- webcam_configs_dict[this_raw_cam.webcam_id] = WebcamConfig(webcam_id=this_raw_cam.webcam_id)+ if webcam_configs_dict is None:+ this_webcam_config = WebcamConfig(webcam_id=this_raw_cam.webcam_id)+ else:+ this_webcam_config = webcam_configs_dict[this_raw_cam.webcam_id]opencv_cam_obj = self._create_single_opencv_cam(webcam_id=this_raw_cam.webcam_id,- webcam_config=webcam_configs_dict[this_raw_cam.webcam_id],- timestamp_logger=this_cam_timestamp_logger,- camera_view_update_function=camera_view_update_function,+ webcam_config=this_webcam_config,+ session_start_time_perf_counter_ns = session_start_time_perf_counter_ns,calibration_video_bool=calibration_videos)open_cv_cameras.append(opencv_cam_obj)return open_cv_cameras@@ -177,8 +170,7 @@ class OpenCVCameraManager:def _create_single_opencv_cam(self,webcam_id: str,webcam_config: WebcamConfig(),- timestamp_logger: TimestampLogger,- camera_view_update_function=None,+ session_start_time_perf_counter_ns:int,calibration_video_bool: bool = False,):#@@ -192,8 +184,7 @@ class OpenCVCameraManager:webcam_config.webcam_id = webcam_idreturn OpenCVCamera(config=webcam_config,session_id=self._session_id,- timestamp_logger=timestamp_logger,- camera_view_update_function=camera_view_update_function,+ session_start_time_perf_counter_ns = session_start_time_perf_counter_ns,calibration_video_bool=calibration_video_bool)def _initialize_timestamp_logger(self):diff --git a/src/core_processor/show_cam_window.py b/src/core_processor/show_cam_window.pyindex c931721d..8a98c2ff 100644--- a/src/core_processor/show_cam_window.py+++ b/src/core_processor/show_cam_window.py@@ -9,6 +9,6 @@ from src.core_processor.utils.image_fps_writer import write_fps_to_imagelogger = logging.getLogger(__name__)-def show_cam_window(webcam_id: str, image: np.array, timestamp_manager_cls: TimestampManager):+def show_cam_window(webcam_id: str, image: np.array,):cv2.imshow(webcam_id, image)return Truediff --git a/src/core_processor/timestamp_manager/timestamp_manager.py b/src/core_processor/timestamp_manager/timestamp_manager.pyindex 633031bc..76760242 100644--- a/src/core_processor/timestamp_manager/timestamp_manager.py+++ b/src/core_processor/timestamp_manager/timestamp_manager.py@@ -1,7 +1,6 @@import logging-import timefrom pathlib import Path-from typing import Dict, List, Union+from typing import Dict, Listimport numpy as npfrom matplotlib import pyplot as plt@@ -23,6 +22,7 @@ class TimestampLogger:self._session_start_time_perf_counter_ns = session_start_time_perf_counter_nsself._num_frames_processed = 0self._timestamps_perf_counter_ns = np.empty(0)+ self._timestamps_from_zero = np.empty(0)@property@@ -54,6 +54,13 @@ class TimestampLogger:self._timestamps_perf_counter_ns = np.append(self._timestamps_perf_counter_ns, timestamp)self._num_frames_processed += 1+ def log_new_timestamp_seconds_from_unspecified_zero(self, timestamp:float):+ self._timestamps_from_zero = np.append(self._timestamps_from_zero, timestamp)+ self._num_frames_processed += 1++ @property+ def timestamps_in_seconds_from_unspecified_zero(self):+ return self._timestamps_from_zeroclass TimestampManager:def __init__(self,@@ -76,39 +83,16 @@ class TimestampManager:- self._multi_frame_timestamps_list = []- self._multi_frame_interval_list = []-- @property- def webcam_timestamp_loggers(self):- return self._webcam_timestamp_loggers-def timestamp_logger_for_webcam_id(self, webcam_id:str)->TimestampLogger:return self._webcam_timestamp_loggers[webcam_id]- @property- def number_of_cameras(self):- return len(self._webcam_timestamp_loggers)@propertydef multi_frame_timestamp_logger(self):return self._multi_frame_timestamp_logger- @property- def multi_frame_timestamps_list(self):- return self._multi_frame_timestamps_list-- @property- def multi_frame_interval_list(self):- return self._multi_frame_interval_list- @property- def latest_multi_frame_timestamp_list(self):- return self._multi_frame_timestamps_list[-1]- @property- def latest_multi_frame_interval(self):- return self._multi_frame_interval_list[-1]def _initialize_webcam_timestamp_loggers(self, webcam_ids: List[str]):dictionary_of_timestamp_logger_objects = {}@@ -117,71 +101,36 @@ class TimestampManager:session_start_time_perf_counter_ns=self._session_start_time_perf_counter_ns)return dictionary_of_timestamp_logger_objects- def get_timestamps_from_camera_in_seconds_from_session_start(self, webcam_id):- return self._webcam_timestamp_loggers[webcam_id].timestamps_in_seconds_from_session_start-- def number_of_frames(self, webcam_id):- return self._webcam_timestamp_loggers[webcam_id].number_of_frames- def median_frames_per_second_for_main_loop(self):- return self._main_loop_timestamp_logger.median_frames_per_second-- def median_frames_per_second_for_webcam(self, webcam_id):- return self._webcam_timestamp_loggers[webcam_id].median_frames_per_second-- def log_new_timestamp_for_main_loop_perf_coutner_ns(self, timestamp):- self._main_loop_timestamp_logger.log_new_timestamp_perf_counter_ns(timestamp)-- def log_new_multi_frame_timestamp_ns(self, timestamp):- self._multi_frame_timestamp_logger.log_new_timestamp_perf_counter_ns(timestamp)-- def log_new_timestamp_for_webcam_ns(self,- webcam_id: str,- timestamp: Union[float, int],- frame_number: int = None):- self._webcam_timestamp_loggers[webcam_id].log_new_timestamp_perf_counter_ns(timestamp)- if frame_number is not None:- if not self._webcam_timestamp_loggers[webcam_id].number_of_frames == frame_number:- logger.error("Numer of timestamps ({self._webcam_timestamp_loggers[webcam_id].number_of_frames}) logged does not match frame_number ({frame_number}) for: camera_{webcam_id}")def _get_each_frame_timestamp(self, this_multi_frame_dict) -> List:-this_multi_frame_timestamps = []for this_cam_id, this_cam_frame in this_multi_frame_dict.items():this_multi_frame_timestamps.append(this_cam_frame.timestamp_in_seconds_from_record_start)return this_multi_frame_timestamps-- def _get_intra_frame_interval_in_seconds(self, this_multi_frame_timestamps) -> float:- timestamp_min = np.min(this_multi_frame_timestamps)- timestamp_max = np.max(this_multi_frame_timestamps)- return (timestamp_max - timestamp_min) / 1e9+ def _get_intra_frame_interval_in_seconds(self, this_multi_frame_timestamps_in_seconds) -> float:+ timestamp_min = np.min(this_multi_frame_timestamps_in_seconds)+ timestamp_max = np.max(this_multi_frame_timestamps_in_seconds)+ return (timestamp_max - timestamp_min)def verify_multi_frame_is_synchronized(self,- this_multi_frame_dict,- expected_framerate: Union[int, None]) -> bool:+ this_multi_frame_dict,) -> bool:this_multi_frame_timestamps = self._get_each_frame_timestamp(this_multi_frame_dict)this_multiframe_timestamp_interval_in_seconds = self._get_intra_frame_interval_in_seconds(this_multi_frame_timestamps)- self._multi_frame_timestamps_list.append(this_multi_frame_timestamps)- self._multi_frame_interval_list.append(this_multiframe_timestamp_interval_in_seconds)- if expected_framerate is None:- # logger.warning('`expected_framerate` not specified, Cannot verify multi_frame synchronization')- print(f"this_multiframe_timestamp_interval: {this_multiframe_timestamp_interval_in_seconds*1e3:.6f} milliseconds")- return True- if this_multiframe_timestamp_interval_in_seconds > 2*(expected_framerate ** -1):- return False+ print(f"this_multiframe_timestamp_interval: {this_multiframe_timestamp_interval_in_seconds*1e3:.6f} milliseconds")return Truedef create_diagnostic_plots(self):"""plot some diagnostics to assess quality of camera sync"""- multi_frame_timestamp_list = self.multi_frame_timestamp_logger.timestamps_in_seconds_from_session_start- multi_frame_interval_list = self.multi_frame_interval_list+ multi_frame_timestamp_list = self.multi_frame_timestamp_logger.timestamps_in_seconds_from_unspecified_zero+ # multi_frame_interval_list = self.multi_frame_interval_listfig = plt.figure(figsize=(18, 10))max_frame_duration = 0.1@@ -226,8 +175,8 @@ class TimestampManager:ylabel="Probability",)- for this_camera_id, this_camera_timestamp_logger in self.webcam_timestamp_loggers.items():- this_camera_timestamps = this_camera_timestamp_logger.timestamps_in_seconds_from_session_start+ for this_camera_id, this_camera_timestamp_logger in self._webcam_timestamp_loggers.items():+ this_camera_timestamps = this_camera_timestamp_logger._timestamps_from_zeroax1.plot(this_camera_timestamps, label=f"Camera# {this_camera_id}")ax1.legend()ax2.plot(np.diff(this_camera_timestamps), ".")@@ -243,31 +192,31 @@ class TimestampManager:label=f"MultiFrame")ax5.plot(- np.diff(multi_frame_timestamp_list),+ np.diff(np.asarray(multi_frame_timestamp_list)),".",color="darkslategrey",label="Multi Frame Duration",)- ax5.plot(- multi_frame_interval_list, ".", color="orangered", label="Frame TimeSpan"- )+ # ax5.plot(+ # multi_frame_interval_list, ".", color="orangered", label="Frame TimeSpan"+ # )ax5.legend()ax6.hist(- np.diff(multi_frame_timestamp_list),+ np.diff(np.asarray(multi_frame_timestamp_list)),bins=np.arange(0, max_frame_duration, 0.0025),density=True,alpha=0.5,color="darkslategrey",label="Frame Duration",)- ax6.hist(- np.diff(multi_frame_interval_list),- bins=np.arange(0, max_frame_duration, 0.0025),- density=True,- alpha=0.5,- color="orangered",- label="Frame Timespan",- )+ # ax6.hist(+ # np.diff(multi_frame_interval_list),+ # bins=np.arange(0, max_frame_duration, 0.0025),+ # density=True,+ # alpha=0.5,+ # color="orangered",+ # label="Frame Timespan",+ # )ax5.legend()fig_save_path = Path(get_session_folder_path(self._session_id)) / "camera_timestamp_diagnostics.png"diff --git a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.pyindex ef00ff2a..ea34efa3 100644--- a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py+++ b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py@@ -37,8 +37,7 @@ class CalibrationPipelineOrchestrator:self._charuco_board_detector = CharucoBoardDetector()self._visualizer_gui = Noneself._expected_framerate = expected_framerate- self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id,- expected_framerate=self._expected_framerate)+ self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id)@propertydef session_id(self):diff --git a/src/pipelines/session_pipeline/data_classes/multi_frame_payload.py b/src/pipelines/session_pipeline/data_classes/multi_frame_payload.pyindex ac741eae..f6ca2d71 100644--- a/src/pipelines/session_pipeline/data_classes/multi_frame_payload.py+++ b/src/pipelines/session_pipeline/data_classes/multi_frame_payload.py@@ -8,6 +8,5 @@ from src.cameras.capture.dataclasses.frame_payload import FramePayloadclass MultiFramePayload(NamedTuple):frames_dict: Dict[str, FramePayload]multi_frame_number: int- intra_frame_interval: Union[int, float]each_frame_timestamp: List[Union[int, float]]- multi_frame_timestamp: Union[int, float]+ multi_frame_timestamp_seconds: Union[int, float]