Free Motion Capture for Everyone πβ¨
TypeScript
timestamp loggers halfway working lol
Jonb9810d41554e
diff --git a/src/api/routes/session/session_router.py b/src/api/routes/session/session_router.pyindex cd25f554..857e241f 100644--- a/src/api/routes/session/session_router.py+++ b/src/api/routes/session/session_router.py@@ -1,4 +1,5 @@import logging+import tracebackfrom pathlib import Pathfrom typing import Optional, Union@@ -53,8 +54,10 @@ class SessionIdModel(BaseModel):@session_router.post("/session/create")async def create_session(session_create_model: SessionCreateModel = SessionCreateModel()) -> SessionResponse:session_id = create_session_id()+ if session_create_model.user_session_tag_str is not None:+ session_id = session_id + session_create_model.user_session_tag_str+create_session_folder(session_id)- session_id_model = SessionIdModel(session_id=session_id)return SessionResponse(session_id=session_id,session_path=get_session_folder_path(session_id))@@ -69,20 +72,29 @@ def calibrate_session(session_calibrate_model: SessionCalibrateModel = SessionCasession_id = get_most_recent_session_id()calibration_orchestrator = CalibrationPipelineOrchestrator(session_id)- calibration_orchestrator.record_videos(show_visualizer_gui=False,- save_video_in_frame_loop=False,- show_camera_views_in_windows=True,- )-- # launch_camera_frame_loop(session_id=session_id,- # webcam_configs_dict=session_calibrate_model.webcam_configs_dict,- # show_camera_views_in_windows=True,- # calibration_videos_bool=True,- # detect_charuco_in_image=True,- # )- calibration_orchestrator.run_anipose_camera_calibration(- charuco_square_size=session_calibrate_model.charuco_square_size,- pin_camera_0_to_origin=True)++ # calibration_orchestrator.record_videos(show_visualizer_gui=False,+ # save_video_in_frame_loop=False,+ # show_camera_views_in_windows=True,+ # )++ launch_camera_frame_loop(session_id=session_id,+ webcam_configs_dict=session_calibrate_model.webcam_configs_dict,+ show_camera_views_in_windows=True,+ calibration_videos_bool=True,+ detect_charuco_in_image=True,+ )++ try:+ calibration_orchestrator.run_anipose_camera_calibration(+ charuco_square_size=session_calibrate_model.charuco_square_size,+ pin_camera_0_to_origin=True)+ except:+ logger.error("Printing Traceback")+ traceback.print_exc()++ return calibration_orchestrator+@session_router.post("/session/record")diff --git a/src/cameras/capture/dataclasses/frame_payload.py b/src/cameras/capture/dataclasses/frame_payload.pyindex 041fa12b..ccef9d6a 100644--- a/src/cameras/capture/dataclasses/frame_payload.py+++ b/src/cameras/capture/dataclasses/frame_payload.py@@ -7,7 +7,7 @@ class FramePayload(NamedTuple):success: bool = Falseimage: np.ndarray = Nonetimestamp_in_seconds_from_record_start: float = None- timestamp_unix_seconds: float = None+ timestamp_unix_time_seconds: float = Noneframe_number: int = Nonewebcam_id: str = Nonediff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.pyindex 388c9d7c..17e67e83 100644--- a/src/cameras/capture/opencv_camera/opencv_camera.py+++ b/src/cameras/capture/opencv_camera/opencv_camera.py@@ -180,29 +180,27 @@ class OpenCVCamera:def get_next_frame(self):- # Why `grab()` not `read()`? see ->- # https://stackoverflow.com/questions/57716962/difference-between-video-capture-read-and- # -grab--try:## uncomment above and below these calls to measure the time it takes to grab a frame## timestamp_ns_pre = time.perf_counter_ns()- # timestamp_ns_pre = time.time_ns()- if not self._opencv_video_capture_object.grab():- return FramePayload(False, None, None)++ ### Q - Why are we using `cv2.VideoCapture.grab();cv2.VideoCapture.retrieve();` and not just `cv2.VideoCapture.read()`?+ ### A - see -> https://stackoverflow.com/questions/57716962/difference-between-video-capture-read-and-grab+ self._opencv_video_capture_object.grab()success, image = self._opencv_video_capture_object.retrieve()- # timestamp_ns_post = time.time_ns()+# timestamp_ns_post = time.perf_counter_ns()# it_took_this_many_seconds_to_grab_the_frame = (timestamp_ns_post-timestamp_ns_pre)/1e9except:logger.error(f"Failed to read frame from Camera: {self.webcam_id_as_str}")+ raise Exception++ self._timestamp_logger.log_new_timestamp_perf_counter_ns(time.perf_counter_ns())self._timestamp_logger.log_new_timestamp_perf_counter_ns(time.perf_counter_ns())- latest_timestamp_in_seconds_from_record_start = self._timestamp_logger.latest_timestamp_in_seconds_from_record_startself._new_frame_ready = successif success:@@ -212,8 +210,8 @@ class OpenCVCamera:return FramePayload(success=success,image=image,- timestamp_in_seconds_from_record_start=latest_timestamp_in_seconds_from_record_start,- timestamp_unix_seconds = time.time_ns(),+ timestamp_in_seconds_from_record_start=self._timestamp_logger.latest_timestamp_in_seconds_from_record_start,+ timestamp_unix_time_seconds = time.time(),frame_number=self.latest_frame_number,webcam_id=self.webcam_id_as_str)diff --git a/src/cameras/launch_camera_frame_loop.py b/src/cameras/launch_camera_frame_loop.pyindex e7a95eae..107e56c3 100644--- a/src/cameras/launch_camera_frame_loop.py+++ b/src/cameras/launch_camera_frame_loop.py@@ -11,7 +11,9 @@ from src.config.webcam_config import WebcamConfigimport asynciofrom src.core_processor.show_cam_window import show_cam_window+from src.core_processor.utils.image_fps_writer import write_fps_to_imagefrom src.pipelines.calibration_pipeline.charuco_board_detection.charuco_board_detector import CharucoBoardDetector+from src.qt_visualizer_and_gui.qt_visualizer_and_gui import QTVisualizerAndGuilogger = logging.getLogger(__name__)@@ -22,9 +24,8 @@ def launch_camera_frame_loop(show_camera_views_in_windows: bool = True,calibration_videos_bool: bool = False,detect_charuco_in_image: bool = True,- camera_view_update_function=None,- # update_gui_function=None,- record_frames_bool: bool = True+ record_frames_bool: bool = True,+ show_visualizer_gui:bool = False,):any_frames_recorded = False@@ -32,41 +33,67 @@ def launch_camera_frame_loop(charuco_board_detector = CharucoBoardDetector()- opencv_camera_manager = OpenCVCameraManager(session_id=session_id,++ opencv_camera_manager = OpenCVCameraManager(session_id=session_id,expected_framerate=None)+ with opencv_camera_manager.start_capture_session_all_cams(+ webcam_configs_dict=webcam_configs_dict,+ calibration_videos=calibration_videos_bool,+ ) as connected_cameras_dict:+++ timestamp_manager = opencv_camera_manager.timestamp_manager# <-this is defines the start-time of the timestamp loggers for each camera- with opencv_camera_manager.start_capture_session_all_cams(webcam_configs_dict=webcam_configs_dict,- camera_view_update_function=camera_view_update_function,- calibration_videos=calibration_videos_bool, ) as connected_cameras_dict:try:++ if show_visualizer_gui:+ visualizer_gui = QTVisualizerAndGui()+ visualizer_gui.setup_and_launch(opencv_camera_manager.available_webcam_ids)+should_continue = Truewhile should_continue:- # if update_gui_function is None:- # update_gui_function()+ timestamp_manager.log_new_timestamp_for_main_loop_perf_coutner_ns(time.perf_counter_ns())++ if not opencv_camera_manager.new_multi_frame_ready():+ continue++ this_multi_frame_payload = opencv_camera_manager.latest_multi_frame++ for this_webcam_id, this_open_cv_camera in connected_cameras_dict.items():++ this_cam_latest_frame = this_multi_frame_payload.frames_dict[this_webcam_id]++ if this_cam_latest_frame is None:+ continue- for this_webcam_id, this_opencv_camera in connected_cameras_dict.items():- this_frame_payload = this_opencv_camera.latest_frame+ # save frame to video file+ if record_frames_bool:+ any_frames_recorded = True+ this_open_cv_camera.video_recorder.append_frame_payload_to_list(this_cam_latest_frame)- image_to_display = this_frame_payload.image+ image_to_display = this_cam_latest_frame.image.copy()if detect_charuco_in_image:+ # detect charuco boardthis_charuco_frame_payload = charuco_board_detector.detect_charuco_board(- this_frame_payload)- image_to_display = this_charuco_frame_payload.annotated_image+ this_cam_latest_frame)+ image_to_display = this_charuco_frame_payload.annotated_image.copy()++ image_to_display = write_fps_to_image(+ image_to_display,+ timestamp_manager.median_frames_per_second_for_webcam(this_webcam_id),+ )if show_camera_views_in_windows:should_continue = show_cam_window(- this_webcam_id,- image_to_display,- opencv_camera_manager.timestamp_manager+ this_webcam_id, image_to_display, timestamp_manager)- if record_frames_bool:- any_frames_recorded = True- this_opencv_camera.record_frames(True)- else:- this_opencv_camera.record_frames(False)+ if show_visualizer_gui:++ visualizer_gui.update_camera_view_image(this_webcam_id,+ image_to_display)# exit loop when user presses ESC keyexit_key = cv2.waitKey(1)@@ -79,9 +106,21 @@ def launch_camera_frame_loop(traceback.print_exc()finally:+ if show_visualizer_gui:+ visualizer_gui.close()+ if show_camera_views_in_windows:+ # logger.info(f"Destroy window {this_open_cv_camera.webcam_id_as_str}")+ # cv2.destroyWindow(this_open_cv_camera.webcam_id_as_str)+ cv2.destroyAllWindows()++for this_open_cv_camera in connected_cameras_dict.values():if any_frames_recorded:+ logger.info(f"Saving {this_open_cv_camera.webcam_id_as_str} video to disk")this_open_cv_camera.video_recorder.save_list_of_frames_to_video_file(this_open_cv_camera.video_recorder.frame_list)this_open_cv_camera.video_recorder.close()- this_open_cv_camera.close()++ logger.info("Creating camera timestamp diagnostic plot")+ timestamp_manager.create_diagnostic_plots()+diff --git a/src/cameras/multicam_manager/cv_camera_manager.py b/src/cameras/multicam_manager/cv_camera_manager.pyindex 05ca0dbf..d9d10cc5 100644--- a/src/cameras/multicam_manager/cv_camera_manager.py+++ b/src/cameras/multicam_manager/cv_camera_manager.py@@ -46,6 +46,7 @@ class OpenCVCameraManager:def timestamp_manager(self):if self._timestamp_manager is None:logger.error('timestamp manager has not been created yet')+ raise Exceptionelse:return self._timestamp_manager@@ -132,6 +133,7 @@ class OpenCVCameraManager:timestamp_manager=self._timestamp_manager,camera_view_update_function=camera_view_update_function,calibration_videos=calibration_videos, )+try:for cv_cam in open_cv_camera_objects:@@ -195,13 +197,18 @@ class OpenCVCameraManager:calibration_video_bool=calibration_video_bool)def _initialize_timestamp_logger(self):+self._session_start_time_unix_ns = time.time_ns()self._session_start_time_perf_counter_ns = time.perf_counter_ns()++ logger.info(f"Initializing timestamp loggers - _session_start_time_unix_ns: {self._session_start_time_unix_ns}, _session_start_time_perf_counter_ns: {self._session_start_time_perf_counter_ns} ")+self._timestamp_manager = TimestampManager(self._session_id,self.available_webcam_ids,self._session_start_time_unix_ns,self._session_start_time_perf_counter_ns)+def _start_frame_capture_on_cam_id(self, opencv_cam: OpenCVCamera):opencv_cam.connect()opencv_cam.start_frame_capture_thread()diff --git a/src/core_processor/timestamp_manager/timestamp_manager.py b/src/core_processor/timestamp_manager/timestamp_manager.pyindex 2c9bd7b6..633031bc 100644--- a/src/core_processor/timestamp_manager/timestamp_manager.py+++ b/src/core_processor/timestamp_manager/timestamp_manager.py@@ -15,53 +15,43 @@ logger = logging.getLogger(__name__)class TimestampLogger:- _num_frames_processed: int- _start_time: float-- def __init__(self, session_start_time_perf_counter_ns: int=None):- if session_start_time_perf_counter_ns is None:- self._session_start_time_perf_counter_ns = time.perf_counter_ns()- else:- self._session_start_time_perf_counter_ns = session_start_time_perf_counter_ns+ def __init__(self,+ logger_name:str,+ session_start_time_perf_counter_ns:int,+ ):+ self._logger_name = logger_name+ self._session_start_time_perf_counter_ns = session_start_time_perf_counter_nsself._num_frames_processed = 0- self._timestamps_ns = np.empty(0)-- def is_started(self):- return self._num_frames_processed > 0+ self._timestamps_perf_counter_ns = np.empty(0)+@propertydef median_frames_per_second(self):if self._num_frames_processed <= 10:return 0- return np.nanmedian(np.diff(self._timestamps_ns) / 1e9) ** -1+ return np.nanmedian(np.diff(self._timestamps_perf_counter_ns) / 1e9) ** -1@propertydef number_of_frames(self):return self._num_frames_processed- @property- def timestamps_from_record_start_ns(self):- """return timestamps in nanoseconds in Unix epoch format (which is both the most useful, ugliest, AND funniest time format around. You can use `self.timestamps_in_seconds_from_zero()` for a more intuitive time format"""- return self._timestamps_ns - self._session_start_time_perf_counter_ns-- @property- def timestamps(self):- return self.timestamps_in_seconds_from_session_start-@propertydef timestamps_in_seconds_from_session_start(self):"""timestamps in seconds counting up from zero (where zero is the start time of the recording session)"""- return (self._timestamps_ns - self._session_start_time_perf_counter_ns) / 1e9+ return (self._timestamps_perf_counter_ns - self._session_start_time_perf_counter_ns) / 1e9@propertydef latest_timestamp_in_seconds_from_record_start(self):"""returns timestamps in seconds from session start"""- if self.timestamps.shape[0] > 0:- return self.timestamps[-1]- return None+ try:+ if self.timestamps_in_seconds_from_session_start.shape[0] > 0:+ return self.timestamps_in_seconds_from_session_start[-1]+ except:+ logger.error(f"Failed to return latest timestamp")+def log_new_timestamp_perf_counter_ns(self, timestamp:int):- self._timestamps_ns = np.append(self._timestamps_ns, timestamp)+ self._timestamps_perf_counter_ns = np.append(self._timestamps_perf_counter_ns, timestamp)self._num_frames_processed += 1@@ -69,24 +59,23 @@ class TimestampManager:def __init__(self,session_id:str,webcam_ids: List[str],- session_start_time_unix_ns: int=None,- _session_start_time_perf_counter_ns:int=None):- self._session_id = session_id+ session_start_time_unix_ns: int,+ _session_start_time_perf_counter_ns:int):- if session_start_time_unix_ns is None:- self._session_start_time_unix_ns = time.time_ns() # UNIX epoch time, nanoseconds- else:- self._session_start_time_unix_ns = session_start_time_unix_ns+ self._session_id = session_id+ self._session_start_time_unix_ns = session_start_time_unix_ns+ self._session_start_time_perf_counter_ns = _session_start_time_perf_counter_ns- if _session_start_time_perf_counter_ns is None:- self._session_start_time_perf_counter_ns = time.perf_counter_ns # arbitrary timebase, nanoseconds - corresponds to self._session_start_time_unix_ns- else:- self._session_start_time_perf_counter_ns = _session_start_time_perf_counter_nsself._webcam_ids = webcam_ids- self._webcam_timestamp_loggers: Dict[str, TimestampLogger] = self._initialize_webcam_timestamp_loggers(- webcam_ids)- self._main_loop_timestamp_logger = TimestampLogger(session_start_time_perf_counter_ns=self._session_start_time_perf_counter_ns)- self._multi_frame_timestamp_logger = TimestampLogger(session_start_time_perf_counter_ns=self._session_start_time_perf_counter_ns)+ self._webcam_timestamp_loggers: Dict[str, TimestampLogger] = self._initialize_webcam_timestamp_loggers(webcam_ids)+ self._main_loop_timestamp_logger = TimestampLogger(logger_name = 'main_loop',+ session_start_time_perf_counter_ns=self._session_start_time_perf_counter_ns)++ self._multi_frame_timestamp_logger = TimestampLogger(logger_name = 'multi_frame',+ session_start_time_perf_counter_ns=self._session_start_time_perf_counter_ns)+++self._multi_frame_timestamps_list = []self._multi_frame_interval_list = []@@ -124,7 +113,8 @@ class TimestampManager:def _initialize_webcam_timestamp_loggers(self, webcam_ids: List[str]):dictionary_of_timestamp_logger_objects = {}for webcam_id in webcam_ids:- dictionary_of_timestamp_logger_objects[webcam_id] = TimestampLogger(session_start_time_perf_counter_ns=self._session_start_time_perf_counter_ns)+ dictionary_of_timestamp_logger_objects[webcam_id] = TimestampLogger(logger_name = 'camera_'+webcam_id,+ session_start_time_perf_counter_ns=self._session_start_time_perf_counter_ns)return dictionary_of_timestamp_logger_objectsdef get_timestamps_from_camera_in_seconds_from_session_start(self, webcam_id):@@ -139,18 +129,20 @@ class TimestampManager:def median_frames_per_second_for_webcam(self, webcam_id):return self._webcam_timestamp_loggers[webcam_id].median_frames_per_second- def log_new_timestamp_for_main_loop_ns(self, timestamp):+ def log_new_timestamp_for_main_loop_perf_coutner_ns(self, timestamp):self._main_loop_timestamp_logger.log_new_timestamp_perf_counter_ns(timestamp)def log_new_multi_frame_timestamp_ns(self, timestamp):self._multi_frame_timestamp_logger.log_new_timestamp_perf_counter_ns(timestamp)- def log_new_timestamp_for_webcam_ns(self, webcam_id: str, timestamp: Union[float, int],+ def log_new_timestamp_for_webcam_ns(self,+ webcam_id: str,+ timestamp: Union[float, int],frame_number: int = None):self._webcam_timestamp_loggers[webcam_id].log_new_timestamp_perf_counter_ns(timestamp)if frame_number is not None:- assert (self._webcam_timestamp_loggers[webcam_id].number_of_frames == frame_number,- f"Numer of timestamps ({self._webcam_timestamp_loggers[webcam_id].number_of_frames}) logged does not match frame_number ({frame_number}) for: camera_{webcam_id}")+ if not self._webcam_timestamp_loggers[webcam_id].number_of_frames == frame_number:+ logger.error("Numer of timestamps ({self._webcam_timestamp_loggers[webcam_id].number_of_frames}) logged does not match frame_number ({frame_number}) for: camera_{webcam_id}")def _get_each_frame_timestamp(self, this_multi_frame_dict) -> List:@@ -177,7 +169,7 @@ class TimestampManager:if expected_framerate is None:# logger.warning('`expected_framerate` not specified, Cannot verify multi_frame synchronization')- print(f"this_multiframe_timestamp_interval: {this_multiframe_timestamp_interval_in_seconds*1e3:.1f} milliseconds")+ print(f"this_multiframe_timestamp_interval: {this_multiframe_timestamp_interval_in_seconds*1e3:.6f} milliseconds")return Trueif this_multiframe_timestamp_interval_in_seconds > 2*(expected_framerate ** -1):diff --git a/src/core_processor/utils/image_fps_writer.py b/src/core_processor/utils/image_fps_writer.pyindex d17fed39..8e926a44 100644--- a/src/core_processor/utils/image_fps_writer.py+++ b/src/core_processor/utils/image_fps_writer.py@@ -4,13 +4,16 @@ import cv2import numpy as np-def write_fps_to_image(image: np.array, fps_number: Union[int,float]):+def write_fps_to_image(image: np.array, fps_number: Union[int,float])->np.ndarray:+ annotated_image = image.copy()cv2.putText(- image,+ annotated_image,f"FPS: {str(fps_number)}",(10, 15),cv2.FONT_HERSHEY_SIMPLEX,- 5,+ 1,(209, 180, 0, 255),1,)++ return annotated_imagediff --git a/src/pipelines/calibration_pipeline/anipose_camera_calibration/freemocap_anipose.py b/src/pipelines/calibration_pipeline/anipose_camera_calibration/freemocap_anipose.pyindex ef83da31..fc252730 100644--- a/src/pipelines/calibration_pipeline/anipose_camera_calibration/freemocap_anipose.py+++ b/src/pipelines/calibration_pipeline/anipose_camera_calibration/freemocap_anipose.py@@ -1554,6 +1554,7 @@ class CameraGroup:assert size is not None, \"Camera with name {} has no specified frame size".format(camera.get_name())+if init_intrinsics:objp, imgp = board.get_all_calibration_points(rows)mixed = [(o, i) for (o, i) in zip(objp, imgp) if len(o) >= 7]diff --git a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.pyindex ebf3b9a2..ef00ff2a 100644--- a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py+++ b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py@@ -69,7 +69,7 @@ class CalibrationPipelineOrchestrator:should_continue = Truewhile should_continue:- timestamp_manager.log_new_timestamp_for_main_loop_ns(time.perf_counter_ns())+ timestamp_manager.log_new_timestamp_for_main_loop_perf_coutner_ns(time.perf_counter_ns())if not self._open_cv_camera_manager.new_multi_frame_ready():continue@@ -83,12 +83,6 @@ class CalibrationPipelineOrchestrator:if this_cam_latest_frame is None:continue- # log timestamp- this_cam_this_frame_timestamp_ns = this_cam_latest_frame.timestamp_in_seconds_from_record_start- this_cam_this_frame_number = this_cam_latest_frame.frame_number- timestamp_manager.log_new_timestamp_for_webcam_ns(this_webcam_id,- this_cam_this_frame_timestamp_ns,- this_cam_this_frame_number)# save frame to video fileif save_video_in_frame_loop:diff --git a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py b/src/pipelines/session_pipeline/session_pipeline_orchestrator.pyindex cd5a89c4..90b3372b 100644--- a/src/pipelines/session_pipeline/session_pipeline_orchestrator.py+++ b/src/pipelines/session_pipeline/session_pipeline_orchestrator.py@@ -190,7 +190,7 @@ class SessionPipelineOrchestrator:self._visualizer_gui.update_mediapipe3d_skeleton(mediapipe3d_multi_frame_payload)dictionary_of_mediapipe_payloads_on_this_multi_frame = {}- timestamp_manager.log_new_timestamp_for_main_loop_ns(time.perf_counter_ns())+ timestamp_manager.log_new_timestamp_for_main_loop_perf_coutner_ns(time.perf_counter_ns())if not self._open_cv_camera_manager.new_multi_frame_ready():continue@@ -209,12 +209,7 @@ class SessionPipelineOrchestrator:image_to_display = this_cam_latest_frame.image.copy()- # log timestamp- this_cam_this_frame_timestamp_ns = this_cam_latest_frame.timestamp_in_seconds_from_record_start- this_cam_this_frame_number = this_cam_latest_frame.frame_number- timestamp_manager.log_new_timestamp_for_webcam_ns(this_webcam_id,- this_cam_this_frame_timestamp_ns,- this_cam_this_frame_number)+if save_video:create_session_folder(self.session_id)