Free Motion Capture for Everyone πβ¨
TypeScript
fiddling with timestamps
Jonec7942e9bccb
diff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.pyindex 6891120c..93a4160a 100644--- a/src/cameras/capture/opencv_camera/opencv_camera.py+++ b/src/cameras/capture/opencv_camera/opencv_camera.py@@ -23,14 +23,14 @@ class OpenCVCamera:self._opencv_video_capture_object: cv2.VideoCapture = Noneself._video_recorder: VideoRecorder = Noneself._running_thread: VideoCaptureThread = None- self._is_there_a_new_frame = False+ self._new_frame_ready = False@propertydef new_frame_ready(self):"""can be called to determine if the frame returned from `self.latest_frame` is new or if it has been called before"""- return self._is_there_a_new_frame+ return self._new_frame_ready@propertydef video_recorder(self):@@ -61,7 +61,7 @@ class OpenCVCamera:@propertydef latest_frame(self):- self._is_there_a_new_frame = False+ self._new_frame_ready = Falsereturn self._running_thread.latest_framedef connect(self):@@ -157,10 +157,10 @@ class OpenCVCamera:if not self._opencv_video_capture_object.grab():return FramePayload(False, None, None)- timestamp_ns = time.perf_counter_ns()+ timestamp_ns = time.time_ns()success, image = self._opencv_video_capture_object.retrieve()- self._is_there_a_new_frame = success+ self._new_frame_ready = successreturn FramePayload(success, image, timestamp_ns)def stop_frame_capture(self):diff --git a/src/core_processor/fps/fps_counter.py b/src/core_processor/fps/timestamp_manager.pysimilarity index 65%rename from src/core_processor/fps/fps_counter.pyrename to src/core_processor/fps/timestamp_manager.pyindex 543d5c70..4b1e6741 100644--- a/src/core_processor/fps/fps_counter.py+++ b/src/core_processor/fps/timestamp_manager.py@@ -1,15 +1,13 @@-import timefrom typing import Dict, Listimport numpy as np-class FPSCounter:+class TimestampLogger:_num_frames_processed: int_start_time: floatdef __init__(self):- self._start_time = 0self._num_frames_processed = 0self._timestamps = np.empty(0)@@ -22,25 +20,39 @@ class FPSCounter:return 0return np.nanmedian(np.diff(self._timestamps)/1e9)**-1- def elapsed(self):- return time.time() - self._start_time+ @property+ def number_of_frames(self):+ return self._num_frames_processed++ @property+ def timestamps(self):+ return self._timestamps++ @property+ def last_timestamp(self):+ if self._timestamps.shape[0] > 0:+ return self._timestamps[-1]+ return Nonedef increment_frame_processed(self, timestamp):self._timestamps = np.append(self._timestamps, timestamp)self._num_frames_processed += 1-class FPSCamCounter:+class TimestampManager:def __init__(self, webcam_ids: List[str]):self._webcam_ids = webcam_ids- self._counters: Dict[str, FPSCounter] = self._init_counters(webcam_ids)+ self._counters: Dict[str, TimestampLogger] = self._init_counters(webcam_ids)def _init_counters(self, webcam_ids: List[str]):d = {}for webcam_id in webcam_ids:- d[webcam_id] = FPSCounter()+ d[webcam_id] = TimestampLogger()return d+ def get_timestamps_from_camera(self, webcam_id):+ return self._counters[webcam_id]+def increment_frame_processed_for(self, webcam_id, timestamp):self._counters[webcam_id].increment_frame_processed(timestamp)diff --git a/src/core_processor/show_cam_window.py b/src/core_processor/show_cam_window.pyindex b70f4a41..d84fa8a2 100644--- a/src/core_processor/show_cam_window.py+++ b/src/core_processor/show_cam_window.py@@ -3,13 +3,13 @@ import loggingimport cv2import numpy as np-from src.core_processor.fps.fps_counter import FPSCamCounter+from src.core_processor.fps.timestamp_manager import TimestampManagerfrom src.core_processor.utils.image_fps_writer import write_fps_to_imagelogger = logging.getLogger(__name__)-def show_cam_window(webcam_id: str, image: np.array, fps_counter_cls: FPSCamCounter):+def show_cam_window(webcam_id: str, image: np.array, fps_counter_cls: TimestampManager):exit_key = cv2.waitKey(1)if exit_key == 27:logger.info("ESC has been pressed.")diff --git a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.pyindex 99d4a653..b18498b9 100644--- a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py+++ b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py@@ -9,6 +9,8 @@ from pyqtgraph.dockarea.Dock import Dockfrom pyqtgraph.dockarea.DockArea import DockAreafrom pyqtgraph.Qt import QtWidgets+from src.core_processor.fps.timestamp_manager import TimestampManager+logger = logging.getLogger(__name__)@@ -62,54 +64,31 @@ class QTVisualizerAndGui:raise ecamera_image_item_widget.setImage(cv2.cvtColor(image_to_display, cv2.COLOR_BGR2RGB))- def update_timestamp_plots(self, dict_of_timestamps):+ def update_timestamp_plots(self, timestamp_manager: TimestampManager):if self.pause_button_pressed:return+ if timestamp_manager.get_timestamps_from_camera(self._camera0_id).number_of_frames <= 0:+ return- try:- camera_0_timestamp = dict_of_timestamps[self._camera0_id]- except:- logger.info('no timestamp for camera0')- camera_0_timestamp = np.nan+ self._update_simple_timestamp_plot(timestamp_manager)+ self._update_timestamp_difference_plot(timestamp_manager)+ self._update_timestamp_difference_histogram(timestamp_manager)+ def _update_simple_timestamp_plot(self, timestamp_manager: TimestampManager):for webcam_id in self._webcam_ids_list:- try:- this_cameras_timestamp = dict_of_timestamps[webcam_id]- time_diff_from_cam0 = np.abs(- this_cameras_timestamp - camera_0_timestamp) # <- this is the actual calculation of timestamp diff-- self._dict_of_cam_timestamps[webcam_id] = np.append(self._dict_of_cam_timestamps[webcam_id],- this_cameras_timestamp)- self._dict_of_cam_timestamp_differences[webcam_id] = np.append(- self._dict_of_cam_timestamp_differences[webcam_id],- time_diff_from_cam0)-- except Exception as e:- logger.warning(f'Could not find timestamp dictionary for camera {webcam_id}')- raise e-- self._update_simple_timestamp_plot()- self._update_timestamp_difference_plot()- self._update_timestamp_difference_histogram()-- def _update_simple_timestamp_plot(self):- for webcam_id, timestamps_array in self._dict_of_cam_timestamps.items():plot_item = self._dict_of_simple_timestamp_line_plots[webcam_id]- plot_item.setData(timestamps_array)-- def _update_timestamp_difference_plot(self):+ plot_item.setData(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps)- for webcam_id in self._dict_of_cam_timestamps.keys():- self._dict_of_timestamp_difference_line_plots[webcam_id].setData(- self._dict_of_cam_timestamp_differences[webcam_id])-- def _update_timestamp_difference_histogram(self):+ def _update_timestamp_difference_plot(self, timestamp_manager: TimestampManager):for webcam_id in self._webcam_ids_list:- if webcam_id == self._camera0_id:- continue+ plot_item = self._dict_of_simple_timestamp_line_plots[webcam_id]+ plot_item.setData(np.diff(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps))- num_samples = self._dict_of_cam_timestamp_differences[webcam_id].shape[0]- counts, bin_edges = np.histogram(self._dict_of_cam_timestamp_differences[webcam_id],+ def _update_timestamp_difference_histogram(self, timestamp_manager: TimestampManager):+ for webcam_id in self._webcam_ids_list:+ this_cam_timestamp_diffs = np.diff(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps)+ num_samples = this_cam_timestamp_diffs.shape[0]+ counts, bin_edges = np.histogram(this_cam_timestamp_diffs,bins=np.linspace(0, 100, 100))counts = counts / num_samplestry:diff --git a/src/session_pipeline/session_pipeline_orchestrator.py b/src/session_pipeline/session_pipeline_orchestrator.pyindex f18b5bf7..ccdfdc0e 100644--- a/src/session_pipeline/session_pipeline_orchestrator.py+++ b/src/session_pipeline/session_pipeline_orchestrator.py@@ -1,37 +1,33 @@import time-START_TIME_UNIX_NS = time.time()-import loggingimport tracebackfrom pathlib import Pathimport cv2-import numpy as npfrom src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManagerfrom src.cameras.persistence.video_writer.save_options_dataclass import SaveOptionsfrom src.config.data_paths import freemocap_data_pathfrom src.core_processor.camera_calibration.camera_calibrator import CameraCalibrator-from src.core_processor.fps.fps_counter import FPSCamCounter+from src.core_processor.fps.timestamp_manager import TimestampManagerfrom src.core_processor.mediapipe_skeleton_detection.mediapipe_skeleton_detection import MediaPipeSkeletonDetectionfrom src.core_processor.show_cam_window import show_cam_windowfrom src.qt_visualizer_and_gui.qt_visualizer_and_gui import QTVisualizerAndGuilogger = logging.getLogger(__name__)-def get_canonical_time_str():return time.strftime("%m-%d-%Y-%H_%M_%S")class SessionPipelineOrchestrator:def __init__(self):- self._session_start_time_unix_ns = START_TIME_UNIX_NS+ self._session_start_time_unix_ns = time.time_ns()self._session_id = 'session_' + time.strftime("%m-%d-%Y-%H_%M_%S")self._visualizer_gui = QTVisualizerAndGui()self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id)self._camera_calibrator = CameraCalibrator()- self._mediapipe_skeleton_detector = MediaPipeSkeletonDetection(model_complexity=0)+ self._mediapipe_skeleton_detector = MediaPipeSkeletonDetection(model_complexity=2)@propertydef session_folder_path(self):@@ -41,7 +37,7 @@ class SessionPipelineOrchestrator:with self._open_cv_camera_manager.start_capture_session_single_cam(webcam_id=webcam_id,) as session_obj:- fps_manager = FPSCamCounter(self._open_cv_camera_manager.available_webcam_ids)+ fps_manager = TimestampManager(self._open_cv_camera_manager.available_webcam_ids)fps_manager.start_all()cv_cam = session_obj.cv_camerawriter = session_obj.video_recorder@@ -92,11 +88,10 @@ class SessionPipelineOrchestrator:if show_visualizer_gui:self._visualizer_gui.setup_and_launch(self._open_cv_camera_manager.available_webcam_ids)- fps_manager = FPSCamCounter(self._open_cv_camera_manager.available_webcam_ids)+ timestamp_manager = TimestampManager(self._open_cv_camera_manager.available_webcam_ids)should_continue = Truewhile should_continue:- this_frame_timestamps_dict = dict.fromkeys(self._open_cv_camera_manager.available_webcam_ids)for this_webcam_id, this_open_cv_camera in connected_cameras_dict.items():@@ -106,13 +101,13 @@ class SessionPipelineOrchestrator:this_cam_latest_frame = this_open_cv_camera.latest_frameif this_cam_latest_frame is None:- image_to_display = this_open_cv_camera.latest_frame # if this camera has no new frame, write the previous one instead- this_frame_timestamps_dict[this_webcam_id] = np.nan # frame drop?- else:- this_frame_timestamps_dict[this_webcam_id] = this_cam_latest_frame.timestamp- image_to_display = this_cam_latest_frame.image+ continue- fps_manager.increment_frame_processed_for(this_webcam_id,this_frame_timestamps_dict[this_webcam_id])+ image_to_display = this_cam_latest_frame.image+ this_cam_this_frame_timestamp = this_cam_latest_frame.timestamp++ timestamp_manager.increment_frame_processed_for(this_webcam_id,+ this_cam_this_frame_timestamp)if save_video:this_open_cv_camera.video_recorder.record(this_cam_latest_frame)@@ -127,12 +122,12 @@ class SessionPipelineOrchestrator:if show_camera_views_in_windows:should_continue = show_cam_window(- this_webcam_id, image_to_display, fps_manager+ this_webcam_id, image_to_display, timestamp_manager)if show_visualizer_gui:self._visualizer_gui.update_camera_view_image(this_webcam_id, image_to_display)- # self._visualizer_gui.update_timestamp_plots(this_frame_timestamps_dict)+ # self._visualizer_gui.update_timestamp_plots(timestamp_manager)except:logger.error("Printing traceback")@@ -150,13 +145,11 @@ class SessionPipelineOrchestrator:self._visualizer_gui.close()--if __name__ == "__main__":print('start main')this_session = SessionPipelineOrchestrator()- this_session.run(save_video=True,+ this_session.run(save_video=False,show_visualizer_gui=True,calibrate_cameras=False,detect_skeleton=True)