Free Motion Capture for Everyone πβ¨
TypeScript
semi-fix framerate/timestamp counter
Jon48503356386d
diff --git a/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py b/src/cameras/capture/opencv_camera/camera_stream_thread_handler.pyindex 4c09e47a..a0c3e9cd 100644--- a/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py+++ b/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py@@ -8,7 +8,7 @@ from src.cameras.capture.dataclasses.frame_payload import FramePayloadlogger = logging.getLogger(__name__)-class VideoCaptureThreadHandler(threading.Thread):+class VideoCaptureThread(threading.Thread):def __init__(self,get_next_frame,@@ -45,12 +45,12 @@ class VideoCaptureThreadHandler(threading.Thread):def _start_frame_loop(self, save_video=False):self._is_capturing_frames = True- start = time.time()+ start = time.perf_counter_ns()try:while self._is_capturing_frames:- frame = self._get_next_frame()+ self._frame = self._get_next_frame()self._num_frames_processed += 1- self._elapsed = time.time() - start+ self._elapsed = time.perf_counter_ns() - startexcept:logger.error("Frame loop thread exited due to error")traceback.print_exc()diff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.pyindex 0c6d047b..6891120c 100644--- a/src/cameras/capture/opencv_camera/opencv_camera.py+++ b/src/cameras/capture/opencv_camera/opencv_camera.py@@ -8,7 +8,7 @@ import cv2from src.cameras.capture.dataclasses.frame_payload import FramePayloadfrom src.cameras.persistence.video_writer.video_recorder import VideoRecorderfrom src.config.webcam_config import WebcamConfig-from src.cameras.capture.opencv_camera.camera_stream_thread_handler import VideoCaptureThreadHandler+from src.cameras.capture.opencv_camera.camera_stream_thread_handler import VideoCaptureThreadlogger = logging.getLogger(__name__)@@ -19,14 +19,14 @@ class OpenCVCamera:"""def __init__(self, config: WebcamConfig, session_id: str = None):self._config = config- self._name = f"Camera_ {self._config.webcam_id}"+ self._name = f"Camera_{self._config.webcam_id}"self._opencv_video_capture_object: cv2.VideoCapture = Noneself._video_recorder: VideoRecorder = None- self._running_thread: VideoCaptureThreadHandler = None+ self._running_thread: VideoCaptureThread = Noneself._is_there_a_new_frame = False@property- def is_there_a_new_frame(self):+ def new_frame_ready(self):"""can be called to determine if the frame returned from `self.latest_frame` is new or if it has been called before"""@@ -113,7 +113,7 @@ class OpenCVCamera:self._running_thread.start()def _create_thread(self):- return VideoCaptureThreadHandler(+ return VideoCaptureThread(get_next_frame=self.get_next_frame,)@@ -149,17 +149,17 @@ class OpenCVCamera:)def get_next_frame(self):- timestamp_ns_pre_grab = time.time_ns()+# Why `grab()` not `read()`? see -># https://stackoverflow.com/questions/57716962/difference-between-video-capture-read-and# -grab+if not self._opencv_video_capture_object.grab():return FramePayload(False, None, None)- timestamp_ns_post_grab = time.time_ns()- timestamp_ns = (timestamp_ns_pre_grab + timestamp_ns_post_grab) / 2-+ timestamp_ns = time.perf_counter_ns()success, image = self._opencv_video_capture_object.retrieve()+self._is_there_a_new_frame = successreturn FramePayload(success, image, timestamp_ns)diff --git a/src/cameras/multicam_manager/cv_camera_manager.py b/src/cameras/multicam_manager/cv_camera_manager.pyindex b994dbdc..0bc7b9a0 100644--- a/src/cameras/multicam_manager/cv_camera_manager.py+++ b/src/cameras/multicam_manager/cv_camera_manager.py@@ -58,8 +58,9 @@ class OpenCVCameraManager:def open_cv_cameras(self):return self._open_cv_camera_objects+ @propertydef new_synchronized_frame_available(self):- new_frame_bool_list = [cam.is_there_a_new_frame for cam in self._open_cv_camera_objects]+ new_frame_bool_list = [cam.new_frame_ready for cam in self._open_cv_camera_objects]if all(new_frame_bool_list):return Truereturn False@@ -67,9 +68,11 @@ class OpenCVCameraManager:def latest_synchronized_frame(self):synchronized_frame_dict = {}for this_cam in self._open_cv_camera_objects:- if not this_cam.is_there_a_new_frame:++ if not self.new_synchronized_frame_available:logger.error(f'New frame not available for camera {this_cam.webcam_id_as_str}')synchronized_frame_dict[this_cam.webcam_id_as_str] = None+synchronized_frame_dict[this_cam.webcam_id_as_str] = this_cam.latest_framereturn synchronized_frame_dictdiff --git a/src/cameras/persistence/video_writer/video_recorder.py b/src/cameras/persistence/video_writer/video_recorder.pyindex fcd8a0bb..998ac8f7 100644--- a/src/cameras/persistence/video_writer/video_recorder.py+++ b/src/cameras/persistence/video_writer/video_recorder.py@@ -29,7 +29,7 @@ class VideoRecorder:self._frames: List[FramePayload] = []self._path_to_save_video = Noneself._video_type_str = ''- self._timestamps_npy = np.array(0)+ self._timestamps_npy = np.empty(0)@propertydef frame_count(self):@@ -39,6 +39,10 @@ class VideoRecorder:self._frames.append(frame_payload)def save_to_disk(self, path_to_save_video: Path, video_type_str: str = 'raw'):+ if len(self._frames) == 0:+ logging.error(f"No frames to save for camera: {self._camera_name}")+ return+self._video_type_str = video_type_strself._path_to_save_video = path_to_save_videoself._path_to_save_video.mkdir(parents=True, exist_ok=True)@@ -51,8 +55,8 @@ class VideoRecorder:video_full_path = self._path_to_save_video / video_file_namecv2_writer = cv2.VideoWriter(- video_full_path,- cv2.VideoWriter_fourcc(self._fourcc),+ str(video_full_path),+ cv2.VideoWriter_fourcc(*self._fourcc),self._median_framerate,(self._image_width, self._image_height))@@ -69,15 +73,18 @@ class VideoRecorder:cv2_writer.release()def _gather_timestamps(self):- for frame in self._frames:- self._timestamps_npy = np.append(self._timestamps_npy, frame.timestamp)- self._median_framerate = np.median(np.diff(self._timestamps_npy))+ try:+ for frame in self._frames:+ self._timestamps_npy = np.append(self._timestamps_npy, frame.timestamp)+ self._median_framerate = (np.nanmedian(np.diff(self._timestamps_npy / 1e9))) ** -1+ except:+ logger.error("Error gathering timestamps")def _save_timestamps(self):- timestamp_file_name_npy = self._camera_name + "_timestamps_ns.npy"+ timestamp_file_name_npy = self._camera_name + "_timestamps.npy"np.save(str(self._path_to_save_video / timestamp_file_name_npy), self._timestamps_npy)logger.info(f"Saved timestamps to path: {timestamp_file_name_npy}")- timestamp_file_name_csv = self._camera_name + "_timestamps_ns.csv"- pd.DataFrame(self._timestamps_npy).to_csv(str(self._path_to_save_video / timestamp_file_name_csv))- logger.info(f"Saved timestamps to path: {timestamp_file_name_csv}")\ No newline at end of file+ timestamp_file_name_csv = self._camera_name + "_timestamps.csv"+ self._timestamps_npy.tofile(str(self._path_to_save_video / timestamp_file_name_csv), sep=',\n')+ logger.info(f"Saved timestamps to path: {timestamp_file_name_csv}")diff --git a/src/core_processor/fps/fps_counter.py b/src/core_processor/fps/fps_counter.pyindex 001d0ca0..543d5c70 100644--- a/src/core_processor/fps/fps_counter.py+++ b/src/core_processor/fps/fps_counter.py@@ -1,6 +1,8 @@import timefrom typing import Dict, List+import numpy as np+class FPSCounter:_num_frames_processed: int@@ -9,25 +11,22 @@ class FPSCounter:def __init__(self):self._start_time = 0self._num_frames_processed = 0+ self._timestamps = np.empty(0)def is_started(self):- return self._start_time > 0-- def start(self):- self._start_time = time.time()+ return self._num_frames_processed > 0@property- def current_fps(self):- if self.elapsed() <= 0:- return 0- if self._num_frames_processed <= 0:+ def median_frames_per_second(self):+ if self._num_frames_processed <= 10:return 0- return self._num_frames_processed / self.elapsed()+ return np.nanmedian(np.diff(self._timestamps)/1e9)**-1def elapsed(self):return time.time() - self._start_time- def increment_frame_processed(self):+ def increment_frame_processed(self, timestamp):+ self._timestamps = np.append(self._timestamps, timestamp)self._num_frames_processed += 1@@ -40,21 +39,12 @@ class FPSCamCounter:d = {}for webcam_id in webcam_ids:d[webcam_id] = FPSCounter()-return d- def increment_frame_processed_for(self, webcam_id):- self._counters[webcam_id].increment_frame_processed()-- def current_fps_for(self, webcam_id):- return self._counters[webcam_id].current_fps+ def increment_frame_processed_for(self, webcam_id, timestamp):+ self._counters[webcam_id].increment_frame_processed(timestamp)- def start_all(self):- for webcam_id in self._webcam_ids:- self.start_for(webcam_id)+ def median_frames_per_second(self, webcam_id):+ return self._counters[webcam_id].median_frames_per_second- def start_for(self, webcam_id):- if self._counters[webcam_id].is_started():- return- self._counters[webcam_id].start()diff --git a/src/core_processor/show_cam_window.py b/src/core_processor/show_cam_window.pyindex bb7f712d..b70f4a41 100644--- a/src/core_processor/show_cam_window.py+++ b/src/core_processor/show_cam_window.py@@ -17,7 +17,7 @@ def show_cam_window(webcam_id: str, image: np.array, fps_counter_cls: FPSCamCounwrite_fps_to_image(image,- fps_counter_cls.current_fps_for(webcam_id),+ fps_counter_cls.median_frames_per_second(webcam_id),)cv2.imshow(webcam_id, image)return Truediff --git a/src/session_pipeline/session_pipeline_orchestrator.py b/src/session_pipeline/session_pipeline_orchestrator.pyindex 45e997c7..0d8cf75c 100644--- a/src/session_pipeline/session_pipeline_orchestrator.py+++ b/src/session_pipeline/session_pipeline_orchestrator.py@@ -1,5 +1,7 @@-import loggingimport time+START_TIME_UNIX_NS = time.time()++import loggingimport tracebackfrom pathlib import Path@@ -24,7 +26,7 @@ def get_canonical_time_str():class SessionPipelineOrchestrator:def __init__(self):- self._session_start_time = time.time_ns()+ self._session_start_time_unix_ns = START_TIME_UNIX_NSself._session_id = 'session_' + time.strftime("%m-%d-%Y-%H_%M_%S")self._visualizer_gui = QTVisualizerAndGui()self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id)@@ -67,7 +69,7 @@ class SessionPipelineOrchestrator:"charuco_board_detection",f"webcam_{cv_cam.webcam_id_as_str}",),- fps=fps_manager.current_fps_for(cv_cam.webcam_id_as_str),+ fps=fps_manager.median_frames_per_second(cv_cam.webcam_id_as_str),frame_width=cv_cam.image_width(),frame_height=cv_cam.image_height(),)@@ -85,47 +87,44 @@ class SessionPipelineOrchestrator:Opens Cameras using OpenCV and begins image processing for charuco boardIf return images is true, the images are returned to the caller"""-with self._open_cv_camera_manager.start_capture_session_all_cams() as connected_cameras_dict:-try:- fps_manager = FPSCamCounter(self._open_cv_camera_manager.available_webcam_ids)- fps_manager.start_all()+ if show_visualizer_gui:+ self._visualizer_gui.setup_and_launch(self._open_cv_camera_manager.available_webcam_ids)- self._visualizer_gui.setup_and_launch(self._open_cv_camera_manager.available_webcam_ids)+ fps_manager = FPSCamCounter(self._open_cv_camera_manager.available_webcam_ids)should_continue = Truewhile should_continue:-- if not self._open_cv_camera_manager.new_synchronized_frame_available():- continue- this_frame_timestamps_dict = {}- incoming_synchronized_frame_dict = self._open_cv_camera_manager.latest_synchronized_frame()+ this_frame_timestamps_dict = dict.fromkeys(self._open_cv_camera_manager.available_webcam_ids)for this_webcam_id, this_open_cv_camera in connected_cameras_dict.items():- this_cam_latest_sync_frame = incoming_synchronized_frame_dict[this_webcam_id]+ if not this_open_cv_camera.new_frame_ready:+ continue++ this_cam_latest_frame = this_open_cv_camera.latest_frame- if this_cam_latest_sync_frame is None:- image_to_display = this_open_cv_camera.latest_frame #if this camera has no new frame, write the previous one instead- this_frame_timestamps_dict[this_webcam_id] = np.nan # a `nan` timestamp denotes a dropped frame+ if this_cam_latest_frame is None:+ image_to_display = this_open_cv_camera.latest_frame # if this camera has no new frame, write the previous one instead+ this_frame_timestamps_dict[this_webcam_id] = np.nan # frame drop?else:- this_frame_timestamps_dict[- this_webcam_id] = (this_cam_latest_sync_frame.timestamp - self._session_start_time)/1e6 #milliseconds, I think- image_to_display = this_cam_latest_sync_frame.image+ this_frame_timestamps_dict[this_webcam_id] = this_cam_latest_frame.timestamp+ image_to_display = this_cam_latest_frame.image++ fps_manager.increment_frame_processed_for(this_webcam_id,this_frame_timestamps_dict[this_webcam_id])if save_video:- # print('sending frame to be written')- this_open_cv_camera.video_recorder.record(this_cam_latest_sync_frame)+ this_open_cv_camera.video_recorder.record(this_cam_latest_frame)if calibrate_cameras:undistorted_annotated_image = self._camera_calibrator.calibrate(this_open_cv_camera)image_to_display = undistorted_annotated_imageif detect_skeleton:- image_to_display = self._mediapipe_skeleton_detector.detect_skeleton_in_image(image_to_display)+ image_to_display = self._mediapipe_skeleton_detector.detect_skeleton_in_image(+ image_to_display)- fps_manager.increment_frame_processed_for(this_webcam_id)if show_camera_views_in_windows:should_continue = show_cam_window(this_webcam_id, image_to_display, fps_manager@@ -133,8 +132,7 @@ class SessionPipelineOrchestrator:if show_visualizer_gui:self._visualizer_gui.update_camera_view_image(this_webcam_id, image_to_display)- if len(this_frame_timestamps_dict.keys()) == self._open_cv_camera_manager.number_of_cameras:- self._visualizer_gui.update_timestamp_plots(this_frame_timestamps_dict)+ # self._visualizer_gui.update_timestamp_plots(this_frame_timestamps_dict)except:logger.error("Printing traceback")@@ -142,16 +140,23 @@ class SessionPipelineOrchestrator:finally:for this_open_cv_camera in connected_cameras_dict.values():if save_video:- this_open_cv_camera.video_recorder.save_to_disk(self.session_folder_path / 'synchronized_videos')+ this_open_cv_camera.video_recorder.save_to_disk(+ self.session_folder_path / 'synchronized_videos')logger.info(f"Destroy window {this_open_cv_camera.webcam_id_as_str}")cv2.destroyWindow(this_open_cv_camera.webcam_id_as_str)cv2.waitKey(1)- self._visualizer_gui.close()+ if show_visualizer_gui:+ self._visualizer_gui.close()++if __name__ == "__main__":print('start main')this_session = SessionPipelineOrchestrator()- this_session.run(calibrate_cameras=True)+ this_session.run(save_video=True,+ show_visualizer_gui=True,+ calibrate_cameras=False,+ detect_skeleton=False)