Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

semi-fix framerate/timestamp counter

Jon48503356386d
diff --git a/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py b/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py
index 4c09e47a..a0c3e9cd 100644
--- a/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py
+++ b/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py
@@ -8,7 +8,7 @@ from src.cameras.capture.dataclasses.frame_payload import FramePayload
logger = logging.getLogger(__name__)
-class VideoCaptureThreadHandler(threading.Thread):
+class VideoCaptureThread(threading.Thread):
def __init__(
self,
get_next_frame,
@@ -45,12 +45,12 @@ class VideoCaptureThreadHandler(threading.Thread):
def _start_frame_loop(self, save_video=False):
self._is_capturing_frames = True
- start = time.time()
+ start = time.perf_counter_ns()
try:
while self._is_capturing_frames:
- frame = self._get_next_frame()
+ self._frame = self._get_next_frame()
self._num_frames_processed += 1
- self._elapsed = time.time() - start
+ self._elapsed = time.perf_counter_ns() - start
except:
logger.error("Frame loop thread exited due to error")
traceback.print_exc()
diff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.py
index 0c6d047b..6891120c 100644
--- a/src/cameras/capture/opencv_camera/opencv_camera.py
+++ b/src/cameras/capture/opencv_camera/opencv_camera.py
@@ -8,7 +8,7 @@ import cv2
from src.cameras.capture.dataclasses.frame_payload import FramePayload
from src.cameras.persistence.video_writer.video_recorder import VideoRecorder
from src.config.webcam_config import WebcamConfig
-from src.cameras.capture.opencv_camera.camera_stream_thread_handler import VideoCaptureThreadHandler
+from src.cameras.capture.opencv_camera.camera_stream_thread_handler import VideoCaptureThread
logger = logging.getLogger(__name__)
@@ -19,14 +19,14 @@ class OpenCVCamera:
"""
def __init__(self, config: WebcamConfig, session_id: str = None):
self._config = config
- self._name = f"Camera_ {self._config.webcam_id}"
+ self._name = f"Camera_{self._config.webcam_id}"
self._opencv_video_capture_object: cv2.VideoCapture = None
self._video_recorder: VideoRecorder = None
- self._running_thread: VideoCaptureThreadHandler = None
+ self._running_thread: VideoCaptureThread = None
self._is_there_a_new_frame = False
@property
- def is_there_a_new_frame(self):
+ def new_frame_ready(self):
"""
can be called to determine if the frame returned from `self.latest_frame` is new or if it has been called before
"""
@@ -113,7 +113,7 @@ class OpenCVCamera:
self._running_thread.start()
def _create_thread(self):
- return VideoCaptureThreadHandler(
+ return VideoCaptureThread(
get_next_frame=self.get_next_frame,
)
@@ -149,17 +149,17 @@ class OpenCVCamera:
)
def get_next_frame(self):
- timestamp_ns_pre_grab = time.time_ns()
+
# Why `grab()` not `read()`? see ->
# https://stackoverflow.com/questions/57716962/difference-between-video-capture-read-and
# -grab
+
if not self._opencv_video_capture_object.grab():
return FramePayload(False, None, None)
- timestamp_ns_post_grab = time.time_ns()
- timestamp_ns = (timestamp_ns_pre_grab + timestamp_ns_post_grab) / 2
-
+ timestamp_ns = time.perf_counter_ns()
success, image = self._opencv_video_capture_object.retrieve()
+
self._is_there_a_new_frame = success
return FramePayload(success, image, timestamp_ns)
diff --git a/src/cameras/multicam_manager/cv_camera_manager.py b/src/cameras/multicam_manager/cv_camera_manager.py
index b994dbdc..0bc7b9a0 100644
--- a/src/cameras/multicam_manager/cv_camera_manager.py
+++ b/src/cameras/multicam_manager/cv_camera_manager.py
@@ -58,8 +58,9 @@ class OpenCVCameraManager:
def open_cv_cameras(self):
return self._open_cv_camera_objects
+ @property
def new_synchronized_frame_available(self):
- new_frame_bool_list = [cam.is_there_a_new_frame for cam in self._open_cv_camera_objects]
+ new_frame_bool_list = [cam.new_frame_ready for cam in self._open_cv_camera_objects]
if all(new_frame_bool_list):
return True
return False
@@ -67,9 +68,11 @@ class OpenCVCameraManager:
def latest_synchronized_frame(self):
synchronized_frame_dict = {}
for this_cam in self._open_cv_camera_objects:
- if not this_cam.is_there_a_new_frame:
+
+ if not self.new_synchronized_frame_available:
logger.error(f'New frame not available for camera {this_cam.webcam_id_as_str}')
synchronized_frame_dict[this_cam.webcam_id_as_str] = None
+
synchronized_frame_dict[this_cam.webcam_id_as_str] = this_cam.latest_frame
return synchronized_frame_dict
diff --git a/src/cameras/persistence/video_writer/video_recorder.py b/src/cameras/persistence/video_writer/video_recorder.py
index fcd8a0bb..998ac8f7 100644
--- a/src/cameras/persistence/video_writer/video_recorder.py
+++ b/src/cameras/persistence/video_writer/video_recorder.py
@@ -29,7 +29,7 @@ class VideoRecorder:
self._frames: List[FramePayload] = []
self._path_to_save_video = None
self._video_type_str = ''
- self._timestamps_npy = np.array(0)
+ self._timestamps_npy = np.empty(0)
@property
def frame_count(self):
@@ -39,6 +39,10 @@ class VideoRecorder:
self._frames.append(frame_payload)
def save_to_disk(self, path_to_save_video: Path, video_type_str: str = 'raw'):
+ if len(self._frames) == 0:
+ logging.error(f"No frames to save for camera: {self._camera_name}")
+ return
+
self._video_type_str = video_type_str
self._path_to_save_video = path_to_save_video
self._path_to_save_video.mkdir(parents=True, exist_ok=True)
@@ -51,8 +55,8 @@ class VideoRecorder:
video_full_path = self._path_to_save_video / video_file_name
cv2_writer = cv2.VideoWriter(
- video_full_path,
- cv2.VideoWriter_fourcc(self._fourcc),
+ str(video_full_path),
+ cv2.VideoWriter_fourcc(*self._fourcc),
self._median_framerate,
(self._image_width, self._image_height))
@@ -69,15 +73,18 @@ class VideoRecorder:
cv2_writer.release()
def _gather_timestamps(self):
- for frame in self._frames:
- self._timestamps_npy = np.append(self._timestamps_npy, frame.timestamp)
- self._median_framerate = np.median(np.diff(self._timestamps_npy))
+ try:
+ for frame in self._frames:
+ self._timestamps_npy = np.append(self._timestamps_npy, frame.timestamp)
+ self._median_framerate = (np.nanmedian(np.diff(self._timestamps_npy / 1e9))) ** -1
+ except:
+ logger.error("Error gathering timestamps")
def _save_timestamps(self):
- timestamp_file_name_npy = self._camera_name + "_timestamps_ns.npy"
+ timestamp_file_name_npy = self._camera_name + "_timestamps.npy"
np.save(str(self._path_to_save_video / timestamp_file_name_npy), self._timestamps_npy)
logger.info(f"Saved timestamps to path: {timestamp_file_name_npy}")
- timestamp_file_name_csv = self._camera_name + "_timestamps_ns.csv"
- pd.DataFrame(self._timestamps_npy).to_csv(str(self._path_to_save_video / timestamp_file_name_csv))
- logger.info(f"Saved timestamps to path: {timestamp_file_name_csv}")
\ No newline at end of file
+ timestamp_file_name_csv = self._camera_name + "_timestamps.csv"
+ self._timestamps_npy.tofile(str(self._path_to_save_video / timestamp_file_name_csv), sep=',\n')
+ logger.info(f"Saved timestamps to path: {timestamp_file_name_csv}")
diff --git a/src/core_processor/fps/fps_counter.py b/src/core_processor/fps/fps_counter.py
index 001d0ca0..543d5c70 100644
--- a/src/core_processor/fps/fps_counter.py
+++ b/src/core_processor/fps/fps_counter.py
@@ -1,6 +1,8 @@
import time
from typing import Dict, List
+import numpy as np
+
class FPSCounter:
_num_frames_processed: int
@@ -9,25 +11,22 @@ class FPSCounter:
def __init__(self):
self._start_time = 0
self._num_frames_processed = 0
+ self._timestamps = np.empty(0)
def is_started(self):
- return self._start_time > 0
-
- def start(self):
- self._start_time = time.time()
+ return self._num_frames_processed > 0
@property
- def current_fps(self):
- if self.elapsed() <= 0:
- return 0
- if self._num_frames_processed <= 0:
+ def median_frames_per_second(self):
+ if self._num_frames_processed <= 10:
return 0
- return self._num_frames_processed / self.elapsed()
+ return np.nanmedian(np.diff(self._timestamps)/1e9)**-1
def elapsed(self):
return time.time() - self._start_time
- def increment_frame_processed(self):
+ def increment_frame_processed(self, timestamp):
+ self._timestamps = np.append(self._timestamps, timestamp)
self._num_frames_processed += 1
@@ -40,21 +39,12 @@ class FPSCamCounter:
d = {}
for webcam_id in webcam_ids:
d[webcam_id] = FPSCounter()
-
return d
- def increment_frame_processed_for(self, webcam_id):
- self._counters[webcam_id].increment_frame_processed()
-
- def current_fps_for(self, webcam_id):
- return self._counters[webcam_id].current_fps
+ def increment_frame_processed_for(self, webcam_id, timestamp):
+ self._counters[webcam_id].increment_frame_processed(timestamp)
- def start_all(self):
- for webcam_id in self._webcam_ids:
- self.start_for(webcam_id)
+ def median_frames_per_second(self, webcam_id):
+ return self._counters[webcam_id].median_frames_per_second
- def start_for(self, webcam_id):
- if self._counters[webcam_id].is_started():
- return
- self._counters[webcam_id].start()
diff --git a/src/core_processor/show_cam_window.py b/src/core_processor/show_cam_window.py
index bb7f712d..b70f4a41 100644
--- a/src/core_processor/show_cam_window.py
+++ b/src/core_processor/show_cam_window.py
@@ -17,7 +17,7 @@ def show_cam_window(webcam_id: str, image: np.array, fps_counter_cls: FPSCamCoun
write_fps_to_image(
image,
- fps_counter_cls.current_fps_for(webcam_id),
+ fps_counter_cls.median_frames_per_second(webcam_id),
)
cv2.imshow(webcam_id, image)
return True
diff --git a/src/session_pipeline/session_pipeline_orchestrator.py b/src/session_pipeline/session_pipeline_orchestrator.py
index 45e997c7..0d8cf75c 100644
--- a/src/session_pipeline/session_pipeline_orchestrator.py
+++ b/src/session_pipeline/session_pipeline_orchestrator.py
@@ -1,5 +1,7 @@
-import logging
import time
+START_TIME_UNIX_NS = time.time()
+
+import logging
import traceback
from pathlib import Path
@@ -24,7 +26,7 @@ def get_canonical_time_str():
class SessionPipelineOrchestrator:
def __init__(self):
- self._session_start_time = time.time_ns()
+ self._session_start_time_unix_ns = START_TIME_UNIX_NS
self._session_id = 'session_' + time.strftime("%m-%d-%Y-%H_%M_%S")
self._visualizer_gui = QTVisualizerAndGui()
self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id)
@@ -67,7 +69,7 @@ class SessionPipelineOrchestrator:
"charuco_board_detection",
f"webcam_{cv_cam.webcam_id_as_str}",
),
- fps=fps_manager.current_fps_for(cv_cam.webcam_id_as_str),
+ fps=fps_manager.median_frames_per_second(cv_cam.webcam_id_as_str),
frame_width=cv_cam.image_width(),
frame_height=cv_cam.image_height(),
)
@@ -85,47 +87,44 @@ class SessionPipelineOrchestrator:
Opens Cameras using OpenCV and begins image processing for charuco board
If return images is true, the images are returned to the caller
"""
-
with self._open_cv_camera_manager.start_capture_session_all_cams() as connected_cameras_dict:
-
try:
- fps_manager = FPSCamCounter(self._open_cv_camera_manager.available_webcam_ids)
- fps_manager.start_all()
+ if show_visualizer_gui:
+ self._visualizer_gui.setup_and_launch(self._open_cv_camera_manager.available_webcam_ids)
- self._visualizer_gui.setup_and_launch(self._open_cv_camera_manager.available_webcam_ids)
+ fps_manager = FPSCamCounter(self._open_cv_camera_manager.available_webcam_ids)
should_continue = True
while should_continue:
-
- if not self._open_cv_camera_manager.new_synchronized_frame_available():
- continue
- this_frame_timestamps_dict = {}
- incoming_synchronized_frame_dict = self._open_cv_camera_manager.latest_synchronized_frame()
+ this_frame_timestamps_dict = dict.fromkeys(self._open_cv_camera_manager.available_webcam_ids)
for this_webcam_id, this_open_cv_camera in connected_cameras_dict.items():
- this_cam_latest_sync_frame = incoming_synchronized_frame_dict[this_webcam_id]
+ if not this_open_cv_camera.new_frame_ready:
+ continue
+
+ this_cam_latest_frame = this_open_cv_camera.latest_frame
- if this_cam_latest_sync_frame is None:
- image_to_display = this_open_cv_camera.latest_frame #if this camera has no new frame, write the previous one instead
- this_frame_timestamps_dict[this_webcam_id] = np.nan # a `nan` timestamp denotes a dropped frame
+ if this_cam_latest_frame is None:
+ image_to_display = this_open_cv_camera.latest_frame # if this camera has no new frame, write the previous one instead
+ this_frame_timestamps_dict[this_webcam_id] = np.nan # frame drop?
else:
- this_frame_timestamps_dict[
- this_webcam_id] = (this_cam_latest_sync_frame.timestamp - self._session_start_time)/1e6 #milliseconds, I think
- image_to_display = this_cam_latest_sync_frame.image
+ this_frame_timestamps_dict[this_webcam_id] = this_cam_latest_frame.timestamp
+ image_to_display = this_cam_latest_frame.image
+
+ fps_manager.increment_frame_processed_for(this_webcam_id,this_frame_timestamps_dict[this_webcam_id])
if save_video:
- # print('sending frame to be written')
- this_open_cv_camera.video_recorder.record(this_cam_latest_sync_frame)
+ this_open_cv_camera.video_recorder.record(this_cam_latest_frame)
if calibrate_cameras:
undistorted_annotated_image = self._camera_calibrator.calibrate(this_open_cv_camera)
image_to_display = undistorted_annotated_image
if detect_skeleton:
- image_to_display = self._mediapipe_skeleton_detector.detect_skeleton_in_image(image_to_display)
+ image_to_display = self._mediapipe_skeleton_detector.detect_skeleton_in_image(
+ image_to_display)
- fps_manager.increment_frame_processed_for(this_webcam_id)
if show_camera_views_in_windows:
should_continue = show_cam_window(
this_webcam_id, image_to_display, fps_manager
@@ -133,8 +132,7 @@ class SessionPipelineOrchestrator:
if show_visualizer_gui:
self._visualizer_gui.update_camera_view_image(this_webcam_id, image_to_display)
- if len(this_frame_timestamps_dict.keys()) == self._open_cv_camera_manager.number_of_cameras:
- self._visualizer_gui.update_timestamp_plots(this_frame_timestamps_dict)
+ # self._visualizer_gui.update_timestamp_plots(this_frame_timestamps_dict)
except:
logger.error("Printing traceback")
@@ -142,16 +140,23 @@ class SessionPipelineOrchestrator:
finally:
for this_open_cv_camera in connected_cameras_dict.values():
if save_video:
- this_open_cv_camera.video_recorder.save_to_disk(self.session_folder_path / 'synchronized_videos')
+ this_open_cv_camera.video_recorder.save_to_disk(
+ self.session_folder_path / 'synchronized_videos')
logger.info(f"Destroy window {this_open_cv_camera.webcam_id_as_str}")
cv2.destroyWindow(this_open_cv_camera.webcam_id_as_str)
cv2.waitKey(1)
- self._visualizer_gui.close()
+ if show_visualizer_gui:
+ self._visualizer_gui.close()
+
+
if __name__ == "__main__":
print('start main')
this_session = SessionPipelineOrchestrator()
- this_session.run(calibrate_cameras=True)
+ this_session.run(save_video=True,
+ show_visualizer_gui=True,
+ calibrate_cameras=False,
+ detect_skeleton=False)