Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

fiddling with timestamps

Jonec7942e9bccb
diff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.py
index 6891120c..93a4160a 100644
--- a/src/cameras/capture/opencv_camera/opencv_camera.py
+++ b/src/cameras/capture/opencv_camera/opencv_camera.py
@@ -23,14 +23,14 @@ class OpenCVCamera:
self._opencv_video_capture_object: cv2.VideoCapture = None
self._video_recorder: VideoRecorder = None
self._running_thread: VideoCaptureThread = None
- self._is_there_a_new_frame = False
+ self._new_frame_ready = False
@property
def new_frame_ready(self):
"""
can be called to determine if the frame returned from `self.latest_frame` is new or if it has been called before
"""
- return self._is_there_a_new_frame
+ return self._new_frame_ready
@property
def video_recorder(self):
@@ -61,7 +61,7 @@ class OpenCVCamera:
@property
def latest_frame(self):
- self._is_there_a_new_frame = False
+ self._new_frame_ready = False
return self._running_thread.latest_frame
def connect(self):
@@ -157,10 +157,10 @@ class OpenCVCamera:
if not self._opencv_video_capture_object.grab():
return FramePayload(False, None, None)
- timestamp_ns = time.perf_counter_ns()
+ timestamp_ns = time.time_ns()
success, image = self._opencv_video_capture_object.retrieve()
- self._is_there_a_new_frame = success
+ self._new_frame_ready = success
return FramePayload(success, image, timestamp_ns)
def stop_frame_capture(self):
diff --git a/src/core_processor/fps/fps_counter.py b/src/core_processor/fps/timestamp_manager.py
similarity index 65%
rename from src/core_processor/fps/fps_counter.py
rename to src/core_processor/fps/timestamp_manager.py
index 543d5c70..4b1e6741 100644
--- a/src/core_processor/fps/fps_counter.py
+++ b/src/core_processor/fps/timestamp_manager.py
@@ -1,15 +1,13 @@
-import time
from typing import Dict, List
import numpy as np
-class FPSCounter:
+class TimestampLogger:
_num_frames_processed: int
_start_time: float
def __init__(self):
- self._start_time = 0
self._num_frames_processed = 0
self._timestamps = np.empty(0)
@@ -22,25 +20,39 @@ class FPSCounter:
return 0
return np.nanmedian(np.diff(self._timestamps)/1e9)**-1
- def elapsed(self):
- return time.time() - self._start_time
+ @property
+ def number_of_frames(self):
+ return self._num_frames_processed
+
+ @property
+ def timestamps(self):
+ return self._timestamps
+
+ @property
+ def last_timestamp(self):
+ if self._timestamps.shape[0] > 0:
+ return self._timestamps[-1]
+ return None
def increment_frame_processed(self, timestamp):
self._timestamps = np.append(self._timestamps, timestamp)
self._num_frames_processed += 1
-class FPSCamCounter:
+class TimestampManager:
def __init__(self, webcam_ids: List[str]):
self._webcam_ids = webcam_ids
- self._counters: Dict[str, FPSCounter] = self._init_counters(webcam_ids)
+ self._counters: Dict[str, TimestampLogger] = self._init_counters(webcam_ids)
def _init_counters(self, webcam_ids: List[str]):
d = {}
for webcam_id in webcam_ids:
- d[webcam_id] = FPSCounter()
+ d[webcam_id] = TimestampLogger()
return d
+ def get_timestamps_from_camera(self, webcam_id):
+ return self._counters[webcam_id]
+
def increment_frame_processed_for(self, webcam_id, timestamp):
self._counters[webcam_id].increment_frame_processed(timestamp)
diff --git a/src/core_processor/show_cam_window.py b/src/core_processor/show_cam_window.py
index b70f4a41..d84fa8a2 100644
--- a/src/core_processor/show_cam_window.py
+++ b/src/core_processor/show_cam_window.py
@@ -3,13 +3,13 @@ import logging
import cv2
import numpy as np
-from src.core_processor.fps.fps_counter import FPSCamCounter
+from src.core_processor.fps.timestamp_manager import TimestampManager
from src.core_processor.utils.image_fps_writer import write_fps_to_image
logger = logging.getLogger(__name__)
-def show_cam_window(webcam_id: str, image: np.array, fps_counter_cls: FPSCamCounter):
+def show_cam_window(webcam_id: str, image: np.array, fps_counter_cls: TimestampManager):
exit_key = cv2.waitKey(1)
if exit_key == 27:
logger.info("ESC has been pressed.")
diff --git a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
index 99d4a653..b18498b9 100644
--- a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
+++ b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
@@ -9,6 +9,8 @@ from pyqtgraph.dockarea.Dock import Dock
from pyqtgraph.dockarea.DockArea import DockArea
from pyqtgraph.Qt import QtWidgets
+from src.core_processor.fps.timestamp_manager import TimestampManager
+
logger = logging.getLogger(__name__)
@@ -62,54 +64,31 @@ class QTVisualizerAndGui:
raise e
camera_image_item_widget.setImage(cv2.cvtColor(image_to_display, cv2.COLOR_BGR2RGB))
- def update_timestamp_plots(self, dict_of_timestamps):
+ def update_timestamp_plots(self, timestamp_manager: TimestampManager):
if self.pause_button_pressed:
return
+ if timestamp_manager.get_timestamps_from_camera(self._camera0_id).number_of_frames <= 0:
+ return
- try:
- camera_0_timestamp = dict_of_timestamps[self._camera0_id]
- except:
- logger.info('no timestamp for camera0')
- camera_0_timestamp = np.nan
+ self._update_simple_timestamp_plot(timestamp_manager)
+ self._update_timestamp_difference_plot(timestamp_manager)
+ self._update_timestamp_difference_histogram(timestamp_manager)
+ def _update_simple_timestamp_plot(self, timestamp_manager: TimestampManager):
for webcam_id in self._webcam_ids_list:
- try:
- this_cameras_timestamp = dict_of_timestamps[webcam_id]
- time_diff_from_cam0 = np.abs(
- this_cameras_timestamp - camera_0_timestamp) # <- this is the actual calculation of timestamp diff
-
- self._dict_of_cam_timestamps[webcam_id] = np.append(self._dict_of_cam_timestamps[webcam_id],
- this_cameras_timestamp)
- self._dict_of_cam_timestamp_differences[webcam_id] = np.append(
- self._dict_of_cam_timestamp_differences[webcam_id],
- time_diff_from_cam0)
-
- except Exception as e:
- logger.warning(f'Could not find timestamp dictionary for camera {webcam_id}')
- raise e
-
- self._update_simple_timestamp_plot()
- self._update_timestamp_difference_plot()
- self._update_timestamp_difference_histogram()
-
- def _update_simple_timestamp_plot(self):
- for webcam_id, timestamps_array in self._dict_of_cam_timestamps.items():
plot_item = self._dict_of_simple_timestamp_line_plots[webcam_id]
- plot_item.setData(timestamps_array)
-
- def _update_timestamp_difference_plot(self):
+ plot_item.setData(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps)
- for webcam_id in self._dict_of_cam_timestamps.keys():
- self._dict_of_timestamp_difference_line_plots[webcam_id].setData(
- self._dict_of_cam_timestamp_differences[webcam_id])
-
- def _update_timestamp_difference_histogram(self):
+ def _update_timestamp_difference_plot(self, timestamp_manager: TimestampManager):
for webcam_id in self._webcam_ids_list:
- if webcam_id == self._camera0_id:
- continue
+ plot_item = self._dict_of_simple_timestamp_line_plots[webcam_id]
+ plot_item.setData(np.diff(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps))
- num_samples = self._dict_of_cam_timestamp_differences[webcam_id].shape[0]
- counts, bin_edges = np.histogram(self._dict_of_cam_timestamp_differences[webcam_id],
+ def _update_timestamp_difference_histogram(self, timestamp_manager: TimestampManager):
+ for webcam_id in self._webcam_ids_list:
+ this_cam_timestamp_diffs = np.diff(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps)
+ num_samples = this_cam_timestamp_diffs.shape[0]
+ counts, bin_edges = np.histogram(this_cam_timestamp_diffs,
bins=np.linspace(0, 100, 100))
counts = counts / num_samples
try:
diff --git a/src/session_pipeline/session_pipeline_orchestrator.py b/src/session_pipeline/session_pipeline_orchestrator.py
index f18b5bf7..ccdfdc0e 100644
--- a/src/session_pipeline/session_pipeline_orchestrator.py
+++ b/src/session_pipeline/session_pipeline_orchestrator.py
@@ -1,37 +1,33 @@
import time
-START_TIME_UNIX_NS = time.time()
-
import logging
import traceback
from pathlib import Path
import cv2
-import numpy as np
from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManager
from src.cameras.persistence.video_writer.save_options_dataclass import SaveOptions
from src.config.data_paths import freemocap_data_path
from src.core_processor.camera_calibration.camera_calibrator import CameraCalibrator
-from src.core_processor.fps.fps_counter import FPSCamCounter
+from src.core_processor.fps.timestamp_manager import TimestampManager
from src.core_processor.mediapipe_skeleton_detection.mediapipe_skeleton_detection import MediaPipeSkeletonDetection
from src.core_processor.show_cam_window import show_cam_window
from src.qt_visualizer_and_gui.qt_visualizer_and_gui import QTVisualizerAndGui
logger = logging.getLogger(__name__)
-
def get_canonical_time_str():
return time.strftime("%m-%d-%Y-%H_%M_%S")
class SessionPipelineOrchestrator:
def __init__(self):
- self._session_start_time_unix_ns = START_TIME_UNIX_NS
+ self._session_start_time_unix_ns = time.time_ns()
self._session_id = 'session_' + time.strftime("%m-%d-%Y-%H_%M_%S")
self._visualizer_gui = QTVisualizerAndGui()
self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id)
self._camera_calibrator = CameraCalibrator()
- self._mediapipe_skeleton_detector = MediaPipeSkeletonDetection(model_complexity=0)
+ self._mediapipe_skeleton_detector = MediaPipeSkeletonDetection(model_complexity=2)
@property
def session_folder_path(self):
@@ -41,7 +37,7 @@ class SessionPipelineOrchestrator:
with self._open_cv_camera_manager.start_capture_session_single_cam(
webcam_id=webcam_id,
) as session_obj:
- fps_manager = FPSCamCounter(self._open_cv_camera_manager.available_webcam_ids)
+ fps_manager = TimestampManager(self._open_cv_camera_manager.available_webcam_ids)
fps_manager.start_all()
cv_cam = session_obj.cv_camera
writer = session_obj.video_recorder
@@ -92,11 +88,10 @@ class SessionPipelineOrchestrator:
if show_visualizer_gui:
self._visualizer_gui.setup_and_launch(self._open_cv_camera_manager.available_webcam_ids)
- fps_manager = FPSCamCounter(self._open_cv_camera_manager.available_webcam_ids)
+ timestamp_manager = TimestampManager(self._open_cv_camera_manager.available_webcam_ids)
should_continue = True
while should_continue:
- this_frame_timestamps_dict = dict.fromkeys(self._open_cv_camera_manager.available_webcam_ids)
for this_webcam_id, this_open_cv_camera in connected_cameras_dict.items():
@@ -106,13 +101,13 @@ class SessionPipelineOrchestrator:
this_cam_latest_frame = this_open_cv_camera.latest_frame
if this_cam_latest_frame is None:
- image_to_display = this_open_cv_camera.latest_frame # if this camera has no new frame, write the previous one instead
- this_frame_timestamps_dict[this_webcam_id] = np.nan # frame drop?
- else:
- this_frame_timestamps_dict[this_webcam_id] = this_cam_latest_frame.timestamp
- image_to_display = this_cam_latest_frame.image
+ continue
- fps_manager.increment_frame_processed_for(this_webcam_id,this_frame_timestamps_dict[this_webcam_id])
+ image_to_display = this_cam_latest_frame.image
+ this_cam_this_frame_timestamp = this_cam_latest_frame.timestamp
+
+ timestamp_manager.increment_frame_processed_for(this_webcam_id,
+ this_cam_this_frame_timestamp)
if save_video:
this_open_cv_camera.video_recorder.record(this_cam_latest_frame)
@@ -127,12 +122,12 @@ class SessionPipelineOrchestrator:
if show_camera_views_in_windows:
should_continue = show_cam_window(
- this_webcam_id, image_to_display, fps_manager
+ this_webcam_id, image_to_display, timestamp_manager
)
if show_visualizer_gui:
self._visualizer_gui.update_camera_view_image(this_webcam_id, image_to_display)
- # self._visualizer_gui.update_timestamp_plots(this_frame_timestamps_dict)
+ # self._visualizer_gui.update_timestamp_plots(timestamp_manager)
except:
logger.error("Printing traceback")
@@ -150,13 +145,11 @@ class SessionPipelineOrchestrator:
self._visualizer_gui.close()
-
-
if __name__ == "__main__":
print('start main')
this_session = SessionPipelineOrchestrator()
- this_session.run(save_video=True,
+ this_session.run(save_video=False,
show_visualizer_gui=True,
calibrate_cameras=False,
detect_skeleton=True)