Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

ok i think i saved christmas πŸŽ…

Joncdbaf8973603
diff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.py
index 17e67e83..5656a825 100644
--- a/src/cameras/capture/opencv_camera/opencv_camera.py
+++ b/src/cameras/capture/opencv_camera/opencv_camera.py
@@ -24,17 +24,11 @@ class OpenCVCamera:
def __init__(self,
config: WebcamConfig,
- session_id: str = None,
- timestamp_logger: TimestampLogger = None,
- camera_view_update_function = None,
+ session_id: str,
+ session_start_time_perf_counter_ns:int,
calibration_video_bool: bool = False):
- if timestamp_logger is None:
- self._timestamp_logger = TimestampLogger()
- else:
- self._timestamp_logger = timestamp_logger
- self._camera_view_update_function = camera_view_update_function
self._config = config
self._name = f"Camera_{self._config.webcam_id}"
@@ -45,6 +39,7 @@ class OpenCVCamera:
self._number_of_frames_recorded = 0
self._calibration_video_bool = calibration_video_bool
self._session_id = session_id
+ self._session_start_time_perf_counter_ns = session_start_time_perf_counter_ns
@property
def session_id(self):
@@ -190,6 +185,7 @@ class OpenCVCamera:
### A - see -> https://stackoverflow.com/questions/57716962/difference-between-video-capture-read-and-grab
self._opencv_video_capture_object.grab()
success, image = self._opencv_video_capture_object.retrieve()
+ this_frame_timestamp_perf_counter_ns = time.perf_counter_ns()-self._session_start_time_perf_counter_ns
# timestamp_ns_post = time.perf_counter_ns()
# it_took_this_many_seconds_to_grab_the_frame = (timestamp_ns_post-timestamp_ns_pre)/1e9
@@ -197,9 +193,6 @@ class OpenCVCamera:
logger.error(f"Failed to read frame from Camera: {self.webcam_id_as_str}")
raise Exception
- self._timestamp_logger.log_new_timestamp_perf_counter_ns(time.perf_counter_ns())
-
- self._timestamp_logger.log_new_timestamp_perf_counter_ns(time.perf_counter_ns())
self._new_frame_ready = success
@@ -210,7 +203,7 @@ class OpenCVCamera:
return FramePayload(success=success,
image=image,
- timestamp_in_seconds_from_record_start=self._timestamp_logger.latest_timestamp_in_seconds_from_record_start,
+ timestamp_in_seconds_from_record_start=this_frame_timestamp_perf_counter_ns/1e9,
timestamp_unix_time_seconds = time.time(),
frame_number=self.latest_frame_number,
webcam_id=self.webcam_id_as_str)
diff --git a/src/cameras/launch_camera_frame_loop.py b/src/cameras/launch_camera_frame_loop.py
index 107e56c3..9d4cd4c0 100644
--- a/src/cameras/launch_camera_frame_loop.py
+++ b/src/cameras/launch_camera_frame_loop.py
@@ -34,8 +34,8 @@ def launch_camera_frame_loop(
- opencv_camera_manager = OpenCVCameraManager(session_id=session_id,
- expected_framerate=None)
+ opencv_camera_manager = OpenCVCameraManager(session_id=session_id)
+
with opencv_camera_manager.start_capture_session_all_cams(
webcam_configs_dict=webcam_configs_dict,
calibration_videos=calibration_videos_bool,
@@ -53,12 +53,12 @@ def launch_camera_frame_loop(
should_continue = True
while should_continue:
- timestamp_manager.log_new_timestamp_for_main_loop_perf_coutner_ns(time.perf_counter_ns())
if not opencv_camera_manager.new_multi_frame_ready():
continue
this_multi_frame_payload = opencv_camera_manager.latest_multi_frame
+ timestamp_manager.multi_frame_timestamp_logger.log_new_timestamp_seconds_from_unspecified_zero(this_multi_frame_payload.multi_frame_timestamp_seconds)
for this_webcam_id, this_open_cv_camera in connected_cameras_dict.items():
@@ -67,6 +67,9 @@ def launch_camera_frame_loop(
if this_cam_latest_frame is None:
continue
+ this_cam_timestamp_logger = timestamp_manager.timestamp_logger_for_webcam_id(this_webcam_id)
+ this_cam_timestamp_logger.log_new_timestamp_seconds_from_unspecified_zero(this_cam_latest_frame.timestamp_in_seconds_from_record_start)
+
# save frame to video file
if record_frames_bool:
any_frames_recorded = True
@@ -82,16 +85,13 @@ def launch_camera_frame_loop(
image_to_display = write_fps_to_image(
image_to_display,
- timestamp_manager.median_frames_per_second_for_webcam(this_webcam_id),
+ timestamp_manager.timestamp_logger_for_webcam_id(this_webcam_id).median_frames_per_second,
)
if show_camera_views_in_windows:
- should_continue = show_cam_window(
- this_webcam_id, image_to_display, timestamp_manager
- )
+ should_continue = show_cam_window(this_webcam_id, image_to_display )
if show_visualizer_gui:
-
visualizer_gui.update_camera_view_image(this_webcam_id,
image_to_display)
diff --git a/src/cameras/multicam_manager/cv_camera_manager.py b/src/cameras/multicam_manager/cv_camera_manager.py
index d9d10cc5..9c43ac6f 100644
--- a/src/cameras/multicam_manager/cv_camera_manager.py
+++ b/src/cameras/multicam_manager/cv_camera_manager.py
@@ -29,7 +29,6 @@ class CamAndWriterResponse(BaseModel):
class OpenCVCameraManager:
def __init__(self,
session_id: str = None,
- expected_framerate: Union[int, float] = None,
shut_down_event_bool:bool = None,
):
self._session_id = session_id
@@ -39,8 +38,9 @@ class OpenCVCameraManager:
self._timestamp_manager: Union[TimestampManager, None] = None
self._connected_cameras_dict = {}
self._available_cameras_dict = {}
- self._expected_framerate = expected_framerate
self._shut_down_event_bool = shut_down_event_bool
+ self._number_of_multi_frames = 0
+
@property
def timestamp_manager(self):
@@ -60,27 +60,24 @@ class OpenCVCameraManager:
logging.error("a multi_frame was requested before it was ready!")
raise Exception
+ this_multi_frame_timestamp_sec = (time.perf_counter_ns() - self._session_start_time_perf_counter_ns)/1e9
+
this_multi_frame_dict = {}
+ each_cam_timestamp = []
+
for this_cam in self._connected_cameras_dict.values():
if not this_cam.new_frame_ready:
logger.error('It shouldnt be able to get here if the `new_multi_frame`')
+
this_multi_frame_dict[this_cam.webcam_id_as_str] = this_cam.latest_frame
- self._timestamp_manager.log_new_multi_frame_timestamp_ns(time.perf_counter_ns())
+ each_cam_timestamp.append(this_cam.latest_frame.timestamp_in_seconds_from_record_start)
- if not self._timestamp_manager.verify_multi_frame_is_synchronized(this_multi_frame_dict,
- self._expected_framerate):
- m_f_interval = self._timestamp_manager.latest_multi_frame_interval
- m_f_number = self._timestamp_manager.multi_frame_timestamp_logger.number_of_frames
- logger.error(
- f"Multi frame is not synchronized!! Skipping this one - multi_frame_frame_number|interval: {m_f_number}|{m_f_interval:.3f}")
- # raise Exception
- return None
+ self._number_of_multi_frames +=1
return MultiFramePayload(frames_dict=this_multi_frame_dict,
- multi_frame_number=self._timestamp_manager.multi_frame_timestamp_logger.number_of_frames,
- intra_frame_interval=self._timestamp_manager.latest_multi_frame_interval,
- each_frame_timestamp=self._timestamp_manager.latest_multi_frame_timestamp_list,
- multi_frame_timestamp=self._timestamp_manager.multi_frame_timestamp_logger.latest_timestamp_in_seconds_from_record_start,
+ multi_frame_number=self._number_of_multi_frames,
+ each_frame_timestamp=each_cam_timestamp,
+ multi_frame_timestamp_seconds=this_multi_frame_timestamp_sec
)
def new_multi_frame_ready(self):
@@ -129,13 +126,11 @@ class OpenCVCameraManager:
self._initialize_timestamp_logger()
open_cv_camera_objects = self._create_opencv_cameras(
+ session_start_time_perf_counter_ns=self._session_start_time_perf_counter_ns,
webcam_configs_dict=webcam_configs_dict,
- timestamp_manager=self._timestamp_manager,
- camera_view_update_function=camera_view_update_function,
calibration_videos=calibration_videos, )
try:
-
for cv_cam in open_cv_camera_objects:
self._connected_cameras_dict[cv_cam.webcam_id_as_str] = cv_cam
self._start_frame_capture_on_cam_id(cv_cam)
@@ -148,28 +143,26 @@ class OpenCVCameraManager:
traceback.print_exc()
def _create_opencv_cameras(self,
- timestamp_manager: TimestampManager,
+ session_start_time_perf_counter_ns: int,
webcam_configs_dict: Dict[str, WebcamConfig] = None,
- camera_view_update_function=None,
- calibration_videos: bool = False,
+ calibration_videos: bool = False,
):
raw_camera_objects = self._detected_cams_data.cameras_found_list
open_cv_cameras: List[OpenCVCamera] = []
- if webcam_configs_dict is None:
- webcam_configs_dict = {}
+
for this_raw_cam in raw_camera_objects:
- this_cam_timestamp_logger = timestamp_manager.timestamp_logger_for_webcam_id(
- webcam_id=this_raw_cam.webcam_id)
- webcam_configs_dict[this_raw_cam.webcam_id] = WebcamConfig(webcam_id=this_raw_cam.webcam_id)
+ if webcam_configs_dict is None:
+ this_webcam_config = WebcamConfig(webcam_id=this_raw_cam.webcam_id)
+ else:
+ this_webcam_config = webcam_configs_dict[this_raw_cam.webcam_id]
opencv_cam_obj = self._create_single_opencv_cam(webcam_id=this_raw_cam.webcam_id,
- webcam_config=webcam_configs_dict[this_raw_cam.webcam_id],
- timestamp_logger=this_cam_timestamp_logger,
- camera_view_update_function=camera_view_update_function,
+ webcam_config=this_webcam_config,
+ session_start_time_perf_counter_ns = session_start_time_perf_counter_ns,
calibration_video_bool=calibration_videos)
open_cv_cameras.append(opencv_cam_obj)
return open_cv_cameras
@@ -177,8 +170,7 @@ class OpenCVCameraManager:
def _create_single_opencv_cam(self,
webcam_id: str,
webcam_config: WebcamConfig(),
- timestamp_logger: TimestampLogger,
- camera_view_update_function=None,
+ session_start_time_perf_counter_ns:int,
calibration_video_bool: bool = False,
):
#
@@ -192,8 +184,7 @@ class OpenCVCameraManager:
webcam_config.webcam_id = webcam_id
return OpenCVCamera(config=webcam_config,
session_id=self._session_id,
- timestamp_logger=timestamp_logger,
- camera_view_update_function=camera_view_update_function,
+ session_start_time_perf_counter_ns = session_start_time_perf_counter_ns,
calibration_video_bool=calibration_video_bool)
def _initialize_timestamp_logger(self):
diff --git a/src/core_processor/show_cam_window.py b/src/core_processor/show_cam_window.py
index c931721d..8a98c2ff 100644
--- a/src/core_processor/show_cam_window.py
+++ b/src/core_processor/show_cam_window.py
@@ -9,6 +9,6 @@ from src.core_processor.utils.image_fps_writer import write_fps_to_image
logger = logging.getLogger(__name__)
-def show_cam_window(webcam_id: str, image: np.array, timestamp_manager_cls: TimestampManager):
+def show_cam_window(webcam_id: str, image: np.array,):
cv2.imshow(webcam_id, image)
return True
diff --git a/src/core_processor/timestamp_manager/timestamp_manager.py b/src/core_processor/timestamp_manager/timestamp_manager.py
index 633031bc..76760242 100644
--- a/src/core_processor/timestamp_manager/timestamp_manager.py
+++ b/src/core_processor/timestamp_manager/timestamp_manager.py
@@ -1,7 +1,6 @@
import logging
-import time
from pathlib import Path
-from typing import Dict, List, Union
+from typing import Dict, List
import numpy as np
from matplotlib import pyplot as plt
@@ -23,6 +22,7 @@ class TimestampLogger:
self._session_start_time_perf_counter_ns = session_start_time_perf_counter_ns
self._num_frames_processed = 0
self._timestamps_perf_counter_ns = np.empty(0)
+ self._timestamps_from_zero = np.empty(0)
@property
@@ -54,6 +54,13 @@ class TimestampLogger:
self._timestamps_perf_counter_ns = np.append(self._timestamps_perf_counter_ns, timestamp)
self._num_frames_processed += 1
+ def log_new_timestamp_seconds_from_unspecified_zero(self, timestamp:float):
+ self._timestamps_from_zero = np.append(self._timestamps_from_zero, timestamp)
+ self._num_frames_processed += 1
+
+ @property
+ def timestamps_in_seconds_from_unspecified_zero(self):
+ return self._timestamps_from_zero
class TimestampManager:
def __init__(self,
@@ -76,39 +83,16 @@ class TimestampManager:
- self._multi_frame_timestamps_list = []
- self._multi_frame_interval_list = []
-
- @property
- def webcam_timestamp_loggers(self):
- return self._webcam_timestamp_loggers
-
def timestamp_logger_for_webcam_id(self, webcam_id:str)->TimestampLogger:
return self._webcam_timestamp_loggers[webcam_id]
- @property
- def number_of_cameras(self):
- return len(self._webcam_timestamp_loggers)
@property
def multi_frame_timestamp_logger(self):
return self._multi_frame_timestamp_logger
- @property
- def multi_frame_timestamps_list(self):
- return self._multi_frame_timestamps_list
-
- @property
- def multi_frame_interval_list(self):
- return self._multi_frame_interval_list
- @property
- def latest_multi_frame_timestamp_list(self):
- return self._multi_frame_timestamps_list[-1]
- @property
- def latest_multi_frame_interval(self):
- return self._multi_frame_interval_list[-1]
def _initialize_webcam_timestamp_loggers(self, webcam_ids: List[str]):
dictionary_of_timestamp_logger_objects = {}
@@ -117,71 +101,36 @@ class TimestampManager:
session_start_time_perf_counter_ns=self._session_start_time_perf_counter_ns)
return dictionary_of_timestamp_logger_objects
- def get_timestamps_from_camera_in_seconds_from_session_start(self, webcam_id):
- return self._webcam_timestamp_loggers[webcam_id].timestamps_in_seconds_from_session_start
-
- def number_of_frames(self, webcam_id):
- return self._webcam_timestamp_loggers[webcam_id].number_of_frames
- def median_frames_per_second_for_main_loop(self):
- return self._main_loop_timestamp_logger.median_frames_per_second
-
- def median_frames_per_second_for_webcam(self, webcam_id):
- return self._webcam_timestamp_loggers[webcam_id].median_frames_per_second
-
- def log_new_timestamp_for_main_loop_perf_coutner_ns(self, timestamp):
- self._main_loop_timestamp_logger.log_new_timestamp_perf_counter_ns(timestamp)
-
- def log_new_multi_frame_timestamp_ns(self, timestamp):
- self._multi_frame_timestamp_logger.log_new_timestamp_perf_counter_ns(timestamp)
-
- def log_new_timestamp_for_webcam_ns(self,
- webcam_id: str,
- timestamp: Union[float, int],
- frame_number: int = None):
- self._webcam_timestamp_loggers[webcam_id].log_new_timestamp_perf_counter_ns(timestamp)
- if frame_number is not None:
- if not self._webcam_timestamp_loggers[webcam_id].number_of_frames == frame_number:
- logger.error("Numer of timestamps ({self._webcam_timestamp_loggers[webcam_id].number_of_frames}) logged does not match frame_number ({frame_number}) for: camera_{webcam_id}")
def _get_each_frame_timestamp(self, this_multi_frame_dict) -> List:
-
this_multi_frame_timestamps = []
for this_cam_id, this_cam_frame in this_multi_frame_dict.items():
this_multi_frame_timestamps.append(this_cam_frame.timestamp_in_seconds_from_record_start)
return this_multi_frame_timestamps
-
- def _get_intra_frame_interval_in_seconds(self, this_multi_frame_timestamps) -> float:
- timestamp_min = np.min(this_multi_frame_timestamps)
- timestamp_max = np.max(this_multi_frame_timestamps)
- return (timestamp_max - timestamp_min) / 1e9
+ def _get_intra_frame_interval_in_seconds(self, this_multi_frame_timestamps_in_seconds) -> float:
+ timestamp_min = np.min(this_multi_frame_timestamps_in_seconds)
+ timestamp_max = np.max(this_multi_frame_timestamps_in_seconds)
+ return (timestamp_max - timestamp_min)
def verify_multi_frame_is_synchronized(self,
- this_multi_frame_dict,
- expected_framerate: Union[int, None]) -> bool:
+ this_multi_frame_dict,) -> bool:
this_multi_frame_timestamps = self._get_each_frame_timestamp(this_multi_frame_dict)
this_multiframe_timestamp_interval_in_seconds = self._get_intra_frame_interval_in_seconds(this_multi_frame_timestamps)
- self._multi_frame_timestamps_list.append(this_multi_frame_timestamps)
- self._multi_frame_interval_list.append(this_multiframe_timestamp_interval_in_seconds)
- if expected_framerate is None:
- # logger.warning('`expected_framerate` not specified, Cannot verify multi_frame synchronization')
- print(f"this_multiframe_timestamp_interval: {this_multiframe_timestamp_interval_in_seconds*1e3:.6f} milliseconds")
- return True
- if this_multiframe_timestamp_interval_in_seconds > 2*(expected_framerate ** -1):
- return False
+ print(f"this_multiframe_timestamp_interval: {this_multiframe_timestamp_interval_in_seconds*1e3:.6f} milliseconds")
return True
def create_diagnostic_plots(self):
"""plot some diagnostics to assess quality of camera sync"""
- multi_frame_timestamp_list = self.multi_frame_timestamp_logger.timestamps_in_seconds_from_session_start
- multi_frame_interval_list = self.multi_frame_interval_list
+ multi_frame_timestamp_list = self.multi_frame_timestamp_logger.timestamps_in_seconds_from_unspecified_zero
+ # multi_frame_interval_list = self.multi_frame_interval_list
fig = plt.figure(figsize=(18, 10))
max_frame_duration = 0.1
@@ -226,8 +175,8 @@ class TimestampManager:
ylabel="Probability",
)
- for this_camera_id, this_camera_timestamp_logger in self.webcam_timestamp_loggers.items():
- this_camera_timestamps = this_camera_timestamp_logger.timestamps_in_seconds_from_session_start
+ for this_camera_id, this_camera_timestamp_logger in self._webcam_timestamp_loggers.items():
+ this_camera_timestamps = this_camera_timestamp_logger._timestamps_from_zero
ax1.plot(this_camera_timestamps, label=f"Camera# {this_camera_id}")
ax1.legend()
ax2.plot(np.diff(this_camera_timestamps), ".")
@@ -243,31 +192,31 @@ class TimestampManager:
label=f"MultiFrame"
)
ax5.plot(
- np.diff(multi_frame_timestamp_list),
+ np.diff(np.asarray(multi_frame_timestamp_list)),
".",
color="darkslategrey",
label="Multi Frame Duration",
)
- ax5.plot(
- multi_frame_interval_list, ".", color="orangered", label="Frame TimeSpan"
- )
+ # ax5.plot(
+ # multi_frame_interval_list, ".", color="orangered", label="Frame TimeSpan"
+ # )
ax5.legend()
ax6.hist(
- np.diff(multi_frame_timestamp_list),
+ np.diff(np.asarray(multi_frame_timestamp_list)),
bins=np.arange(0, max_frame_duration, 0.0025),
density=True,
alpha=0.5,
color="darkslategrey",
label="Frame Duration",
)
- ax6.hist(
- np.diff(multi_frame_interval_list),
- bins=np.arange(0, max_frame_duration, 0.0025),
- density=True,
- alpha=0.5,
- color="orangered",
- label="Frame Timespan",
- )
+ # ax6.hist(
+ # np.diff(multi_frame_interval_list),
+ # bins=np.arange(0, max_frame_duration, 0.0025),
+ # density=True,
+ # alpha=0.5,
+ # color="orangered",
+ # label="Frame Timespan",
+ # )
ax5.legend()
fig_save_path = Path(get_session_folder_path(self._session_id)) / "camera_timestamp_diagnostics.png"
diff --git a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py
index ef00ff2a..ea34efa3 100644
--- a/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py
+++ b/src/pipelines/calibration_pipeline/calibration_pipeline_orchestrator.py
@@ -37,8 +37,7 @@ class CalibrationPipelineOrchestrator:
self._charuco_board_detector = CharucoBoardDetector()
self._visualizer_gui = None
self._expected_framerate = expected_framerate
- self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id,
- expected_framerate=self._expected_framerate)
+ self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id)
@property
def session_id(self):
diff --git a/src/pipelines/session_pipeline/data_classes/multi_frame_payload.py b/src/pipelines/session_pipeline/data_classes/multi_frame_payload.py
index ac741eae..f6ca2d71 100644
--- a/src/pipelines/session_pipeline/data_classes/multi_frame_payload.py
+++ b/src/pipelines/session_pipeline/data_classes/multi_frame_payload.py
@@ -8,6 +8,5 @@ from src.cameras.capture.dataclasses.frame_payload import FramePayload
class MultiFramePayload(NamedTuple):
frames_dict: Dict[str, FramePayload]
multi_frame_number: int
- intra_frame_interval: Union[int, float]
each_frame_timestamp: List[Union[int, float]]
- multi_frame_timestamp: Union[int, float]
+ multi_frame_timestamp_seconds: Union[int, float]