Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

various bug fixes - functional camera view

Jon513b733b8961
diff --git a/src/api/services/user_config.py b/src/api/services/user_config.py
index 3c9a5eaa..5737c77e 100644
--- a/src/api/services/user_config.py
+++ b/src/api/services/user_config.py
@@ -7,18 +7,18 @@ from src.config.webcam_config import WebcamConfig
@singleton
class UserConfigService:
- _webcam_config: Dict[str, WebcamConfig] = {}
+ _dict_of_webcam_configs: Dict[str, WebcamConfig] = {}
@property
def webcam_configs(self):
- return self._webcam_config
+ return self._dict_of_webcam_configs
- def webcam_config_by_id(self, webcam_id: str):
- if webcam_id in self._webcam_config:
- return self._webcam_config[webcam_id]
+ def webcam_config_by_id(self, webcam_id_as_str: str) -> WebcamConfig:
+ if webcam_id_as_str in self._dict_of_webcam_configs:
+ return self._dict_of_webcam_configs[webcam_id_as_str]
else:
- return WebcamConfig(webcam_id=int(webcam_id))
+ return WebcamConfig(webcam_id=int(webcam_id_as_str))
def set_webcam_config(self, webcam_config: WebcamConfig):
as_str = str(webcam_config.webcam_id)
- self._webcam_config[as_str] = webcam_config
+ self._dict_of_webcam_configs[as_str] = webcam_config
diff --git a/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py b/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py
index a0c3e9cd..a2ff1479 100644
--- a/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py
+++ b/src/cameras/capture/opencv_camera/camera_stream_thread_handler.py
@@ -21,13 +21,6 @@ class VideoCaptureThread(threading.Thread):
self._frame: FramePayload = FramePayload()
self.setDaemon(True)
- @property
- def average_fps(self):
- if self._elapsed <= 0:
- return 0
- if self._num_frames_processed <= 0:
- return 0
- return self._num_frames_processed / self._elapsed
@property
def latest_frame(self):
diff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.py
index 93a4160a..7e1c56ea 100644
--- a/src/cameras/capture/opencv_camera/opencv_camera.py
+++ b/src/cameras/capture/opencv_camera/opencv_camera.py
@@ -76,13 +76,13 @@ class OpenCVCamera:
self._apply_configuration()
success, image = self._opencv_video_capture_object.read()
- if not success:
+ if not success or image is None:
logger.error(
"Could not connect to a camera at port# {}".format(
self._config.webcam_id
)
)
- return success
+ return False
try:
image_height = image.shape[0]
@@ -157,11 +157,14 @@ class OpenCVCamera:
if not self._opencv_video_capture_object.grab():
return FramePayload(False, None, None)
- timestamp_ns = time.time_ns()
+ timestamp_ns_pre = time.perf_counter_ns()
success, image = self._opencv_video_capture_object.retrieve()
+ timestamp_ns_post = time.perf_counter_ns()
+
+ timestamp_ns = (timestamp_ns_pre + timestamp_ns_post)/2
self._new_frame_ready = success
- return FramePayload(success, image, timestamp_ns)
+ return FramePayload(success, image, timestamp_ns, self.webcam_id_as_str)
def stop_frame_capture(self):
self.close()
diff --git a/src/cameras/detection/cam_detection.py b/src/cameras/detection/cam_detection.py
index 14d3e7c8..2d7d81d0 100644
--- a/src/cameras/detection/cam_detection.py
+++ b/src/cameras/detection/cam_detection.py
@@ -19,7 +19,7 @@ class DetectPossibleCameras:
cap = cv2.VideoCapture(cam_id, cv2_backend)
success, image = cap.read()
- if success:
+ if success and image is not None:
try:
cams_to_use_list.append(
RawCamera(
diff --git a/src/cameras/multicam_manager/cv_camera_manager.py b/src/cameras/multicam_manager/cv_camera_manager.py
index 0bc7b9a0..a0efce0a 100644
--- a/src/cameras/multicam_manager/cv_camera_manager.py
+++ b/src/cameras/multicam_manager/cv_camera_manager.py
@@ -33,7 +33,7 @@ class OpenCVCameraManager:
if _open_cv_cameras is None:
logger.info("Creating cams.")
# we create the _cv_cams /once/, and reuse it for the lifetime of the session
- _open_cv_cameras = self._create_opencv_cameras(calibrate_cameras=True)
+ _open_cv_cameras = self._create_opencv_cameras()
self._open_cv_camera_objects = _open_cv_cameras
# self._timestamp_logger = TimestampLogger(self.available_webcam_ids, self.camera0_id)
self._number_of_cameras = len(self._open_cv_camera_objects)
@@ -82,7 +82,7 @@ class OpenCVCameraManager:
return cam
return None
- def _create_opencv_cameras(self, calibrate_cameras=True):
+ def _create_opencv_cameras(self):
raw_camera_objects = self._detected_cams_data.cameras_found_list
open_cv_cameras: List[OpenCVCamera] = []
for this_raw_cam in raw_camera_objects:
diff --git a/src/config/webcam_config.py b/src/config/webcam_config.py
index 0443d0be..76a823d9 100644
--- a/src/config/webcam_config.py
+++ b/src/config/webcam_config.py
@@ -2,9 +2,11 @@ from pathlib import Path
from pydantic import BaseModel
+
def _get_home_dir():
return str(Path.home())
+
class WebcamConfig(BaseModel):
webcam_id: int = 0
exposure: int = -6
@@ -13,4 +15,4 @@ class WebcamConfig(BaseModel):
save_video: bool = True
# fourcc: str = "MP4V"
fourcc: str = "MJPG"
- base_save_video_dir = _get_home_dir()
\ No newline at end of file
+ base_save_video_dir = _get_home_dir()
diff --git a/src/core_processor/fps/timestamp_manager.py b/src/core_processor/fps/timestamp_manager.py
index 4b1e6741..ef6dd1cc 100644
--- a/src/core_processor/fps/timestamp_manager.py
+++ b/src/core_processor/fps/timestamp_manager.py
@@ -7,7 +7,8 @@ class TimestampLogger:
_num_frames_processed: int
_start_time: float
- def __init__(self):
+ def __init__(self, _session_start_time_unix_ns: int):
+ self._session_start_time_unix_ns = _session_start_time_unix_ns
self._num_frames_processed = 0
self._timestamps = np.empty(0)
@@ -18,7 +19,7 @@ class TimestampLogger:
def median_frames_per_second(self):
if self._num_frames_processed <= 10:
return 0
- return np.nanmedian(np.diff(self._timestamps)/1e9)**-1
+ return np.nanmedian(np.diff(self._timestamps) / 1e9) ** -1
@property
def number_of_frames(self):
@@ -26,37 +27,52 @@ class TimestampLogger:
@property
def timestamps(self):
+ """return timestamps in nanoseconds in Unix epoch format (which is both the most useful, ugliest, AND funniest time format around. You can use `self.timestamps_in_seconds_from_zero()` for a more intuitive time format"""
return self._timestamps
+ @property
+ def timestamps_in_seconds_from_session_start(self):
+ """timestamps in seconds counting up from zero (where zero is the start time of the recording session)"""
+ return (self._timestamps - self._session_start_time_unix_ns) / 1e9
+
@property
def last_timestamp(self):
if self._timestamps.shape[0] > 0:
return self._timestamps[-1]
return None
- def increment_frame_processed(self, timestamp):
+ def log_new_timestamp(self, timestamp):
self._timestamps = np.append(self._timestamps, timestamp)
self._num_frames_processed += 1
class TimestampManager:
- def __init__(self, webcam_ids: List[str]):
+ def __init__(self, webcam_ids: List[str], session_start_time_unix_ns: int):
+ self._session_start_time_unix_ns = session_start_time_unix_ns
self._webcam_ids = webcam_ids
- self._counters: Dict[str, TimestampLogger] = self._init_counters(webcam_ids)
+ self._webcam_timestamp_loggers: Dict[str, TimestampLogger] = self._initialize_timestamp_loggers(webcam_ids)
+ self._main_loop_timestamp_logger = TimestampLogger(self._session_start_time_unix_ns)
- def _init_counters(self, webcam_ids: List[str]):
- d = {}
+ def _initialize_timestamp_loggers(self, webcam_ids: List[str]):
+ dictionary_of_timestamp_logger_objects = {}
for webcam_id in webcam_ids:
- d[webcam_id] = TimestampLogger()
- return d
+ dictionary_of_timestamp_logger_objects[webcam_id] = TimestampLogger(self._session_start_time_unix_ns)
+ return dictionary_of_timestamp_logger_objects
+
+ def get_timestamps_from_camera_in_seconds_from_session_start(self, webcam_id):
+ return self._webcam_timestamp_loggers[webcam_id].timestamps_in_seconds_from_session_start
- def get_timestamps_from_camera(self, webcam_id):
- return self._counters[webcam_id]
+ def get_number_of_frames(self, webcam_id):
+ return self._webcam_timestamp_loggers[webcam_id].number_of_frames
- def increment_frame_processed_for(self, webcam_id, timestamp):
- self._counters[webcam_id].increment_frame_processed(timestamp)
+ def increment_main_loop_timestamp_logger(self, timestamp):
+ self._main_loop_timestamp_logger.log_new_timestamp(timestamp)
- def median_frames_per_second(self, webcam_id):
- return self._counters[webcam_id].median_frames_per_second
+ def median_frames_per_second_for_main_loop(self):
+ return self._main_loop_timestamp_logger.median_frames_per_second
+ def increment_frame_processed_for_webcam(self, webcam_id, timestamp):
+ self._webcam_timestamp_loggers[webcam_id].log_new_timestamp(timestamp)
+ def median_frames_per_second_for_webcam(self, webcam_id):
+ return self._webcam_timestamp_loggers[webcam_id].median_frames_per_second
diff --git a/src/core_processor/show_cam_window.py b/src/core_processor/show_cam_window.py
index d84fa8a2..779b0ab8 100644
--- a/src/core_processor/show_cam_window.py
+++ b/src/core_processor/show_cam_window.py
@@ -9,15 +9,6 @@ from src.core_processor.utils.image_fps_writer import write_fps_to_image
logger = logging.getLogger(__name__)
-def show_cam_window(webcam_id: str, image: np.array, fps_counter_cls: TimestampManager):
- exit_key = cv2.waitKey(1)
- if exit_key == 27:
- logger.info("ESC has been pressed.")
- return False
-
- write_fps_to_image(
- image,
- fps_counter_cls.median_frames_per_second(webcam_id),
- )
+def show_cam_window(webcam_id: str, image: np.array, timestamp_manager_cls: TimestampManager):
cv2.imshow(webcam_id, image)
return True
diff --git a/src/core_processor/utils/image_fps_writer.py b/src/core_processor/utils/image_fps_writer.py
index bdfcba14..e1716379 100644
--- a/src/core_processor/utils/image_fps_writer.py
+++ b/src/core_processor/utils/image_fps_writer.py
@@ -1,8 +1,10 @@
+from typing import Union
+
import cv2
import numpy as np
-def write_fps_to_image(image: np.array, fps_number: int):
+def write_fps_to_image(image: np.array, fps_number: Union[int,float]):
cv2.putText(
image,
f"FPS: {str(fps_number)}",
diff --git a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
index 5b4cbd46..e34def08 100644
--- a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
+++ b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
@@ -47,9 +47,9 @@ class QTVisualizerAndGui:
self._setup_main_window()
self._setup_control_panel()
self._setup_camera_views_dock()
- self._setup_timestamp_plot()
- self._setup_time_difference_from_cam0_line_plot()
- self._setup_time_difference_from_cam0_histogram_plot()
+ # self._setup_timestamp_plot()
+ # self._setup_time_difference_from_cam0_line_plot()
+ # self._setup_time_difference_from_cam0_histogram_plot()
self._setup_3d_viewport()
logger.info('launching QT Visualizer and GUI window')
self._main_window_widget.show()
@@ -68,26 +68,26 @@ class QTVisualizerAndGui:
def update_timestamp_plots(self, timestamp_manager: TimestampManager):
if self.pause_button_pressed:
return
- if timestamp_manager.get_timestamps_from_camera(self._camera0_id).number_of_frames <= 0:
+ if timestamp_manager.get_number_of_frames(self._camera0_id) <= 0:
return
- self._update_simple_timestamp_plot(timestamp_manager)
- self._update_timestamp_difference_plot(timestamp_manager)
- self._update_timestamp_difference_histogram(timestamp_manager)
+ # self._update_simple_timestamp_plot(timestamp_manager)
+ # self._update_timestamp_difference_plot(timestamp_manager)
+ # self._update_timestamp_difference_histogram(timestamp_manager)
def _update_simple_timestamp_plot(self, timestamp_manager: TimestampManager):
for webcam_id in self._webcam_ids_list:
plot_item = self._dict_of_simple_timestamp_line_plots[webcam_id]
- plot_item.setData(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps)
+ plot_item.setData(timestamp_manager.get_timestamps_from_camera_in_seconds_from_session_start(webcam_id))
def _update_timestamp_difference_plot(self, timestamp_manager: TimestampManager):
for webcam_id in self._webcam_ids_list:
- plot_item = self._dict_of_simple_timestamp_line_plots[webcam_id]
- plot_item.setData(np.diff(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps))
+ plot_item = self._dict_of_timestamp_difference_line_plots[webcam_id]
+ plot_item.setData(np.diff(timestamp_manager.get_timestamps_from_camera_in_seconds_from_session_start(webcam_id)))
def _update_timestamp_difference_histogram(self, timestamp_manager: TimestampManager):
for webcam_id in self._webcam_ids_list:
- this_cam_timestamp_diffs = np.diff(timestamp_manager.get_timestamps_from_camera(webcam_id).timestamps)
+ this_cam_timestamp_diffs = np.diff(timestamp_manager.get_timestamps_from_camera_in_seconds_from_session_start(webcam_id))
num_samples = this_cam_timestamp_diffs.shape[0]
counts, bin_edges = np.histogram(this_cam_timestamp_diffs,
bins=np.linspace(0, 100, 100))
@@ -100,7 +100,7 @@ class QTVisualizerAndGui:
logger.warning(f'Could not find timestamp difference line plot widget for camera {webcam_id}')
raise e
- def _setup_main_window(self, window_width: int = 1000, window_height: int = 1000):
+ def _setup_main_window(self, window_width: int = 1280, window_height: int = 720):
"""
This is the main main window for the GUI.
Its structure is based loosely on the 'Dock Widgets' example from `python -m pyqtgraph.examples`
@@ -270,4 +270,4 @@ class QTVisualizerAndGui:
self.opengl_3d_plot_dock = Dock("3d View Port")
self.opengl_3d_plot_dock.addWidget(self.opengl_3d_plot_widget)
- self._main_dock_area.addDock(self.opengl_3d_plot_dock, 'right', self._timestamp_diff_histgoram_dock )
+ self._main_dock_area.addDock(self.opengl_3d_plot_dock, 'right', self._camera_views_dock )
diff --git a/src/session_pipeline/session_pipeline_orchestrator.py b/src/session_pipeline/session_pipeline_orchestrator.py
index ccdfdc0e..05b1de9d 100644
--- a/src/session_pipeline/session_pipeline_orchestrator.py
+++ b/src/session_pipeline/session_pipeline_orchestrator.py
@@ -12,23 +12,35 @@ from src.core_processor.camera_calibration.camera_calibrator import CameraCalibr
from src.core_processor.fps.timestamp_manager import TimestampManager
from src.core_processor.mediapipe_skeleton_detection.mediapipe_skeleton_detection import MediaPipeSkeletonDetection
from src.core_processor.show_cam_window import show_cam_window
+from src.core_processor.utils.image_fps_writer import write_fps_to_image
from src.qt_visualizer_and_gui.qt_visualizer_and_gui import QTVisualizerAndGui
logger = logging.getLogger(__name__)
+
def get_canonical_time_str():
return time.strftime("%m-%d-%Y-%H_%M_%S")
class SessionPipelineOrchestrator:
- def __init__(self):
+
+ def __init__(self, session_id: str = None):
+
+ if session_id is not None:
+ self._session_id = session_id
+ else:
+ self._session_id = 'session_' + time.strftime("%m-%d-%Y-%H_%M_%S")
+
self._session_start_time_unix_ns = time.time_ns()
- self._session_id = 'session_' + time.strftime("%m-%d-%Y-%H_%M_%S")
self._visualizer_gui = QTVisualizerAndGui()
self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id)
self._camera_calibrator = CameraCalibrator()
self._mediapipe_skeleton_detector = MediaPipeSkeletonDetection(model_complexity=2)
+ @property
+ def session_id(self):
+ return self._session_id
+
@property
def session_folder_path(self):
return freemocap_data_path / self._session_id
@@ -65,7 +77,7 @@ class SessionPipelineOrchestrator:
"charuco_board_detection",
f"webcam_{cv_cam.webcam_id_as_str}",
),
- fps=fps_manager.median_frames_per_second(cv_cam.webcam_id_as_str),
+ fps=fps_manager.median_frames_per_second_for_webcam(cv_cam.webcam_id_as_str),
frame_width=cv_cam.image_width(),
frame_height=cv_cam.image_height(),
)
@@ -88,11 +100,14 @@ class SessionPipelineOrchestrator:
if show_visualizer_gui:
self._visualizer_gui.setup_and_launch(self._open_cv_camera_manager.available_webcam_ids)
- timestamp_manager = TimestampManager(self._open_cv_camera_manager.available_webcam_ids)
+ timestamp_manager = TimestampManager(self._open_cv_camera_manager.available_webcam_ids,
+ self._session_start_time_unix_ns)
should_continue = True
while should_continue:
+ timestamp_manager.increment_main_loop_timestamp_logger(time.perf_counter_ns())
+
for this_webcam_id, this_open_cv_camera in connected_cameras_dict.items():
if not this_open_cv_camera.new_frame_ready:
@@ -103,11 +118,11 @@ class SessionPipelineOrchestrator:
if this_cam_latest_frame is None:
continue
- image_to_display = this_cam_latest_frame.image
- this_cam_this_frame_timestamp = this_cam_latest_frame.timestamp
+ image_to_display = this_cam_latest_frame.image.copy()
+ this_cam_this_frame_timestamp_ns = this_cam_latest_frame.timestamp
- timestamp_manager.increment_frame_processed_for(this_webcam_id,
- this_cam_this_frame_timestamp)
+ timestamp_manager.increment_frame_processed_for_webcam(this_webcam_id,
+ this_cam_this_frame_timestamp_ns)
if save_video:
this_open_cv_camera.video_recorder.record(this_cam_latest_frame)
@@ -126,8 +141,20 @@ class SessionPipelineOrchestrator:
)
if show_visualizer_gui:
+ write_fps_to_image(
+ image_to_display,
+ timestamp_manager.median_frames_per_second_for_webcam(this_webcam_id),
+ )
self._visualizer_gui.update_camera_view_image(this_webcam_id, image_to_display)
- # self._visualizer_gui.update_timestamp_plots(timestamp_manager)
+ self._visualizer_gui.update_timestamp_plots(timestamp_manager)
+
+ # exit loop when user presses ESC key
+ exit_key = cv2.waitKey(1)
+ if exit_key == 27:
+ logger.info("ESC has been pressed.")
+ should_continue = False
+
+
except:
logger.error("Printing traceback")
@@ -148,8 +175,8 @@ class SessionPipelineOrchestrator:
if __name__ == "__main__":
print('start main')
- this_session = SessionPipelineOrchestrator()
- this_session.run(save_video=False,
- show_visualizer_gui=True,
- calibrate_cameras=False,
- detect_skeleton=True)
+ this_session_orchestrator = SessionPipelineOrchestrator()
+ this_session_orchestrator.run(save_video=False,
+ show_visualizer_gui=True,
+ calibrate_cameras=False,
+ detect_skeleton=True)