Free Motion Capture for Everyone πβ¨
TypeScript
timestamp vis and very basic skel tracking working
Joncde2edf2e8b8
diff --git a/requirements.txt b/requirements.txtindex 91a54d89..d1095aea 100644--- a/requirements.txt+++ b/requirements.txt@@ -1,7 +1,7 @@aniposelib==0.4.3h5py>=3.1.0imutils~=0.5.4-matplotlib~=3.1.3+matplotlib~=3.5.1# mediapipe==0.8.8opencv-contrib-python==4.5.5.62pandas~=1.2.5@@ -30,4 +30,6 @@ pre-commit==2.17.0# Requirements for core-processorpyqtgraph==0.12.4pyqt5==5.15.6-pyopengl==3.1.6\ No newline at end of file+pyopengl==3.1.6+mediapipe~=0.8.9.1+setuptools~=58.0.4\ No newline at end of filediff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.pyindex f7240b48..8302d7b5 100644--- a/src/cameras/capture/opencv_camera/opencv_camera.py+++ b/src/cameras/capture/opencv_camera/opencv_camera.py@@ -46,7 +46,7 @@ class OpenCVCamera:@propertydef is_capturing_frames(self):if not self._running_thread:- logger.error("Frame Capture thread not running yet")+ logger.info("Frame Capture thread not running yet")return Falsereturn self._running_thread.is_capturing_framesdiff --git a/src/cameras/multicam_manager/cv_camera_manager.py b/src/cameras/multicam_manager/cv_camera_manager.pyindex db96f3b4..57750ef4 100644--- a/src/cameras/multicam_manager/cv_camera_manager.py+++ b/src/cameras/multicam_manager/cv_camera_manager.py@@ -3,6 +3,7 @@ import tracebackfrom contextlib import contextmanagerfrom typing import ContextManager, Dict, List+import numpy as npfrom pydantic import BaseModelfrom src.api.services.user_config import UserConfigService@@ -24,7 +25,7 @@ class CamAndWriterResponse(BaseModel):class OpenCVCameraManager:- def __init__(self, session_id = str):+ def __init__(self, session_id=str):self._session_id = session_idself._config_service = UserConfigService()self._detected_cams_data = get_or_create_cams()@@ -34,18 +35,44 @@ class OpenCVCameraManager:# we create the _cv_cams /once/, and reuse it for the lifetime of the session_open_cv_cameras = self._create_opencv_cameras(calibrate_cameras=True)self._open_cv_camera_objects = _open_cv_cameras+ # self._timestamp_logger = TimestampLogger(self.available_webcam_ids, self.camera0_id)+ self._number_of_cameras = len(self._open_cv_camera_objects)else:logger.info("Reusing already created resources cam resources.")self._open_cv_camera_objects = _open_cv_cameras+ @property+ def number_of_cameras(self):+ return self._number_of_cameras+@propertydef available_webcam_ids(self):return [cv_cam.webcam_id_as_str for cv_cam in self._open_cv_camera_objects]+ @property+ def camera0_id(self):+ """ ID of the first camera found. This camera will be used as the reference point for time sync and camera position and rotation"""+ return self.available_webcam_ids[0]+@propertydef open_cv_cameras(self):return self._open_cv_camera_objects+ def new_synchronized_frame_available(self):+ new_frame_bool_list = [cam.new_frame for cam in self._open_cv_camera_objects]+ if all(new_frame_bool_list):+ return True+ return False++ def latest_synchronized_frame(self):+ synchronized_frame_dict = {}+ for this_cam in self._open_cv_camera_objects:+ if not this_cam.new_frame:+ logger.error(f'New frame not available for camera {this_cam.webcam_id_as_str}')+ synchronized_frame_dict[this_cam.webcam_id_as_str] = None+ synchronized_frame_dict[this_cam.webcam_id_as_str] = this_cam.latest_frame+ return synchronized_frame_dict+def cv_cam_by_id(self, webcam_id: str):for cam in self._open_cv_camera_objects:if cam.webcam_id_as_str == str(webcam_id):@@ -63,7 +90,7 @@ class OpenCVCameraManager:@contextmanagerdef start_capture_session_single_cam(- self, webcam_id: str = None+ self, webcam_id: str = None) -> ContextManager[CamAndWriterResponse]:"""Context manager for easy start up, usage, and cleanup of camera resources.@@ -80,16 +107,16 @@ class OpenCVCameraManager:@contextmanagerdef start_capture_session_all_cams(- self,- ) -> ContextManager[List[CamAndWriterResponse]]:+ self,+ ) -> ContextManager[Dict[str, CamAndWriterResponse]]:try:+ available_cam_and_writer_dict = {}writer_dict = self._start_frame_capture_all_cams()- responses = [- CamAndWriterResponse(cv_cam=self.cv_cam_by_id(webcam_id), writer=writer)- for webcam_id, writer in writer_dict.items()- ]- yield responses+ for webcam_id, writer in writer_dict.items():+ available_cam_and_writer_dict[webcam_id] = CamAndWriterResponse(cv_cam=self.cv_cam_by_id(webcam_id),+ writer=writer)+ yield available_cam_and_writer_dictself._stop_frame_capture_all_cams()except:logger.error("Printing traceback from starting capture session by cam")@@ -99,9 +126,12 @@ class OpenCVCameraManager:d = {}for cv_cam in self._open_cv_camera_objects:cv_cam.connect()- cv_cam.start_frame_capture()d[cv_cam.webcam_id_as_str] = VideoWriter()+ # wait until all cameras are connected before starting frame capture+ for cv_cam in self._open_cv_camera_objects:+ cv_cam.start_frame_capture()+return ddef _start_frame_capture_on_cam_id(self, webcam_id: str) -> VideoWriter:@@ -109,7 +139,7 @@ class OpenCVCameraManager:filter(lambda c: c.webcam_id_as_str == str(webcam_id), self._open_cv_camera_objects))assert (- len(filtered_cams) == 1+ len(filtered_cams) == 1), "The CV Cams list should only have 1 cam per webcam_id"cv_cam = filtered_cams[0]cv_cam.connect()diff --git a/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py b/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.pyindex b0f34168..69971e30 100644--- a/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py+++ b/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py@@ -1,167 +1,65 @@-import logging-import time-import traceback-from pathlib import Path-from typing import Any+from dataclasses import dataclass-import cv2-import mediapipe as mpimport numpy as np+import mediapipe as mp-from src.cameras.capture.dataclasses.frame_payload import FramePayload-from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManager-from src.cameras.persistence.video_writer.video_writer import SaveOptions-from src.config.data_paths import create_home_data_directory-from src.core_processor.fps.fps_counter import FPSCamCounter-from src.core_processor.utils.image_fps_writer import write_fps_to_image--mp_holistic = mp.solutions.holistic-mp_drawing = mp.solutions.drawing_utils-mp_drawing_styles = mp.solutions.drawing_styles--logger = logging.getLogger(__name__)---class ProcessResponse:- frame_payload: FramePayload- processed_image: Any-- def __init__(self, frame_payload: FramePayload, processed_image: np.array):- self.frame_payload = frame_payload- self.processed_image = processed_image---class MediapipeSkeletonDetection:- def __init__(self, cam_manager: OpenCVCameraManager):- self._cam_manager = cam_manager-- async def process_as_frame_loop(self, webcam_id, cb, model_complexity: int = 1):- with self._cam_manager.start_capture_session_single_cam(- webcam_id=webcam_id- ) as session_obj:- with mp_holistic.Holistic(- min_detection_confidence=0.5,- min_tracking_confidence=0.5,- model_complexity=model_complexity,- ) as holistic:- try:- while True:- response = self._process_single_cam_frame(- holistic, session_obj.cv_cam- )- session_obj.writer.write(- FramePayload(- image=response.processed_image, timestamp=time.time_ns()- )- )- if cb and response.processed_image is not None:- await cb(response.processed_image)- except Exception as e:- logger.error("Printing traceback")- traceback.print_exc()- raise e-- def process(self, model_complexity: int = 1):- with self._cam_manager.start_capture_session_all_cams() as session_obj:- fps_manager = FPSCamCounter(self._cam_manager.available_webcam_ids)- with mp_holistic.Holistic(- min_detection_confidence=0.5,- min_tracking_confidence=0.5,- model_complexity=model_complexity,- ) as holistic:- fps_manager.start_all()- try:- while True:- exit_key = cv2.waitKey(1)- if exit_key == 27:- logger.info("ESC has been pressed.")- break- for response in session_obj:- cv_cam = response.cv_cam- writer = response.writer- response = self._process_single_cam_frame(holistic, cv_cam)- if not response:- continue- fps_manager.increment_frame_processed_for(- cv_cam.webcam_id_as_str- )- write_fps_to_image(- response.processed_image,- fps_manager.current_fps_for(cv_cam.webcam_id_as_str),- )- writer.write(- FramePayload(- image=response.processed_image,- timestamp=time.time_ns(),- )- )-- cv2.imshow(- cv_cam.webcam_id_as_str, response.processed_image- )- except:- logger.error("Printing traceback")- traceback.print_exc()- finally:- for response in session_obj:- cv_cam = response.cv_cam- options = SaveOptions(- writer_dir=Path().joinpath(- cv_cam.session_writer_base_path,- "skeleton_detection",- f"webcam_{cv_cam.webcam_id_as_str}",- ),- fps=fps_manager.current_fps_for(cv_cam.webcam_id_as_str),- frame_width=cv_cam.image_width(),- frame_height=cv_cam.image_height(),- )- response.writer.save(options)- logger.info(f"Destroy window {cv_cam.webcam_id_as_str}")- cv2.destroyWindow(cv_cam.webcam_id_as_str)- cv2.waitKey(1)-- def _process_single_cam_frame(self, holistic, cv_cam):- frame_payload = cv_cam.latest_frame- if not frame_payload.success:- return- if frame_payload.image is None:- return-- processed_image = self.detect_mediapipe_skeleton(holistic, frame_payload.image)- return ProcessResponse(- frame_payload=frame_payload, processed_image=processed_image- )-- def detect_mediapipe_skeleton(self, holistic, image):- # adapted from 'webcam' part of demo code here -- # https://google.github.io/mediapipe/solutions/holistic.html- # For webcam input:- # To improve performance, optionally mark the image as not writeable to- # pass by reference.- image.flags.writeable = False- image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)- results = holistic.process(image)+@dataclass+class MediaPipeSkeletonDetection:+ model_complexity: int = 1 # can be 0,1, or 2 - higher numbers are more accurate but heavier computationally+ min_detection_confidence: float = .5+ min_tracking_confidence: float = .5++ def __post_init__(self):+ self._mp_drawing = mp.solutions.drawing_utils+ self._mp_drawing_styles = mp.solutions.drawing_styles+ self._mp_holistic = mp.solutions.holistic++ self._body_drawing_spec = self._mp_drawing.DrawingSpec(thickness=1, circle_radius=.2)+ self._hand_drawing_spec = self._mp_drawing.DrawingSpec(thickness=1, circle_radius=.2)+ self._face_drawing_spec = self._mp_drawing.DrawingSpec(thickness=1, circle_radius=.2)++ self._holistic_tracker = self._mp_holistic.Holistic(model_complexity=self.model_complexity,+ min_detection_confidence=self.min_detection_confidence,+ min_tracking_confidence=self.min_tracking_confidence)++ def detect_skeleton_in_image(self, raw_image, annotate_image=True):+ mediapipe_results = self._holistic_tracker.process(raw_image) # <-this is where the magic happens+ if annotate_image:+ return self._annotate_image(raw_image, mediapipe_results)+++ def _annotate_image(self, image, mediapipe_results):+ self._mp_drawing.draw_landmarks(image=image,+ landmark_list=mediapipe_results.face_landmarks,+ connections=self._mp_holistic.FACEMESH_CONTOURS,+ landmark_drawing_spec=None,+ connection_drawing_spec=self._mp_drawing_styles+ .get_default_face_mesh_contours_style())+ self._mp_drawing.draw_landmarks(image=image,+ landmark_list=mediapipe_results.face_landmarks,+ connections=self._mp_holistic.FACEMESH_TESSELATION,+ landmark_drawing_spec=None,+ connection_drawing_spec=self._mp_drawing_styles+ .get_default_face_mesh_tesselation_style())+ self._mp_drawing.draw_landmarks(image=image,+ landmark_list=mediapipe_results.pose_landmarks,+ connections=self._mp_holistic.POSE_CONNECTIONS,+ landmark_drawing_spec=self._mp_drawing_styles+ .get_default_pose_landmarks_style())+ self._mp_drawing.draw_landmarks(+ image=image,+ landmark_list=mediapipe_results.left_hand_landmarks,+ connections=self._mp_holistic.HAND_CONNECTIONS,+ landmark_drawing_spec=None,+ connection_drawing_spec=self._mp_drawing_styles+ .get_default_hand_connections_style())- # Draw landmark annotation on the image.- image.flags.writeable = True- image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)- mp_drawing.draw_landmarks(- image,- results.face_landmarks,- mp_holistic.FACEMESH_CONTOURS,+ self._mp_drawing.draw_landmarks(+ image=image,+ landmark_list=mediapipe_results.right_hand_landmarks,+ connections=self._mp_holistic.HAND_CONNECTIONS,landmark_drawing_spec=None,- connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_contours_style(),- )- mp_drawing.draw_landmarks(- image,- results.pose_landmarks,- mp_holistic.POSE_CONNECTIONS,- landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style(),- )+ connection_drawing_spec=self._mp_drawing_styles+ .get_default_hand_connections_style())return image---if __name__ == "__main__":- create_home_data_directory()- MediapipeSkeletonDetection(OpenCVCameraManager()).process()diff --git a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.pyindex a153169d..af14aa5f 100644--- a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py+++ b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py@@ -19,11 +19,13 @@ class QTVisualizerAndGui:self.pyqtgraph_app = pg.mkQApp("FreeMoCap! :O ")self._dict_of_camera_image_item_widgets = {}- self._dict_of_timestamp_line_plots = {}+ self._dict_of_simple_timestamp_line_plots = {}self._dict_of_cam_timestamps = {}self._dict_of_timestamp_difference_line_plots = {}self._dict_of_cam_timestamp_differences = {}self._dict_of_timestamp_difference_histograms = {}+ self._number_of_cameras = None+ self._webcam_ids_list = []self._is_paused = False@@ -34,18 +36,23 @@ class QTVisualizerAndGui:def close(self):self._main_window_widget.close()- def setup_and_launch(self, cam_and_writer_response_list):+ def setup_and_launch(self, webcam_ids_list):logger.info('setting up QT Visualizer and GUI')+ self._number_of_cameras = len(webcam_ids_list)+ self._webcam_ids_list = webcam_ids_listself._setup_main_window()self._setup_control_panel()- self._setup_camera_views_dock(cam_and_writer_response_list)- self._setup_timestamp_plot(cam_and_writer_response_list)- self._setup_time_difference_from_cam0_line_plot(cam_and_writer_response_list)- self._setup_time_difference_from_cam0_histogram_plot(cam_and_writer_response_list)+ self._setup_camera_views_dock()+ self._setup_timestamp_plot()+ self._setup_time_difference_from_cam0_line_plot()+ self._setup_time_difference_from_cam0_histogram_plot()logger.info('launching QT Visualizer and GUI window')self._main_window_widget.show()def update_camera_view_image(self, webcam_id, image_to_display):+ if self.pause_button_pressed:+ return+try:camera_image_item_widget = self._dict_of_camera_image_item_widgets[webcam_id]except Exception as e:@@ -53,38 +60,58 @@ class QTVisualizerAndGui:raise ecamera_image_item_widget.setImage(cv2.cvtColor(image_to_display, cv2.COLOR_BGR2RGB))- def update_timestamp_plot(self, webcam_id, new_timestamp):- try:- timestamp_line_plot = self._dict_of_timestamp_line_plots[webcam_id]- self._dict_of_cam_timestamps[webcam_id] = np.append(self._dict_of_cam_timestamps[webcam_id], new_timestamp)- except Exception as e:- logger.warning(f'Could not find timestamp line plot widget for camera {webcam_id}')- raise e-- timestamp_line_plot.setData(self._dict_of_cam_timestamps[webcam_id])+ def update_timestamp_plots(self, dict_of_timestamps):+ if self.pause_button_pressed:+ return- def update_timestamp_difference_plot(self, webcam_id, new_timestamp_difference):try:- self._dict_of_cam_timestamp_differences[webcam_id] = np.append(- self._dict_of_cam_timestamp_differences[webcam_id],- new_timestamp_difference)+ camera_0_timestamp = dict_of_timestamps['0']+ except:+ logger.info('no timestamp for camera0')+ camera_0_timestamp = np.nan++ for webcam_id in self._webcam_ids_list:+ try:+ this_cameras_timestamp = dict_of_timestamps[webcam_id]+ time_diff_from_cam0 = this_cameras_timestamp - camera_0_timestamp++ self._dict_of_cam_timestamps[webcam_id] = np.append(self._dict_of_cam_timestamps[webcam_id],+ this_cameras_timestamp)+ self._dict_of_cam_timestamp_differences[webcam_id] = np.append(+ self._dict_of_cam_timestamp_differences[webcam_id],+ time_diff_from_cam0)++ except Exception as e:+ logger.warning(f'Could not find timestamp dictionary for camera {webcam_id}')+ raise e++ self._update_simple_timestamp_plot()+ self._update_timestamp_difference_plot()+ self._update_timestamp_difference_histogram()++ def _update_simple_timestamp_plot(self):+ for webcam_id, timestamps_array in self._dict_of_cam_timestamps.items():+ plot_item = self._dict_of_simple_timestamp_line_plots[webcam_id]+ plot_item.setData(timestamps_array)++ def _update_timestamp_difference_plot(self):+ camera_0_id = '0'++ for webcam_id in self._dict_of_cam_timestamps.keys():self._dict_of_timestamp_difference_line_plots[webcam_id].setData(self._dict_of_cam_timestamp_differences[webcam_id])- self._update_timestamp_difference_histogram(webcam_id)- except Exception as e:- logger.warning(f'Could not find timestamp difference line plot widget for camera {webcam_id}')- raise e- def _update_timestamp_difference_histogram(self, webcam_id):- counts, bin_edges = np.histogram(self._dict_of_cam_timestamp_differences[webcam_id],- bins=np.linspace(-5, 5, 500))- try:- self._dict_of_timestamp_difference_histograms[webcam_id].setData(- x=bin_edges,- y=counts)- except Exception as e:- logger.warning(f'Could not find timestamp difference line plot widget for camera {webcam_id}')- raise e+ def _update_timestamp_difference_histogram(self):+ for webcam_id in self._webcam_ids_list:+ counts, bin_edges = np.histogram(self._dict_of_cam_timestamp_differences[webcam_id],+ bins=np.linspace(-50, 50, 100))+ try:+ self._dict_of_timestamp_difference_histograms[webcam_id].setData(+ x=bin_edges,+ y=counts)+ except Exception as e:+ logger.warning(f'Could not find timestamp difference line plot widget for camera {webcam_id}')+ raise edef _setup_main_window(self, window_width: int = 1000, window_height: int = 1000):"""@@ -106,68 +133,63 @@ class QTVisualizerAndGui:label = QtWidgets.QLabel("Blah Blah Blah put words here ")control_panel_layout_widget.addWidget(label, row=0, col=0)- self._pause_button = QtWidgets.QPushButton('Pause Frame Loop (Don\'t press this lol)')+ self._pause_button = QtWidgets.QPushButton('Pause')self._pause_button.setEnabled(True)self._pause_button.clicked.connect(self._pause)control_panel_layout_widget.addWidget(self._pause_button, row=1, col=0)- self._play_button = QtWidgets.QPushButton('Resume Frame Loop')+ self._play_button = QtWidgets.QPushButton('Resume')self._play_button.setEnabled(False)self._play_button.clicked.connect(self._play)control_panel_layout_widget.addWidget(self._play_button, row=2, col=0)self._main_dock_area.addDock(self._control_panel_dock, position='left')- def _setup_camera_views_dock(self, cam_and_writer_response_list):+ def _setup_camera_views_dock(self):self._camera_graphics_layout_window = pg.GraphicsLayoutWidget()self._camera_views_dock = Dock('Camera Views')self._camera_views_dock.addWidget(self._camera_graphics_layout_window)self._main_dock_area.addDock(self._camera_views_dock, 'right', self._control_panel_dock)- for this_response in cam_and_writer_response_list:- this_webcam_id = this_response.cv_cam.webcam_id_as_str- self._dict_of_camera_image_item_widgets[this_webcam_id] = self._create_camera_view_widget(this_webcam_id)+ for this_webcam_id in self._webcam_ids_list:+ self._dict_of_camera_image_item_widgets[this_webcam_id] = self._create_camera_view_widget()- def _create_camera_view_widget(self, webcam_id):+ def _create_camera_view_widget(self):camera_view_box_widget = pg.ViewBox(invertY=True, lockAspect=True)camera_image_item = pg.ImageItem()camera_view_box_widget.addItem(camera_image_item)self._camera_graphics_layout_window.addItem(camera_view_box_widget)return camera_image_item- def _setup_timestamp_plot(self, cam_and_writer_response_list):+ def _setup_timestamp_plot(self):# create widget/dock for reprojection error subplottimestamp_plot_widget = pg.PlotWidget(title="Timestamp vs Frame#")timestamp_plot_widget.setLabel('left', "timestamp", units='seconds')timestamp_plot_widget.setLabel('bottom', "Frame#", units='Frame')timestamp_plot_widget.addLegend()- for this_cam_num, this_response in enumerate(cam_and_writer_response_list):- num_cams = len(cam_and_writer_response_list)- this_webcam_id = this_response.cv_cam.webcam_id_as_str+ for this_cam_num, this_webcam_id in enumerate(self._webcam_ids_list):this_timestamp_plot_line = timestamp_plot_widget.plot(np.empty(0),- pen=(this_cam_num, num_cams),+ pen=(this_cam_num, self._number_of_cameras),name="camera " + this_webcam_id)- self._dict_of_timestamp_line_plots[this_webcam_id] = this_timestamp_plot_line+ self._dict_of_simple_timestamp_line_plots[this_webcam_id] = this_timestamp_plot_lineself._dict_of_cam_timestamps[this_webcam_id] = np.ndarray(0)self._timestamp_plot_dock = Dock("Camera Timestamps")self._timestamp_plot_dock.addWidget(timestamp_plot_widget)self._main_dock_area.addDock(self._timestamp_plot_dock, 'bottom')- def _setup_time_difference_from_cam0_line_plot(self, cam_and_writer_response_list):+ def _setup_time_difference_from_cam0_line_plot(self):# create widget/dock for reprojection error subplottimestamp_difference_plot_widget = pg.PlotWidget(title="Timestamp difference from Camera0 on each frame")timestamp_difference_plot_widget.setLabel('left', "this_camera timestamp - cam0 timestamp", units='seconds')timestamp_difference_plot_widget.setLabel('bottom', "Frame#", units='Frame')timestamp_difference_plot_widget.addLegend()- for this_cam_num, this_response in enumerate(cam_and_writer_response_list):- num_cams = len(cam_and_writer_response_list)- this_webcam_id = this_response.cv_cam.webcam_id_as_str+ for this_cam_num, this_webcam_id in enumerate(self._webcam_ids_list):this_plot_line = timestamp_difference_plot_widget.plot(np.empty(0),- pen=(this_cam_num, num_cams),+ pen=(this_cam_num, self._number_of_cameras),name="camera " + this_webcam_id)self._dict_of_timestamp_difference_line_plots[this_webcam_id] = this_plot_line@@ -177,23 +199,22 @@ class QTVisualizerAndGui:self._timestamp_difference_plot_dock.addWidget(timestamp_difference_plot_widget)self._main_dock_area.addDock(self._timestamp_difference_plot_dock, 'right', self._timestamp_plot_dock)- def _setup_time_difference_from_cam0_histogram_plot(self, cam_and_writer_response_list):+ def _setup_time_difference_from_cam0_histogram_plot(self):timestamp_difference_histogram_plot_widget = pg.PlotWidget(title="Timestamp difference from Camera0 on each frame")timestamp_difference_histogram_plot_widget.setLabel('left', "Proportion")timestamp_difference_histogram_plot_widget.setLabel('bottom', "milliseconds")timestamp_difference_histogram_plot_widget.addLegend()- for this_cam_num, this_response in enumerate(cam_and_writer_response_list):- num_cams = len(cam_and_writer_response_list)- this_webcam_id = this_response.cv_cam.webcam_id_as_str+ for this_cam_num, this_webcam_id in enumerate(self._webcam_ids_list):this_histogram_plot_item = timestamp_difference_histogram_plot_widget.plot(np.empty(0),np.empty(0),stepMode="center",fillLevel=0,fillOutline=True,brush=pg.mkBrush(color=(- this_cam_num, num_cams),+ this_cam_num,+ self._number_of_cameras),alpha=.5),name="camera " + this_webcam_id)@@ -206,7 +227,7 @@ class QTVisualizerAndGui:def _pause(self):self._is_paused = Trueself._play_button.setEnabled(True)- self._play_button.setText("I warned you!")+ self._play_button.setText("GUI Paused - Main loop still active")self._pause_button.setEnabled(False)def _play(self):diff --git a/src/session_pipeline/__init__.py b/src/session_pipeline/__init__.pynew file mode 100644index 00000000..e69de29bdiff --git a/src/core_processor/session/session_pipeline_orchestrator.py b/src/session_pipeline/session_pipeline_orchestrator.pysimilarity index 66%rename from src/core_processor/session/session_pipeline_orchestrator.pyrename to src/session_pipeline/session_pipeline_orchestrator.pyindex bd8f85da..c4ba6a5e 100644--- a/src/core_processor/session/session_pipeline_orchestrator.py+++ b/src/session_pipeline/session_pipeline_orchestrator.py@@ -10,6 +10,7 @@ from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManagerfrom src.cameras.persistence.video_writer.save_options import SaveOptionsfrom src.core_processor.camera_calibration.camera_calibrator import CameraCalibratorfrom src.core_processor.fps.fps_counter import FPSCamCounter+from src.core_processor.mediapipe_skeleton_detection.mediapipe_skeleton_detection import MediaPipeSkeletonDetectionfrom src.core_processor.show_cam_window import show_cam_windowfrom src.qt_visualizer_and_gui.qt_visualizer_and_gui import QTVisualizerAndGui@@ -27,6 +28,7 @@ class SessionPipelineOrchestrator:self._visualizer_gui = QTVisualizerAndGui()self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id)self._camera_calibrator = CameraCalibrator()+ self._mediapipe_skeleton_detector = MediaPipeSkeletonDetection(model_complexity=0)async def process_by_cam_id(self, webcam_id: str, cb):with self._open_cv_camera_manager.start_capture_session_single_cam(@@ -71,6 +73,7 @@ class SessionPipelineOrchestrator:show_camera_views_in_windows=True,show_visualizer_gui=True,calibrate_cameras=True,+ detect_skeleton=True,save_video=True,):"""@@ -78,75 +81,66 @@ class SessionPipelineOrchestrator:If return images is true, the images are returned to the caller"""- with self._open_cv_camera_manager.start_capture_session_all_cams() as cam_and_writer_response_list:+ with self._open_cv_camera_manager.start_capture_session_all_cams() as available_cam_and_writer_dict:- self._visualizer_gui.setup_and_launch(cam_and_writer_response_list)+ try:+ fps_manager = FPSCamCounter(self._open_cv_camera_manager.available_webcam_ids)+ fps_manager.start_all()- fps_manager = FPSCamCounter(self._open_cv_camera_manager.available_webcam_ids)- fps_manager.start_all()+ self._visualizer_gui.setup_and_launch(self._open_cv_camera_manager.available_webcam_ids)- try:should_continue = Truewhile should_continue:- each_camera_timestamp_on_this_frame_dict = {}- for this_response in cam_and_writer_response_list:- this_open_cv_camera = this_response.cv_cam- this_video_writer_object = this_response.writer-- if self._visualizer_gui.pause_button_pressed:- continue-- new_frame = this_open_cv_camera.new_frame- if not new_frame:- continue+ if not self._open_cv_camera_manager.new_synchronized_frame_available():+ continue+ this_frame_timestamps_dict = {}+ incoming_synchronized_frame_dict = self._open_cv_camera_manager.latest_synchronized_frame()- this_webcam_id_as_str = this_open_cv_camera.webcam_id_as_str- this_cam_latest_frame = this_open_cv_camera.latest_frame+ for this_webcam_id, this_cam_and_writer in available_cam_and_writer_dict.items():- each_camera_timestamp_on_this_frame_dict[- this_webcam_id_as_str] = this_cam_latest_frame.timestamp+ this_open_cv_camera = this_cam_and_writer.cv_cam+ this_video_writer_object = this_cam_and_writer.writer- if this_cam_latest_frame.image is None:- continue+ this_cam_latest_frame = incoming_synchronized_frame_dict[this_webcam_id]- image_to_display = this_cam_latest_frame.image+ if this_cam_latest_frame is None:+ image_to_display = this_open_cv_camera.latest_frame+ this_frame_timestamps_dict[this_webcam_id] = np.nan+ else:+ this_frame_timestamps_dict[+ this_webcam_id] = (this_cam_latest_frame.timestamp - self._session_start_time)/1e6 #milliseconds, I think+ image_to_display = this_cam_latest_frame.imageif save_video:+ print('sending frame to be written')this_video_writer_object.write(this_cam_latest_frame)if calibrate_cameras:undistorted_annotated_image = self._camera_calibrator.calibrate(this_open_cv_camera)image_to_display = undistorted_annotated_image- fps_manager.increment_frame_processed_for(this_webcam_id_as_str)+ if detect_skeleton:+ image_to_display = self._mediapipe_skeleton_detector.detect_skeleton_in_image(image_to_display)++ fps_manager.increment_frame_processed_for(this_webcam_id)if show_camera_views_in_windows:should_continue = show_cam_window(- this_webcam_id_as_str, image_to_display, fps_manager+ this_webcam_id, image_to_display, fps_manager)if show_visualizer_gui:- self._visualizer_gui.update_camera_view_image(this_webcam_id_as_str, image_to_display)- self._visualizer_gui.update_timestamp_plot(this_webcam_id_as_str,- this_cam_latest_frame.timestamp - self._session_start_time)- if show_visualizer_gui and new_frame:- cam0_timestamp = np.nan- try:- cam0_timestamp = each_camera_timestamp_on_this_frame_dict['0']- except:- logger.debug('no timestamp logged for camera 0')-- for this_cam_id, this_cam_timestamp in each_camera_timestamp_on_this_frame_dict.items():- self._visualizer_gui.update_timestamp_difference_plot(this_cam_id,- this_cam_timestamp - cam0_timestamp - self._session_start_time)+ self._visualizer_gui.update_camera_view_image(this_webcam_id, image_to_display)+ if len(this_frame_timestamps_dict.keys()) == self._open_cv_camera_manager.number_of_cameras:+ self._visualizer_gui.update_timestamp_plots(this_frame_timestamps_dict)except:logger.error("Printing traceback")traceback.print_exc()finally:- for this_response in cam_and_writer_response_list:- this_open_cv_camera = this_response.cv_cam- # this_video_writer_object = this_response.writer+ for this_webcam_id, this_cam_and_writer in available_cam_and_writer_dict.items():+ this_open_cv_camera = this_cam_and_writer.cv_cam+ # this_video_writer_object = this_cam_and_writer.writer# options = SaveOptions(# path_to_save_video=Path().joinpath(# this_open_cv_camera.session_writer_base_path,@@ -169,4 +163,4 @@ if __name__ == "__main__":print('start main')this_session = SessionPipelineOrchestrator()- this_session.run(calibrate_cameras=False)+ this_session.run(calibrate_cameras=True)diff --git a/src/session_pipeline/timestamp_handler.py b/src/session_pipeline/timestamp_handler.pynew file mode 100644index 00000000..98eae34f--- /dev/null+++ b/src/session_pipeline/timestamp_handler.py@@ -0,0 +1,22 @@+import numpy as np+++class TimestampLogger:+ pass+ # def __init__(self, _camera_id_list: list[str], camera0_id: str = '0'):+ # self._camera0_id = camera0_id+ # self._number_of_cameras = len(_camera_id_list)+ # self._camera_timestamp_dict = {}+ #+ # for this_cam_id in _camera_id_list:+ # self._camera_timestamp_dict[this_cam_id] = np.ndarray(0)+ #+ # def update(self, this_webcam_id, new_timestamp):+ # self._camera_timestamp_dict[this_webcam_id] = np.append(self._camera_timestamp_dict[this_webcam_id],+ # new_timestamp)+ #+ # timestamp_array_lengths = [this_timestamp_array.shape[0] for this_timestamp_array in self._camera_timestamp_dict.values()]+ #+ # if np.sum(np.diff(timestamp_array_lengths)) == 0: # if timestamp arrays are all the same length+ # self._update_timestamp_differences_from_camera()+ #