Skip to content

Free Motion Capture for Everyone πŸ’€βœ¨

TypeScript

timestamp vis and very basic skel tracking working

Joncde2edf2e8b8
diff --git a/requirements.txt b/requirements.txt
index 91a54d89..d1095aea 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,7 @@
aniposelib==0.4.3
h5py>=3.1.0
imutils~=0.5.4
-matplotlib~=3.1.3
+matplotlib~=3.5.1
# mediapipe==0.8.8
opencv-contrib-python==4.5.5.62
pandas~=1.2.5
@@ -30,4 +30,6 @@ pre-commit==2.17.0
# Requirements for core-processor
pyqtgraph==0.12.4
pyqt5==5.15.6
-pyopengl==3.1.6
\ No newline at end of file
+pyopengl==3.1.6
+mediapipe~=0.8.9.1
+setuptools~=58.0.4
\ No newline at end of file
diff --git a/src/cameras/capture/opencv_camera/opencv_camera.py b/src/cameras/capture/opencv_camera/opencv_camera.py
index f7240b48..8302d7b5 100644
--- a/src/cameras/capture/opencv_camera/opencv_camera.py
+++ b/src/cameras/capture/opencv_camera/opencv_camera.py
@@ -46,7 +46,7 @@ class OpenCVCamera:
@property
def is_capturing_frames(self):
if not self._running_thread:
- logger.error("Frame Capture thread not running yet")
+ logger.info("Frame Capture thread not running yet")
return False
return self._running_thread.is_capturing_frames
diff --git a/src/cameras/multicam_manager/cv_camera_manager.py b/src/cameras/multicam_manager/cv_camera_manager.py
index db96f3b4..57750ef4 100644
--- a/src/cameras/multicam_manager/cv_camera_manager.py
+++ b/src/cameras/multicam_manager/cv_camera_manager.py
@@ -3,6 +3,7 @@ import traceback
from contextlib import contextmanager
from typing import ContextManager, Dict, List
+import numpy as np
from pydantic import BaseModel
from src.api.services.user_config import UserConfigService
@@ -24,7 +25,7 @@ class CamAndWriterResponse(BaseModel):
class OpenCVCameraManager:
- def __init__(self, session_id = str):
+ def __init__(self, session_id=str):
self._session_id = session_id
self._config_service = UserConfigService()
self._detected_cams_data = get_or_create_cams()
@@ -34,18 +35,44 @@ class OpenCVCameraManager:
# we create the _cv_cams /once/, and reuse it for the lifetime of the session
_open_cv_cameras = self._create_opencv_cameras(calibrate_cameras=True)
self._open_cv_camera_objects = _open_cv_cameras
+ # self._timestamp_logger = TimestampLogger(self.available_webcam_ids, self.camera0_id)
+ self._number_of_cameras = len(self._open_cv_camera_objects)
else:
logger.info("Reusing already created resources cam resources.")
self._open_cv_camera_objects = _open_cv_cameras
+ @property
+ def number_of_cameras(self):
+ return self._number_of_cameras
+
@property
def available_webcam_ids(self):
return [cv_cam.webcam_id_as_str for cv_cam in self._open_cv_camera_objects]
+ @property
+ def camera0_id(self):
+ """ ID of the first camera found. This camera will be used as the reference point for time sync and camera position and rotation"""
+ return self.available_webcam_ids[0]
+
@property
def open_cv_cameras(self):
return self._open_cv_camera_objects
+ def new_synchronized_frame_available(self):
+ new_frame_bool_list = [cam.new_frame for cam in self._open_cv_camera_objects]
+ if all(new_frame_bool_list):
+ return True
+ return False
+
+ def latest_synchronized_frame(self):
+ synchronized_frame_dict = {}
+ for this_cam in self._open_cv_camera_objects:
+ if not this_cam.new_frame:
+ logger.error(f'New frame not available for camera {this_cam.webcam_id_as_str}')
+ synchronized_frame_dict[this_cam.webcam_id_as_str] = None
+ synchronized_frame_dict[this_cam.webcam_id_as_str] = this_cam.latest_frame
+ return synchronized_frame_dict
+
def cv_cam_by_id(self, webcam_id: str):
for cam in self._open_cv_camera_objects:
if cam.webcam_id_as_str == str(webcam_id):
@@ -63,7 +90,7 @@ class OpenCVCameraManager:
@contextmanager
def start_capture_session_single_cam(
- self, webcam_id: str = None
+ self, webcam_id: str = None
) -> ContextManager[CamAndWriterResponse]:
"""
Context manager for easy start up, usage, and cleanup of camera resources.
@@ -80,16 +107,16 @@ class OpenCVCameraManager:
@contextmanager
def start_capture_session_all_cams(
- self,
- ) -> ContextManager[List[CamAndWriterResponse]]:
+ self,
+ ) -> ContextManager[Dict[str, CamAndWriterResponse]]:
try:
+ available_cam_and_writer_dict = {}
writer_dict = self._start_frame_capture_all_cams()
- responses = [
- CamAndWriterResponse(cv_cam=self.cv_cam_by_id(webcam_id), writer=writer)
- for webcam_id, writer in writer_dict.items()
- ]
- yield responses
+ for webcam_id, writer in writer_dict.items():
+ available_cam_and_writer_dict[webcam_id] = CamAndWriterResponse(cv_cam=self.cv_cam_by_id(webcam_id),
+ writer=writer)
+ yield available_cam_and_writer_dict
self._stop_frame_capture_all_cams()
except:
logger.error("Printing traceback from starting capture session by cam")
@@ -99,9 +126,12 @@ class OpenCVCameraManager:
d = {}
for cv_cam in self._open_cv_camera_objects:
cv_cam.connect()
- cv_cam.start_frame_capture()
d[cv_cam.webcam_id_as_str] = VideoWriter()
+ # wait until all cameras are connected before starting frame capture
+ for cv_cam in self._open_cv_camera_objects:
+ cv_cam.start_frame_capture()
+
return d
def _start_frame_capture_on_cam_id(self, webcam_id: str) -> VideoWriter:
@@ -109,7 +139,7 @@ class OpenCVCameraManager:
filter(lambda c: c.webcam_id_as_str == str(webcam_id), self._open_cv_camera_objects)
)
assert (
- len(filtered_cams) == 1
+ len(filtered_cams) == 1
), "The CV Cams list should only have 1 cam per webcam_id"
cv_cam = filtered_cams[0]
cv_cam.connect()
diff --git a/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py b/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py
index b0f34168..69971e30 100644
--- a/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py
+++ b/src/core_processor/mediapipe_skeleton_detection/mediapipe_skeleton_detection.py
@@ -1,167 +1,65 @@
-import logging
-import time
-import traceback
-from pathlib import Path
-from typing import Any
+from dataclasses import dataclass
-import cv2
-import mediapipe as mp
import numpy as np
+import mediapipe as mp
-from src.cameras.capture.dataclasses.frame_payload import FramePayload
-from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManager
-from src.cameras.persistence.video_writer.video_writer import SaveOptions
-from src.config.data_paths import create_home_data_directory
-from src.core_processor.fps.fps_counter import FPSCamCounter
-from src.core_processor.utils.image_fps_writer import write_fps_to_image
-
-mp_holistic = mp.solutions.holistic
-mp_drawing = mp.solutions.drawing_utils
-mp_drawing_styles = mp.solutions.drawing_styles
-
-logger = logging.getLogger(__name__)
-
-
-class ProcessResponse:
- frame_payload: FramePayload
- processed_image: Any
-
- def __init__(self, frame_payload: FramePayload, processed_image: np.array):
- self.frame_payload = frame_payload
- self.processed_image = processed_image
-
-
-class MediapipeSkeletonDetection:
- def __init__(self, cam_manager: OpenCVCameraManager):
- self._cam_manager = cam_manager
-
- async def process_as_frame_loop(self, webcam_id, cb, model_complexity: int = 1):
- with self._cam_manager.start_capture_session_single_cam(
- webcam_id=webcam_id
- ) as session_obj:
- with mp_holistic.Holistic(
- min_detection_confidence=0.5,
- min_tracking_confidence=0.5,
- model_complexity=model_complexity,
- ) as holistic:
- try:
- while True:
- response = self._process_single_cam_frame(
- holistic, session_obj.cv_cam
- )
- session_obj.writer.write(
- FramePayload(
- image=response.processed_image, timestamp=time.time_ns()
- )
- )
- if cb and response.processed_image is not None:
- await cb(response.processed_image)
- except Exception as e:
- logger.error("Printing traceback")
- traceback.print_exc()
- raise e
-
- def process(self, model_complexity: int = 1):
- with self._cam_manager.start_capture_session_all_cams() as session_obj:
- fps_manager = FPSCamCounter(self._cam_manager.available_webcam_ids)
- with mp_holistic.Holistic(
- min_detection_confidence=0.5,
- min_tracking_confidence=0.5,
- model_complexity=model_complexity,
- ) as holistic:
- fps_manager.start_all()
- try:
- while True:
- exit_key = cv2.waitKey(1)
- if exit_key == 27:
- logger.info("ESC has been pressed.")
- break
- for response in session_obj:
- cv_cam = response.cv_cam
- writer = response.writer
- response = self._process_single_cam_frame(holistic, cv_cam)
- if not response:
- continue
- fps_manager.increment_frame_processed_for(
- cv_cam.webcam_id_as_str
- )
- write_fps_to_image(
- response.processed_image,
- fps_manager.current_fps_for(cv_cam.webcam_id_as_str),
- )
- writer.write(
- FramePayload(
- image=response.processed_image,
- timestamp=time.time_ns(),
- )
- )
-
- cv2.imshow(
- cv_cam.webcam_id_as_str, response.processed_image
- )
- except:
- logger.error("Printing traceback")
- traceback.print_exc()
- finally:
- for response in session_obj:
- cv_cam = response.cv_cam
- options = SaveOptions(
- writer_dir=Path().joinpath(
- cv_cam.session_writer_base_path,
- "skeleton_detection",
- f"webcam_{cv_cam.webcam_id_as_str}",
- ),
- fps=fps_manager.current_fps_for(cv_cam.webcam_id_as_str),
- frame_width=cv_cam.image_width(),
- frame_height=cv_cam.image_height(),
- )
- response.writer.save(options)
- logger.info(f"Destroy window {cv_cam.webcam_id_as_str}")
- cv2.destroyWindow(cv_cam.webcam_id_as_str)
- cv2.waitKey(1)
-
- def _process_single_cam_frame(self, holistic, cv_cam):
- frame_payload = cv_cam.latest_frame
- if not frame_payload.success:
- return
- if frame_payload.image is None:
- return
-
- processed_image = self.detect_mediapipe_skeleton(holistic, frame_payload.image)
- return ProcessResponse(
- frame_payload=frame_payload, processed_image=processed_image
- )
-
- def detect_mediapipe_skeleton(self, holistic, image):
- # adapted from 'webcam' part of demo code here -
- # https://google.github.io/mediapipe/solutions/holistic.html
- # For webcam input:
- # To improve performance, optionally mark the image as not writeable to
- # pass by reference.
- image.flags.writeable = False
- image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
- results = holistic.process(image)
+@dataclass
+class MediaPipeSkeletonDetection:
+ model_complexity: int = 1 # can be 0,1, or 2 - higher numbers are more accurate but heavier computationally
+ min_detection_confidence: float = .5
+ min_tracking_confidence: float = .5
+
+ def __post_init__(self):
+ self._mp_drawing = mp.solutions.drawing_utils
+ self._mp_drawing_styles = mp.solutions.drawing_styles
+ self._mp_holistic = mp.solutions.holistic
+
+ self._body_drawing_spec = self._mp_drawing.DrawingSpec(thickness=1, circle_radius=.2)
+ self._hand_drawing_spec = self._mp_drawing.DrawingSpec(thickness=1, circle_radius=.2)
+ self._face_drawing_spec = self._mp_drawing.DrawingSpec(thickness=1, circle_radius=.2)
+
+ self._holistic_tracker = self._mp_holistic.Holistic(model_complexity=self.model_complexity,
+ min_detection_confidence=self.min_detection_confidence,
+ min_tracking_confidence=self.min_tracking_confidence)
+
+ def detect_skeleton_in_image(self, raw_image, annotate_image=True):
+ mediapipe_results = self._holistic_tracker.process(raw_image) # <-this is where the magic happens
+ if annotate_image:
+ return self._annotate_image(raw_image, mediapipe_results)
+
+
+ def _annotate_image(self, image, mediapipe_results):
+ self._mp_drawing.draw_landmarks(image=image,
+ landmark_list=mediapipe_results.face_landmarks,
+ connections=self._mp_holistic.FACEMESH_CONTOURS,
+ landmark_drawing_spec=None,
+ connection_drawing_spec=self._mp_drawing_styles
+ .get_default_face_mesh_contours_style())
+ self._mp_drawing.draw_landmarks(image=image,
+ landmark_list=mediapipe_results.face_landmarks,
+ connections=self._mp_holistic.FACEMESH_TESSELATION,
+ landmark_drawing_spec=None,
+ connection_drawing_spec=self._mp_drawing_styles
+ .get_default_face_mesh_tesselation_style())
+ self._mp_drawing.draw_landmarks(image=image,
+ landmark_list=mediapipe_results.pose_landmarks,
+ connections=self._mp_holistic.POSE_CONNECTIONS,
+ landmark_drawing_spec=self._mp_drawing_styles
+ .get_default_pose_landmarks_style())
+ self._mp_drawing.draw_landmarks(
+ image=image,
+ landmark_list=mediapipe_results.left_hand_landmarks,
+ connections=self._mp_holistic.HAND_CONNECTIONS,
+ landmark_drawing_spec=None,
+ connection_drawing_spec=self._mp_drawing_styles
+ .get_default_hand_connections_style())
- # Draw landmark annotation on the image.
- image.flags.writeable = True
- image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
- mp_drawing.draw_landmarks(
- image,
- results.face_landmarks,
- mp_holistic.FACEMESH_CONTOURS,
+ self._mp_drawing.draw_landmarks(
+ image=image,
+ landmark_list=mediapipe_results.right_hand_landmarks,
+ connections=self._mp_holistic.HAND_CONNECTIONS,
landmark_drawing_spec=None,
- connection_drawing_spec=mp_drawing_styles.get_default_face_mesh_contours_style(),
- )
- mp_drawing.draw_landmarks(
- image,
- results.pose_landmarks,
- mp_holistic.POSE_CONNECTIONS,
- landmark_drawing_spec=mp_drawing_styles.get_default_pose_landmarks_style(),
- )
+ connection_drawing_spec=self._mp_drawing_styles
+ .get_default_hand_connections_style())
return image
-
-
-if __name__ == "__main__":
- create_home_data_directory()
- MediapipeSkeletonDetection(OpenCVCameraManager()).process()
diff --git a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
index a153169d..af14aa5f 100644
--- a/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
+++ b/src/qt_visualizer_and_gui/qt_visualizer_and_gui.py
@@ -19,11 +19,13 @@ class QTVisualizerAndGui:
self.pyqtgraph_app = pg.mkQApp("FreeMoCap! :O ")
self._dict_of_camera_image_item_widgets = {}
- self._dict_of_timestamp_line_plots = {}
+ self._dict_of_simple_timestamp_line_plots = {}
self._dict_of_cam_timestamps = {}
self._dict_of_timestamp_difference_line_plots = {}
self._dict_of_cam_timestamp_differences = {}
self._dict_of_timestamp_difference_histograms = {}
+ self._number_of_cameras = None
+ self._webcam_ids_list = []
self._is_paused = False
@@ -34,18 +36,23 @@ class QTVisualizerAndGui:
def close(self):
self._main_window_widget.close()
- def setup_and_launch(self, cam_and_writer_response_list):
+ def setup_and_launch(self, webcam_ids_list):
logger.info('setting up QT Visualizer and GUI')
+ self._number_of_cameras = len(webcam_ids_list)
+ self._webcam_ids_list = webcam_ids_list
self._setup_main_window()
self._setup_control_panel()
- self._setup_camera_views_dock(cam_and_writer_response_list)
- self._setup_timestamp_plot(cam_and_writer_response_list)
- self._setup_time_difference_from_cam0_line_plot(cam_and_writer_response_list)
- self._setup_time_difference_from_cam0_histogram_plot(cam_and_writer_response_list)
+ self._setup_camera_views_dock()
+ self._setup_timestamp_plot()
+ self._setup_time_difference_from_cam0_line_plot()
+ self._setup_time_difference_from_cam0_histogram_plot()
logger.info('launching QT Visualizer and GUI window')
self._main_window_widget.show()
def update_camera_view_image(self, webcam_id, image_to_display):
+ if self.pause_button_pressed:
+ return
+
try:
camera_image_item_widget = self._dict_of_camera_image_item_widgets[webcam_id]
except Exception as e:
@@ -53,38 +60,58 @@ class QTVisualizerAndGui:
raise e
camera_image_item_widget.setImage(cv2.cvtColor(image_to_display, cv2.COLOR_BGR2RGB))
- def update_timestamp_plot(self, webcam_id, new_timestamp):
- try:
- timestamp_line_plot = self._dict_of_timestamp_line_plots[webcam_id]
- self._dict_of_cam_timestamps[webcam_id] = np.append(self._dict_of_cam_timestamps[webcam_id], new_timestamp)
- except Exception as e:
- logger.warning(f'Could not find timestamp line plot widget for camera {webcam_id}')
- raise e
-
- timestamp_line_plot.setData(self._dict_of_cam_timestamps[webcam_id])
+ def update_timestamp_plots(self, dict_of_timestamps):
+ if self.pause_button_pressed:
+ return
- def update_timestamp_difference_plot(self, webcam_id, new_timestamp_difference):
try:
- self._dict_of_cam_timestamp_differences[webcam_id] = np.append(
- self._dict_of_cam_timestamp_differences[webcam_id],
- new_timestamp_difference)
+ camera_0_timestamp = dict_of_timestamps['0']
+ except:
+ logger.info('no timestamp for camera0')
+ camera_0_timestamp = np.nan
+
+ for webcam_id in self._webcam_ids_list:
+ try:
+ this_cameras_timestamp = dict_of_timestamps[webcam_id]
+ time_diff_from_cam0 = this_cameras_timestamp - camera_0_timestamp
+
+ self._dict_of_cam_timestamps[webcam_id] = np.append(self._dict_of_cam_timestamps[webcam_id],
+ this_cameras_timestamp)
+ self._dict_of_cam_timestamp_differences[webcam_id] = np.append(
+ self._dict_of_cam_timestamp_differences[webcam_id],
+ time_diff_from_cam0)
+
+ except Exception as e:
+ logger.warning(f'Could not find timestamp dictionary for camera {webcam_id}')
+ raise e
+
+ self._update_simple_timestamp_plot()
+ self._update_timestamp_difference_plot()
+ self._update_timestamp_difference_histogram()
+
+ def _update_simple_timestamp_plot(self):
+ for webcam_id, timestamps_array in self._dict_of_cam_timestamps.items():
+ plot_item = self._dict_of_simple_timestamp_line_plots[webcam_id]
+ plot_item.setData(timestamps_array)
+
+ def _update_timestamp_difference_plot(self):
+ camera_0_id = '0'
+
+ for webcam_id in self._dict_of_cam_timestamps.keys():
self._dict_of_timestamp_difference_line_plots[webcam_id].setData(
self._dict_of_cam_timestamp_differences[webcam_id])
- self._update_timestamp_difference_histogram(webcam_id)
- except Exception as e:
- logger.warning(f'Could not find timestamp difference line plot widget for camera {webcam_id}')
- raise e
- def _update_timestamp_difference_histogram(self, webcam_id):
- counts, bin_edges = np.histogram(self._dict_of_cam_timestamp_differences[webcam_id],
- bins=np.linspace(-5, 5, 500))
- try:
- self._dict_of_timestamp_difference_histograms[webcam_id].setData(
- x=bin_edges,
- y=counts)
- except Exception as e:
- logger.warning(f'Could not find timestamp difference line plot widget for camera {webcam_id}')
- raise e
+ def _update_timestamp_difference_histogram(self):
+ for webcam_id in self._webcam_ids_list:
+ counts, bin_edges = np.histogram(self._dict_of_cam_timestamp_differences[webcam_id],
+ bins=np.linspace(-50, 50, 100))
+ try:
+ self._dict_of_timestamp_difference_histograms[webcam_id].setData(
+ x=bin_edges,
+ y=counts)
+ except Exception as e:
+ logger.warning(f'Could not find timestamp difference line plot widget for camera {webcam_id}')
+ raise e
def _setup_main_window(self, window_width: int = 1000, window_height: int = 1000):
"""
@@ -106,68 +133,63 @@ class QTVisualizerAndGui:
label = QtWidgets.QLabel("Blah Blah Blah put words here ")
control_panel_layout_widget.addWidget(label, row=0, col=0)
- self._pause_button = QtWidgets.QPushButton('Pause Frame Loop (Don\'t press this lol)')
+ self._pause_button = QtWidgets.QPushButton('Pause')
self._pause_button.setEnabled(True)
self._pause_button.clicked.connect(self._pause)
control_panel_layout_widget.addWidget(self._pause_button, row=1, col=0)
- self._play_button = QtWidgets.QPushButton('Resume Frame Loop')
+ self._play_button = QtWidgets.QPushButton('Resume')
self._play_button.setEnabled(False)
self._play_button.clicked.connect(self._play)
control_panel_layout_widget.addWidget(self._play_button, row=2, col=0)
self._main_dock_area.addDock(self._control_panel_dock, position='left')
- def _setup_camera_views_dock(self, cam_and_writer_response_list):
+ def _setup_camera_views_dock(self):
self._camera_graphics_layout_window = pg.GraphicsLayoutWidget()
self._camera_views_dock = Dock('Camera Views')
self._camera_views_dock.addWidget(self._camera_graphics_layout_window)
self._main_dock_area.addDock(self._camera_views_dock, 'right', self._control_panel_dock)
- for this_response in cam_and_writer_response_list:
- this_webcam_id = this_response.cv_cam.webcam_id_as_str
- self._dict_of_camera_image_item_widgets[this_webcam_id] = self._create_camera_view_widget(this_webcam_id)
+ for this_webcam_id in self._webcam_ids_list:
+ self._dict_of_camera_image_item_widgets[this_webcam_id] = self._create_camera_view_widget()
- def _create_camera_view_widget(self, webcam_id):
+ def _create_camera_view_widget(self):
camera_view_box_widget = pg.ViewBox(invertY=True, lockAspect=True)
camera_image_item = pg.ImageItem()
camera_view_box_widget.addItem(camera_image_item)
self._camera_graphics_layout_window.addItem(camera_view_box_widget)
return camera_image_item
- def _setup_timestamp_plot(self, cam_and_writer_response_list):
+ def _setup_timestamp_plot(self):
# create widget/dock for reprojection error subplot
timestamp_plot_widget = pg.PlotWidget(title="Timestamp vs Frame#")
timestamp_plot_widget.setLabel('left', "timestamp", units='seconds')
timestamp_plot_widget.setLabel('bottom', "Frame#", units='Frame')
timestamp_plot_widget.addLegend()
- for this_cam_num, this_response in enumerate(cam_and_writer_response_list):
- num_cams = len(cam_and_writer_response_list)
- this_webcam_id = this_response.cv_cam.webcam_id_as_str
+ for this_cam_num, this_webcam_id in enumerate(self._webcam_ids_list):
this_timestamp_plot_line = timestamp_plot_widget.plot(np.empty(0),
- pen=(this_cam_num, num_cams),
+ pen=(this_cam_num, self._number_of_cameras),
name="camera " + this_webcam_id)
- self._dict_of_timestamp_line_plots[this_webcam_id] = this_timestamp_plot_line
+ self._dict_of_simple_timestamp_line_plots[this_webcam_id] = this_timestamp_plot_line
self._dict_of_cam_timestamps[this_webcam_id] = np.ndarray(0)
self._timestamp_plot_dock = Dock("Camera Timestamps")
self._timestamp_plot_dock.addWidget(timestamp_plot_widget)
self._main_dock_area.addDock(self._timestamp_plot_dock, 'bottom')
- def _setup_time_difference_from_cam0_line_plot(self, cam_and_writer_response_list):
+ def _setup_time_difference_from_cam0_line_plot(self):
# create widget/dock for reprojection error subplot
timestamp_difference_plot_widget = pg.PlotWidget(title="Timestamp difference from Camera0 on each frame")
timestamp_difference_plot_widget.setLabel('left', "this_camera timestamp - cam0 timestamp", units='seconds')
timestamp_difference_plot_widget.setLabel('bottom', "Frame#", units='Frame')
timestamp_difference_plot_widget.addLegend()
- for this_cam_num, this_response in enumerate(cam_and_writer_response_list):
- num_cams = len(cam_and_writer_response_list)
- this_webcam_id = this_response.cv_cam.webcam_id_as_str
+ for this_cam_num, this_webcam_id in enumerate(self._webcam_ids_list):
this_plot_line = timestamp_difference_plot_widget.plot(np.empty(0),
- pen=(this_cam_num, num_cams),
+ pen=(this_cam_num, self._number_of_cameras),
name="camera " + this_webcam_id)
self._dict_of_timestamp_difference_line_plots[this_webcam_id] = this_plot_line
@@ -177,23 +199,22 @@ class QTVisualizerAndGui:
self._timestamp_difference_plot_dock.addWidget(timestamp_difference_plot_widget)
self._main_dock_area.addDock(self._timestamp_difference_plot_dock, 'right', self._timestamp_plot_dock)
- def _setup_time_difference_from_cam0_histogram_plot(self, cam_and_writer_response_list):
+ def _setup_time_difference_from_cam0_histogram_plot(self):
timestamp_difference_histogram_plot_widget = pg.PlotWidget(
title="Timestamp difference from Camera0 on each frame")
timestamp_difference_histogram_plot_widget.setLabel('left', "Proportion")
timestamp_difference_histogram_plot_widget.setLabel('bottom', "milliseconds")
timestamp_difference_histogram_plot_widget.addLegend()
- for this_cam_num, this_response in enumerate(cam_and_writer_response_list):
- num_cams = len(cam_and_writer_response_list)
- this_webcam_id = this_response.cv_cam.webcam_id_as_str
+ for this_cam_num, this_webcam_id in enumerate(self._webcam_ids_list):
this_histogram_plot_item = timestamp_difference_histogram_plot_widget.plot(np.empty(0),
np.empty(0),
stepMode="center",
fillLevel=0,
fillOutline=True,
brush=pg.mkBrush(color=(
- this_cam_num, num_cams),
+ this_cam_num,
+ self._number_of_cameras),
alpha=.5),
name="camera " + this_webcam_id)
@@ -206,7 +227,7 @@ class QTVisualizerAndGui:
def _pause(self):
self._is_paused = True
self._play_button.setEnabled(True)
- self._play_button.setText("I warned you!")
+ self._play_button.setText("GUI Paused - Main loop still active")
self._pause_button.setEnabled(False)
def _play(self):
diff --git a/src/session_pipeline/__init__.py b/src/session_pipeline/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/core_processor/session/session_pipeline_orchestrator.py b/src/session_pipeline/session_pipeline_orchestrator.py
similarity index 66%
rename from src/core_processor/session/session_pipeline_orchestrator.py
rename to src/session_pipeline/session_pipeline_orchestrator.py
index bd8f85da..c4ba6a5e 100644
--- a/src/core_processor/session/session_pipeline_orchestrator.py
+++ b/src/session_pipeline/session_pipeline_orchestrator.py
@@ -10,6 +10,7 @@ from src.cameras.multicam_manager.cv_camera_manager import OpenCVCameraManager
from src.cameras.persistence.video_writer.save_options import SaveOptions
from src.core_processor.camera_calibration.camera_calibrator import CameraCalibrator
from src.core_processor.fps.fps_counter import FPSCamCounter
+from src.core_processor.mediapipe_skeleton_detection.mediapipe_skeleton_detection import MediaPipeSkeletonDetection
from src.core_processor.show_cam_window import show_cam_window
from src.qt_visualizer_and_gui.qt_visualizer_and_gui import QTVisualizerAndGui
@@ -27,6 +28,7 @@ class SessionPipelineOrchestrator:
self._visualizer_gui = QTVisualizerAndGui()
self._open_cv_camera_manager = OpenCVCameraManager(session_id=self._session_id)
self._camera_calibrator = CameraCalibrator()
+ self._mediapipe_skeleton_detector = MediaPipeSkeletonDetection(model_complexity=0)
async def process_by_cam_id(self, webcam_id: str, cb):
with self._open_cv_camera_manager.start_capture_session_single_cam(
@@ -71,6 +73,7 @@ class SessionPipelineOrchestrator:
show_camera_views_in_windows=True,
show_visualizer_gui=True,
calibrate_cameras=True,
+ detect_skeleton=True,
save_video=True,
):
"""
@@ -78,75 +81,66 @@ class SessionPipelineOrchestrator:
If return images is true, the images are returned to the caller
"""
- with self._open_cv_camera_manager.start_capture_session_all_cams() as cam_and_writer_response_list:
+ with self._open_cv_camera_manager.start_capture_session_all_cams() as available_cam_and_writer_dict:
- self._visualizer_gui.setup_and_launch(cam_and_writer_response_list)
+ try:
+ fps_manager = FPSCamCounter(self._open_cv_camera_manager.available_webcam_ids)
+ fps_manager.start_all()
- fps_manager = FPSCamCounter(self._open_cv_camera_manager.available_webcam_ids)
- fps_manager.start_all()
+ self._visualizer_gui.setup_and_launch(self._open_cv_camera_manager.available_webcam_ids)
- try:
should_continue = True
while should_continue:
- each_camera_timestamp_on_this_frame_dict = {}
- for this_response in cam_and_writer_response_list:
- this_open_cv_camera = this_response.cv_cam
- this_video_writer_object = this_response.writer
-
- if self._visualizer_gui.pause_button_pressed:
- continue
-
- new_frame = this_open_cv_camera.new_frame
- if not new_frame:
- continue
+ if not self._open_cv_camera_manager.new_synchronized_frame_available():
+ continue
+ this_frame_timestamps_dict = {}
+ incoming_synchronized_frame_dict = self._open_cv_camera_manager.latest_synchronized_frame()
- this_webcam_id_as_str = this_open_cv_camera.webcam_id_as_str
- this_cam_latest_frame = this_open_cv_camera.latest_frame
+ for this_webcam_id, this_cam_and_writer in available_cam_and_writer_dict.items():
- each_camera_timestamp_on_this_frame_dict[
- this_webcam_id_as_str] = this_cam_latest_frame.timestamp
+ this_open_cv_camera = this_cam_and_writer.cv_cam
+ this_video_writer_object = this_cam_and_writer.writer
- if this_cam_latest_frame.image is None:
- continue
+ this_cam_latest_frame = incoming_synchronized_frame_dict[this_webcam_id]
- image_to_display = this_cam_latest_frame.image
+ if this_cam_latest_frame is None:
+ image_to_display = this_open_cv_camera.latest_frame
+ this_frame_timestamps_dict[this_webcam_id] = np.nan
+ else:
+ this_frame_timestamps_dict[
+ this_webcam_id] = (this_cam_latest_frame.timestamp - self._session_start_time)/1e6 #milliseconds, I think
+ image_to_display = this_cam_latest_frame.image
if save_video:
+ print('sending frame to be written')
this_video_writer_object.write(this_cam_latest_frame)
if calibrate_cameras:
undistorted_annotated_image = self._camera_calibrator.calibrate(this_open_cv_camera)
image_to_display = undistorted_annotated_image
- fps_manager.increment_frame_processed_for(this_webcam_id_as_str)
+ if detect_skeleton:
+ image_to_display = self._mediapipe_skeleton_detector.detect_skeleton_in_image(image_to_display)
+
+ fps_manager.increment_frame_processed_for(this_webcam_id)
if show_camera_views_in_windows:
should_continue = show_cam_window(
- this_webcam_id_as_str, image_to_display, fps_manager
+ this_webcam_id, image_to_display, fps_manager
)
if show_visualizer_gui:
- self._visualizer_gui.update_camera_view_image(this_webcam_id_as_str, image_to_display)
- self._visualizer_gui.update_timestamp_plot(this_webcam_id_as_str,
- this_cam_latest_frame.timestamp - self._session_start_time)
- if show_visualizer_gui and new_frame:
- cam0_timestamp = np.nan
- try:
- cam0_timestamp = each_camera_timestamp_on_this_frame_dict['0']
- except:
- logger.debug('no timestamp logged for camera 0')
-
- for this_cam_id, this_cam_timestamp in each_camera_timestamp_on_this_frame_dict.items():
- self._visualizer_gui.update_timestamp_difference_plot(this_cam_id,
- this_cam_timestamp - cam0_timestamp - self._session_start_time)
+ self._visualizer_gui.update_camera_view_image(this_webcam_id, image_to_display)
+ if len(this_frame_timestamps_dict.keys()) == self._open_cv_camera_manager.number_of_cameras:
+ self._visualizer_gui.update_timestamp_plots(this_frame_timestamps_dict)
except:
logger.error("Printing traceback")
traceback.print_exc()
finally:
- for this_response in cam_and_writer_response_list:
- this_open_cv_camera = this_response.cv_cam
- # this_video_writer_object = this_response.writer
+ for this_webcam_id, this_cam_and_writer in available_cam_and_writer_dict.items():
+ this_open_cv_camera = this_cam_and_writer.cv_cam
+ # this_video_writer_object = this_cam_and_writer.writer
# options = SaveOptions(
# path_to_save_video=Path().joinpath(
# this_open_cv_camera.session_writer_base_path,
@@ -169,4 +163,4 @@ if __name__ == "__main__":
print('start main')
this_session = SessionPipelineOrchestrator()
- this_session.run(calibrate_cameras=False)
+ this_session.run(calibrate_cameras=True)
diff --git a/src/session_pipeline/timestamp_handler.py b/src/session_pipeline/timestamp_handler.py
new file mode 100644
index 00000000..98eae34f
--- /dev/null
+++ b/src/session_pipeline/timestamp_handler.py
@@ -0,0 +1,22 @@
+import numpy as np
+
+
+class TimestampLogger:
+ pass
+ # def __init__(self, _camera_id_list: list[str], camera0_id: str = '0'):
+ # self._camera0_id = camera0_id
+ # self._number_of_cameras = len(_camera_id_list)
+ # self._camera_timestamp_dict = {}
+ #
+ # for this_cam_id in _camera_id_list:
+ # self._camera_timestamp_dict[this_cam_id] = np.ndarray(0)
+ #
+ # def update(self, this_webcam_id, new_timestamp):
+ # self._camera_timestamp_dict[this_webcam_id] = np.append(self._camera_timestamp_dict[this_webcam_id],
+ # new_timestamp)
+ #
+ # timestamp_array_lengths = [this_timestamp_array.shape[0] for this_timestamp_array in self._camera_timestamp_dict.values()]
+ #
+ # if np.sum(np.diff(timestamp_array_lengths)) == 0: # if timestamp arrays are all the same length
+ # self._update_timestamp_differences_from_camera()
+ #